]> git.uio.no Git - u/mrichter/AliRoot.git/blame - PWGPP/benchmark/benchmark.sh
git://git.uio.no / u / mrichter / AliRoot.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
RelVal: produce JSON summary handy for webapps
[u/mrichter/AliRoot.git] / PWGPP / benchmark / benchmark.sh
CommitLineData
fc7e1e45 1#!/bin/bash
71353a5f 2#include benchmark.config
3
b802abe4 4# blame: Mikolaj Krzewicki, mkrzewic@cern.ch
fc7e1e45 5# this script runs the CPass0/CPass1 train
6# produced OCDB updates are local
7
8main()
9{
10 #run in proper mode depending on the selection
71353a5f 11 if [[ $# -lt 1 ]]; then
12 if [[ ! "${0}" =~ "bash" ]]; then
13 echo "uses makeflow:"
14 echo " ${0} \"run\" productionID inputList configFile [extraOpts]"
15 echo "uses a batch system (SGE):"
16 echo " ${0} \"submit\" productionID inputList configFile [extraOpts]"
17 echo "extraOpts if non-empty override the config file, e.g.:"
9510565c 18 echo " ${0} submit test1 benchmark.list benchmark.config runNumber=169123 nEvents=10"
71353a5f 19 fi
20 return
21 fi
22
23 #define some aliases - default is to call one of the functions directly
24 runMode=${1}
fc7e1e45 25 umask 0002
26 shift
71353a5f 27 case ${runMode} in
fc7e1e45 28 "CPass0") goCPass0 "$@";;
29 "CPass1") goCPass1 "$@";;
71353a5f 30 "MakeLocalOCDBaccessConfig") goMakeLocalOCDBaccessConfig "$@";;
fc7e1e45 31 "MergeCPass0") goMergeCPass0 "$@";;
32 "MergeCPass1") goMergeCPass1 "$@";;
cb473870 33 "MakeFilteredTrees") goMakeFilteredTrees "$@";;
71353a5f 34 "MakeSummary") goMakeSummary "$@";;
cb473870 35 "run") goSubmitMakeflow "$@";;
36 "submit") goSubmitBatch "$@";;
fc7e1e45 37 "test") goTest "$@";;
71353a5f 38 "GenerateMakeflow") goGenerateMakeflow "$@";;
39 "PrintValues") goPrintValues "$@";;
cb473870 40 "CreateQAplots") goCreateQAplots "$@";;
41 "WaitForOutput") goWaitForOutput "$@";;
71353a5f 42 "Merge") goMerge "$@";;
43 *)
444b9916 44 ${runMode} "$@"
fc7e1e45 45 ;;
46 esac
39527a15 47 return 0
fc7e1e45 48}
49
71353a5f 50generateMC()
51{
52 #generate one raw chunk in current directory
53 SEED=${JOB_ID}${SGE_TASK_ID}
54 export CONFIG_SEED=${SEED}
55 runNumber=${1}
56 OCDBpath=${2}
57 nEventsim=${3}
58 if [[ -n ${pretend} ]]; then
0eaf38c7 59 sleep ${pretendDelay}
71353a5f 60 touch galice.root
61 else
62 if [[ -f sim.C && -f Config.C ]] ; then
63 time aliroot -b -q -x sim.C\(${runNumber},\"${OCDBpath}\",${nEventsim}\) >sim.log 2>&1
64 mv syswatch.log simwatch.log
65 fi
66 fi
67}
68
fc7e1e45 69goCPass0()
78d0bbfd 70(
fc7e1e45 71 umask 0002
72
71353a5f 73 targetDirectory=${1}
74 inputList=${2}
75 nEvents=${3}
76 ocdbPath=${4}
77 configFile=${5}
78 runNumber=${6}
79 jobindex=${7}
80 shift 7
6fc48f9e 81 if ! parseConfig ${configFile} "$@"; then return 1; fi
71353a5f 82
39527a15 83 #record the working directory provided by the batch system
84 batchWorkingDirectory=${PWD}
85
71353a5f 86 #use the jobindex only if set and non-negative
87 if [[ -z ${jobindex} || ${jobindex} -lt 0 ]]; then
88 [[ -n "${LSB_JOBINDEX}" ]] && jobindex=${LSB_JOBINDEX}
89 [[ -n "${SGE_TASK_ID}" ]] && jobindex=${SGE_TASK_ID}
999a843d 90 if [[ -z ${jobindex} ]]; then
91 echo "no jobindex!"
92 return 1
93 fi
71353a5f 94 fi
95
96 [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
ef77d296 97
98 # This file signals that/if everything went fine
99 doneFileBase="cpass0.job${jobindex}.run${runNumber}.done"
100 [[ -n ${useProfilingCommand} ]] && doneFileBase="profiling.cpass0.job${jobindex}.run${runNumber}.done"
101
102 # We will have two copies of the file
103 mkdir -p "${commonOutputPath}/meta" || return 1
104 doneFileTmp="${batchWorkingDirectory}/${doneFileBase}"
105 doneFile="${commonOutputPath}/meta/${doneFileBase}"
71353a5f 106
107 [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
108
109 if [[ -n ${ALIROOT_FORCE_COREDUMP} ]]; then
110 ulimit -c unlimited
111 export ALIROOT_FORCE_COREDUMP
112 fi
fc7e1e45 113
cb473870 114 #the contents of this is stored in the tree and used later (e.g. AliAnalysisTaskPIDResponse)!
115 #at the QA stage the pass number is guessed from the path stored here.
116 #The Format is:
117 #Packages= ;OutputDir= ;LPMPass= ;TriggerAlias= ;LPMRunNumber= ;LPMProductionType= ;LPMInteractionType= ;LPMProductionTag= ;LPMAnchorRun= ;LPMAnchorProduction= ;LPMAnchorYear=
118 export PRODUCTION_METADATA="OutputDir=cpass0"
119
5de3064b 120 if [[ "${inputList}" =~ \.root$ ]]; then
121 infile=${inputList}
122 else
123 infile=$(sed -ne "${jobindex}p" ${inputList} | egrep '\s*\w*/\w*')
124 fi
71353a5f 125 chunkName=${infile##*/}
71353a5f 126
127 outputDir=${targetDirectory}/${jobindex}_${chunkName%.*}
128 mkdir -p ${outputDir}
1795556a 129 if [[ ! -d ${outputDir} ]]; then
ef77d296 130 touch ${doneFileTmp}
131 echo "cannot make ${outputDir}" >> ${doneFileTmp}
132 cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
133 [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
1795556a 134 return 1
135 fi
71353a5f 136
71353a5f 137 runpath=${outputDir}
0eaf38c7 138 [[ ${reconstructInTemporaryDir} -eq 1 ]] && runpath=$(mktemp -d -t cpass0.XXXXXX)
e91a9c78 139 [[ ${reconstructInTemporaryDir} -eq 2 ]] && runpath=${PWD}/rundir_cpass0_${runNumber}_${jobindex}
71353a5f 140 mkdir -p ${runpath}
1795556a 141 if [[ ! -d ${runpath} ]]; then
ef77d296 142 touch ${doneFileTmp}
143 echo "cannot make runpath ${runpath}" >> ${doneFileTmp}
144 cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
145 [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
1795556a 146 return 1
147 fi
0eaf38c7 148 if ! cd ${runpath}; then
ef77d296 149 touch ${doneFileTmp}
150 echo "PWD=$PWD is not the runpath=${runpath}" >> ${doneFileTmp}
151 cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
152 [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
0eaf38c7 153 return 1
154 fi
fc7e1e45 155
5de3064b 156 #runCPassX/C expects the raw chunk to be linked in the run dir
157 #despite it being accessed by the full path
2ac23087 158 if [[ $copyInputData == 0 ]]; then
159 ln -s ${infile} ${runpath}/${chunkName}
160 else
161 copyFileToLocal ${infile} ${runpath}/${chunkName}
162 fi
fc7e1e45 163
71353a5f 164 #####MC
165 if [[ -n ${generateMC} ]]; then
166 olddir=${PWD}
167 outputDirMC=${commonOutputPath}/000${runNumber}/sim/${jobindex}
168 simrunpath=${outputDirMC}
0eaf38c7 169 #[[ ${simulateInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && simrunpath=${TMPDIR}
170 [[ ${simulateInTemporaryDir} -eq 1 ]] && simrunpath=$(mktemp -d -t cpass0MC.XXXXXX)
71353a5f 171 mkdir -p ${outputDirMC}
172 mkdir -p ${simrunpath}
0eaf38c7 173 if cd ${simrunpath}; then
fc7e1e45 174
0eaf38c7 175 filesMC=(
176 "${batchWorkingDirectory}/sim.C"
177 "${batchWorkingDirectory}/rec.C"
178 "${batchWorkingDirectory}/Config.C"
179 "${batchWorkingDirectory}/OCDB_*.root"
180 )
181 for file in ${filesMC[*]}; do
182 [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
183 done
fc7e1e45 184
0eaf38c7 185 generateMC ${runNumber} ${ocdbPath} ${nEvents}
186
187 [[ ! "${simrunpath}" =~ "${outputDirMC}" ]] && mv * ${outputDirMC} #TODO check if it works
188 cd ${olddir}
189
190 ln -s ${outputDirMC}/* ${runpath}/
191
192 inputList=${outputDirMC}/galice.root #TODO not valid outside shell !!!
193 infile=""
194 fi
71353a5f 195 fi
196 ######
197
02871a69 198 if [[ "${inputList}" == "${inputList%%://*}" && ! -f "${inputList}" && -z ${pretend} ]]; then
ef77d296 199 touch ${doneFileTmp}
200 echo "input file ${inputList} not found, exiting..." >> ${doneFileTmp}
201 cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
202 [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
1795556a 203 return 1
204 fi
71353a5f 205
206 logOutputDir=${runpath}
207 [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir}
0eaf38c7 208 [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/stdout
209 #[[ -z ${dontRedirectStdOutToLog} ]] && exec 2> ${logOutputDir}/stderr
71353a5f 210 echo "${0} $*"
fc7e1e45 211
212 echo "#####################"
213 echo CPass0:
214 echo JOB setup
ef77d296 215 echo nEvents ${nEvents}
216 echo runNumber ${runNumber}
217 echo ocdbPath ${ocdbPath}
218 echo infile ${infile}
219 echo chunkName ${chunkName}
220 echo jobindex ${jobindex}
221 echo recoTriggerOptions ${recoTriggerOptions}
222 echo targetDirectory ${targetDirectory}
223 echo commonOutputPath ${commonOutputPath}
224 echo doneFile ${doneFile}
225 echo batchWorkingDirectory ${batchWorkingDirectory}
226 echo runpath ${runpath}
227 echo outputDir ${outputDir}
228 echo PWD ${PWD}
229 echo ALICE_ROOT ${ALICE_ROOT}
fc7e1e45 230 echo "########## ###########"
231
71353a5f 232 alirootInfo > ALICE_ROOT.log
fc7e1e45 233
234 filesCPass0=(
39527a15 235 "${batchWorkingDirectory}/runCPass0.sh"
236 "${batchWorkingDirectory}/recCPass0.C"
237 "${batchWorkingDirectory}/runCalibTrain.C"
238 "${batchWorkingDirectory}/localOCDBaccessConfig.C"
239 "${batchWorkingDirectory}/OCDB.root"
71353a5f 240 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/runCPass0.sh"
241 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/recCPass0.C"
242 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/runCalibTrain.C"
fc7e1e45 243 )
244
245 for file in ${filesCPass0[*]}; do
246 [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
1a0e38e3 247 [[ ${file##*/} =~ .*\.sh ]] && chmod +x ${file##*/}
fc7e1e45 248 done
249
71353a5f 250 echo "this directory (${PWD}) contents:"
251 /bin/ls
fc7e1e45 252 echo
253 chmod u+x runCPass0.sh
254
71353a5f 255 sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh
256
257 if [[ -n ${postSetUpActionCPass0} ]]; then
258 echo "running ${postSetUpActionCPass0}"
259 eval ${postSetUpActionCPass0}
fc7e1e45 260 fi
261
262 #run CPass0
2ac23087 263 echo "${runpath}/runCPass0.sh /${infile} ${nEvents} ${runNumber} ${ocdbPath} ${recoTriggerOptions}"
71353a5f 264 if [[ -n ${pretend} ]]; then
0eaf38c7 265 sleep ${pretendDelay}
cb473870 266 touch AliESDs.root
267 touch AliESDfriends.root
fc7e1e45 268 touch AliESDfriends_v1.root
269 touch rec.log
270 touch calib.log
271 else
2ac23087 272 #caveat: in the local case, first arg must start with a slash
273 ./runCPass0.sh "/${infile}" "${nEvents}" "${runNumber}" "${ocdbPath}" "${recoTriggerOptions}"
fc7e1e45 274 fi
275
fc7e1e45 276 #move stuff to final destination
71353a5f 277 echo "this directory (${PWD}) contents:"
278 /bin/ls
fc7e1e45 279 echo
280
ef77d296 281 # [dberzano] OK this is fine!
71353a5f 282 echo rm -f ./${chunkName}
283 rm -f ./${chunkName}
13abb339 284 echo "paranoidCp ${runpath}/* ${outputDir}"
285 paranoidCp ${runpath}/* ${outputDir}
fc7e1e45 286 echo
5ed7c82c 287
288 #validate CPass0
289 cd ${outputDir}
ef77d296 290 if summarizeLogs >> ${doneFileTmp}; then
291 [[ -f ${outputDirMC}/galice.root ]] && echo "sim ${outputDirMC}/galice.root" >> ${doneFileTmp}
292 [[ -f AliESDfriends_v1.root ]] && echo "calibfile ${outputDir}/AliESDfriends_v1.root" >> ${doneFileTmp}
293 [[ -f AliESDs.root ]] && echo "esd ${outputDir}/AliESDs.root" >> ${doneFileTmp}
ab946185 294 fi
fc7e1e45 295
0eaf38c7 296 [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath} && echo "removing ${runpath}"
ef77d296 297 cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
298 [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
5de3064b 299 return 0
78d0bbfd 300)
fc7e1e45 301
302goCPass1()
78d0bbfd 303(
fc7e1e45 304 umask 0002
305
71353a5f 306 targetDirectory=${1}
307 inputList=${2}
308 nEvents=${3}
309 ocdbPath=${4}
310 configFile=${5}
311 runNumber=${6}
312 jobindex=${7}
313 shift 7
444b9916 314 extraOpts=("$@")
6fc48f9e 315 if ! parseConfig ${configFile} "$@"; then return 1; fi
71353a5f 316
39527a15 317 #record the working directory provided by the batch system
318 batchWorkingDirectory=${PWD}
319
71353a5f 320 #use the jobindex only if set and non-negative
321 if [[ -z ${jobindex} || ${jobindex} -lt 0 ]]; then
322 [[ -n "${LSB_JOBINDEX}" ]] && jobindex=${LSB_JOBINDEX}
323 [[ -n "${SGE_TASK_ID}" ]] && jobindex=${SGE_TASK_ID}
999a843d 324 if [[ -z ${jobindex} ]]; then
325 echo "no jobindex!"
326 return 1
327 fi
71353a5f 328 fi
329
330 [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
ef77d296 331
332 # This file signals that/if everything went fine
333 doneFileBase="cpass1.job${jobindex}.run${runNumber}.done"
334 [[ -n ${useProfilingCommand} ]] && doneFileBase="profiling.cpass0.job${jobindex}.run${runNumber}.done"
335
336 # We will have two copies of the file
337 mkdir -p "${commonOutputPath}/meta" || return 1
338 doneFileTmp="${batchWorkingDirectory}/${doneFileBase}"
339 doneFile="${commonOutputPath}/meta/${doneFileBase}"
71353a5f 340
341 [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
342
343 if [[ -n ${ALIROOT_FORCE_COREDUMP} ]]; then
344 ulimit -c unlimited
345 export ALIROOT_FORCE_COREDUMP
346 fi
fc7e1e45 347
cb473870 348 #the contents of this is stored in the tree and used later (e.g. AliAnalysisTaskPIDResponse)!
349 #at the QA stage the pass number is guessed from the path stored here.
350 #The Format is:
351 #Packages= ;OutputDir= ;LPMPass= ;TriggerAlias= ;LPMRunNumber= ;LPMProductionType= ;LPMInteractionType= ;LPMProductionTag= ;LPMAnchorRun= ;LPMAnchorProduction= ;LPMAnchorYear=
352 export PRODUCTION_METADATA="OutputDir=cpass1"
353
02871a69 354 if [[ "${inputList}" == "${inputList%%://*}" && ! -f "${inputList}" && -z ${pretend} ]]; then
ef77d296 355 touch ${doneFileTmp}
356 echo "input file ${inputList} not found, exiting..." >> ${doneFileTmp}
357 cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
358 [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
1795556a 359 return 1
360 fi
fc7e1e45 361 if [[ "${inputList}" =~ \.root$ ]]; then
71353a5f 362 infile=${inputList}
fc7e1e45 363 else
71353a5f 364 infile=$(sed -ne "${jobindex}p" ${inputList} | egrep '\s*\w*/\w*')
fc7e1e45 365 fi
fc7e1e45 366 chunkName=${infile##*/}
5de3064b 367
71353a5f 368 outputDir=${targetDirectory}/${jobindex}_${chunkName%.*}
369 mkdir -p ${outputDir}
1795556a 370 if [[ ! -d ${outputDir} ]];then
ef77d296 371 touch ${doneFileTmp}
372 echo "cannot make ${outputDir}" >> ${doneFileTmp}
373 cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
374 [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
1795556a 375 return 1
376 fi
fc7e1e45 377
71353a5f 378 runpath=${outputDir}
0eaf38c7 379 [[ ${reconstructInTemporaryDir} -eq 1 ]] && runpath=$(mktemp -d -t cpass1.XXXXXX)
e91a9c78 380 [[ ${reconstructInTemporaryDir} -eq 2 ]] && runpath=${PWD}/rundir_cpass1_${runNumber}_${jobindex}
71353a5f 381
382 #MC
383 if [[ "${infile}" =~ galice\.root ]]; then
384 ln -s ${inputList%/*}/* ${runpath}
385 infile=""
386 fi
fc7e1e45 387
388 #init the running path
71353a5f 389 mkdir -p ${runpath}
1795556a 390 if [[ ! -d ${runpath} ]]; then
ef77d296 391 touch ${doneFileTmp}
392 echo "cannot make runpath ${runpath}" >> ${doneFileTmp}
393 cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
394 [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
1795556a 395 return 1
396 fi
0eaf38c7 397 if ! cd ${runpath}; then
ef77d296 398 touch ${doneFileTmp}
399 echo "PWD=$PWD is not the runpath=${runpath}" >> ${doneFileTmp}
400 cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
401 [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
0eaf38c7 402 return 1
403 fi
fc7e1e45 404
71353a5f 405 #this is needed for runCPass1.sh
2ac23087 406 if [[ $copyInputData == 0 ]]; then
407 ln -s ${infile} ${runpath}/${chunkName}
408 else
409 copyFileToLocal ${infile} ${runpath}/${chunkName}
410 fi
fc7e1e45 411
71353a5f 412 logOutputDir=${runpath}
413 [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir}
0eaf38c7 414 [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/stdout
415 #[[ -z ${dontRedirectStdOutToLog} ]] && exec 2> ${logOutputDir}/stderr
71353a5f 416 echo "${0} $*"
fc7e1e45 417
418 echo "#####################"
419 echo CPass1:
420 echo JOB setup
ef77d296 421 echo nEvents ${nEvents}
422 echo runNumber ${runNumber}
423 echo ocdbPath ${ocdbPath}
424 echo infile ${infile}
425 echo chunkName ${chunkName}
426 echo jobindex ${jobindex}
427 echo recoTriggerOptions ${recoTriggerOptions}
428 echo targetDirectory ${targetDirectory}
429 echo commonOutputPath ${commonOutputPath}
430 echo doneFile ${doneFile}
431 echo runpath ${runpath}
432 echo outputDir ${outputDir}
1795556a 433 echo batchWorkingDirectory ${batchWorkingDirectory}
ef77d296 434 echo ALICE_ROOT ${ALICE_ROOT}
435 echo PWD ${PWD}
436 echo "#####################"
fc7e1e45 437
71353a5f 438 alirootInfo > ALICE_ROOT.log
fc7e1e45 439
440 filesCPass1=(
39527a15 441 "${batchWorkingDirectory}/runCPass1.sh"
442 "${batchWorkingDirectory}/recCPass1.C"
443 "${batchWorkingDirectory}/recCPass1_OuterDet.C"
444 "${batchWorkingDirectory}/runCalibTrain.C"
445 "${batchWorkingDirectory}/QAtrain_duo.C"
446 "${batchWorkingDirectory}/localOCDBaccessConfig.C"
0eaf38c7 447 "${batchWorkingDirectory}/${configFile}"
617f8886 448 "${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz"
39527a15 449 "${batchWorkingDirectory}/OCDB.root"
71353a5f 450 "${trustedQAtrainMacro}"
451 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/runCPass1.sh"
452 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/recCPass1.C"
453 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/recCPass1_OuterDet.C"
454 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/runCalibTrain.C"
455 "${ALICE_ROOT}/ANALYSIS/macros/QAtrain_duo.C"
fc7e1e45 456 )
457
78d0bbfd 458 for file in "${filesCPass1[@]}"; do
fc7e1e45 459 [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
78d0bbfd 460 [[ ${file##*/} =~ .*\.sh ]] && echo "making ${file##*/} executable" && chmod +x ${file##*/}
fc7e1e45 461 done
462
71353a5f 463 echo "this directory (${PWD}) contents:"
464 /bin/ls
fc7e1e45 465 echo
466
78d0bbfd 467 #remove spaces around commas from calls to root
71353a5f 468 sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh
469
470 if [[ -n ${postSetUpActionCPass1} ]]; then
471 echo "running ${postSetUpActionCPass1}"
472 eval ${postSetUpActionCPass1}
fc7e1e45 473 echo
474 fi
475
476 #configure local OCDB storage from CPass0 (creates the localOCDBaccessConfig.C script)
477 if [[ -f cpass0.localOCDB.${runNumber}.tgz ]]; then
71353a5f 478 echo goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz"
479 goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz"
cb473870 480 else
481 echo "WARNING: file cpass0.localOCDB.${runNumber}.tgz not found!"
482 fi
483
71353a5f 484 if [[ ! $(/bin/ls -1 OCDB/*/*/*/*.root 2>/dev/null) ]]; then
ef77d296 485 touch ${doneFileTmp}
486 echo "cpass0 produced no calibration! exiting..." >> ${doneFileTmp}
487 cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
488 [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
71353a5f 489 return 1
cb473870 490 fi
fc7e1e45 491
71353a5f 492 #create the Barrel and OuterDet directories for CPass1 and link the local OCDB directory
493 #there to make the localOCDBaccessConfig.C file work, since it may point to the OCDB
494 #entries using a relative path, e.g. local://./OCDB
78d0bbfd 495 echo "linking the OCDB/ for Barrel and OuterDet directories"
71353a5f 496 mkdir Barrel OuterDet
78d0bbfd 497 ls -l
71353a5f 498 ln -s ../OCDB Barrel/OCDB
499 ln -s ../OCDB OuterDet/OCDB
500
444b9916 501 #setup the filtering
502 #the following option enables the filtering task inside the QAtrain_duo.C
503 [[ -n $runESDfiltering ]] && export QA_TaskFilteredTree=1
504 #set the downscaling factors during the filtering fro expert QA (overrides the previous values)
505 if [[ -n ${filteringFactorHighPt} ]]; then
506 export AliAnalysisTaskFilteredTree_fLowPtTrackDownscaligF=${filteringFactorHighPt}
507 fi
508 if [[ -n ${filteringFactorV0s} ]]; then
509 export AliAnalysisTaskFilteredTree_fLowPtV0DownscaligF=${filteringFactorV0s}
510 fi
511
fc7e1e45 512 #run CPass1
513 chmod u+x runCPass1.sh
2ac23087 514 echo "${runpath}/runCPass1.sh /${infile} ${nEvents} ${runNumber} ${ocdbPath} ${recoTriggerOptions}"
71353a5f 515 if [[ -n ${pretend} ]]; then
0eaf38c7 516 sleep ${pretendDelay}
cb473870 517 touch AliESDs_Barrel.root
518 touch AliESDfriends_Barrel.root
fc7e1e45 519 touch AliESDfriends_v1.root
cb473870 520 touch QAresults_barrel.root
521 touch EventStat_temp_barrel.root
522 touch AODtpITS.root
523 touch AliESDs_Outer.root
524 touch AliESDfriends_Outer.root
525 touch QAresults_outer.root
526 touch EventStat_temp_outer.root
fc7e1e45 527 touch rec.log
528 touch calib.log
529 touch qa.log
444b9916 530 touch filtering.log FilterEvents_Trees.root
fc7e1e45 531 else
2ac23087 532 #caveat: in the local case, first arg must start with a slash
533 ./runCPass1.sh "/${infile}" "${nEvents}" "${runNumber}" "${ocdbPath}" "${recoTriggerOptions}"
444b9916 534
535 [[ ! -f AliESDs_Barrel.root && -f Barrel/AliESDs.root ]] && mv Barrel/AliESDs.root AliESDs_Barrel.root
536 [[ ! -f AliESDfriends_Barrel.root && -f Barrel/AliESDfriends.root ]] && mv Barrel/AliESDfriends.root AliESDfriends_Barrel.root
537 [[ ! -f AliESDfriends_v1.root && -f Barrel/AliESDfriends_v1.root ]] && mv Barrel/AliESDfriends_v1.root .
538 [[ ! -f QAresults_barrel.root && -f Barrel/QAresults_barrel.root ]] && mv Barrel/QAresults_barrel.root .
539 [[ ! -f AliESDs_Outer.root && -f OuterDet/AliESDs.root ]] && mv OuterDet/AliESDs.root AliESDs_Outer.root
540 [[ ! -f AliESDfriends_Outer.root && -f OuterDet/AliESDfriends.root ]] && mv OuterDet/AliESDfriends.root AliESDfriends_Outer.root
541 [[ ! -f QAresults_outer.root && -f OuterDet/QAresults_outer.root ]] && mv OuterDet/QAresults_outer.root .
542 [[ ! -f FilterEvents_Trees.root && -f Barrel/FilterEvents_Trees.root ]] && mv Barrel/FilterEvents_Trees.root .
543
544 #make the filtered tree (if requested and not already produced by QA
545 [[ -f AliESDs_Barrel.root ]] && echo "AliESDs_Barrel.root" > filtered.list
fab31805 546 if [[ -n ${runESDfiltering} && ! -f FilterEvents_Trees.root && -f filtered.list ]]; then
547 goMakeFilteredTrees ${PWD} ${runNumber} "${PWD}/filtered.list" ${filteringFactorHighPt} ${filteringFactorV0s} ${ocdbPath} 1000000 0 10000000 0 ${configFile} AliESDs_Barrel.root "${extraOpts[@]}" >filtering.log
548 else
549 echo ""
444b9916 550 fi
551
fc7e1e45 552 fi
cb473870 553
554 ##handle possible crashes in QA (happens often in trunk)
555 ##rerun QA with a trusted aliroot version
556 #if [[ $(validateLog qa_barrel.log) ]]; then
557 # echo "qa_barrel.log not validated!"
558 #fi
559 #if [[ ! -f QAresults_barrel.root && -f ${setupTrustedAliROOTenvInCurrentShell} || $(validateLog qa_barrel.log) ]]; then
71353a5f 560 # echo "WARNING: using trusted QA aliroot ${ALICE_ROOT}"
cb473870 561 # source ${setupTrustedAliROOTenvInCurrentShell}
562 # cd Barrel
563 # rm QAresults_barrel.root
564 # rm EventStat_temp_barrel.root
565 # rm AODtpITS.root
566 # [[ ! -f AliESDs.root ]] && ln -s ../AliESDs_Barrel.root AliESDs.root
567 # [[ ! -f AliESDfriends.root ]] && ln -s ../AliESDfriends_Barrel.root AliESDfriends.root
71353a5f 568 # if [[ -n ${trustedQAtrainMacro} ]]; then
569 # eval "cp ${trustedQAtrainMacro} QAtrain_duo_trusted.C"
cb473870 570 # fi
71353a5f 571 # echo executing aliroot -b -q "QAtrain_duo_trusted.C(\"_barrel\",${runNumber},\"wn.xml\",0,\"${ocdbPath}\")"
572 # time aliroot -b -q "QAtrain_duo.C(\"_barrel\",${runNumber},\"wn.xml\",0,\"${ocdbPath}\")" &> ../qa_barrel_trusted.log
cb473870 573 # cd ../
574 #fi
cb473870 575
fc7e1e45 576 #move stuff to final destination
71353a5f 577 echo "this directory (${PWD}) contents:"
578 /bin/ls
579 echo rm -f ./${chunkName}
580 rm -f ./${chunkName}
13abb339 581 echo "paranoidCp ${runpath}/* ${outputDir}"
582 paranoidCp ${runpath}/* ${outputDir}
fc7e1e45 583 echo
584
5ed7c82c 585 #validate CPass1
586 cd ${outputDir}
ef77d296 587 if summarizeLogs >> ${doneFileTmp}; then
588 [[ -f AliESDs_Barrel.root ]] && echo "esd ${outputDir}/AliESDs_Barrel.root" >> ${doneFileTmp}
589 [[ -f AliESDfriends_v1.root ]] && echo "calibfile ${outputDir}/AliESDfriends_v1.root" >> ${doneFileTmp}
590 [[ -f QAresults_Barrel.root ]] && echo "qafile ${outputDir}/QAresults_Barrel.root" >> ${doneFileTmp}
591 [[ -f QAresults_Outer.root ]] && echo "qafile ${outputDir}/QAresults_Outer.root" >> ${doneFileTmp}
592 [[ -f QAresults_barrel.root ]] && echo "qafile ${outputDir}/QAresults_barrel.root" >> ${doneFileTmp}
593 [[ -f QAresults_outer.root ]] && echo "qafile ${outputDir}/QAresults_outer.root" >> ${doneFileTmp}
594 [[ -f FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFileTmp}
922d0b10 595 else
ef77d296 596 if grep "qa_outer.log.*OK" ${doneFileTmp} > /dev/null; then
597 [[ -f QAresults_Outer.root ]] && echo "qafile ${outputDir}/QAresults_Outer.root" >> ${doneFileTmp}
598 [[ -f QAresults_outer.root ]] && echo "qafile ${outputDir}/QAresults_outer.root" >> ${doneFileTmp}
922d0b10 599 fi
ef77d296 600 if grep "qa_barrel.log.*OK" ${doneFileTmp} > /dev/null; then
601 [[ -f QAresults_Barrel.root ]] && echo "qafile ${outputDir}/QAresults_Barrel.root" >> ${doneFileTmp}
602 [[ -f QAresults_barrel.root ]] && echo "qafile ${outputDir}/QAresults_barrel.root" >> ${doneFileTmp}
922d0b10 603 fi
ef77d296 604 if grep "filtering.log.*OK" ${doneFileTmp} > /dev/null; then
605 [[ -f FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFileTmp}
922d0b10 606 fi
ab946185 607 fi
608
71353a5f 609 [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
ef77d296 610 cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
611 [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
5de3064b 612 return 0
78d0bbfd 613)
fc7e1e45 614
615
616goMergeCPass0()
78d0bbfd 617(
fc7e1e45 618 #
619 # find the output files and merge them
620 #
621
71353a5f 622 outputDir=${1}
623 ocdbStorage=${2}
624 configFile=${3}
625 runNumber=${4}
626 calibrationFilesToMerge=${5} #can be a non-existent file, will then be produced on the fly
627 shift 5
6fc48f9e 628 if ! parseConfig ${configFile} "$@"; then return 1; fi
71353a5f 629
39527a15 630 #record the working directory provided by the batch system
631 batchWorkingDirectory=${PWD}
632
71353a5f 633 [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
ef77d296 634
635 # This file signals that everything went fine
636 doneFileBase="merge.cpass0.run${runNumber}.done"
637
638 # We will have two copies of the file
639 mkdir -p "${commonOutputPath}/meta" || return 1
640 doneFileTmp="${batchWorkingDirectory}/${doneFileBase}"
641 doneFile="${commonOutputPath}/meta/${doneFileBase}"
71353a5f 642
643 umask 0002
644 ulimit -c unlimited
fc7e1e45 645
71353a5f 646 [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
fc7e1e45 647
71353a5f 648 runpath=${outputDir}
0eaf38c7 649 [[ ${reconstructInTemporaryDir} -eq 1 ]] && runpath=$(mktemp -d -t mergeCPass0.XXXXXX)
e91a9c78 650 [[ ${reconstructInTemporaryDir} -eq 2 ]] && runpath=${PWD}/rundir_mergeCPass0_${runNumber}
fc7e1e45 651
71353a5f 652 mkdir -p ${runpath}
1795556a 653 if [[ ! -d ${runpath} ]]; then
ef77d296 654 touch ${doneFileTmp}
655 echo "not able to make the runpath ${runpath}" >> ${doneFileTmp}
656 cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
657 [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
1795556a 658 return 1
659 fi
0eaf38c7 660 if ! cd ${runpath}; then
ef77d296 661 touch ${doneFileTmp}
662 echo "PWD=$PWD is not the runpath=${runpath}" >> ${doneFileTmp}
663 cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
664 [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
0eaf38c7 665 return 1
666 fi
fc7e1e45 667
71353a5f 668 logOutputDir=${runpath}
669 [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir}
1795556a 670 [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/stdout
71353a5f 671 echo "${0} $*"
fc7e1e45 672
fc7e1e45 673 mergingScript="mergeMakeOCDB.byComponent.sh"
fc7e1e45 674
675 echo goMergeCPass0 SETUP:
71353a5f 676 echo runNumber=${runNumber}
677 echo outputDir=${outputDir}
678 echo ocdbStorage=${ocdbStorage}
679 echo calibrationFilesToMerge=${calibrationFilesToMerge}
680 echo mergingScript=${mergingScript}
681 echo commonOutputPath=${commonOutputPath}
682 echo runpath=${runpath}
fc7e1e45 683
684 # copy files in case they are not already there
685 filesMergeCPass0=(
39527a15 686 "${batchWorkingDirectory}/${calibrationFilesToMerge}"
687 "${batchWorkingDirectory}/OCDB.root"
688 "${batchWorkingDirectory}/localOCDBaccessConfig.C"
71353a5f 689 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/mergeMakeOCDB.byComponent.sh"
690 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/mergeByComponent.C"
691 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/makeOCDB.C"
692 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/merge.C"
693 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/mergeMakeOCDB.sh"
fc7e1e45 694 )
695 for file in ${filesMergeCPass0[*]}; do
696 [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
1a0e38e3 697 [[ ${file##*/} =~ .*\.sh ]] && chmod +x ${file##*/}
fc7e1e45 698 done
699
71353a5f 700 sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh
701
702 alirootInfo > ALICE_ROOT.log
fc7e1e45 703
704 #
cb473870 705 echo "PWD"
71353a5f 706 /bin/ls
cb473870 707 echo "PWD/.."
71353a5f 708 /bin/ls ../
cb473870 709
fc7e1e45 710
711 #merge calibration
71353a5f 712 chmod u+x ${mergingScript}
fc7e1e45 713 mkdir -p ./OCDB
71353a5f 714 if [[ ! -f ${calibrationFilesToMerge} ]]; then
715 echo "/bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root > ${calibrationFilesToMerge}"
716 /bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root 2>/dev/null > ${calibrationFilesToMerge}
fc7e1e45 717 fi
718
ef15db95 719 echo "${mergingScript} ${calibrationFilesToMerge} ${runNumber} local://./OCDB defaultOCDB=${ocdbStorage} fileAccessMethod=nocopy"
71353a5f 720 if [[ -n ${pretend} ]]; then
0eaf38c7 721 sleep ${pretendDelay}
fc7e1e45 722 touch CalibObjects.root
723 touch ocdb.log
724 touch merge.log
617f8886 725 touch dcsTime.root
71353a5f 726 mkdir -p ./OCDB/TPC/Calib/TimeGain/
727 mkdir -p ./OCDB/TPC/Calib/TimeDrift/
1795556a 728 echo "some calibration" >> ./OCDB/TPC/Calib/TimeGain/someCalibObject_0-999999_cpass0.root
729 echo "some calibration" >> ./OCDB/TPC/Calib/TimeDrift/otherCalibObject_0-999999_cpass0.root
fc7e1e45 730 else
ef15db95 731 ./${mergingScript} ${calibrationFilesToMerge} ${runNumber} "local://./OCDB" defaultOCDB=${ocdbStorage} fileAccessMethod=nocopy >> "mergeMakeOCDB.log"
fc7e1e45 732
617f8886 733 #produce the calib trees for expert QA (dcsTime.root)
734 goMakeLocalOCDBaccessConfig ./OCDB
735 echo aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")"
736 aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")"
737 fi
738
fc7e1e45 739 ### produce the output
740 #tar the produced OCDB for reuse
ef77d296 741 #tar czf ${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz ./OCDB
742
743 # Create tarball with OCDB, store on the shared directory, create signal file on batch directory
744 mkdir -p ${commonOutputPath}/meta
745 baseTar="cpass0.localOCDB.${runNumber}.tgz"
746 tar czf ${batchWorkingDirectory}/${baseTar} ./OCDB && \
747 mv ${batchWorkingDirectory}/${baseTar} ${commonOutputPath}/meta/${baseTar} && \
748 touch ${batchWorkingDirectory}/${baseTar}.done
fc7e1e45 749
71353a5f 750 /bin/ls
5ed7c82c 751
752 #copy all to output dir
13abb339 753 echo "paranoidCp ${runpath}/* ${outputDir}"
754 paranoidCp ${runpath}/* ${outputDir}
617f8886 755
71353a5f 756 if [[ -n ${generateMC} ]]; then
757 goPrintValues sim ${commonOutputPath}/meta/sim.run${runNumber}.list ${commonOutputPath}/meta/cpass0.job*.run${runNumber}.done
758 fi
759
fc7e1e45 760 #validate merging cpass0
5ed7c82c 761 cd ${outputDir}
ef77d296 762 if summarizeLogs >> ${doneFileTmp}; then
763 [[ -f CalibObjects.root ]] && echo "calibfile ${outputDir}/CalibObjects.root" >> ${doneFileTmp}
764 [[ -f dcsTime.root ]] && echo "dcsTree ${outputDir}/dcsTime.root" >> ${doneFileTmp}
ab946185 765 fi
fc7e1e45 766
71353a5f 767 [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
ef77d296 768 cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
769 [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
5de3064b 770 return 0
78d0bbfd 771)
fc7e1e45 772
773goMergeCPass1()
78d0bbfd 774(
fc7e1e45 775 #
776 # find the output files and merge them
777 #
778
71353a5f 779 outputDir=${1}
780 ocdbStorage=${2}
781 configFile=${3}
782 runNumber=${4}
783 calibrationFilesToMerge=${5}
784 qaFilesToMerge=${6}
444b9916 785 filteredFilesToMerge=${7}
786 shift 7
6fc48f9e 787 if ! parseConfig ${configFile} "$@"; then return 1; fi
71353a5f 788
39527a15 789 #record the working directory provided by the batch system
790 batchWorkingDirectory=${PWD}
791
71353a5f 792 [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
ef77d296 793
794 # This file signals that everything went fine
795 doneFileBase="merge.cpass1.run${runNumber}.done"
796
797 # We will have two copies of the file
798 mkdir -p "${commonOutputPath}/meta" || return 1
799 doneFileTmp="${batchWorkingDirectory}/${doneFileBase}"
800 doneFile="${commonOutputPath}/meta/${doneFileBase}"
71353a5f 801
802 umask 0002
803 ulimit -c unlimited
fc7e1e45 804
805 #clean up first:
71353a5f 806 rm -f ${outputDir}/*.log
807 rm -f ${outputDir}/*.root
808 rm -f ${outputDir}/*done
809
810 [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
811
71353a5f 812 runpath=${outputDir}
0eaf38c7 813 [[ ${reconstructInTemporaryDir} -eq 1 ]] && runpath=$(mktemp -d -t mergeCPass1.XXXXXX)
e91a9c78 814 [[ ${reconstructInTemporaryDir} -eq 2 ]] && runpath=${PWD}/rundir_mergeCPass1_${runNumber}
71353a5f 815
816 mkdir -p ${runpath}
1795556a 817 if [[ ! -d ${runpath} ]]; then
ef77d296 818 touch ${doneFileTmp}
819 echo "not able to make the runpath ${runpath}" >> ${doneFileTmp}
820 cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
821 [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
1795556a 822 return 1
823 fi
0eaf38c7 824 if ! cd ${runpath}; then
ef77d296 825 touch ${doneFileTmp}
826 echo "PWD=$PWD is not the runpath=${runpath}" >> ${doneFileTmp}
827 cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
828 [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
0eaf38c7 829 return 1
830 fi
71353a5f 831
832 logOutputDir=${runpath}
833 [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir}
834 [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/mergeMakeOCDB.log
835 echo "${0} $*"
836
837 calibrationOutputFileName='AliESDfriends_v1.root'
838 qaOutputFileName='QAresults*.root'
fc7e1e45 839 mergingScript="mergeMakeOCDB.byComponent.sh"
cb473870 840 #important to have the string "Stage.txt" in the filename to trigger the merging
841 #it has to be a list of directories containing the files
fc7e1e45 842 qaMergedOutputFileName="QAresults_merged.root"
843
844 echo goMergeCPass1 SETUP:
71353a5f 845 echo runNumber=${runNumber}
846 echo outputDir=${outputDir}
847 echo ocdbStorage=${ocdbStorage}
848 echo calibrationFilesToMerge=$calibrationFilesToMerge
849 echo qaFilesToMerge=$qaFilesToMerge
850 echo calibrationOutputFileName=${calibrationOutputFileName}
851 echo mergingScript=${mergingScript}
fc7e1e45 852
853 # copy files in case they are not already there
854 filesMergeCPass1=(
39527a15 855 "${batchWorkingDirectory}/${calibrationFilesToMerge}"
856 "${batchWorkingDirectory}/${qaFilesToMerge}"
857 "${batchWorkingDirectory}/OCDB.root"
858 "${batchWorkingDirectory}/localOCDBaccessConfig.C"
3fbef3b9 859 "${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz"
39527a15 860 "${batchWorkingDirectory}/QAtrain_duo.C"
71353a5f 861 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/mergeMakeOCDB.byComponent.sh"
862 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/mergeByComponent.C"
863 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/makeOCDB.C"
864 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/merge.C"
865 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/mergeMakeOCDB.sh"
866 "${trustedQAtrainMacro}"
867 "${ALICE_ROOT}/ANALYSIS/macros/QAtrain_duo.C"
fc7e1e45 868 )
869 for file in ${filesMergeCPass1[*]}; do
870 [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
1a0e38e3 871 [[ ${file##*/} =~ .*\.sh ]] && chmod +x ${file##*/}
fc7e1e45 872 done
873
71353a5f 874 sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh
875
cb473870 876 #configure local OCDB storage from CPass0 (creates the localOCDBaccessConfig.C script)
877 if [[ -f cpass0.localOCDB.${runNumber}.tgz ]]; then
71353a5f 878 echo goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz"
879 goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz"
cb473870 880 else
881 echo "WARNING: file cpass0.localOCDB.${runNumber}.tgz not found!"
882 fi
883
71353a5f 884 alirootInfo > ALICE_ROOT.log
fc7e1e45 885
886 #
71353a5f 887 /bin/ls
fc7e1e45 888
889 #merge calibration
71353a5f 890 chmod u+x ${mergingScript}
fc7e1e45 891 mkdir -p OCDB
444b9916 892
893 #if not provided, create the lists of files to merge
894 if [[ ! -f ${filteredFilesToMerge} ]]; then
895 echo "/bin/ls -1 ${outputDir}/*/FilterEvents_Trees.root > ${filteredFilesToMerge}"
896 /bin/ls -1 ${outputDir}/*/FilterEvents_Trees.root 2>/dev/null > ${filteredFilesToMerge}
897 fi
71353a5f 898 if [[ ! -f ${calibrationFilesToMerge} ]]; then
444b9916 899 echo "/bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root > ${calibrationFilesToMerge}"
900 /bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root 2>/dev/null > ${calibrationFilesToMerge}
901 fi
902 if [[ ! -f ${qaFilesToMerge} ]]; then
903 #find the files, but only store the directories (QAtrain_duo.C requires this)
904 echo "/bin/ls -1 ${outputDir}/*/QAresults*.root | while read x; do echo ${x%/*}; done | sort | uniq > ${qaFilesToMerge}"
905 /bin/ls -1 ${outputDir}/*/QAresults*.root | while read x; do echo ${x%/*}; done | sort | uniq > ${qaFilesToMerge}
fc7e1e45 906 fi
ef15db95 907
908 echo "${mergingScript} ${calibrationFilesToMerge} ${runNumber} local://./OCDB defaultOCDB=${ocdbStorage} fileAccessMethod=nocopy"
71353a5f 909 if [[ -n ${pretend} ]]; then
0eaf38c7 910 sleep ${pretendDelay}
fc7e1e45 911 touch ocdb.log
71353a5f 912 touch cpass1.localOCDB.${runNumber}.tgz
71353a5f 913 touch ${qaMergedOutputFileName}
fc7e1e45 914 touch merge.log
cb473870 915 touch trending.root
444b9916 916 touch FilterEvents_Trees.root
ab946185 917 touch CalibObjects.root
918 touch dcsTime.root
919 touch ${qaMergedOutputFileName}
617f8886 920 mkdir -p OCDB
fc7e1e45 921 else
ef15db95 922 ./${mergingScript} ${calibrationFilesToMerge} ${runNumber} "local://./OCDB" defaultOCDB=${ocdbStorage} fileAccessMethod=nocopy
617f8886 923
924 #merge QA (and filtered trees)
925 [[ -n ${AliAnalysisTaskFilteredTree_fLowPtTrackDownscaligF} ]] && export AliAnalysisTaskFilteredTree_fLowPtTrackDownscaligF
926 [[ -n ${AliAnalysisTaskFilteredTree_fLowPtV0DownscaligF} ]] && export AliAnalysisTaskFilteredTree_fLowPtV0DownscaligF
927
928 #echo aliroot -l -b -q "merge.C(\"${qaFilesToMerge}\",\"\",kFALSE,\"${qaMergedOutputFileName}\")"
929 echo aliroot -b -q "QAtrain_duo.C(\"_barrel\",${runNumber},\"${qaFilesToMerge}\",1,\"${ocdbStorage}\")"
71353a5f 930 #aliroot -l -b -q "merge.C(\"${qaFilesToMerge}\",\"\",kFALSE,\"${qaMergedOutputFileName}\")"
931 aliroot -b -q "QAtrain_duo.C(\"_barrel\",${runNumber},\"${qaFilesToMerge}\",1,\"${ocdbStorage}\")" > mergeQA.log
932 mv QAresults_barrel.root ${qaMergedOutputFileName}
cb473870 933 mv trending_barrel.root trending.root
444b9916 934
935 #merge filtered trees
936 echo aliroot -l -b -q "merge.C(\"${qaFilesToMerge}\",\"\",kFALSE,\"${qaMergedOutputFileName}\")"
922d0b10 937 aliroot -l -b -q "merge.C(\"${filteredFilesToMerge}\",\"\",kFALSE,\"FilterEvents_Trees.root\")" > mergeFilteredTrees.log
617f8886 938
939 #produce the calib trees for expert QA
940 echo aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")"
922d0b10 941 aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")" > calibTree.log
fc7e1e45 942 fi
617f8886 943
ef77d296 944 # Create tarball with OCDB, store on the shared directory, create signal file on batch directory
945 mkdir -p ${commonOutputPath}/meta
946 baseTar="cpass1.localOCDB.${runNumber}.tgz"
947 tar czf ${batchWorkingDirectory}/${baseTar} ./OCDB && \
948 mv ${batchWorkingDirectory}/${baseTar} ${commonOutputPath}/meta/${baseTar} && \
949 touch ${batchWorkingDirectory}/${baseTar}.done
71353a5f 950
951 /bin/ls
5ed7c82c 952
953 #copy all to output dir
13abb339 954 echo "paranoidCp ${runpath}/* ${outputDir}"
955 paranoidCp ${runpath}/* ${outputDir}
5ed7c82c 956
fc7e1e45 957 #validate merge cpass1
5ed7c82c 958 cd ${outputDir}
ef77d296 959 if summarizeLogs >> ${doneFileTmp}; then
960 [[ -f CalibObjects.root ]] && echo "calibfile ${outputDir}/CalibObjects.root" >> ${doneFileTmp}
961 [[ -f ${qaMergedOutputFileName} ]] && echo "qafile ${outputDir}/${qaMergedOutputFileName}" >> ${doneFileTmp}
962 [[ -f trending.root ]] && echo "trendingfile ${outputDir}/trending.root" >> ${doneFileTmp}
963 [[ -f dcsTime.root ]] && echo "dcsTree ${outputDir}/dcsTime.root" >> ${doneFileTmp}
964 [[ -f FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFileTmp}
922d0b10 965 else
ef77d296 966 if grep "mergeQA.log.*OK" ${doneFileTmp} > /dev/null; then
967 [[ -f ${qaMergedOutputFileName} ]] && echo "qafile ${outputDir}/${qaMergedOutputFileName}" >> ${doneFileTmp}
922d0b10 968 fi
ef77d296 969 if grep "mergeFilteredTrees.log.*OK" ${doneFileTmp} > /dev/null; then
970 [[ -f FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFileTmp}
922d0b10 971 fi
ab946185 972 fi
71353a5f 973
974 [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
ef77d296 975 cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
976 [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
5de3064b 977 return 0
78d0bbfd 978)
cb473870 979
980goMerge()
78d0bbfd 981(
cb473870 982 #generic root merge using CPass1 merge.C script
71353a5f 983 inputList=${1}
984 outputFile=${2}
0eaf38c7 985 configFile=${3-"benchmark.config"}
71353a5f 986 shift 3
6fc48f9e 987 if ! parseConfig ${configFile} "$@"; then return 1; fi
39527a15 988
989 #record the working directory provided by the batch system
990 batchWorkingDirectory=${PWD}
fc7e1e45 991
71353a5f 992 [[ ! -f ${inputList} ]] && echo "inputList ${inputList} does not exist!" && return 1
993 [[ ! -f ${configFile} ]] && echo "configFile ${configFile} does not exist!" && return 1
994 umask 0002
995 [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
996 rm -f ${outputFile}
997 aliroot -b -q "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/merge.C(\"${inputList}\",\"\",kFALSE,\"${outputFile}\")" > merge_${inputList}.log
5de3064b 998 return 0
78d0bbfd 999)
fc7e1e45 1000
cb473870 1001goSubmitMakeflow()
5dfac970 1002{
cb473870 1003 #run
71353a5f 1004 productionID=${1}
1005 inputFileList=${2}
1006 configFile=${3}
1007 shift 3
444b9916 1008 extraOpts=("$@")
6fc48f9e 1009 if ! parseConfig ${configFile} "${extraOpts[@]}"; then return 1; fi
39527a15 1010
1011 #record the working directory provided by the batch system
1012 batchWorkingDirectory=${PWD}
fc7e1e45 1013
5dfac970 1014 [[ -z ${configFile} ]] && configFile="benchmark.config"
1015 [[ ! -f ${configFile} ]] && echo "no config file found (${configFile})" && return 1
fc7e1e45 1016
71353a5f 1017 if [[ ! $(which makeflow &>/dev/null) && -n ${makeflowPath} ]]; then
cb473870 1018 echo "setting the makflow path from the config: "
1019 echo " export PATH=${makeflowPath}:${PATH}"
1020 export PATH=${makeflowPath}:${PATH}
1021 fi
1022
39527a15 1023 #create the common output dir and the meta dir
1024 commonOutputPath=${baseOutputDirectory}/${productionID}
1025 if [[ -d ${commonOutputPath} ]]; then
1026 echo "output dir ${commonOutputPath} exists!"
0eaf38c7 1027 #return 1
39527a15 1028 else
1029 mkdir -p ${commonOutputPath}
1030 fi
1031 mkdir -p ${commonOutputPath}/meta
839b07ca 1032
0eaf38c7 1033 self=${0}
1034 #if which greadlink; then self=$(greadlink -f "${0}"); fi
839b07ca 1035
0eaf38c7 1036 #for reference copy the setup to the output dir
7db04251 1037 paranoidCp ${self} ${commonOutputPath}
1038 paranoidCp ${configFile} ${commonOutputPath}
1039 paranoidCp ${inputFileList} ${commonOutputPath}
39527a15 1040
cb473870 1041 #submit - use makeflow if available, fall back to old stuff when makeflow not there
1042 if which makeflow; then
39527a15 1043 goGenerateMakeflow ${productionID} ${inputFileList} ${configFile} "${extraOpts[@]}" commonOutputPath=${commonOutputPath} > benchmark.makeflow
1795556a 1044 cp benchmark.makeflow ${commonOutputPath}
cb473870 1045 makeflow ${makeflowOptions} benchmark.makeflow
cb473870 1046 else
1047 echo "no makeflow!"
1048 fi
054fdec5 1049
1050 #summarize the run based on the makeflow log
1051 #and add it to the end of summary log
1052 awk '/STARTED/ {startTime=$3}
1053 /COMPLETED/ {endTime=$3}
1054 END {print "makeflow running time: "(endTime-startTime)/1000000/3600" hours"}' \
1055 benchmark.makeflow.makeflowlog | tee -a summary.log
1056 paranoidCp summary.log ${commonOutputPath}
1057
5de3064b 1058 return 0
5dfac970 1059}
fc7e1e45 1060
5dfac970 1061goGenerateMakeflow()
78d0bbfd 1062(
5dfac970 1063 #generate the makeflow file
ab946185 1064 [[ $# -lt 3 ]] && echo "args: id inputFileList configFile" && return 1
71353a5f 1065 productionID=${1}
1066 inputFileList=${2}
1067 configFile=${3}
1068 shift 3
444b9916 1069 extraOpts=("$@")
0eaf38c7 1070
1071 #batch systems/makeflow sometimes handle spaces in arguments poorly, so encode them
1072 for (( i=0;i<${#extraOpts[@]};i++ )); do
1073 extraOpts[i]=$(encSpaces "${extraOpts[i]}")
1074 done
1075 extraOpts+=("encodedSpaces=1")
1076
39527a15 1077 if ! parseConfig ${configFile} "${extraOpts[@]}" &>/dev/null; then return 1; fi
1078
1a0e38e3 1079 #extra safety
1080 if [[ -z ${commonOutputPath} ]]; then
1081 commonOutputPath=${baseOutputDirectory}/${productionID}
1082 extraOpts=( "${extraOpts[@]}" "commonOutputPath=${commonOutputPath}" )
1083 fi
bff71da0 1084
39527a15 1085 #record the working directory provided by the batch system
1086 batchWorkingDirectory=${PWD}
fc7e1e45 1087
5dfac970 1088 [[ -z ${configFile} ]] && configFile="benchmark.config"
1089 [[ ! -f ${configFile} ]] && echo "no config file found (${configFile})" && return 1
fc7e1e45 1090
5dfac970 1091 #these files will be made a dependency - will be copied to the working dir of the jobs
1092 declare -a copyFiles
1093 inputFiles=(
1094 "OCDB.root"
1095 "localOCDBaccessConfig.C"
cb473870 1096 "QAtrain_duo.C"
1097 "runCPass1.sh"
1098 "recCPass1.C"
1099 "recCPass1_OuterDet.C"
1100 "runCalibTrain.C"
1101 "runCPass0.sh"
1102 "recCPass0.C"
ef77d296 1103 "runQA.sh"
5dfac970 1104 )
1105 for file in ${inputFiles[*]}; do
1106 [[ -f ${file} ]] && copyFiles+=("${file}")
1107 done
1108
1109 #create the makeflow file
cb473870 1110 [[ -n ${batchFlags} ]] && echo "BATCH_OPTIONS = ${batchFlags}"
444b9916 1111 declare -A arr_cpass0_merged arr_cpass1_merged
1112 declare -A arr_cpass0_calib_list arr_cpass1_calib_list
1113 declare -A arr_cpass1_QA_list arr_cpass1_ESD_list arr_cpass1_filtered_list
6fc48f9e 1114 declare -A arr_cpass0_profiled_outputs
444b9916 1115 declare -A listOfRuns
1116 [[ -n ${runNumber} ]] && listOfRuns[${runNumber}]=1
617f8886 1117 while read x; do tmpRun=$(guessRunNumber ${x}); [[ -n ${tmpRun} ]] && listOfRuns[${tmpRun}]=1; done < ${inputFileList}
444b9916 1118 for runNumber in "${!listOfRuns[@]}"; do
71353a5f 1119 [[ -z ${runNumber} ]] && continue
fc7e1e45 1120 [[ ! ${runNumber} =~ ^[0-9]*[0-9]$ ]] && continue
fc7e1e45 1121
444b9916 1122 unset arr_cpass0_outputs
1123 unset arr_cpass1_outputs
1124 declare -a arr_cpass0_outputs
1125 declare -a arr_cpass1_outputs
fc7e1e45 1126
ef77d296 1127 #Header
1128 echo "### Automatically generated on $(LANG=C date) ###"
1129 echo ; echo
1130
cb473870 1131 jobindex=0
6fc48f9e 1132 inputFile=""
5dfac970 1133 while read inputFile; do
1134 currentDefaultOCDB=${defaultOCDB}
cb473870 1135 [[ -z ${autoOCDB} ]] && autoOCDB=1
1136 if [[ ${autoOCDB} -ne 0 ]]; then
71353a5f 1137 currentDefaultOCDB=$(setYear ${inputFile} ${defaultOCDB})
cb473870 1138 fi
ef77d296 1139 guessRunData ${inputFile}
1140
1141 #Set variables
1142 echo "### Variables ###"
1143 echo "OUTPATH=\"${commonOutputPath}/${year}/${period}\""
1144 echo ; echo
fc7e1e45 1145
5dfac970 1146 #CPass0
ef77d296 1147 #arr_cpass0_outputs[${jobindex}]="${commonOutputPath}/meta/cpass0.job${jobindex}.run${runNumber}.done"
1148 arr_cpass0_outputs[${jobindex}]="cpass0.job${jobindex}.run${runNumber}.done"
1149 echo "### CPass0 ###"
1150 echo "${arr_cpass0_outputs[${jobindex}]}: benchmark.sh ${configFile} ${copyFiles[@]}"
1151 echo " ${alirootEnv} ./benchmark.sh CPass0 \$OUTPATH/000${runNumber}/cpass0 ${inputFile} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} ${jobindex} ${extraOpts[@]}"" "
1152 echo ; echo
5dfac970 1153
1154 #CPass1
ef77d296 1155 #arr_cpass1_outputs[${jobindex}]="${commonOutputPath}/meta/cpass1.job${jobindex}.run${runNumber}.done"
1156 arr_cpass1_outputs[${jobindex}]="cpass1.job${jobindex}.run${runNumber}.done"
1157 echo "### CPass1 ###"
1158 echo "${arr_cpass1_outputs[${jobindex}]}: benchmark.sh ${configFile} cpass0.localOCDB.${runNumber}.tgz.done ${copyFiles[@]}"
1159 echo " ${alirootEnv} ./benchmark.sh CPass1 \$OUTPATH/000${runNumber}/cpass1 ${inputFile} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} ${jobindex} ${extraOpts[@]}"" "
1160 echo ; echo
5dfac970 1161 ((jobindex++))
fc7e1e45 1162
71353a5f 1163 done< <(grep "/000${runNumber}/" ${inputFileList})
cb473870 1164
5dfac970 1165 #CPass0 list of Calib files to merge
ef77d296 1166 #arr_cpass0_calib_list[${runNumber}]="${commonOutputPath}/meta/cpass0.calib.run${runNumber}.list"
1167 arr_cpass0_calib_list[${runNumber}]="cpass0.calib.run${runNumber}.list"
1168 echo "### Produces the list of CPass0 files to merge (executes locally) ###"
1169 echo "${arr_cpass0_calib_list[${runNumber}]}: benchmark.sh ${arr_cpass0_outputs[*]}"
1170 echo " LOCAL ./benchmark.sh PrintValues calibfile ${arr_cpass0_calib_list[${runNumber}]} ${arr_cpass0_outputs[*]} && mkdir -p \$OUTPATH/meta && cp ${arr_cpass0_calib_list[${runNumber}]} \$OUTPATH/meta/${arr_cpass0_calib_list[${runNumber}]}"
1171 echo ; echo
71353a5f 1172
1173 #CPass0 merging
ef77d296 1174 echo "### Merges CPass0 files ###"
1175 #arr_cpass0_merged[${runNumber}]="${commonOutputPath}/meta/merge.cpass0.run${runNumber}.done"
1176 arr_cpass0_merged[${runNumber}]="merge.cpass0.run${runNumber}.done"
1177 echo "cpass0.localOCDB.${runNumber}.tgz.done ${arr_cpass0_merged[${runNumber}]}: benchmark.sh ${configFile} ${arr_cpass0_calib_list[${runNumber}]} ${copyFiles[@]}"
1178 echo " ${alirootEnv} ./benchmark.sh MergeCPass0 \$OUTPATH/000${runNumber}/cpass0 ${currentDefaultOCDB} ${configFile} ${runNumber} ${arr_cpass0_calib_list[${runNumber}]} ${extraOpts[@]}"" "
1179 echo ; echo
fc7e1e45 1180
ab946185 1181 #CPass1 list of Calib/QA/ESD/filtered files
1182 # the trick with QA is to have the string "Stage.txt" in the file name of the list of directories with QA output to trigger
1183 # the production of the QA trending tree (only then the task->Finish() will be called in QAtrain_duo.C, on the grid
8d6a0a43 1184 # this corresponds to the last merging stage)
ef77d296 1185 #arr_cpass1_QA_list[${runNumber}]="${commonOutputPath}/meta/cpass1.QA.run${runNumber}.lastMergingStage.txt.list"
1186 arr_cpass1_QA_list[${runNumber}]="cpass1.QA.run${runNumber}.lastMergingStage.txt.list"
1187 echo "### Lists CPass1 QA ###"
444b9916 1188 echo "${arr_cpass1_QA_list[${runNumber}]}: benchmark.sh ${arr_cpass1_outputs[*]}"
ef77d296 1189 echo " LOCAL ./benchmark.sh PrintValues dir ${arr_cpass1_QA_list[${runNumber}]} ${arr_cpass1_outputs[*]} && mkdir -p \$OUTPATH/meta && cp ${arr_cpass1_QA_list[${runNumber}]} \$OUTPATH/meta/${arr_cpass1_QA_list[${runNumber}]}"
1190 echo ; echo
1191
1192 #arr_cpass1_calib_list[${runNumber}]="${commonOutputPath}/meta/cpass1.calib.run${runNumber}.list"
1193 arr_cpass1_calib_list[${runNumber}]="cpass1.calib.run${runNumber}.list"
1194 echo "### Lists CPass1 Calib ###"
1195 echo "${arr_cpass1_calib_list[${runNumber}]}: benchmark.sh ${arr_cpass1_outputs[*]}"
1196 echo " LOCAL ./benchmark.sh PrintValues calibfile ${arr_cpass1_calib_list[${runNumber}]} ${arr_cpass1_outputs[*]} && mkdir -p \$OUTPATH/meta && cp ${arr_cpass1_calib_list[${runNumber}]} \$OUTPATH/meta/${arr_cpass1_calib_list[${runNumber}]}"
1197 echo ; echo
1198
1199 #arr_cpass1_ESD_list[${runNumber}]="${commonOutputPath}/meta/cpass1.ESD.run${runNumber}.list"
1200 arr_cpass1_ESD_list[${runNumber}]="cpass1.ESD.run${runNumber}.list"
1201 echo "### Lists CPass1 ESDs ###"
1202 echo "${arr_cpass1_ESD_list[${runNumber}]}: benchmark.sh ${arr_cpass1_outputs[*]}"
1203 echo " LOCAL ./benchmark.sh PrintValues esd ${arr_cpass1_ESD_list[${runNumber}]} ${arr_cpass1_outputs[*]} && mkdir -p \$OUTPATH/meta && cp ${arr_cpass1_ESD_list[${runNumber}]} \$OUTPATH/meta/${arr_cpass1_ESD_list[${runNumber}]}"
1204 echo ; echo
1205
1206 #arr_cpass1_filtered_list[${runNumber}]="${commonOutputPath}/meta/cpass1.filtered.run${runNumber}.list"
1207 arr_cpass1_filtered_list[${runNumber}]="cpass1.filtered.run${runNumber}.list"
1208 echo "### Lists CPass1 filtered ###"
1209 echo "${arr_cpass1_filtered_list[${runNumber}]}: benchmark.sh ${arr_cpass1_outputs[*]}"
1210 echo " LOCAL ./benchmark.sh PrintValues filteredTree ${arr_cpass1_filtered_list[${runNumber}]} ${arr_cpass1_outputs[*]} && mkdir -p \$OUTPATH/meta && cp ${arr_cpass1_filtered_list[${runNumber}]} \$OUTPATH/meta/${arr_cpass1_filtered_list[${runNumber}]}"
1211 echo ; echo
444b9916 1212
71353a5f 1213 #CPass1 merging
ef77d296 1214 #arr_cpass1_merged[${runNumber}]="${commonOutputPath}/meta/merge.cpass1.run${runNumber}.done"
1215 arr_cpass1_merged[${runNumber}]="merge.cpass1.run${runNumber}.done"
1216 echo "### Merges CPass1 files ###"
1217 echo "cpass1.localOCDB.${runNumber}.tgz.done ${arr_cpass1_merged[${runNumber}]}: benchmark.sh ${configFile} ${arr_cpass1_calib_list[${runNumber}]} ${arr_cpass1_QA_list[${runNumber}]} ${copyFiles[@]}"
1218 echo " ${alirootEnv} ./benchmark.sh MergeCPass1 \$OUTPATH/000${runNumber}/cpass1 ${currentDefaultOCDB} ${configFile} ${runNumber} ${arr_cpass1_calib_list[${runNumber}]} ${arr_cpass1_QA_list[${runNumber}]} ${arr_cpass1_filtered_list[${runNumber}]} ${extraOpts[@]}"" "
1219 echo ; echo
fc7e1e45 1220
6fc48f9e 1221 #CPass0 wrapped in a profiling tool (valgrind,....)
39527a15 1222 if [[ -n ${profilingCommand} ]]; then
0eaf38c7 1223 inputFile=$(grep -m1 "${runNumber}/" ${inputFileList})
1224 [[ -z ${nEventsProfiling} ]] && nEventsProfiling=2
1225 currentDefaultOCDB=$(setYear ${inputFile} ${defaultOCDB})
1226 jobindex="profiling"
1227
ef77d296 1228 #arr_cpass0_profiled_outputs[${runNumber}]="${commonOutputPath}/meta/profiling.cpass0.job${jobindex}.run${runNumber}.done"
1229 arr_cpass0_profiled_outputs[${runNumber}]="profiling.cpass0.job${jobindex}.run${runNumber}.done"
1230 echo "### CPass0 in a profiler ###"
1231 echo "${arr_cpass0_profiled_outputs[${runNumber}]}: benchmark.sh ${configFile} ${copyFiles[@]}"
0eaf38c7 1232 profilingCommand=$(encSpaces "${profilingCommand}")
ef77d296 1233 echo " ${alirootEnv} ./benchmark.sh CPass0 \$OUTPATH/000${runNumber}/${jobindex} ${inputFile} ${nEventsProfiling} ${currentDefaultOCDB} ${configFile} ${runNumber} ${jobindex} ${extraOpts[@]} useProfilingCommand=${profilingCommand}"
1234 echo ; echo
6fc48f9e 1235 fi
1236
444b9916 1237 done #runs
fc7e1e45 1238
5dfac970 1239 #Summary
ef77d296 1240 echo "### Summary ###"
1241 echo "summary.log: benchmark.sh ${configFile} ${arr_cpass1_merged[*]}"
1242 echo " ${alirootEnv} ./benchmark.sh MakeSummary ${configFile} ${extraOpts[@]}"
1243 echo ; echo
cb473870 1244
5de3064b 1245 return 0
78d0bbfd 1246)
cb473870 1247
1248goPrintValues()
78d0bbfd 1249(
cb473870 1250 #print the values given the key from any number of files (store in output file on second argument)
71353a5f 1251 if [[ $# -lt 3 ]]; then
1252 echo "goPrintValues key outputFile inputFiles"
ef77d296 1253 echo "if outputFile is \"-\" don't redirect to a file"
71353a5f 1254 return
1255 fi
1256 key=${1}
1257 outputFile=${2}
1258 [[ ${outputFile} =~ "-" ]] && outputFile=""
cb473870 1259 shift 2 #remove 2 first arguments from arg list to only pass the input files to awk
71353a5f 1260 awk -v key=${key} '$0 ~ key" " {print $2}' "$@" | tee ${outputFile}
5de3064b 1261 return 0
78d0bbfd 1262)
fc7e1e45 1263
5dfac970 1264goCreateQAplots()
78d0bbfd 1265(
5dfac970 1266 umask 0002
71353a5f 1267 mergedQAfileList=${1}
1268 productionID=${2}
1269 outputDir=${3}
1270 configFile=${4}
1271 shift 4
e48b4924 1272 if ! parseConfig ${configFile} "$@"; then return 1; fi
71353a5f 1273
39527a15 1274 #record the working directory provided by the batch system
1275 batchWorkingDirectory=${PWD}
1276
71353a5f 1277 [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
fc7e1e45 1278
71353a5f 1279 [[ -z ${logOutputDir} ]] && logOutputDir=${PWD}
0eaf38c7 1280 [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/makeQAplots.log
71353a5f 1281 echo "${0} $*"
1282
1283 olddir=${PWD}
1284 mkdir -p ${outputDir}
1285 cd ${outputDir}
839b07ca 1286 [[ ! "${PWD}" =~ "${outputDir}" ]] && echo "PWD is not equal to outputDir=${outputDir}" && cd ${olddir} && return 1
fc7e1e45 1287
ef77d296 1288 inputFiles=(
1289 "${batchWorkingDirectory}/runQA.sh"
1290 "${ALICE_ROOT}/PWGPP/QA/scripts/runQA.sh"
1291 )
1292 for file in ${inputFiles[*]}; do
1293 [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
1294 done
1295
1296 echo "running QA with command:"
1297 echo ./runQA.sh inputList="${mergedQAfileList}" inputListHighPtTrees="${filteringList}" ocdbStorage="${defaultOCDB}"
1298 ./runQA.sh inputList="${mergedQAfileList}" inputListHighPtTrees="${filteringList}" ocdbStorage="${defaultOCDB}"
71353a5f 1299 cd ${olddir}
5de3064b 1300 return 0
78d0bbfd 1301)
fc7e1e45 1302
5dfac970 1303goTest()
78d0bbfd 1304(
1305 echo AA
1306)
fc7e1e45 1307
5dfac970 1308alirootInfo()
78d0bbfd 1309(
5dfac970 1310 umask 0002
71353a5f 1311 # save aliroot repository info
1312 [[ -z "${ALICE_ROOT}" ]] && return 1
1313
1314 echo "\${ALICE_ROOT}=${ALICE_ROOT}"
1315 echo "\${ROOTSYS}=${ROOTSYS}"
1316 echo "\${PATH}=${PATH}"
1317 echo "\${LD_LIBRARY_PATH}=${LD_LIBRARY_PATH}"
0eaf38c7 1318 echo
71353a5f 1319
1320 pushd ${PWD}
1321 cd ${ALICE_ROOT}
71353a5f 1322
1323 currentBranch=$(git rev-parse --abbrev-ref HEAD)
1324 git status
5dfac970 1325 echo ""
1326 echo ""
71353a5f 1327 git diff ${currentBranch}
1328 popd
5de3064b 1329 return 0
78d0bbfd 1330)
fc7e1e45 1331
5dfac970 1332setYear()
78d0bbfd 1333(
5dfac970 1334 #set the year
71353a5f 1335 # ${1} - year to be set
1336 # ${2} - where to set the year
1337 year1=$(guessYear ${1})
1338 year2=$(guessYear ${2})
1339 local path=${2}
1340 [[ ${year1} -ne ${year2} && -n ${year2} && -n ${year1} ]] && path=${2/\/${year2}\//\/${year1}\/}
1341 echo ${path}
5de3064b 1342 return 0
78d0bbfd 1343)
fc7e1e45 1344
5dfac970 1345guessPeriod()
78d0bbfd 1346(
5dfac970 1347 #guess the period from the path, pick the rightmost one
1348 local IFS="/"
71353a5f 1349 declare -a path=( ${1} )
5dfac970 1350 local dirDepth=${#path[*]}
1351 for ((x=${dirDepth}-1;x>=0;x--)); do
1352 local field=${path[${x}]}
71353a5f 1353 [[ ${field} =~ ^LHC[0-9][0-9][a-z]$ ]] && period=${field} && break
5dfac970 1354 done
71353a5f 1355 echo ${period}
5de3064b 1356 return 0
78d0bbfd 1357)
fc7e1e45 1358
5dfac970 1359guessYear()
78d0bbfd 1360(
5dfac970 1361 #guess the year from the path, pick the rightmost one
1362 local IFS="/"
71353a5f 1363 declare -a path=( ${1} )
5dfac970 1364 local dirDepth=${#path[*]}
1365 for ((x=${dirDepth}-1;x>=0;x--)); do
1366 local field=${path[${x}]}
71353a5f 1367 [[ ${field} =~ ^20[0-9][0-9]$ ]] && year=${field} && break
5dfac970 1368 done
71353a5f 1369 echo ${year}
5de3064b 1370 return 0
78d0bbfd 1371)
fc7e1e45 1372
5dfac970 1373guessRunNumber()
78d0bbfd 1374(
5dfac970 1375 #guess the run number from the path, pick the rightmost one
71353a5f 1376 #works for /path/foo/000123456/bar/...
1377 #and /path/foo.run123456.bar
1378 local IFS="/."
1379 declare -a path=( ${1} )
5dfac970 1380 local dirDepth=${#path[*]}
1381 for ((x=${dirDepth}-1;x>=0;x--)); do
1382 local field=${path[${x}]}
71353a5f 1383 field=${field/run/000}
617f8886 1384 [[ ${field} =~ [0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && runNumber=${field#000} && break
5dfac970 1385 done
71353a5f 1386 echo ${runNumber}
5de3064b 1387 return 0
78d0bbfd 1388)
fc7e1e45 1389
cb473870 1390validateLog()
78d0bbfd 1391(
71353a5f 1392 log=${1}
cb473870 1393 errorConditions=(
71353a5f 1394 'There was a crash'
1395 'floating'
1396 'error while loading shared libraries'
1397 'std::bad_alloc'
1398 's_err_syswatch_'
1399 'Thread [0-9]* (Thread'
1400 'AliFatal'
1401 'core dumped'
1402 '\.C.*error:.*\.h: No such file'
839b07ca 1403 'line.*Aborted'
71353a5f 1404 )
1405
1406 warningConditions=(
1407 'This is serious !'
1408 'rocVoltage out of range:'
cb473870 1409 )
1410
71353a5f 1411 local logstatus=0
1412 local errorSummary=""
1413 local warningSummary=""
1414
1415 for ((i=0; i<${#errorConditions[@]};i++)); do
1416 local tmp=$(grep -m1 -e "${errorConditions[${i}]}" ${log})
1417 [[ -n ${tmp} ]] && tmp+=" : "
1418 errorSummary+=${tmp}
1419 done
1420
1421 for ((i=0; i<${#warningConditions[@]};i++)); do
1422 local tmp=$(grep -m1 -e "${warningConditions[${i}]}" ${log})
1423 [[ -n ${tmp} ]] && tmp+=" : "
1424 warningSummary+=${tmp}
1425 done
1426
1427 if [[ -n ${errorSummary} ]]; then
1428 echo "${errorSummary}"
1429 return 1
1430 fi
cb473870 1431
71353a5f 1432 if [[ -n ${warningSummary} ]]; then
1433 echo "${warningSummary}"
1434 return 2
1435 fi
1436
1437 return 0
78d0bbfd 1438)
cb473870 1439
5dfac970 1440summarizeLogs()
78d0bbfd 1441(
5dfac970 1442 #print a summary of logs
1443 logFiles=(
1444 "*.log"
cb473870 1445 "stdout"
1446 "stderr"
5dfac970 1447 )
fc7e1e45 1448
9c124791 1449 #put dir information in the output
1450 echo "dir $PWD"
1451
cb473870 1452 #check logs
71353a5f 1453 local logstatus=0
5dfac970 1454 for log in ${logFiles[*]}; do
71353a5f 1455 finallog=${PWD%/}/${log}
1456 [[ ! -f ${log} ]] && continue
1457 errorSummary=$(validateLog ${log})
1458 validationStatus=$?
1459 if [[ ${validationStatus} -eq 0 ]]; then
5dfac970 1460 #in pretend mode randomly report an error in rec.log some cases
71353a5f 1461 if [[ -n ${pretend} && "${log}" == "rec.log" ]]; then
617f8886 1462 #[[ $(( ${RANDOM}%2 )) -ge 1 ]] && echo "${finallog} BAD random error" || echo "${finallog} OK"
1463 echo "${finallog} OK"
fc7e1e45 1464 else
71353a5f 1465 echo "${finallog} OK"
fc7e1e45 1466 fi
71353a5f 1467 elif [[ ${validationStatus} -eq 1 ]]; then
1468 echo "${finallog} BAD ${errorSummary}"
ab946185 1469 logstatus=1
71353a5f 1470 elif [[ ${validationStatus} -eq 2 ]]; then
1471 echo "${finallog} OK MWAH ${errorSummary}"
5dfac970 1472 fi
1473 done
cb473870 1474
1475 #report core files
71353a5f 1476 while read x; do
1477 echo ${x}
1478 chmod 644 ${x}
1479 gdb --batch --quiet -ex "bt" -ex "quit" aliroot ${x} > stacktrace_${x//\//_}.log
1480 done < <(/bin/ls ${PWD}/*/core 2>/dev/null; /bin/ls ${PWD}/core 2>/dev/null)
cb473870 1481
71353a5f 1482 return ${logstatus}
78d0bbfd 1483)
fc7e1e45 1484
5dfac970 1485spitOutLocalOCDBaccessConfig()
1486{
1487 umask 0002
71353a5f 1488 #find ${1} -name "*root" | \
1489 /bin/ls -1 ${1}/*/*/*/*.root 2>/dev/null | \
5dfac970 1490 while read line
1491 do
71353a5f 1492 local tmp=${line#${1}}
5dfac970 1493 echo ${tmp%/*} | \
71353a5f 1494 awk -v ocdb=${1} '{print " man->SetSpecificStorage(\""$1"\",\"local://"ocdb"\");"}'
5dfac970 1495 done
5de3064b 1496 return 0
5dfac970 1497}
fc7e1e45 1498
71353a5f 1499goMakeLocalOCDBaccessConfig()
5dfac970 1500{
1501 umask 0002
1502 # make a script that sets the specific storages form all the root files produced by CPass0
71353a5f 1503 local localOCDBpathCPass0=${1}
1504 local OCDBpathPrefix=${2}
1505 [[ -z ${OCDBpathPrefix} ]] && OCDBpathPrefix="."
1506
1507 if [[ -f ${localOCDBpathCPass0} && ${localOCDBpathCPass0} =~ \.tgz$ ]]; then
1508 tar xzf ${localOCDBpathCPass0}
1509 local localOCDBpathCPass0="${OCDBpathPrefix}/OCDB"
5dfac970 1510 fi
fc7e1e45 1511
5dfac970 1512 echo
1513 echo creating the specific storage script
1514 echo localOCDBaccessConfig.C
71353a5f 1515 echo based on OCDB: ${localOCDBaccessConfig}
5dfac970 1516 echo
fc7e1e45 1517
5dfac970 1518 local tempLocalOCDB=""
1519 if [[ -f localOCDBaccessConfig.C ]]; then
0eaf38c7 1520 tempLocalOCDB=$(mktemp -t tempLocalOCDB.XXXXXX)
71353a5f 1521 echo "egrep "SetSpecificStorage" localOCDBaccessConfig.C > ${tempLocalOCDB}"
1522 egrep "SetSpecificStorage" localOCDBaccessConfig.C > ${tempLocalOCDB}
5dfac970 1523 fi
fc7e1e45 1524
5dfac970 1525 echo "localOCDBaccessConfig()" > localOCDBaccessConfig.C
1526 echo "{" >> localOCDBaccessConfig.C
1527 echo " AliCDBManager* man = AliCDBManager::Instance();" >> localOCDBaccessConfig.C
71353a5f 1528 spitOutLocalOCDBaccessConfig ${localOCDBpathCPass0}|sort|uniq >> localOCDBaccessConfig.C
1529 [[ -f "${tempLocalOCDB}" ]] && cat ${tempLocalOCDB} >> localOCDBaccessConfig.C
5dfac970 1530 echo "}" >> localOCDBaccessConfig.C
fc7e1e45 1531
71353a5f 1532 [[ -f "${tempLocalOCDB}" ]] && rm -f ${tempLocalOCDB}
fc7e1e45 1533
5dfac970 1534 if ! grep SetSpecificStorage localOCDBaccessConfig.C; then
1535 echo
1536 echo "!!!!!!! CPass0 produced no OCDB entries"
1537 return 1
1538 fi
5de3064b 1539 return 0
5dfac970 1540}
fc7e1e45 1541
cb473870 1542goMakeFilteredTrees()
78d0bbfd 1543(
71353a5f 1544 outputDir=${1}
1545 runNumber=${2}
cb473870 1546 #get path to input list
71353a5f 1547 inputListfiles=${3}
cb473870 1548 #get scale number for tracks
71353a5f 1549 filterT=${4}
cb473870 1550 #get scale number for V0s
71353a5f 1551 filterV=${5}
cb473870 1552 #get OCDB path (optional)
1553 OCDBpath=${6}
1554 #get max number of files
1555 maxFiles=${7-"1000000"}
1556 #get offset of first file
1557 offsetFile=${8-"0"}
1558 #get max number of events
1559 maxEvents=${9-"30000000"}
1560 #get offset of first event
1561 offsetEvent=${10-"0"}
1562 configFile=${11-"benchmark.config"}
1563 esdFileName=${12-"AliESDs_Barrel.root"}
71353a5f 1564 shift 12
6fc48f9e 1565 if ! parseConfig ${configFile} "$@"; then return 1; fi
39527a15 1566
1567 #record the working directory provided by the batch system
1568 batchWorkingDirectory=${PWD}
1569
78d0bbfd 1570 [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
ef77d296 1571 doneFileBase=filtering.cpass1.run${runNumber}.done
1572 doneFileTmp=${batchWorkingDirectory}/${doneFileBase}
1573 doneFile=${commonOutputPath}/meta/${doneFileBase}
71353a5f 1574
cb473870 1575 cat > filtering.log << EOF
1576 goMakeFilteredTrees config:
71353a5f 1577 runpath=${runpath}
1578 outputDir=${outputDir}
1579 commonOutputPath=${commonOutputPath}
1580 ALICE_ROOT=${ALICE_ROOT}
1581 PATH=${PATH}
1582 offsetEvent=$offsetEvent
1583 configFile=$configFile
1584 esdFileName=$esdFileName
fab31805 1585 inputListfiles=$inputListfiles
1586 doneFile=$doneFile
cb473870 1587EOF
1588
fab31805 1589 #runpath=${outputDir}
1590 #[[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
1591 #[[ ${reconstructInTemporaryDir} -eq 1 ]] && runpath=$(mktemp -d -t goMakeFilteredTrees.XXXXXX)
1592 #mkdir -p ${outputDir}
1593 #mkdir -p ${runpath}
1594 #if ! cd ${runpath}; then
1595 # echo "PWD=$PWD is not the runpath=${runpath}"
1596 # touch ${doneFile}
1597 # return 1
1598 #fi
1599
cb473870 1600 if [[ -z ${pretend} ]];then
71353a5f 1601 aliroot -l -b -q "${ALICE_ROOT}/PWGPP/macros/runFilteringTask.C(\"${inputListfiles}\",${filterT},${filterV},\"${OCDBpath}\",${maxFiles},${offsetFile},${maxEvents},${offsetEvent},\"${esdFileName}\")" &>> filtering.log
cb473870 1602 else
0eaf38c7 1603 sleep ${pretendDelay}
71353a5f 1604 touch filtering.log FilterEvents_Trees.root
cb473870 1605 fi
1606 pwd
71353a5f 1607 /bin/ls
71353a5f 1608 summarizeLogs >> ${doneFile}
fab31805 1609
1610 #echo mv -f * ${outputDir}
1611 #mv -f * ${outputDir}
1612 #[[ -f ${outputDir}/FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFile}
1613 #cd ${commonOutputPath}
1614 #[[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
e344162c 1615
ef77d296 1616 cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
1617 [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
e344162c 1618
5de3064b 1619 return 0
78d0bbfd 1620)
cb473870 1621
1622submit()
1623{
1624 umask 0002
444b9916 1625 [[ $# -lt 5 ]] && echo "at least 5 args needed, you supplied $#" && return 1
71353a5f 1626 JobID=${1}
1627 startID=${2}
1628 endID=${3}
1629 waitForJOBID=${4}
1630 command=${5}
444b9916 1631 shift 5
78d0bbfd 1632 local commandArgs=("$@")
444b9916 1633
1634 #add quote strings around the extra arguments
1635 for ((i=0; i<${#commandArgs[@]}; i++)); do
1636 commandArgs[i]=\"${commandArgs[i]}\"
1637 done
cb473870 1638
71353a5f 1639 [[ -z ${waitForJOBID} ]] && waitForJOBID=0
cb473870 1640
1641 newFarm=$(which qsub|grep "^/usr/bin/qsub")
1642
1643 batchSystem="SGE"
1644
71353a5f 1645 if [[ -z "${newFarm}" ]]
cb473870 1646 then
1647 #old LSF
1648 # submit it (as job array)
71353a5f 1649 nFiles=$(( ${endID}-${startID}+1 ))
1650 while [ ${startID} -le ${nFiles} ] ; do
1651 if [ $(expr ${nFiles} - ${startID}) -gt 999 ] ; then
1652 endID=$(expr ${startID} + 999)
cb473870 1653 else
71353a5f 1654 endID=${nFiles}
cb473870 1655 fi
1656 if [[ ${waitForJOBID} -eq 0 ]]; then
71353a5f 1657 echo ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}"
1658 ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}"
cb473870 1659 else
71353a5f 1660 echo ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -w "ended(${waitForJOBID})" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}"
1661 ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -w "ended(${waitForJOBID})" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}"
cb473870 1662 fi
71353a5f 1663 startID=$(expr ${endID} + 1)
cb473870 1664 done
1665 else
1666 #new SGE farm
1667 if [[ ${waitForJOBID} =~ "000" ]]; then
444b9916 1668 echo ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}"
1669 ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}"
cb473870 1670 else
444b9916 1671 echo ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -hold_jid "${waitForJOBID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}"
1672 ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -hold_jid "${waitForJOBID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}"
cb473870 1673 fi
1674 fi
5de3064b 1675 return 0
cb473870 1676}
1677
1678goSubmitBatch()
5dfac970 1679{
71353a5f 1680 if [[ $# -lt 3 ]]; then
1681 echo "minimal use:"
1682 echo " ${0} submit fileList productionID configFile"
1683 return 0
1684 fi
1685
1686 productionID=${1}
1687 inputList=${2}
1688 configFile=${3:-"benchmark.config"}
0eaf38c7 1689 #if which greadlink; then configFile=$(greadlink -f ${configFile}); fi
71353a5f 1690 shift 3
444b9916 1691 extraOpts=("$@")
6fc48f9e 1692 if ! parseConfig ${configFile} "${extraOpts[@]}"; then return 1; fi
39527a15 1693
b0e401b4 1694 #batch systems/makeflow sometimes handle spaces in arguments poorly, so encode them
1695 for (( i=0;i<${#extraOpts[@]};i++ )); do
1696 extraOpts[i]=$(encSpaces "${extraOpts[i]}")
1697 done
1698 extraOpts+=("encodedSpaces=1")
ef77d296 1699 #this removes the copy of the done file used by makeflow (in the running dir)
1700 extraOpts+=("removeTMPdoneFile=1")
b0e401b4 1701
39527a15 1702 #record the working directory provided by the batch system
1703 batchWorkingDirectory=${PWD}
fc7e1e45 1704
5dfac970 1705 #redirect all output to submit.log
1706 echo "redirecting all output to ${PWD}/submit_${productionID//"/"/_}.log"
1707 exec 7>&1
1708 exec 1>submit_${productionID//"/"/_}.log 2>&1
fc7e1e45 1709
5dfac970 1710 umask 0002
71353a5f 1711 echo ${0}" submit $*"
1712 if [[ -z "${inputList}" || -z "${productionID}" ]]
5dfac970 1713 then
fc7e1e45 1714 echo
71353a5f 1715 echo " Usage: ${0} submit inputList productionID [configFile=benchmark.config]"
fc7e1e45 1716 echo
71353a5f 1717 return
fc7e1e45 1718 fi
fc7e1e45 1719
5dfac970 1720 # check if config file is there
71353a5f 1721 if [ ! -f ${configFile} ]; then
1722 echo "ERROR! Config File '${configFile}' not found" >&2
1723 return
5dfac970 1724 else
71353a5f 1725 echo "Using Config File: '${configFile}'"
fc7e1e45 1726 fi
1727
9510565c 1728 [[ ! -f ${alirootEnv} ]] && echo "alirootEnv script ${alirootEnv} not found!..." && return 1
fc7e1e45 1729
71353a5f 1730 #move the script, config and some other stuff to ${commonOutputPath} first, then use them from there
0eaf38c7 1731 self=${0}
1732 #if which greadlink; then self=$(greadlink -f "${0}"); fi
71353a5f 1733 configPath=$(dirname ${configFile})
5dfac970 1734 export commonOutputPath=${baseOutputDirectory}/${productionID}
71353a5f 1735
cb473870 1736 mkdir -p ${commonOutputPath}
71353a5f 1737 mkdir -p ${commonOutputPath}/logs
1738 mkdir -p ${commonOutputPath}/meta
1739
1740 cp ${self} ${commonOutputPath}
1741 cp ${configFile} ${commonOutputPath}
1742 cp ${inputList} ${commonOutputPath}
cb473870 1743 self=${commonOutputPath}/${self##*/}
71353a5f 1744 chmod u+x ${self}
cb473870 1745 configFile=${commonOutputPath}/${configFile##*/}
1746 inputList=${commonOutputPath}/${inputList##*/}
fc7e1e45 1747
5dfac970 1748 #convert to absolut pathnames
0eaf38c7 1749 #if which greadlink; then inputList=$(greadlink -f "${inputList}"); fi
5dfac970 1750 #make list of runs
71353a5f 1751 if [[ -z ${runNumber} ]]; then
1752 listOfRuns=($(while read x; do guessRunNumber ${x}; done < ${inputList} | sort | uniq))
5dfac970 1753 else
71353a5f 1754 listOfRuns=${runNumber}
5dfac970 1755 fi
fc7e1e45 1756
0eaf38c7 1757 #if which greadlink; then alirootSource=$(greadlink -f "${alirootSource}"); fi
fc7e1e45 1758
5dfac970 1759 echo ""
1760 echo "### BEGIN CONFIGURATION ###"
1761 echo ""
1762 echo "GENERAL:"
1763 echo ""
71353a5f 1764 echo " productionID: ${productionID}"
1765 echo " batchCommand: ${batchCommand}"
1766 echo " batchFlags: ${batchFlags}"
9510565c 1767 echo " alirootEnv: ${alirootEnv}"
1768 ${alirootEnv} echo ' ALICE_ROOT: ${ALICE_ROOT}'
1769 ${alirootEnv} echo ' ALIROOT_RELEASE: ${ALICE_RELEASE}'
71353a5f 1770 echo " inputList: ${inputList}"
1771 echo " configPath: ${configPath}"
1772 echo " commonOutputPath: ${commonOutputPath}"
1773 echo " defaultOCDB: ${defaultOCDB}"
1774 echo " autoOCDB: ${autoOCDB}"
1775 echo " recoTriggerOptions: ${recoTriggerOptions}"
5dfac970 1776 echo " runs:"
1777 echo " ${listOfRuns[*]}"
1778 echo ""
1779 echo "THE TRAIN WILL RUN:"
fc7e1e45 1780
71353a5f 1781 if [ ${runCPass0reco} -eq 1 ]; then
5dfac970 1782 echo " Pass0 - Recontruction"
1783 fi
fc7e1e45 1784
71353a5f 1785 if [ ${runCPass0MergeMakeOCDB} -eq 1 ]; then
5dfac970 1786 echo " Pass0 - merging and OCDB export"
1787 fi
fc7e1e45 1788
71353a5f 1789 if [ ${runCPass1reco} -eq 1 ]; then
5dfac970 1790 echo " Pass1 - Recontruction"
1791 fi
71353a5f 1792 if [ ${runCPass1MergeMakeOCDB} -eq 1 ]; then
5dfac970 1793 echo " Pass1 - merging and OCDB export"
1794 fi
fc7e1e45 1795
5dfac970 1796 echo ""
1797 echo "LIMITS:"
71353a5f 1798 echo " max. Events/Chunk: ${nEvents}"
1799 echo " max. Number of Chunks per Run: ${nMaxChunks}"
5dfac970 1800 echo ""
1801 echo "### END CONFIGURATION ###"
1802 echo ""
fc7e1e45 1803
36b5de1a 1804
5dfac970 1805 # check if input file is there
71353a5f 1806 if [ ! -f ${inputList} ]; then
1807 echo "ERROR! Input List '${inputList}' not found" >&2
1808 return
5dfac970 1809 fi
1810
1811 # define jobid (for dependent jobs)
cb473870 1812 date=$(date +%Y_%m_%d_%H%M%S)
5dfac970 1813 #for each run we submit one jobarray:
fc7e1e45 1814 for runNumber in ${listOfRuns[*]}; do
cb473870 1815
71353a5f 1816 [[ -z ${runNumber} ]] && continue
fc7e1e45 1817 [[ ! ${runNumber} =~ ^[0-9]*[0-9]$ ]] && continue
cb473870 1818
1819 JOBpostfix="${productionID//"/"/_}_${runNumber}_${date}"
1820 JOBID1="p0_${JOBpostfix}"
1821 JOBID1wait="w0_${JOBpostfix}"
1822 JOBID2="m0_${JOBpostfix}"
1823 JOBID2wait="wm0_${JOBpostfix}"
1824 JOBID3="op0_${JOBpostfix}"
1825 JOBID3wait="wop0_${JOBpostfix}"
1826 JOBID4="p1_${JOBpostfix}"
1827 JOBID4wait="w1_${JOBpostfix}"
1828 JOBID5="m1_${JOBpostfix}"
1829 JOBID5wait="wm1_${JOBpostfix}"
1830 JOBID6="s1_${JOBpostfix}"
1831 JOBID6wait="ws1_${JOBpostfix}"
1832 JOBID7="QA_${JOBpostfix}"
1833 JOBmakeESDlistCPass1="lp1_${JOBpostfix}"
1834 JOBfilterESDcpass1="fp1_${JOBpostfix}"
1835 LASTJOB="000"
1836
0eaf38c7 1837 oneInputFile=$(egrep -m1 "${runNumber}/" ${inputList})
cb473870 1838
1839 currentDefaultOCDB=${defaultOCDB}
1840 [[ -z ${autoOCDB} ]] && autoOCDB=1
1841 if [[ ${autoOCDB} -ne 0 ]]; then
71353a5f 1842 currentDefaultOCDB=$(setYear ${oneInputFile} ${defaultOCDB})
cb473870 1843 fi
ef77d296 1844 period=$(guessPeriod ${oneInputFile})
1845 year=$(guessYear ${oneInputFile})
cb473870 1846
71353a5f 1847 echo "submitting run ${runNumber} with OCDB ${currentDefaultOCDB}"
1848
1849 ###############################################################################
1850 #run one chunk with valgrind:
6fc48f9e 1851 if [[ -n ${profilingCommand} ]]; then
1852 [[ -z ${nEventsProfiling} ]] && nEventsProfiling=2
1853 [[ -z ${profilingCommand} ]] && profilingCommand="/usr/bin/valgrind --tool=callgrind --num-callers=40 -v --trace-children=yes"
ef77d296 1854 submit "profile-${JOBpostfix}" 1 1 000 "${alirootEnv} ${self}" CPass0 ${commonOutputPath}/${year}/${period}/000${runNumber}/${jobindex} ${oneInputFile} ${nEventsProfiling} ${currentDefaultOCDB} ${configFile} ${runNumber} ${jobindex} useProfilingCommand=$(encSpaces "${profilingCommand}") "${extraOpts[@]}"
71353a5f 1855 fi
fc7e1e45 1856
5dfac970 1857 ################################################################################
1858 ################################################################################
1859 # run the CPass0 if requested
fc7e1e45 1860
71353a5f 1861 if [ ${runCPass0reco} -eq 1 ]; then
fc7e1e45 1862
fc7e1e45 1863 echo
71353a5f 1864 echo "starting CPass0... for run ${runNumber}"
fc7e1e45 1865 echo
fc7e1e45 1866
5dfac970 1867 # create directory and copy all files that are needed
ef77d296 1868 targetDirectory="${commonOutputPath}/${year}/${period}/000${runNumber}/cpass0"
71353a5f 1869 mkdir -p ${targetDirectory}
1870
cb473870 1871 filesCPass0=(
71353a5f 1872 "${configPath}/runCPass0.sh"
1873 "${configPath}/recCPass0.C"
1874 "${configPath}/runCalibTrain.C"
1875 "${configPath}/localOCDBaccessConfig.C"
1876 "${configPath}/OCDB*.root"
1877 "${configPath}/sim.C"
1878 "${configPath}/rec.C"
1879 "${configPath}/Config.C"
cb473870 1880 )
1881 for file in ${filesCPass0[*]}; do
1882 [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
1883 done
fc7e1e45 1884
71353a5f 1885 localInputList=${targetDirectory}/${inputList##*/}
1886 [[ ! -f ${localInputList} ]] && egrep "\/000${runNumber}\/" ${inputList} > ${localInputList}
5dfac970 1887 # limit nFiles to nMaxChunks
71353a5f 1888 nFiles=$(wc -l < ${localInputList})
1889 [[ ${nFiles} -eq 0 ]] && echo "list contains ZERO files! exiting..." && return 1
1890 echo "raw files in list: ${nFiles}"
1891 if [[ ${nMaxChunks} -gt 0 && ${nMaxChunks} -le ${nFiles} ]]; then
1892 nFiles=${nMaxChunks}
5dfac970 1893 fi
71353a5f 1894 echo "raw files to process: ${nFiles}"
1895 [[ -z "${percentProcessedFilesToContinue}" ]] && percentProcessedFilesToContinue=100
1896 if [[ ${percentProcessedFilesToContinue} -eq 100 ]]; then
1897 nFilesToWaitFor=${nFiles}
5dfac970 1898 else
71353a5f 1899 nFilesToWaitFor=$(( ${nFiles}-${nFiles}/(100/(100-${percentProcessedFilesToContinue})) ))
5dfac970 1900 fi
71353a5f 1901 echo "requested success rate is ${percentProcessedFilesToContinue}%"
1902 echo "merging will start after ${nFilesToWaitFor} jobs are done"
fc7e1e45 1903
444b9916 1904 submit ${JOBID1} 1 ${nFiles} 000 "${alirootEnv} ${self}" CPass0 ${targetDirectory} ${localInputList} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} -1 "${extraOpts[@]}"
fc7e1e45 1905
5dfac970 1906 ## submit a monitoring job that will run until a certain number of jobs are done with reconstruction
0ff60de9 1907 submit "${JOBID1wait}" 1 1 000 "${alirootEnv} ${self}" WaitForOutput ${commonOutputPath} "meta/cpass0.job*.run${runNumber}.done" ${nFilesToWaitFor} ${maxSecondsToWait}
71353a5f 1908 LASTJOB=${JOBID1wait}
fc7e1e45 1909
5dfac970 1910 fi #end running CPass0
1911 ################################################################################
fc7e1e45 1912
fc7e1e45 1913
5dfac970 1914 ################################################################################
1915 # submit merging of CPass0, depends on the reconstruction
fc7e1e45 1916
71353a5f 1917 if [ ${runCPass0MergeMakeOCDB} -eq 1 ]; then
fc7e1e45 1918
5dfac970 1919 echo
71353a5f 1920 echo "submit CPass0 merging for run ${runNumber}"
5dfac970 1921 echo
fc7e1e45 1922
ef77d296 1923 targetDirectory="${commonOutputPath}/${year}/${period}/000${runNumber}/cpass0"
71353a5f 1924 mkdir -p ${targetDirectory}
fc7e1e45 1925
cb473870 1926 #copy the scripts
1927 filesMergeCPass0=(
71353a5f 1928 "${configPath}/OCDB.root"
1929 "${configPath}/mergeMakeOCDB.byComponent.sh"
1930 "${configPath}/mergeMakeOCDB.sh"
1931 "${configPath}/localOCDBaccessConfig.C"
1932 "${configPath}/mergeByComponent.C"
1933 "${configPath}/makeOCDB.C"
1934 "${configPath}/merge.C"
cb473870 1935 )
1936 for file in ${filesMergeCPass0[*]}; do
1937 [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
1938 done
1939
444b9916 1940 submit ${JOBID2} 1 1 "${LASTJOB}" "${alirootEnv} ${self}" MergeCPass0 ${targetDirectory} ${currentDefaultOCDB} ${configFile} ${runNumber} cpass0.calib.run${runNumber}.list "${extraOpts[@]}"
71353a5f 1941 LASTJOB=${JOBID2}
fc7e1e45 1942
71353a5f 1943 if [[ -n ${generateMC} ]]; then
444b9916 1944 submit "mrl${JOBpostfix}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" PrintValues sim ${commonOutputPath}/meta/sim.run${runNumber}.list ${commonOutputPath}/meta/cpass0.job*.run${runNumber}.done
71353a5f 1945 LASTJOB="mrl${JOBpostfix}"
1946 fi
fc7e1e45 1947
5dfac970 1948 echo
fc7e1e45 1949 fi
5dfac970 1950 # end of merging CPass0
1951 ################################################################################
fc7e1e45 1952
5dfac970 1953 ################################################################################
1954 ################################################################################
1955 # run the CPass1 if requested
fc7e1e45 1956
71353a5f 1957 if [ ${runCPass1reco} -eq 1 ]; then
fc7e1e45 1958
ef77d296 1959 targetDirectory="${commonOutputPath}/${year}/${period}/000${runNumber}/cpass1"
71353a5f 1960 rm -f ${commonOutputPath}/meta/cpass1.job*.run${runNumber}.done
fc7e1e45 1961
5dfac970 1962 # safety feature: if we are re-running for any reason we want to delete the previous output first.
71353a5f 1963 [[ -d ${targetDirectory} ]] && rm -rf ${targetDirectory}/* && echo "removed old output at ${targetDirectory}/*"
fc7e1e45 1964
5dfac970 1965 echo
71353a5f 1966 echo "starting CPass1... for run ${runNumber}"
5dfac970 1967 echo
1968
1969 # create directory and copy all files that are needed
71353a5f 1970 mkdir -p ${targetDirectory}
5dfac970 1971
cb473870 1972 filesCPass1=(
71353a5f 1973 "${configPath}/runCPass1.sh"
1974 "${configPath}/recCPass1.C"
1975 "${configPath}/recCPass1_OuterDet.C"
1976 "${configPath}/runCalibTrain.C"
1977 "${configPath}/QAtrain_duo.C"
1978 "${configPath}/localOCDBaccessConfig.C"
1979 "${configPath}/OCDB.root"
cb473870 1980 )
1981 for file in ${filesCPass1[*]}; do
1982 [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
1983 done
5dfac970 1984
71353a5f 1985 if [[ -n ${generateMC} ]]; then
1986 localInputList=${commonOutputPath}/meta/sim.run${runNumber}.list
1987 else
1988 localInputList=${targetDirectory}/${inputList##*/}
1989 [[ ! -f ${localInputList} ]] && egrep "\/000${runNumber}\/" ${inputList} > ${localInputList}
1990 fi
5dfac970 1991 # limit nFiles to nMaxChunks
71353a5f 1992 nFiles=$(wc -l < ${localInputList})
1993 [[ ${nFiles} -eq 0 ]] && echo "list contains ZERO files! continuing..." && continue
1994 echo "raw files in list: ${nFiles}"
1995 if [[ ${nMaxChunks} -gt 0 && ${nMaxChunks} -le ${nFiles} ]]; then
1996 nFiles=${nMaxChunks}
5dfac970 1997 fi
71353a5f 1998 echo "raw files to process: ${nFiles}"
1999 [[ -z "${percentProcessedFilesToContinue}" ]] && percentProcessedFilesToContinue=100
2000 if [[ ${percentProcessedFilesToContinue} -eq 100 ]]; then
2001 nFilesToWaitFor=${nFiles}
fc7e1e45 2002 else
71353a5f 2003 nFilesToWaitFor=$(( ${nFiles}-${nFiles}/(100/(100-${percentProcessedFilesToContinue})) ))
fc7e1e45 2004 fi
71353a5f 2005 echo "requested success rate is ${percentProcessedFilesToContinue}%"
2006 echo "merging will start after ${nFilesToWaitFor} jobs are done"
fc7e1e45 2007
444b9916 2008 submit ${JOBID4} 1 ${nFiles} "${LASTJOB}" "${alirootEnv} ${self}" CPass1 ${targetDirectory} ${localInputList} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} -1 "${extraOpts[@]}"
fc7e1e45 2009
5dfac970 2010 ################################################################################
2011 ## submit a monitoring job that will run until a certain number of jobs are done with reconstruction
0ff60de9 2012 submit "${JOBID4wait}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" WaitForOutput ${commonOutputPath} "meta/cpass1.job*.run${runNumber}.done" ${nFilesToWaitFor} ${maxSecondsToWait}
71353a5f 2013 LASTJOB=${JOBID4wait}
5dfac970 2014 ################################################################################
fc7e1e45 2015
5dfac970 2016 echo
2017 fi #end running CPass1
fc7e1e45 2018
5dfac970 2019 ################################################################################
2020 # submit merging of CPass1, depends on the reconstruction
71353a5f 2021 if [ ${runCPass1MergeMakeOCDB} -eq 1 ]; then
fc7e1e45 2022
5dfac970 2023 echo
71353a5f 2024 echo "submit CPass1 merging for run ${runNumber}"
5dfac970 2025 echo
fc7e1e45 2026
ef77d296 2027 targetDirectory="${commonOutputPath}/${year}/${period}/000${runNumber}/cpass1"
71353a5f 2028 rm -f ${commonOutputPath}/meta/merge.cpass1.run${runNumber}.done
2029 mkdir -p ${targetDirectory}
fc7e1e45 2030
cb473870 2031 # copy files
2032 filesMergeCPass1=(
71353a5f 2033 "${configPath}/OCDB.root"
2034 "${configPath}/localOCDBaccessConfig.C"
2035 "${configPath}/mergeMakeOCDB.byComponent.sh"
2036 "${configPath}/mergeByComponent.C"
2037 "${configPath}/makeOCDB.C"
2038 "${configPath}/merge.C"
2039 "${configPath}/mergeMakeOCDB.sh"
2040 "${configPath}/QAtrain_duo.C"
cb473870 2041 )
2042 for file in ${filesMergeCPass1[*]}; do
2043 [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
2044 done
2045
444b9916 2046 submit "${JOBID5}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" MergeCPass1 ${targetDirectory} ${currentDefaultOCDB} ${configFile} ${runNumber} cpass1.calib.run${runNumber}.list cpass1.QA.run${runNumber}.lastMergingStage.txt.list cpass1.filtered.run${runNumber}.list "${extraOpts[@]}"
71353a5f 2047 LASTJOB=${JOBID5}
5dfac970 2048 echo
2049 fi
2050
444b9916 2051 ###############################
2052 #if [ ${runESDfiltering} -eq 1 ]; then
2053 # rm -f ${commonOutputPath}/cpass1.ESD.run${runNumber}.list
2054 # rm -f ${commonOutputPath}/meta/filtering.cpass1.run*.done
2055 # echo
2056 # echo submitting filtering for run ${runNumber}
2057 # echo
2058 # submit "${JOBmakeESDlistCPass1}" 1 1 "${LASTJOB}" "${self}" PrintValues esd ${commonOutputPath}/meta/cpass1.ESD.run${runNumber}.list ${commonOutputPath}/meta/cpass1.job*.run${runNumber}.done
ef77d296 2059 # submit "${JOBfilterESDcpass1}" 1 1 "${JOBmakeESDlistCPass1}" "${alirootEnv} ${self}" MakeFilteredTrees ${commonOutputPath}/${year}/${period}/000${runNumber}/cpass1 ${runNumber} ${commonOutputPath}/meta/cpass1.ESD.run${runNumber}.list ${filteringFactorHighPt} ${filteringFactorV0s} ${currentDefaultOCDB} 1000000 0 10000000 0 ${configFile} AliESDs_Barrel.root "${extraOpts[@]}"
444b9916 2060 # LASTJOB=${JOBfilterESDcpass1}
2061 #fi
cb473870 2062
5dfac970 2063 done
2064
71353a5f 2065 #################################################################################
2066 #################################################################################
444b9916 2067 #if [ ${runESDfiltering} -eq 1 ]; then
2068 # submit "${JOBID5wait}" 1 1 "${LASTJOB}" "${self}" WaitForOutput ${commonOutputPath} "meta/filtering.cpass1.run*.done" "${#listOfRuns[@]}" ${maxSecondsToWait}
2069 #else
2070 submit "${JOBID5wait}" 1 1 "${LASTJOB}" "${self}" WaitForOutput ${commonOutputPath} "meta/merge.cpass1.run*.done" ${#listOfRuns[@]} ${maxSecondsToWait}
2071 #fi
71353a5f 2072 LASTJOB=${JOBID5wait}
cb473870 2073
5de3064b 2074 #################################################################################
2075 echo
2076 echo "submit make a summary"
2077 echo
5dfac970 2078
3d530ff9 2079 [[ -z ${alirootEnvQA} ]] && alirootEnvQA=$(encSpaces "${alirootEnv}")
4f8ecc60 2080 submit "${JOBID6}" 1 1 "${LASTJOB}" "${alirootEnvQA} ${self}" MakeSummary ${configFile} "commonOutputPath=${commonOutputPath}"
5de3064b 2081 LASTJOB=${JOBID6}
71353a5f 2082 #################################################################################
5dfac970 2083
5dfac970 2084 #restore stdout
2085 exec 1>&7 7>&-
2086 echo "jobs submitted."
5de3064b 2087 return 0
fc7e1e45 2088}
2089
cb473870 2090goWaitForOutput()
78d0bbfd 2091(
cb473870 2092 umask 0002
71353a5f 2093 [[ $# -lt 3 ]] && echo "goWaitForOutput() wrong number of arguments, exiting.." && return 1
2094 echo searchPath=${1}
2095 echo fileName=${2}
2096 echo numberOfFiles=${3}
2097 echo maxSecondsToWait=${4}
2098 searchPath=${1}
2099 fileName=${2}
2100 numberOfFiles=${3}
2101 maxSecondsToWait=${4}
0ff60de9 2102 echo "command to be executed: /bin/ls -1 ${searchPath}/${fileName}"
71353a5f 2103 [[ -z "${maxSecondsToWait}" ]] && maxSecondsToWait=$(( 3600*12 ))
cb473870 2104 while true; do
cb473870 2105 n=$(/bin/ls -1 ${searchPath}/${fileName} 2>/dev/null | wc -l)
71353a5f 2106 [[ ${n} -gt 0 ]] && echo "found ${n} X ${fileName}"
2107 [[ ${n} -ge ${numberOfFiles} ]] && break
2108 [[ ${SECONDS} -gt ${maxSecondsToWait} ]] && echo "timeout of ${maxSecondsToWait}s!" && break
cb473870 2109 sleep 60
2110 done
2111 echo "DONE! exiting..."
5de3064b 2112 return 0
78d0bbfd 2113)
cb473870 2114
71353a5f 2115mergeSysLogs()
78d0bbfd 2116(
71353a5f 2117 outputFile=${1}
2118 shift
2119 inputFiles="$@"
2120 i=0
2121 if ! ls -1 ${inputFiles} &>/dev/null; then echo "the files dont exist!: ${inputFiles}"; return 1; fi
2122 while read x; do
2123 runNumber=$(guessRunNumber ${x})
2124 [[ -z ${runNumber} ]] && echo "run number cannot be guessed for ${x}" && continue
2125 awk -v run=${runNumber} -v i=${i} 'NR > 1 {print run" "$0} NR==1 && i==0 {print "run/I:"$0}' ${x}
2126 (( i++ ))
2127 done < <(ls -1 ${inputFiles}) > ${outputFile}
5de3064b 2128 return 0
78d0bbfd 2129)
71353a5f 2130
2131goMakeMergedSummaryTree()
78d0bbfd 2132(
71353a5f 2133 # create list of calibration entries
2134 # takes no arguments, just run it in the base output
2135 # directory with the following files in the working directory
2136 #
2137 # Calibration file lists:
2138 # cpass0.dcsTree.list, cpass1.dcsTree.list
2139 # QA trending root file:
2140 # trending.root
2141 #
2142 # Production infoo ascii files:
2143 # summary_pass0.tree
2144 # summary_pass1.tree
2145 #
617f8886 2146
71353a5f 2147 [[ ! -f cpass0.dcsTree.list ]] && echo "no cpass0.dcsTree.list" && return 1
2148 [[ ! -f cpass1.dcsTree.list ]] && echo "no cpass1.dcsTree.list" && return 1
2149 [[ ! -f trending.root ]] && echo "no trending.root" && return 1
2150 [[ ! -f summary_pass0.tree ]] && echo "no summary_pass0.tree" && return 1
2151 [[ ! -f summary_pass1.tree ]] && echo "no summary_pass1.tree" && return 1
2152
2153 #first, dump the C macro to file
2154 cat << EOF > mergeTree.C
2155 //
2156 // Merge summary information
2157 // Following files are expected to be in the working directory
2158 //
2159 // Calibration file lists:
2160 // cpass0.dcsTree.list, cpass1.dcsTree.list
2161 // QA trending root file:
2162 // trending.root
2163 //
2164 // Production infoo ascii files:
2165 // summary_pass0.tree
2166 // summary_pass1.tree
2167 //
2168 void mergeTree(){
2169 //
2170 //
2171 //
2172 // Calibration values dump
2173 //
839b07ca 2174 //Printf("MakeTreeFromList cpass0.dcsTree.list");
71353a5f 2175 AliXRDPROOFtoolkit::MakeTreeFromList("Calib.TPC.CPass0.root", "dcs","dcs","cpass0.dcsTree.list",1);
839b07ca 2176 //Printf("MakeTreeFromList cpass1.dcsTree.list");
71353a5f 2177 AliXRDPROOFtoolkit::MakeTreeFromList("Calib.TPC.CPass1.root", "dcs","dcs","cpass1.dcsTree.list",1);
2178 //
2179 // Calibration status dump
2180 //
2181 TFile *fprod = TFile::Open("fproduction.root","recreate");
2182 TTree tree0, tree1;
839b07ca 2183 //Printf("reading summary_pass0.tree");
71353a5f 2184 tree0.ReadFile("summary_pass0.tree");
839b07ca 2185 //Printf("reading summary_pass1.tree");
71353a5f 2186 tree1.ReadFile("summary_pass1.tree");
2187 tree0.Write("CPass0");
2188 tree1.Write("CPass1");
2189 fprod->Close();
2190 //
2191 //
2192 //
2193 TString stringSetup="";
2194 stringSetup+="1#QA.TPC#run#SummaryTPCQA/tpcQA#trending.root+"; //
2195 stringSetup+="1#Calib.TPC.CPass0#run#dcs#Calib.TPC.CPass0.root+"; //
2196 stringSetup+="1#Calib.TPC.CPass1#run#dcs#Calib.TPC.CPass1.root+"; //
2197 //
2198 stringSetup+="1#CPass0#runnumber#CPass0#fproduction.root+"; //
2199 stringSetup+="1#CPass1#runnumber#CPass1#fproduction.root+"; //
2200 //
839b07ca 2201 //Printf("stringSetup: %s", stringSetup.Data());
71353a5f 2202 AliXRDPROOFtoolkit::JoinTreesIndex("outAll.root","joinAll","run",stringSetup.Data(), 1);
2203 }
2204EOF
2205
2206 aliroot -b -q "mergeTree.C" > mergeTrees.log
2207 return $?
78d0bbfd 2208)
71353a5f 2209
177b9ebc 2210stackTraceTree()
2211(
e00eb304 2212 if [[ $# -lt 1 ]]; then
2213 echo 'make stacktrace processing in case of standard root crash log'
2214 echo 'input is a (list of) text files with the stack trace (either gdb aoutput'
2215 echo 'produced with e.g. gdb --batch --quiet -ex "bt" -ex "quit" aliroot core,'
2216 echo 'or the root crash log), output is a TTree formatted table.'
2217 echo 'example usage:'
2218 echo 'benchmark.sh stackTraceTree /foo/*/rec.log'
2219 echo 'benchmark.sh stackTraceTree $(cat file.list)'
0685f982 2220 echo 'benchmark.sh stackTraceTree `cat file.list`'
e00eb304 2221 return 0
2222 fi
4a3c2422 2223 gawk '
177b9ebc 2224 BEGIN {
80c75839 2225 print "frame/I:method/C:line/C:cpass/I:aliroot/I:file/C";
177b9ebc 2226 RS="#[0-9]*";
2227 aliroot=0;
4a3c2422 2228 read=1;
177b9ebc 2229 }
4a3c2422 2230 /There was a crash/ {read=1;}
2231 /The lines below might hint at the cause of the crash/ {read=0;}
2232 read==1 {
177b9ebc 2233 if ($3 ~ /Ali*/) aliroot=1; else aliroot=0;
2234 gsub("#","",RT);
80c75839 2235 if ($NF!="" && RT!="" && $3!="") print RT" "$3" "$NF" "0" "aliroot" "FILENAME
177b9ebc 2236 }
2237 ' "$@" 2>/dev/null
2238)
2239
71353a5f 2240goMakeSummary()
78d0bbfd 2241(
71353a5f 2242 #all the final stuff goes in here for ease of use:
2243 # summary logs
2244 # qa plot making
2245 # final file lists
ef77d296 2246 # runs in current dir - in makeflow mode it can run LOCAL, then the QA plots and summaries
2247 # will appear in the submission dir.
444b9916 2248 #some defaults:
2249 log="summary.log"
d95a5a53 2250 jsonLog="summary.json"
444b9916 2251 productionID="qa"
71353a5f 2252
2253 configFile=${1}
2254 shift 1
444b9916 2255 extraOpts=("$@")
6fc48f9e 2256 if ! parseConfig ${configFile} "${extraOpts[@]}"; then return 1; fi
71353a5f 2257
0eaf38c7 2258 #if which greadlink; then configFile=$(greadlink -f ${configFile}); fi
839b07ca 2259
39527a15 2260 #record the working directory provided by the batch system
2261 batchWorkingDirectory=${PWD}
ef77d296 2262
d95a5a53 2263 logTmp="${batchWorkingDirectory}/${log}"
2264 jsonLogTmp="${batchWorkingDirectory}/${jsonLog}"
39527a15 2265
71353a5f 2266 [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
2267
71353a5f 2268 [[ ! -f ${configFile} ]] && echo "no config file ${configFile}!" && return
2269
2270 [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
2271
39527a15 2272 #copy some useful stuff
d95a5a53 2273 [ -f "${commonOutputPath}/${configFile}" ] || paranoidCp "${configFile}" "${commonOutputPath}"
39527a15 2274
ef77d296 2275 exec &> >(tee ${logTmp})
39527a15 2276
71353a5f 2277 #summarize the global stuff
2278 echo "env script: ${alirootSource} ${alirootEnv}"
2279 echo "\$ALICE_ROOT=${ALICE_ROOT}"
2280 echo "commonOutputPath=${commonOutputPath}"
2281
2282 #summarize the stacktraces
ef77d296 2283 stackTraceTree ${commonOutputPath}/*/*/000*/cpass0/*/stacktrace* > stacktrace_cpass0.tree
2284 stackTraceTree ${commonOutputPath}/*/*/000*/cpass1/*/stacktrace* > stacktrace_cpass1.tree
71353a5f 2285
d95a5a53 2286 # json header: open array of objects
2287 echo '[' > "${jsonLogTmp}"
2288
71353a5f 2289 echo total numbers for the production:
2290 echo
2291 awk 'BEGIN {nFiles=0;nCore=0;}
2292 /^calibfile/ {nFiles++;}
2293 /core dumped/ {nCore++i;}
ef77d296 2294 END {print "cpass0 produced "nFiles" calib files, "nCore" core files";}' ${commonOutputPath}/meta/cpass0.job*done 2>/dev/null
71353a5f 2295 awk 'BEGIN {nOK=0; nBAD=0; }
2296 /\/rec.log OK/ {nOK++;}
2297 /\/rec.log BAD/ {nBAD++;}
2298 /stderr BAD/ {if ($0 ~ /rec.log/){nBAD++;}}
ef77d296 2299 END {print "cpass0 reco: OK: "nOK"\tBAD: "nBAD;}' ${commonOutputPath}/meta/cpass0.job*done 2>/dev/null
71353a5f 2300 awk 'BEGIN {nOK=0; nBAD=0; }
2301 /\/calib.log OK/ {nOK++;}
2302 /\/calib.log BAD/ {nBAD++;}
ef77d296 2303 END {print "cpass0 calib: OK: "nOK"\tBAD: "nBAD;}' ${commonOutputPath}/meta/cpass0.job*done 2>/dev/null
71353a5f 2304
2305 awk 'BEGIN {nOK=0; nBAD=0; }
2306 /merge.log OK/ {nOK++;}
2307 /merge.log BAD/ {nBAD++;}
ef77d296 2308 END {print "cpass0 merge: OK: "nOK"\tBAD: "nBAD;}' ${commonOutputPath}/meta/merge.cpass0*done 2>/dev/null
71353a5f 2309 awk 'BEGIN {nOK=0; nBAD=0; }
2310 /ocdb.log OK/ {nOK++;}
2311 /ocdb.log BAD/ {nBAD++;}
ef77d296 2312 END {print "cpass0 OCDB: OK: "nOK"\tBAD: "nBAD;}' ${commonOutputPath}/meta/merge.cpass0*done 2>/dev/null
71353a5f 2313
2314 echo
2315 awk 'BEGIN {nFiles=0;nCore=0;}
2316 /^calibfile/ {nFiles++;}
2317 /core dumped/ {nCore++;}
ef77d296 2318 END {print "cpass1 produced "nFiles" calib files, "nCore" core files";}' ${commonOutputPath}/meta/cpass1.job*done 2>/dev/null
71353a5f 2319 awk 'BEGIN {nOK=0; nBAD=0; }
2320 /\/rec.log OK/ {nOK++;}
2321 /\/rec.log BAD/ {nBAD++;}
2322 /stderr BAD/ {if ($0 ~ /rec.log/){nBAD++;}}
ef77d296 2323 END {print "cpass1 reco: OK: "nOK"\tBAD: "nBAD;}' ${commonOutputPath}/meta/cpass1.job*done 2>/dev/null
71353a5f 2324 awk 'BEGIN {nOK=0; nBAD=0; }
2325 /\/calib.log OK/ {nOK++;}
2326 /\/calib.log BAD/ {nBAD++;}
ef77d296 2327 END {print "cpass1 calib: OK: "nOK"\tBAD: "nBAD;}' ${commonOutputPath}/meta/cpass1.job*done 2>/dev/null
71353a5f 2328
2329 awk 'BEGIN {nOK=0; nBAD=0; }
2330 /merge.log OK/ {nOK++;}
2331 /merge.log BAD/ {nBAD++;}
ef77d296 2332 END {print "cpass1 merge: OK: "nOK"\tBAD: "nBAD;}' ${commonOutputPath}/meta/merge.cpass1*done 2>/dev/null
71353a5f 2333 awk 'BEGIN {nOK=0; nBAD=0; }
2334 /ocdb.log OK/ {nOK++;}
2335 /ocdb.log BAD/ {nBAD++;}
ef77d296 2336 END {print "cpass1 OCDB: OK: "nOK"\tBAD: "nBAD;}' ${commonOutputPath}/meta/merge.cpass1*done 2>/dev/null
71353a5f 2337
2338 echo
2339 echo per run stats:
ef77d296 2340 /bin/ls -1 ${commonOutputPath}/meta/merge.cpass0.run*.done | while read x
71353a5f 2341do
5a2ee6ff 2342 dir=$(goPrintValues dir - ${x})
71353a5f 2343 runNumber=$(guessRunNumber ${dir})
2344 [[ -z ${runNumber} ]] && continue
2345
ef77d296 2346 if $(/bin/ls ${commonOutputPath}/meta/cpass0.job*.run${runNumber}.done &> /dev/null); then
71353a5f 2347 statusCPass0=( $(
2348 awk 'BEGIN {nOKrec=0;nBADrec=0;nOKcalib=0;nBADcalib=0;nOKstderr=0;nBADstderr=0;}
2349 /\/rec.log OK/ {nOKrec++;}
2350 /\/rec.log BAD/ {nBADrec++;}
2351 /stderr BAD/ {if ($0 ~ /rec.log/) {nBADrec++;} nBADstderr++;}
2352 /stderr OK/ {nOKstderr++;}
2353 /\/calib.log OK/ {nOKcalib++;}
2354 /\/calib.log BAD/ {nBADcalib++}
ef77d296 2355 END {print ""nOKrec" "nBADrec" "nOKstderr" "nBADstderr" "nOKcalib" "nBADcalib;}' ${commonOutputPath}/meta/cpass0.job*.run${runNumber}.done 2>/dev/null
71353a5f 2356 ) )
2357 fi
2358
ef77d296 2359 if $(/bin/ls ${commonOutputPath}/meta/cpass1.job*.run${runNumber}.done &>/dev/null); then
71353a5f 2360 statusCPass1=( $(
2361 awk 'BEGIN {nOKrec=0;nBADrec=0;nOKcalib=0;nBADcalib=0;nOKstderr=0;nBADstderr=0;nQAbarrelOK=0;nQAbarrelBAD=0;nQAouterOK=0;nQAouterBAD=0;}
2362 /\/rec.log OK/ {nOKrec++;}
2363 /\/rec.log BAD/ {nBADrec++;}
2364 /stderr BAD/ {if ($0 ~ /rec.log/) nBADrec++;nBADstderr++;}
2365 /stderr OK/ {nOKstderr++;}
2366 /\/calib.log OK/ {nOKcalib++;}
2367 /\/calib.log BAD/ {nBADcalib++}
2368 /\/qa_barrel.log OK/ {nQAbarrelOK++;}
2369 /\/qa_barrel.log BAD/ {nQAbarrelBAD++;}
2370 /\/qa_outer.log OK/ {nQAouterOK++;}
2371 /\/qa_outer.log BAD/ {nQAouterBAD++;}
ef77d296 2372 END {print ""nOKrec" "nBADrec" "nOKstderr" "nBADstderr" "nOKcalib" "nBADcalib" "nQAbarrelOK" "nQAbarrelBAD" "nQAouterOK" "nQAouterBAD;}' ${commonOutputPath}/meta/cpass1.job*.run${runNumber}.done 2>/dev/null
71353a5f 2373 ) )
2374 fi
2375
2376 statusOCDBcpass0=$(awk '/ocdb.log/ {print $2} ' ${x} 2>/dev/null)
2377 statusOCDBcpass1=$(awk '/ocdb.log/ {print $2}' ${x/cpass0/cpass1} 2>/dev/null)
2378 statusQA=$(awk '/mergeMakeOCDB.log/ {print $2}' ${x/cpass0/cpass1} 2>/dev/null)
2379
2380 printf "%s\t ocdb.log cpass0: %s\t ocdb.log cpass1: %s\tqa.log:%s\t| cpass0: rec:%s/%s stderr:%s/%s calib:%s/%s cpass1: rec:%s/%s stderr:%s/%s calib:%s/%s QAbarrel:%s/%s QAouter:%s/%s\n" ${runNumber} ${statusOCDBcpass0} ${statusOCDBcpass1} ${statusQA} ${statusCPass0[0]} ${statusCPass0[1]} ${statusCPass0[2]} ${statusCPass0[3]} ${statusCPass0[4]} ${statusCPass0[5]} ${statusCPass1[0]} ${statusCPass1[1]} ${statusCPass1[2]} ${statusCPass1[3]} ${statusCPass1[4]} ${statusCPass1[5]} ${statusCPass1[6]} ${statusCPass1[7]} ${statusCPass1[8]} ${statusCPass1[9]}
d95a5a53 2381
2382 # produce json summary
2383 statusOCDBcpass0json=false
2384 statusOCDBcpass1json=false
2385 statusQAjson=false
2386 [[ "$statusOCDBcpass0" == 'OK' ]] && statusOCDBcpass0json=true
2387 [[ "$statusOCDBcpass1" == 'OK' ]] && statusOCDBcpass1json=true
2388 [[ "$statusQA" == 'OK' ]] && statusQAjson=true
2389 cat >> "$jsonLogTmp" <<EOF
2390 {
2391 run: ${runNumber},
2392 status: { ocdb_pass0: ${statusOCDBcpass0json}, ocdb_pass1: ${statusOCDBcpass1json}, qa: ${statusQAjson} },
2393 cpass0: {
2394 reco: { n_ok: ${statusCPass0[0]}, n_bad: ${statusCPass0[1]} },
2395 stderr: { n_ok: ${statusCPass0[2]}, n_bad: ${statusCPass0[3]} },
2396 calib: { n_ok: ${statusCPass0[4]}, n_bad: ${statusCPass0[5]} }
2397 },
2398 cpass1: {
2399 reco: { n_ok: ${statusCPass1[0]}, n_bad: ${statusCPass1[1]} },
2400 stderr: { n_ok: ${statusCPass1[2]}, n_bad: ${statusCPass1[3]} },
2401 calib: { n_ok: ${statusCPass1[4]}, n_bad: ${statusCPass1[5]} },
2402 qabarrel: { n_ok: ${statusCPass1[6]}, n_bad: ${statusCPass1[7]} },
2403 qarouter: { n_ok: ${statusCPass1[8]}, n_bad: ${statusCPass1[9]} }
2404 }
2405 },
2406EOF
2407
71353a5f 2408done
2409
d95a5a53 2410 # json footer: close array of objects
2411 echo ']' >> "${jsonLogTmp}"
2412
71353a5f 2413 #make lists with output files - QA, trending, filtering and calibration
2414 ### wait for the merging of all runs to be over ###
2415 rm -f qa.list
2416 goPrintValues qafile qa.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2417 rm -f calib.list
2418 goPrintValues calibfile calib.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2419 rm -f trending.list
2420 goPrintValues trendingfile trending.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2421 rm -f filtering.list
444b9916 2422 goPrintValues filteredTree filtering.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
71353a5f 2423 rm -f cpass0.dcsTree.list
2424 goPrintValues dcsTree cpass0.dcsTree.list ${commonOutputPath}/meta/merge.cpass0.run*.done &>/dev/null
71353a5f 2425 rm -f cpass1.dcsTree.list
2426 goPrintValues dcsTree cpass1.dcsTree.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2427
2428 #merge trending
fecdbe75 2429 rm -f trending.root
2430 goMerge trending.list trending.root ${configFile} "${extraOpts[@]}" &> mergeTrending.log
71353a5f 2431
2432 goMakeSummaryTree ${commonOutputPath} 0
2433 goMakeSummaryTree ${commonOutputPath} 1
2434
839b07ca 2435 goCreateQAplots "${PWD}/qa.list" "${productionID}" "QAplots" "${configFile}" "${extraOpts[@]}" filteringList="${PWD}/filtering.list" &>createQAplots.log
71353a5f 2436
2437 #make a merged summary tree out of the QA trending, dcs trees and log summary trees
2438 goMakeMergedSummaryTree
2439
2440 #if set, email the summary
ef77d296 2441 [[ -n ${MAILTO} ]] && cat ${logTmp} | mail -s "benchmark ${productionID} done" ${MAILTO}
2442
d95a5a53 2443 #copy logs to destination
2444 paranoidCp "$logTmp" "${commonOutputPath}"
2445 paranoidCp "$jsonLogTmp" "${commonOutputPath}"
fecdbe75 2446
2447 #copy output files
054fdec5 2448 exec &> >(tee fileCopy.log)
13abb339 2449 paranoidCp QAplots ${commonOutputPath}
2450 paranoidCp *.list ${commonOutputPath}
2451 paranoidCp *.root ${commonOutputPath}
2452 paranoidCp *.log ${commonOutputPath}
054fdec5 2453 paranoidCp fileCopy.log ${commonOutputPath}
71353a5f 2454
2455 return 0
78d0bbfd 2456)
71353a5f 2457
2458goMakeSummaryTree()
78d0bbfd 2459(
71353a5f 2460 if [[ $# -lt 1 ]] ; then
2461 return
2462 fi
2463 #1. define vars/arrays
2464 DIR=${1} #use input or exec in current dir
2465 pass=${2-"0"} #pass from input
2466 outfile="summary_pass${pass}.tree"
2467 Ncolumns=0
2468 test -f ${outfile} && : >${outfile}
2469 errfile=${outfile/tree/err}
2470 test -f ${errfile} && : >${errfile}
2471
2472 declare -a counterString=(TOFevents TOFtracks TPCevents TPCtracks TRDevents TRDtracks T0events SDDevents SDDtracks MeanVertexevents)
2473 Ncounter=${#counterString[@]}
2474
2475 declare -a statusString=(TRDStatus TOFStatus TPCStatus T0Status MeanVertexStatus)
2476 Nstatus=${#statusString[@]}
2477
2478
2479 declare -a ratesString=(rec stderr calib qa_barrel qa_outer)
2480 Nrates=${#ratesString[@]}
2481
2482 runs=( $(ls -1 ${DIR}/meta/merge.cpass0* | while read x; do guessRunNumber $x; done) )
2483 Nruns=${#runs[@]}
2484
2485 echo -n runnumber/I >>${outfile}
2486 echo -n :cpass${pass}status/I >>${outfile}
2487 echo -n :cpass${pass}QAstatus/I >>${outfile}
2488 for i in ${ratesString[@]}; do
2489 echo -n :${i}OK/I >>${outfile}
2490 echo -n :${i}BAD/I >>${outfile}
2491
2492 done
2493 for i in ${counterString[@]} ${statusString[@]} ; do
2494 echo -n :${i}/I >>${outfile}
2495 done
2496 Ncolumns=$((2 + 2*Nrates + Ncounter + Nstatus))
2497 echo >> ${outfile}
2498
2499 #2. loop runs
2500
2501 for runnumber in ${runs[@]} ; do
2502
2503
2504
2505 filejob="${DIR}/meta/cpass${pass}.job*.run${runnumber}.done"
2506 filemerge="${DIR}/meta/merge.cpass${pass}.run${runnumber}.done"
2507 fileOCDB=$(grep /ocdb.log ${filemerge} | awk '{print $1}')
2508 if ! $(/bin/ls ${filemerge} &>/dev/null) ; then
2509 echo "${filemerge} does not exist!" >>${errfile}
2510 continue
2511 elif ! $(/bin/ls ${filejob} &>/dev/null) ; then
2512 echo "${filejob} does not exist!" >>${errfile}
2513 echo -n ${runnumber} >> ${outfile}
2514 for i in $(seq ${Ncolumns}) ; do
2515 echo -n "-1" >> ${outfile}
2516 done
2517 echo >> ${outfile}
2518 continue
2519 fi
2520 echo -n ${runnumber} >> ${outfile}
2521 #pass0status= grep '/ocdb.log' ${filemerge} | cut -d' ' -f2 | tr OK x1 | tr BAD xx0 | tr -d 'x'
2522 passStatus=$(grep '/ocdb.log' ${filemerge} | grep OK | wc -l)
2523 echo -n " ${passStatus}" >> ${outfile}
2524 qaStatus=$(grep '/mergeMakeOCDB.log' ${filemerge} | grep OK | wc -l)
2525 echo -n " ${qaStatus}" >> ${outfile}
2526
2527
2528 #fill OK/BAD rates
2529 for i in $(seq 0 $((${Nrates}-1))) ; do
2530 var1=$(grep "/${ratesString[${i}]}.log" ${filejob} | grep OK | wc -l)
2531 var2=$(grep "/${ratesString[${i}]}.log" ${filejob} | grep BAD | wc -l)
2532
2533 if [[ ${ratesString[${i}]} == "stderr" ]] ; then
2534 var1=$(grep "stderr" ${filejob} | grep OK | wc -l)
2535 var2=$(grep "stderr" ${filejob} | grep "rec.log" | grep BAD | wc -l)
2536 fi
2537 echo -n " ${var1}" >> ${outfile}
2538 echo -n " ${var2}" >> ${outfile}
2539 done
2540
2541 if [[ -f ${fileOCDB} ]] ; then
2542 #fill counter
2543 for i in $(seq 0 $((${Ncounter}-1))) ; do
2544 var1=$(grep Monalisa ${fileOCDB} | grep ${counterString[${i}]} | cut -f2)
2545 echo -n " ${var1:-"-1"}" >> ${outfile}
2546 done
2547
2548 #fill status
2549 for i in $(seq 0 $((${Nstatus}-1))) ; do
2550 var1=$(grep "calibration status=" ${fileOCDB} | grep ${statusString[${i}]/Status/} | cut -d'=' -f2)
2551 echo -n " ${var1:-"-1"}" >> ${outfile}
2552 done
2553 fi
2554 echo >> ${outfile}
2555 done
2556
5de3064b 2557 return 0
78d0bbfd 2558)
71353a5f 2559
2560parseConfig()
2561{
2562 configFile=${1}
2563 shift
444b9916 2564 args=("$@")
2565
0eaf38c7 2566
444b9916 2567 #some defaults
6fc48f9e 2568 #autoOCDB=0
2569 defaultOCDB="raw://"
2570 #runNumber=167123
2571 #makeflowPath="/hera/alice/aux/cctools/bin"
2572 #makeflowOptions="-T wq -N alice -d all -C ali-copilot.cern.ch:9097"
2573 #makeflowOptions="-T wq -N alice -C ali-copilot.cern.ch:9097"
2574 makeflowOptions=""
39527a15 2575 #batchCommand="/usr/bin/qsub"
ef77d296 2576 batchFlags=""
6fc48f9e 2577 baseOutputDirectory="$PWD/output"
2578 #alirootEnv="/cvmfs/alice.cern.ch/bin/alienv setenv AliRoot/v5-04-34-AN -c"
2579 #alirootEnv="/home/mkrzewic/alisoft/balice_master.sh"
2580 #trustedQAtrainMacro='/hera/alice/mkrzewic/gsisvn/Calibration/QAtrain_duo.C'
2581 reconstructInTemporaryDir=0
2582 recoTriggerOptions="\"\""
2583 percentProcessedFilesToContinue=100
2584 maxSecondsToWait=$(( 3600*24 ))
2585 nEvents=-1
2586 nMaxChunks=0
2587 postSetUpActionCPass0=""
2588 postSetUpActionCPass1=""
2589 runCPass0reco=1
2590 runCPass0MergeMakeOCDB=1
2591 runCPass1reco=1
2592 runCPass1MergeMakeOCDB=1
2593 runESDfiltering=1
444b9916 2594 filteringFactorHighPt=1e2
2595 filteringFactorV0s=1e1
6fc48f9e 2596 MAILTO=""
2597 #pretend=1
2598 #dontRedirectStdOutToLog=1
2599 logToFinalDestination=1
2600 ALIROOT_FORCE_COREDUMP=1
0eaf38c7 2601 pretendDelay=0
2ac23087 2602 copyInputData=0
71353a5f 2603
2604 #first, source the config file
2605 if [ -f ${configFile} ]; then
2606 source ${configFile}
2607 else
0eaf38c7 2608 echo "config file ${configFile} not found!"
2609 return 1
71353a5f 2610 fi
2611
839b07ca 2612 unset encodedSpaces
2613 for opt in "${args[@]}"; do
0eaf38c7 2614 [[ "${opt}" =~ encodedSpaces=.* ]] && encodedSpaces=1 && echo "encodedSpaces!" && break
839b07ca 2615 done
2616
d56eeaab 2617 #then, parse the options as they override the options from file
2618 for opt in "${args[@]}"; do
0eaf38c7 2619 [[ -z ${opt} ]] && continue
2620 [[ -n ${encodedSpaces} ]] && opt="$(decSpaces ${opt})"
2621 [[ "${opt}" =~ ^[[:space:]]*$ ]] && continue
d56eeaab 2622 if [[ ! "${opt}" =~ .*=.* ]]; then
0eaf38c7 2623 echo "badly formatted option \"${opt}\" should be: option=value, stopping..."
d56eeaab 2624 return 1
2625 fi
2626 local var="${opt%%=*}"
2627 local value="${opt#*=}"
9d0c204f 2628 echo "${var}=${value}"
444b9916 2629 export ${var}="${value}"
71353a5f 2630 done
2631
6fc48f9e 2632 #do some checking
2633 [[ -z ${alirootEnv} ]] && echo "alirootEnv not defined!" && return 1
2634
71353a5f 2635 #export the aliroot function if defined to override normal behaviour
fecdbe75 2636 [[ $(type -t aliroot) =~ "function" ]] && export -f aliroot && echo "exporting aliroot() function..."
6fc48f9e 2637
5de3064b 2638 return 0
71353a5f 2639}
2640
2641aliroot()
2642{
e48b4924 2643 args=("$@")
6fc48f9e 2644 if [[ -n ${useProfilingCommand} ]]; then
e48b4924 2645 profilerLogFile="cpu.txt"
1a0fcd62 2646 [[ "${args[@]}" =~ rec ]] && profilerLogFile="cpu_rec.txt"
2647 [[ "${args[@]}" =~ Calib ]] && profilerLogFile="cpu_calib.txt"
2f2d99b8 2648 echo running "${useProfilingCommand} aliroot ${args[@]} &> ${profilerLogFile}"
e48b4924 2649 ${useProfilingCommand} aliroot "${args[@]}" &> ${profilerLogFile}
71353a5f 2650 else
2651 #to prevent an infinite recursion use "command aliroot" to disable
2652 #aliases and functions
e48b4924 2653 echo running command aliroot "${args[@]}"
2654 command aliroot "${args[@]}"
71353a5f 2655 fi
5de3064b 2656 return 0
71353a5f 2657}
2658
2ac23087 2659copyFileToLocal()
2660(
2661 #copies a file from either a remote or local location to a local destination
2662 src="$1"
2663 dst="$2"
2664
2665 proto="${src%%://*}"
2666 if [[ "$proto" == "$src" ]]; then
2667 cp "$src" "$dst"
2668 else
2669 case "$proto" in
2670 root)
2671 xrdcp -f "$src" "$dst"
2672 ;;
2673 http)
2674 curl -L "$src" -O "$dst"
2675 ;;
2676 *)
2677 echo "protocol not supported: $proto"
2678 return 1
2679 ;;
2680 esac
2681 fi
2682)
2683
13abb339 2684paranoidCp()
2685(
2686 #recursively copy files and directories
2687 #to avoid using find and the like as they kill
2688 #the performance on some cluster file systems
2689 #does not copy links to avoid problems
2690 sourceFiles=("${@}")
2691 destination="${sourceFiles[@]:(-1)}" #last element
2692 unset sourceFiles[${#sourceFiles[@]}-1] #remove last element (dst)
2693 for src in "${sourceFiles[@]}"; do
2694 if [[ -f "${src}" && ! -h "${src}" ]]; then
2695 paranoidCopyFile "${src}" "${destination}"
2696 elif [[ -d "${src}" && ! -h "${src}" ]]; then
2697 src="${src%/}"
2698 dst="${destination}/${src##*/}"
2699 mkdir -p "${dst}"
2700 paranoidCp "${src}"/* "${dst}"
2701 fi
2702 done
2703)
2704
2705paranoidCopyFile()
2706(
2707 #copy a single file to a target in an existing dir
2708 #repeat a few times if copy fails
2709 src="${1}"
2710 dst="${2}"
2711 [[ -d "${dst}" ]] && dst="${dst}/${src##*/}"
2712 [[ -z "${maxCopyTries}" ]] && maxCopyTries=5
054fdec5 2713 #echo "maxCopyTries=${maxCopyTries}"
13abb339 2714 echo "cp ${src} ${dst}"
2715 cp "${src}" "${dst}"
2716 i=0
2717 until cmp -s "${src}" "${dst}"; do
2718 echo "try: ${i}"
2719 [[ -f "${dst}" ]] && rm "${dst}"
2720 cp "${src}" "${dst}"
2721 [[ ${i} -gt ${maxCopyTries} ]] && ret=1 && return 1
2722 (( i++ ))
2723 done
2724 return 0
2725)
2726
71353a5f 2727guessRunData()
2728{
2729 #guess the period from the path, pick the rightmost one
2730 period=""
2731 runNumber=""
2732 year=""
2733 pass=""
2734 legoTrainRunNumber=""
2735 dataType=""
2736
2737 local shortRunNumber=""
2738 local IFS="/"
2739 declare -a path=( $1 )
2740 local dirDepth=$(( ${#path[*]}-1 ))
2741 i=0
2742 #for ((x=${dirDepth};x>=0;x--)); do
2743 for ((x=0;x<=${dirDepth};x++)); do
2744
2745 [[ $((x-1)) -ge 0 ]] && local fieldPrev=${path[$((x-1))]}
2746 local field=${path[${x}]}
2747 local fieldNext=${path[$((x+1))]}
2748
2749 [[ ${field} =~ ^[0-9]*$ && ${fieldNext} =~ (.*\.zip$|.*\.root$) ]] && legoTrainRunNumber=${field}
2750 [[ -n ${legoTrainRunNumber} && -z ${pass} ]] && pass=${fieldPrev}
2751 [[ ${field} =~ ^LHC[0-9][0-9][a-z].*$ ]] && period=${field%_*}
2752 [[ ${field} =~ ^000[0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && runNumber=${field#000}
2753 [[ ${field} =~ ^[0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && shortRunNumber=${field}
2754 [[ ${field} =~ ^20[0-9][0-9]$ ]] && year=${field}
2755 [[ ${field} =~ ^(^sim$|^data$) ]] && dataType=${field}
2756 (( i++ ))
2757 done
2758 [[ -z ${legoTrainRunNumber} ]] && pass=${path[$((dirDepth-1))]}
2759 [[ "${dataType}" =~ ^sim$ ]] && pass="passMC" && runNumber=${shortRunNumber}
2760
2761 #if [[ -z ${dataType} || -z ${year} || -z ${period} || -z ${runNumber}} || -z ${pass} ]];
2762 if [[ -z ${runNumber}} ]];
2763 then
2764 #error condition
2765 return 1
2766 else
2767 #ALL OK
2768 return 0
2769 fi
5de3064b 2770 return 0
71353a5f 2771}
2772
839b07ca 2773#these functions encode strings to and from a space-less form
2774#use when spaces are not well handled (e.g. in arguments to
2775#commands in makeflow files, etc.
0eaf38c7 2776encSpaces()(echo "${1// /@@@@}")
2777decSpaces()(echo "${1//@@@@/ }")
839b07ca 2778
fc7e1e45 2779main "$@"