this contains the fixes by Dario to workaround some of the quirks of EOS
[u/mrichter/AliRoot.git] / PWGPP / benchmark / benchmark.sh
CommitLineData
fc7e1e45 1#!/bin/bash
71353a5f 2#include benchmark.config
3
fc7e1e45 4# this script runs the CPass0/CPass1 train
5# produced OCDB updates are local
6
7main()
8{
9 #run in proper mode depending on the selection
71353a5f 10 if [[ $# -lt 1 ]]; then
11 if [[ ! "${0}" =~ "bash" ]]; then
12 echo "uses makeflow:"
13 echo " ${0} \"run\" productionID inputList configFile [extraOpts]"
14 echo "uses a batch system (SGE):"
15 echo " ${0} \"submit\" productionID inputList configFile [extraOpts]"
16 echo "extraOpts if non-empty override the config file, e.g.:"
9510565c 17 echo " ${0} submit test1 benchmark.list benchmark.config runNumber=169123 nEvents=10"
71353a5f 18 fi
19 return
20 fi
21
22 #define some aliases - default is to call one of the functions directly
23 runMode=${1}
fc7e1e45 24 umask 0002
25 shift
71353a5f 26 case ${runMode} in
fc7e1e45 27 "CPass0") goCPass0 "$@";;
28 "CPass1") goCPass1 "$@";;
71353a5f 29 "MakeLocalOCDBaccessConfig") goMakeLocalOCDBaccessConfig "$@";;
fc7e1e45 30 "MergeCPass0") goMergeCPass0 "$@";;
31 "MergeCPass1") goMergeCPass1 "$@";;
cb473870 32 "MakeFilteredTrees") goMakeFilteredTrees "$@";;
71353a5f 33 "MakeSummary") goMakeSummary "$@";;
cb473870 34 "run") goSubmitMakeflow "$@";;
35 "submit") goSubmitBatch "$@";;
fc7e1e45 36 "test") goTest "$@";;
71353a5f 37 "GenerateMakeflow") goGenerateMakeflow "$@";;
38 "PrintValues") goPrintValues "$@";;
cb473870 39 "CreateQAplots") goCreateQAplots "$@";;
40 "WaitForOutput") goWaitForOutput "$@";;
71353a5f 41 "Merge") goMerge "$@";;
42 *)
444b9916 43 ${runMode} "$@"
fc7e1e45 44 ;;
45 esac
39527a15 46 return 0
fc7e1e45 47}
48
71353a5f 49generateMC()
50{
51 #generate one raw chunk in current directory
52 SEED=${JOB_ID}${SGE_TASK_ID}
53 export CONFIG_SEED=${SEED}
54 runNumber=${1}
55 OCDBpath=${2}
56 nEventsim=${3}
57 if [[ -n ${pretend} ]]; then
0eaf38c7 58 sleep ${pretendDelay}
71353a5f 59 touch galice.root
60 else
61 if [[ -f sim.C && -f Config.C ]] ; then
62 time aliroot -b -q -x sim.C\(${runNumber},\"${OCDBpath}\",${nEventsim}\) >sim.log 2>&1
63 mv syswatch.log simwatch.log
64 fi
65 fi
66}
67
fc7e1e45 68goCPass0()
78d0bbfd 69(
fc7e1e45 70 umask 0002
71
71353a5f 72 targetDirectory=${1}
73 inputList=${2}
74 nEvents=${3}
75 ocdbPath=${4}
76 configFile=${5}
77 runNumber=${6}
78 jobindex=${7}
79 shift 7
6fc48f9e 80 if ! parseConfig ${configFile} "$@"; then return 1; fi
71353a5f 81
39527a15 82 #record the working directory provided by the batch system
83 batchWorkingDirectory=${PWD}
84
71353a5f 85 #use the jobindex only if set and non-negative
86 if [[ -z ${jobindex} || ${jobindex} -lt 0 ]]; then
87 [[ -n "${LSB_JOBINDEX}" ]] && jobindex=${LSB_JOBINDEX}
88 [[ -n "${SGE_TASK_ID}" ]] && jobindex=${SGE_TASK_ID}
999a843d 89 if [[ -z ${jobindex} ]]; then
90 echo "no jobindex!"
91 return 1
92 fi
71353a5f 93 fi
94
95 [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
ef77d296 96
97 # This file signals that/if everything went fine
98 doneFileBase="cpass0.job${jobindex}.run${runNumber}.done"
99 [[ -n ${useProfilingCommand} ]] && doneFileBase="profiling.cpass0.job${jobindex}.run${runNumber}.done"
100
101 # We will have two copies of the file
102 mkdir -p "${commonOutputPath}/meta" || return 1
103 doneFileTmp="${batchWorkingDirectory}/${doneFileBase}"
104 doneFile="${commonOutputPath}/meta/${doneFileBase}"
71353a5f 105
106 [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
107
108 if [[ -n ${ALIROOT_FORCE_COREDUMP} ]]; then
109 ulimit -c unlimited
110 export ALIROOT_FORCE_COREDUMP
111 fi
fc7e1e45 112
cb473870 113 #the contents of this is stored in the tree and used later (e.g. AliAnalysisTaskPIDResponse)!
114 #at the QA stage the pass number is guessed from the path stored here.
115 #The Format is:
116 #Packages= ;OutputDir= ;LPMPass= ;TriggerAlias= ;LPMRunNumber= ;LPMProductionType= ;LPMInteractionType= ;LPMProductionTag= ;LPMAnchorRun= ;LPMAnchorProduction= ;LPMAnchorYear=
117 export PRODUCTION_METADATA="OutputDir=cpass0"
118
5de3064b 119 if [[ "${inputList}" =~ \.root$ ]]; then
120 infile=${inputList}
121 else
122 infile=$(sed -ne "${jobindex}p" ${inputList} | egrep '\s*\w*/\w*')
123 fi
71353a5f 124 chunkName=${infile##*/}
71353a5f 125
126 outputDir=${targetDirectory}/${jobindex}_${chunkName%.*}
127 mkdir -p ${outputDir}
1795556a 128 if [[ ! -d ${outputDir} ]]; then
ef77d296 129 touch ${doneFileTmp}
130 echo "cannot make ${outputDir}" >> ${doneFileTmp}
131 cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
132 [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
1795556a 133 return 1
134 fi
71353a5f 135
136 #runpath=${PWD}/rundir_cpass0_${runNumber}_${jobindex}
137 runpath=${outputDir}
0eaf38c7 138 #[[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
139 [[ ${reconstructInTemporaryDir} -eq 1 ]] && runpath=$(mktemp -d -t cpass0.XXXXXX)
71353a5f 140 mkdir -p ${runpath}
1795556a 141 if [[ ! -d ${runpath} ]]; then
ef77d296 142 touch ${doneFileTmp}
143 echo "cannot make runpath ${runpath}" >> ${doneFileTmp}
144 cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
145 [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
1795556a 146 return 1
147 fi
0eaf38c7 148 if ! cd ${runpath}; then
ef77d296 149 touch ${doneFileTmp}
150 echo "PWD=$PWD is not the runpath=${runpath}" >> ${doneFileTmp}
151 cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
152 [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
0eaf38c7 153 return 1
154 fi
fc7e1e45 155
5de3064b 156 #runCPassX/C expects the raw chunk to be linked in the run dir
157 #despite it being accessed by the full path
158 ln -s ${infile} ${runpath}/${chunkName}
fc7e1e45 159
71353a5f 160 #####MC
161 if [[ -n ${generateMC} ]]; then
162 olddir=${PWD}
163 outputDirMC=${commonOutputPath}/000${runNumber}/sim/${jobindex}
164 simrunpath=${outputDirMC}
0eaf38c7 165 #[[ ${simulateInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && simrunpath=${TMPDIR}
166 [[ ${simulateInTemporaryDir} -eq 1 ]] && simrunpath=$(mktemp -d -t cpass0MC.XXXXXX)
71353a5f 167 mkdir -p ${outputDirMC}
168 mkdir -p ${simrunpath}
0eaf38c7 169 if cd ${simrunpath}; then
fc7e1e45 170
0eaf38c7 171 filesMC=(
172 "${batchWorkingDirectory}/sim.C"
173 "${batchWorkingDirectory}/rec.C"
174 "${batchWorkingDirectory}/Config.C"
175 "${batchWorkingDirectory}/OCDB_*.root"
176 )
177 for file in ${filesMC[*]}; do
178 [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
179 done
fc7e1e45 180
0eaf38c7 181 generateMC ${runNumber} ${ocdbPath} ${nEvents}
182
183 [[ ! "${simrunpath}" =~ "${outputDirMC}" ]] && mv * ${outputDirMC} #TODO check if it works
184 cd ${olddir}
185
186 ln -s ${outputDirMC}/* ${runpath}/
187
188 inputList=${outputDirMC}/galice.root #TODO not valid outside shell !!!
189 infile=""
190 fi
71353a5f 191 fi
192 ######
193
1795556a 194 if [[ ! -f ${inputList} && -z ${pretend} ]]; then
ef77d296 195 touch ${doneFileTmp}
196 echo "input file ${inputList} not found, exiting..." >> ${doneFileTmp}
197 cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
198 [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
1795556a 199 return 1
200 fi
71353a5f 201
202 logOutputDir=${runpath}
203 [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir}
0eaf38c7 204 [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/stdout
205 #[[ -z ${dontRedirectStdOutToLog} ]] && exec 2> ${logOutputDir}/stderr
71353a5f 206 echo "${0} $*"
fc7e1e45 207
208 echo "#####################"
209 echo CPass0:
210 echo JOB setup
ef77d296 211 echo nEvents ${nEvents}
212 echo runNumber ${runNumber}
213 echo ocdbPath ${ocdbPath}
214 echo infile ${infile}
215 echo chunkName ${chunkName}
216 echo jobindex ${jobindex}
217 echo recoTriggerOptions ${recoTriggerOptions}
218 echo targetDirectory ${targetDirectory}
219 echo commonOutputPath ${commonOutputPath}
220 echo doneFile ${doneFile}
221 echo batchWorkingDirectory ${batchWorkingDirectory}
222 echo runpath ${runpath}
223 echo outputDir ${outputDir}
224 echo PWD ${PWD}
225 echo ALICE_ROOT ${ALICE_ROOT}
fc7e1e45 226 echo "########## ###########"
227
71353a5f 228 alirootInfo > ALICE_ROOT.log
fc7e1e45 229
230 filesCPass0=(
39527a15 231 "${batchWorkingDirectory}/runCPass0.sh"
232 "${batchWorkingDirectory}/recCPass0.C"
233 "${batchWorkingDirectory}/runCalibTrain.C"
234 "${batchWorkingDirectory}/localOCDBaccessConfig.C"
235 "${batchWorkingDirectory}/OCDB.root"
71353a5f 236 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/runCPass0.sh"
237 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/recCPass0.C"
238 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/runCalibTrain.C"
fc7e1e45 239 )
240
241 for file in ${filesCPass0[*]}; do
242 [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
1a0e38e3 243 [[ ${file##*/} =~ .*\.sh ]] && chmod +x ${file##*/}
fc7e1e45 244 done
245
71353a5f 246 echo "this directory (${PWD}) contents:"
247 /bin/ls
fc7e1e45 248 echo
249 chmod u+x runCPass0.sh
250
71353a5f 251 sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh
252
253 if [[ -n ${postSetUpActionCPass0} ]]; then
254 echo "running ${postSetUpActionCPass0}"
255 eval ${postSetUpActionCPass0}
fc7e1e45 256 fi
257
258 #run CPass0
71353a5f 259 echo "${runpath}/runCPass0.sh ${infile} ${nEvents} ${runNumber} ${ocdbPath} ${recoTriggerOptions}"
260 if [[ -n ${pretend} ]]; then
0eaf38c7 261 sleep ${pretendDelay}
cb473870 262 touch AliESDs.root
263 touch AliESDfriends.root
fc7e1e45 264 touch AliESDfriends_v1.root
265 touch rec.log
266 touch calib.log
267 else
71353a5f 268 echo ./runCPass0.sh "${infile}" "${nEvents}" "${runNumber}" "${ocdbPath}" "${recoTriggerOptions}"
269 ./runCPass0.sh "${infile}" "${nEvents}" "${runNumber}" "${ocdbPath}" "${recoTriggerOptions}"
fc7e1e45 270 fi
271
fc7e1e45 272 #move stuff to final destination
71353a5f 273 echo "this directory (${PWD}) contents:"
274 /bin/ls
fc7e1e45 275 echo
276
ef77d296 277 # [dberzano] OK this is fine!
71353a5f 278 echo rm -f ./${chunkName}
279 rm -f ./${chunkName}
0eaf38c7 280 echo "cp -R ${runpath}/* ${outputDir}"
281 cp -p -R ${runpath}/* ${outputDir}
fc7e1e45 282 echo
5ed7c82c 283
284 #validate CPass0
285 cd ${outputDir}
ef77d296 286 touch ${doneFileTmp}
287 echo "dir ${outputDir}" >> ${doneFileTmp}
288 if summarizeLogs >> ${doneFileTmp}; then
289 [[ -f ${outputDirMC}/galice.root ]] && echo "sim ${outputDirMC}/galice.root" >> ${doneFileTmp}
290 [[ -f AliESDfriends_v1.root ]] && echo "calibfile ${outputDir}/AliESDfriends_v1.root" >> ${doneFileTmp}
291 [[ -f AliESDs.root ]] && echo "esd ${outputDir}/AliESDs.root" >> ${doneFileTmp}
ab946185 292 fi
fc7e1e45 293
0eaf38c7 294 [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath} && echo "removing ${runpath}"
ef77d296 295 cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
296 [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
5de3064b 297 return 0
78d0bbfd 298)
fc7e1e45 299
300goCPass1()
78d0bbfd 301(
fc7e1e45 302 umask 0002
303
71353a5f 304 targetDirectory=${1}
305 inputList=${2}
306 nEvents=${3}
307 ocdbPath=${4}
308 configFile=${5}
309 runNumber=${6}
310 jobindex=${7}
311 shift 7
444b9916 312 extraOpts=("$@")
6fc48f9e 313 if ! parseConfig ${configFile} "$@"; then return 1; fi
71353a5f 314
39527a15 315 #record the working directory provided by the batch system
316 batchWorkingDirectory=${PWD}
317
71353a5f 318 #use the jobindex only if set and non-negative
319 if [[ -z ${jobindex} || ${jobindex} -lt 0 ]]; then
320 [[ -n "${LSB_JOBINDEX}" ]] && jobindex=${LSB_JOBINDEX}
321 [[ -n "${SGE_TASK_ID}" ]] && jobindex=${SGE_TASK_ID}
999a843d 322 if [[ -z ${jobindex} ]]; then
323 echo "no jobindex!"
324 return 1
325 fi
71353a5f 326 fi
327
328 [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
ef77d296 329
330 # This file signals that/if everything went fine
331 doneFileBase="cpass1.job${jobindex}.run${runNumber}.done"
332 [[ -n ${useProfilingCommand} ]] && doneFileBase="profiling.cpass0.job${jobindex}.run${runNumber}.done"
333
334 # We will have two copies of the file
335 mkdir -p "${commonOutputPath}/meta" || return 1
336 doneFileTmp="${batchWorkingDirectory}/${doneFileBase}"
337 doneFile="${commonOutputPath}/meta/${doneFileBase}"
71353a5f 338
339 [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
340
341 if [[ -n ${ALIROOT_FORCE_COREDUMP} ]]; then
342 ulimit -c unlimited
343 export ALIROOT_FORCE_COREDUMP
344 fi
fc7e1e45 345
cb473870 346 #the contents of this is stored in the tree and used later (e.g. AliAnalysisTaskPIDResponse)!
347 #at the QA stage the pass number is guessed from the path stored here.
348 #The Format is:
349 #Packages= ;OutputDir= ;LPMPass= ;TriggerAlias= ;LPMRunNumber= ;LPMProductionType= ;LPMInteractionType= ;LPMProductionTag= ;LPMAnchorRun= ;LPMAnchorProduction= ;LPMAnchorYear=
350 export PRODUCTION_METADATA="OutputDir=cpass1"
351
1795556a 352 if [[ ! -f ${inputList} && -z ${pretend} ]]; then
ef77d296 353 touch ${doneFileTmp}
354 echo "input file ${inputList} not found, exiting..." >> ${doneFileTmp}
355 cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
356 [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
1795556a 357 return 1
358 fi
fc7e1e45 359 if [[ "${inputList}" =~ \.root$ ]]; then
71353a5f 360 infile=${inputList}
fc7e1e45 361 else
71353a5f 362 infile=$(sed -ne "${jobindex}p" ${inputList} | egrep '\s*\w*/\w*')
fc7e1e45 363 fi
fc7e1e45 364 chunkName=${infile##*/}
5de3064b 365
71353a5f 366 outputDir=${targetDirectory}/${jobindex}_${chunkName%.*}
367 mkdir -p ${outputDir}
1795556a 368 if [[ ! -d ${outputDir} ]];then
ef77d296 369 touch ${doneFileTmp}
370 echo "cannot make ${outputDir}" >> ${doneFileTmp}
371 cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
372 [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
1795556a 373 return 1
374 fi
fc7e1e45 375
71353a5f 376 #runpath=${PWD}/rundir_cpass1_${runNumber}_${jobindex}
377 runpath=${outputDir}
0eaf38c7 378 #[[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
379 [[ ${reconstructInTemporaryDir} -eq 1 ]] && runpath=$(mktemp -d -t cpass1.XXXXXX)
71353a5f 380
381 #MC
382 if [[ "${infile}" =~ galice\.root ]]; then
383 ln -s ${inputList%/*}/* ${runpath}
384 infile=""
385 fi
fc7e1e45 386
387 #init the running path
71353a5f 388 mkdir -p ${runpath}
1795556a 389 if [[ ! -d ${runpath} ]]; then
ef77d296 390 touch ${doneFileTmp}
391 echo "cannot make runpath ${runpath}" >> ${doneFileTmp}
392 cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
393 [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
1795556a 394 return 1
395 fi
0eaf38c7 396 if ! cd ${runpath}; then
ef77d296 397 touch ${doneFileTmp}
398 echo "PWD=$PWD is not the runpath=${runpath}" >> ${doneFileTmp}
399 cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
400 [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
0eaf38c7 401 return 1
402 fi
fc7e1e45 403
71353a5f 404 #this is needed for runCPass1.sh
405 ln -s ${infile} ${runpath}/${chunkName}
fc7e1e45 406
71353a5f 407 logOutputDir=${runpath}
408 [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir}
0eaf38c7 409 [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/stdout
410 #[[ -z ${dontRedirectStdOutToLog} ]] && exec 2> ${logOutputDir}/stderr
71353a5f 411 echo "${0} $*"
fc7e1e45 412
413 echo "#####################"
414 echo CPass1:
415 echo JOB setup
ef77d296 416 echo nEvents ${nEvents}
417 echo runNumber ${runNumber}
418 echo ocdbPath ${ocdbPath}
419 echo infile ${infile}
420 echo chunkName ${chunkName}
421 echo jobindex ${jobindex}
422 echo recoTriggerOptions ${recoTriggerOptions}
423 echo targetDirectory ${targetDirectory}
424 echo commonOutputPath ${commonOutputPath}
425 echo doneFile ${doneFile}
426 echo runpath ${runpath}
427 echo outputDir ${outputDir}
1795556a 428 echo batchWorkingDirectory ${batchWorkingDirectory}
ef77d296 429 echo ALICE_ROOT ${ALICE_ROOT}
430 echo PWD ${PWD}
431 echo "#####################"
fc7e1e45 432
71353a5f 433 alirootInfo > ALICE_ROOT.log
fc7e1e45 434
435 filesCPass1=(
39527a15 436 "${batchWorkingDirectory}/runCPass1.sh"
437 "${batchWorkingDirectory}/recCPass1.C"
438 "${batchWorkingDirectory}/recCPass1_OuterDet.C"
439 "${batchWorkingDirectory}/runCalibTrain.C"
440 "${batchWorkingDirectory}/QAtrain_duo.C"
441 "${batchWorkingDirectory}/localOCDBaccessConfig.C"
0eaf38c7 442 "${batchWorkingDirectory}/${configFile}"
617f8886 443 "${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz"
39527a15 444 "${batchWorkingDirectory}/OCDB.root"
71353a5f 445 "${trustedQAtrainMacro}"
446 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/runCPass1.sh"
447 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/recCPass1.C"
448 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/recCPass1_OuterDet.C"
449 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/runCalibTrain.C"
450 "${ALICE_ROOT}/ANALYSIS/macros/QAtrain_duo.C"
fc7e1e45 451 )
452
78d0bbfd 453 for file in "${filesCPass1[@]}"; do
fc7e1e45 454 [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
78d0bbfd 455 [[ ${file##*/} =~ .*\.sh ]] && echo "making ${file##*/} executable" && chmod +x ${file##*/}
fc7e1e45 456 done
457
71353a5f 458 echo "this directory (${PWD}) contents:"
459 /bin/ls
fc7e1e45 460 echo
461
78d0bbfd 462 #remove spaces around commas from calls to root
71353a5f 463 sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh
464
465 if [[ -n ${postSetUpActionCPass1} ]]; then
466 echo "running ${postSetUpActionCPass1}"
467 eval ${postSetUpActionCPass1}
fc7e1e45 468 echo
469 fi
470
471 #configure local OCDB storage from CPass0 (creates the localOCDBaccessConfig.C script)
472 if [[ -f cpass0.localOCDB.${runNumber}.tgz ]]; then
71353a5f 473 echo goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz"
474 goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz"
cb473870 475 else
476 echo "WARNING: file cpass0.localOCDB.${runNumber}.tgz not found!"
477 fi
478
71353a5f 479 if [[ ! $(/bin/ls -1 OCDB/*/*/*/*.root 2>/dev/null) ]]; then
ef77d296 480 touch ${doneFileTmp}
481 echo "cpass0 produced no calibration! exiting..." >> ${doneFileTmp}
482 cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
483 [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
71353a5f 484 return 1
cb473870 485 fi
fc7e1e45 486
71353a5f 487 #create the Barrel and OuterDet directories for CPass1 and link the local OCDB directory
488 #there to make the localOCDBaccessConfig.C file work, since it may point to the OCDB
489 #entries using a relative path, e.g. local://./OCDB
78d0bbfd 490 echo "linking the OCDB/ for Barrel and OuterDet directories"
71353a5f 491 mkdir Barrel OuterDet
78d0bbfd 492 ls -l
71353a5f 493 ln -s ../OCDB Barrel/OCDB
494 ln -s ../OCDB OuterDet/OCDB
495
444b9916 496 #setup the filtering
497 #the following option enables the filtering task inside the QAtrain_duo.C
498 [[ -n $runESDfiltering ]] && export QA_TaskFilteredTree=1
499 #set the downscaling factors during the filtering fro expert QA (overrides the previous values)
500 if [[ -n ${filteringFactorHighPt} ]]; then
501 export AliAnalysisTaskFilteredTree_fLowPtTrackDownscaligF=${filteringFactorHighPt}
502 fi
503 if [[ -n ${filteringFactorV0s} ]]; then
504 export AliAnalysisTaskFilteredTree_fLowPtV0DownscaligF=${filteringFactorV0s}
505 fi
506
fc7e1e45 507 #run CPass1
508 chmod u+x runCPass1.sh
71353a5f 509 echo "${runpath}/runCPass1.sh ${infile} ${nEvents} ${runNumber} ${ocdbPath} ${recoTriggerOptions}"
510 if [[ -n ${pretend} ]]; then
0eaf38c7 511 sleep ${pretendDelay}
cb473870 512 touch AliESDs_Barrel.root
513 touch AliESDfriends_Barrel.root
fc7e1e45 514 touch AliESDfriends_v1.root
cb473870 515 touch QAresults_barrel.root
516 touch EventStat_temp_barrel.root
517 touch AODtpITS.root
518 touch AliESDs_Outer.root
519 touch AliESDfriends_Outer.root
520 touch QAresults_outer.root
521 touch EventStat_temp_outer.root
fc7e1e45 522 touch rec.log
523 touch calib.log
524 touch qa.log
444b9916 525 touch filtering.log FilterEvents_Trees.root
fc7e1e45 526 else
71353a5f 527 ./runCPass1.sh "${infile}" "${nEvents}" "${runNumber}" "${ocdbPath}" "${recoTriggerOptions}"
444b9916 528
529 [[ ! -f AliESDs_Barrel.root && -f Barrel/AliESDs.root ]] && mv Barrel/AliESDs.root AliESDs_Barrel.root
530 [[ ! -f AliESDfriends_Barrel.root && -f Barrel/AliESDfriends.root ]] && mv Barrel/AliESDfriends.root AliESDfriends_Barrel.root
531 [[ ! -f AliESDfriends_v1.root && -f Barrel/AliESDfriends_v1.root ]] && mv Barrel/AliESDfriends_v1.root .
532 [[ ! -f QAresults_barrel.root && -f Barrel/QAresults_barrel.root ]] && mv Barrel/QAresults_barrel.root .
533 [[ ! -f AliESDs_Outer.root && -f OuterDet/AliESDs.root ]] && mv OuterDet/AliESDs.root AliESDs_Outer.root
534 [[ ! -f AliESDfriends_Outer.root && -f OuterDet/AliESDfriends.root ]] && mv OuterDet/AliESDfriends.root AliESDfriends_Outer.root
535 [[ ! -f QAresults_outer.root && -f OuterDet/QAresults_outer.root ]] && mv OuterDet/QAresults_outer.root .
536 [[ ! -f FilterEvents_Trees.root && -f Barrel/FilterEvents_Trees.root ]] && mv Barrel/FilterEvents_Trees.root .
537
538 #make the filtered tree (if requested and not already produced by QA
539 [[ -f AliESDs_Barrel.root ]] && echo "AliESDs_Barrel.root" > filtered.list
fab31805 540 if [[ -n ${runESDfiltering} && ! -f FilterEvents_Trees.root && -f filtered.list ]]; then
541 goMakeFilteredTrees ${PWD} ${runNumber} "${PWD}/filtered.list" ${filteringFactorHighPt} ${filteringFactorV0s} ${ocdbPath} 1000000 0 10000000 0 ${configFile} AliESDs_Barrel.root "${extraOpts[@]}" >filtering.log
542 else
543 echo ""
444b9916 544 fi
545
fc7e1e45 546 fi
cb473870 547
548 ##handle possible crashes in QA (happens often in trunk)
549 ##rerun QA with a trusted aliroot version
550 #if [[ $(validateLog qa_barrel.log) ]]; then
551 # echo "qa_barrel.log not validated!"
552 #fi
553 #if [[ ! -f QAresults_barrel.root && -f ${setupTrustedAliROOTenvInCurrentShell} || $(validateLog qa_barrel.log) ]]; then
71353a5f 554 # echo "WARNING: using trusted QA aliroot ${ALICE_ROOT}"
cb473870 555 # source ${setupTrustedAliROOTenvInCurrentShell}
556 # cd Barrel
557 # rm QAresults_barrel.root
558 # rm EventStat_temp_barrel.root
559 # rm AODtpITS.root
560 # [[ ! -f AliESDs.root ]] && ln -s ../AliESDs_Barrel.root AliESDs.root
561 # [[ ! -f AliESDfriends.root ]] && ln -s ../AliESDfriends_Barrel.root AliESDfriends.root
71353a5f 562 # if [[ -n ${trustedQAtrainMacro} ]]; then
563 # eval "cp ${trustedQAtrainMacro} QAtrain_duo_trusted.C"
cb473870 564 # fi
71353a5f 565 # echo executing aliroot -b -q "QAtrain_duo_trusted.C(\"_barrel\",${runNumber},\"wn.xml\",0,\"${ocdbPath}\")"
566 # time aliroot -b -q "QAtrain_duo.C(\"_barrel\",${runNumber},\"wn.xml\",0,\"${ocdbPath}\")" &> ../qa_barrel_trusted.log
cb473870 567 # cd ../
568 #fi
cb473870 569
fc7e1e45 570 #move stuff to final destination
71353a5f 571 echo "this directory (${PWD}) contents:"
572 /bin/ls
573 echo rm -f ./${chunkName}
574 rm -f ./${chunkName}
0eaf38c7 575 echo "cp -R ${runpath}/* ${outputDir}"
576 cp -pf -R ${runpath}/* ${outputDir}
fc7e1e45 577 echo
578
5ed7c82c 579 #validate CPass1
580 cd ${outputDir}
ef77d296 581 touch ${doneFileTmp}
582 echo "dir ${outputDir}" >> ${doneFileTmp}
583 if summarizeLogs >> ${doneFileTmp}; then
584 [[ -f AliESDs_Barrel.root ]] && echo "esd ${outputDir}/AliESDs_Barrel.root" >> ${doneFileTmp}
585 [[ -f AliESDfriends_v1.root ]] && echo "calibfile ${outputDir}/AliESDfriends_v1.root" >> ${doneFileTmp}
586 [[ -f QAresults_Barrel.root ]] && echo "qafile ${outputDir}/QAresults_Barrel.root" >> ${doneFileTmp}
587 [[ -f QAresults_Outer.root ]] && echo "qafile ${outputDir}/QAresults_Outer.root" >> ${doneFileTmp}
588 [[ -f QAresults_barrel.root ]] && echo "qafile ${outputDir}/QAresults_barrel.root" >> ${doneFileTmp}
589 [[ -f QAresults_outer.root ]] && echo "qafile ${outputDir}/QAresults_outer.root" >> ${doneFileTmp}
590 [[ -f FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFileTmp}
922d0b10 591 else
ef77d296 592 if grep "qa_outer.log.*OK" ${doneFileTmp} > /dev/null; then
593 [[ -f QAresults_Outer.root ]] && echo "qafile ${outputDir}/QAresults_Outer.root" >> ${doneFileTmp}
594 [[ -f QAresults_outer.root ]] && echo "qafile ${outputDir}/QAresults_outer.root" >> ${doneFileTmp}
922d0b10 595 fi
ef77d296 596 if grep "qa_barrel.log.*OK" ${doneFileTmp} > /dev/null; then
597 [[ -f QAresults_Barrel.root ]] && echo "qafile ${outputDir}/QAresults_Barrel.root" >> ${doneFileTmp}
598 [[ -f QAresults_barrel.root ]] && echo "qafile ${outputDir}/QAresults_barrel.root" >> ${doneFileTmp}
922d0b10 599 fi
ef77d296 600 if grep "filtering.log.*OK" ${doneFileTmp} > /dev/null; then
601 [[ -f FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFileTmp}
922d0b10 602 fi
ab946185 603 fi
604
71353a5f 605 [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
ef77d296 606 cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
607 [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
5de3064b 608 return 0
78d0bbfd 609)
fc7e1e45 610
611
612goMergeCPass0()
78d0bbfd 613(
fc7e1e45 614 #
615 # find the output files and merge them
616 #
617
71353a5f 618 outputDir=${1}
619 ocdbStorage=${2}
620 configFile=${3}
621 runNumber=${4}
622 calibrationFilesToMerge=${5} #can be a non-existent file, will then be produced on the fly
623 shift 5
6fc48f9e 624 if ! parseConfig ${configFile} "$@"; then return 1; fi
71353a5f 625
39527a15 626 #record the working directory provided by the batch system
627 batchWorkingDirectory=${PWD}
628
71353a5f 629 [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
ef77d296 630
631 # This file signals that everything went fine
632 doneFileBase="merge.cpass0.run${runNumber}.done"
633
634 # We will have two copies of the file
635 mkdir -p "${commonOutputPath}/meta" || return 1
636 doneFileTmp="${batchWorkingDirectory}/${doneFileBase}"
637 doneFile="${commonOutputPath}/meta/${doneFileBase}"
71353a5f 638
639 umask 0002
640 ulimit -c unlimited
fc7e1e45 641
71353a5f 642 [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
fc7e1e45 643
71353a5f 644 #runpath=${PWD}/rundir_cpass0_Merge_${runNumber}
645 runpath=${outputDir}
0eaf38c7 646 #[[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
647 [[ ${reconstructInTemporaryDir} -eq 1 ]] && runpath=$(mktemp -d -t mergeCPass0.XXXXXX)
fc7e1e45 648
71353a5f 649 mkdir -p ${runpath}
1795556a 650 if [[ ! -d ${runpath} ]]; then
ef77d296 651 touch ${doneFileTmp}
652 echo "not able to make the runpath ${runpath}" >> ${doneFileTmp}
653 cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
654 [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
1795556a 655 return 1
656 fi
0eaf38c7 657 if ! cd ${runpath}; then
ef77d296 658 touch ${doneFileTmp}
659 echo "PWD=$PWD is not the runpath=${runpath}" >> ${doneFileTmp}
660 cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
661 [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
0eaf38c7 662 return 1
663 fi
fc7e1e45 664
71353a5f 665 logOutputDir=${runpath}
666 [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir}
1795556a 667 [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/stdout
71353a5f 668 echo "${0} $*"
fc7e1e45 669
fc7e1e45 670 mergingScript="mergeMakeOCDB.byComponent.sh"
fc7e1e45 671
672 echo goMergeCPass0 SETUP:
71353a5f 673 echo runNumber=${runNumber}
674 echo outputDir=${outputDir}
675 echo ocdbStorage=${ocdbStorage}
676 echo calibrationFilesToMerge=${calibrationFilesToMerge}
677 echo mergingScript=${mergingScript}
678 echo commonOutputPath=${commonOutputPath}
679 echo runpath=${runpath}
fc7e1e45 680
681 # copy files in case they are not already there
682 filesMergeCPass0=(
39527a15 683 "${batchWorkingDirectory}/${calibrationFilesToMerge}"
684 "${batchWorkingDirectory}/OCDB.root"
685 "${batchWorkingDirectory}/localOCDBaccessConfig.C"
71353a5f 686 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/mergeMakeOCDB.byComponent.sh"
687 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/mergeByComponent.C"
688 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/makeOCDB.C"
689 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/merge.C"
690 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/mergeMakeOCDB.sh"
fc7e1e45 691 )
692 for file in ${filesMergeCPass0[*]}; do
693 [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
1a0e38e3 694 [[ ${file##*/} =~ .*\.sh ]] && chmod +x ${file##*/}
fc7e1e45 695 done
696
71353a5f 697 sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh
698
699 alirootInfo > ALICE_ROOT.log
fc7e1e45 700
701 #
cb473870 702 echo "PWD"
71353a5f 703 /bin/ls
cb473870 704 echo "PWD/.."
71353a5f 705 /bin/ls ../
cb473870 706
fc7e1e45 707
708 #merge calibration
71353a5f 709 chmod u+x ${mergingScript}
fc7e1e45 710 mkdir -p ./OCDB
71353a5f 711 if [[ ! -f ${calibrationFilesToMerge} ]]; then
712 echo "/bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root > ${calibrationFilesToMerge}"
713 /bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root 2>/dev/null > ${calibrationFilesToMerge}
fc7e1e45 714 fi
715
71353a5f 716 echo "${mergingScript} ${calibrationFilesToMerge} ${runNumber} local://./OCDB ${ocdbStorage}"
717 if [[ -n ${pretend} ]]; then
0eaf38c7 718 sleep ${pretendDelay}
fc7e1e45 719 touch CalibObjects.root
720 touch ocdb.log
721 touch merge.log
617f8886 722 touch dcsTime.root
71353a5f 723 mkdir -p ./OCDB/TPC/Calib/TimeGain/
724 mkdir -p ./OCDB/TPC/Calib/TimeDrift/
1795556a 725 echo "some calibration" >> ./OCDB/TPC/Calib/TimeGain/someCalibObject_0-999999_cpass0.root
726 echo "some calibration" >> ./OCDB/TPC/Calib/TimeDrift/otherCalibObject_0-999999_cpass0.root
fc7e1e45 727 else
1795556a 728 ./${mergingScript} ${calibrationFilesToMerge} ${runNumber} "local://./OCDB" ${ocdbStorage} >> "mergeMakeOCDB.log"
fc7e1e45 729
617f8886 730 #produce the calib trees for expert QA (dcsTime.root)
731 goMakeLocalOCDBaccessConfig ./OCDB
732 echo aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")"
733 aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")"
734 fi
735
fc7e1e45 736 ### produce the output
737 #tar the produced OCDB for reuse
ef77d296 738 #tar czf ${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz ./OCDB
739
740 # Create tarball with OCDB, store on the shared directory, create signal file on batch directory
741 mkdir -p ${commonOutputPath}/meta
742 baseTar="cpass0.localOCDB.${runNumber}.tgz"
743 tar czf ${batchWorkingDirectory}/${baseTar} ./OCDB && \
744 mv ${batchWorkingDirectory}/${baseTar} ${commonOutputPath}/meta/${baseTar} && \
745 touch ${batchWorkingDirectory}/${baseTar}.done
fc7e1e45 746
71353a5f 747 /bin/ls
5ed7c82c 748
749 #copy all to output dir
0eaf38c7 750 cp -pf -R ${runpath}/* ${outputDir}
617f8886 751
71353a5f 752 if [[ -n ${generateMC} ]]; then
753 goPrintValues sim ${commonOutputPath}/meta/sim.run${runNumber}.list ${commonOutputPath}/meta/cpass0.job*.run${runNumber}.done
754 fi
755
fc7e1e45 756 #validate merging cpass0
5ed7c82c 757 cd ${outputDir}
ef77d296 758 touch ${doneFileTmp}
759 echo "dir ${outputDir}" >> ${doneFileTmp}
760 if summarizeLogs >> ${doneFileTmp}; then
761 [[ -f CalibObjects.root ]] && echo "calibfile ${outputDir}/CalibObjects.root" >> ${doneFileTmp}
762 [[ -f dcsTime.root ]] && echo "dcsTree ${outputDir}/dcsTime.root" >> ${doneFileTmp}
ab946185 763 fi
fc7e1e45 764
71353a5f 765 [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
ef77d296 766 cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
767 [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
5de3064b 768 return 0
78d0bbfd 769)
fc7e1e45 770
771goMergeCPass1()
78d0bbfd 772(
fc7e1e45 773 #
774 # find the output files and merge them
775 #
776
71353a5f 777 outputDir=${1}
778 ocdbStorage=${2}
779 configFile=${3}
780 runNumber=${4}
781 calibrationFilesToMerge=${5}
782 qaFilesToMerge=${6}
444b9916 783 filteredFilesToMerge=${7}
784 shift 7
6fc48f9e 785 if ! parseConfig ${configFile} "$@"; then return 1; fi
71353a5f 786
39527a15 787 #record the working directory provided by the batch system
788 batchWorkingDirectory=${PWD}
789
71353a5f 790 [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
ef77d296 791
792 # This file signals that everything went fine
793 doneFileBase="merge.cpass1.run${runNumber}.done"
794
795 # We will have two copies of the file
796 mkdir -p "${commonOutputPath}/meta" || return 1
797 doneFileTmp="${batchWorkingDirectory}/${doneFileBase}"
798 doneFile="${commonOutputPath}/meta/${doneFileBase}"
71353a5f 799
800 umask 0002
801 ulimit -c unlimited
fc7e1e45 802
803 #clean up first:
71353a5f 804 rm -f ${outputDir}/*.log
805 rm -f ${outputDir}/*.root
806 rm -f ${outputDir}/*done
807
808 [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
809
810 #runpath=${PWD}/rundir_cpass1_Merge_${runNumber}
811 runpath=${outputDir}
0eaf38c7 812 #[[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
813 [[ ${reconstructInTemporaryDir} -eq 1 ]] && runpath=$(mktemp -d -t mergeCPass1.XXXXXX)
71353a5f 814
815 mkdir -p ${runpath}
1795556a 816 if [[ ! -d ${runpath} ]]; then
ef77d296 817 touch ${doneFileTmp}
818 echo "not able to make the runpath ${runpath}" >> ${doneFileTmp}
819 cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
820 [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
1795556a 821 return 1
822 fi
0eaf38c7 823 if ! cd ${runpath}; then
ef77d296 824 touch ${doneFileTmp}
825 echo "PWD=$PWD is not the runpath=${runpath}" >> ${doneFileTmp}
826 cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
827 [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
0eaf38c7 828 return 1
829 fi
71353a5f 830
831 logOutputDir=${runpath}
832 [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir}
833 [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/mergeMakeOCDB.log
834 echo "${0} $*"
835
836 calibrationOutputFileName='AliESDfriends_v1.root'
837 qaOutputFileName='QAresults*.root'
fc7e1e45 838 mergingScript="mergeMakeOCDB.byComponent.sh"
cb473870 839 #important to have the string "Stage.txt" in the filename to trigger the merging
840 #it has to be a list of directories containing the files
fc7e1e45 841 qaMergedOutputFileName="QAresults_merged.root"
842
843 echo goMergeCPass1 SETUP:
71353a5f 844 echo runNumber=${runNumber}
845 echo outputDir=${outputDir}
846 echo ocdbStorage=${ocdbStorage}
847 echo calibrationFilesToMerge=$calibrationFilesToMerge
848 echo qaFilesToMerge=$qaFilesToMerge
849 echo calibrationOutputFileName=${calibrationOutputFileName}
850 echo mergingScript=${mergingScript}
fc7e1e45 851
852 # copy files in case they are not already there
853 filesMergeCPass1=(
39527a15 854 "${batchWorkingDirectory}/${calibrationFilesToMerge}"
855 "${batchWorkingDirectory}/${qaFilesToMerge}"
856 "${batchWorkingDirectory}/OCDB.root"
857 "${batchWorkingDirectory}/localOCDBaccessConfig.C"
3fbef3b9 858 "${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz"
39527a15 859 "${batchWorkingDirectory}/QAtrain_duo.C"
71353a5f 860 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/mergeMakeOCDB.byComponent.sh"
861 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/mergeByComponent.C"
862 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/makeOCDB.C"
863 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/merge.C"
864 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/mergeMakeOCDB.sh"
865 "${trustedQAtrainMacro}"
866 "${ALICE_ROOT}/ANALYSIS/macros/QAtrain_duo.C"
fc7e1e45 867 )
868 for file in ${filesMergeCPass1[*]}; do
869 [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
1a0e38e3 870 [[ ${file##*/} =~ .*\.sh ]] && chmod +x ${file##*/}
fc7e1e45 871 done
872
71353a5f 873 sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh
874
cb473870 875 #configure local OCDB storage from CPass0 (creates the localOCDBaccessConfig.C script)
876 if [[ -f cpass0.localOCDB.${runNumber}.tgz ]]; then
71353a5f 877 echo goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz"
878 goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz"
cb473870 879 else
880 echo "WARNING: file cpass0.localOCDB.${runNumber}.tgz not found!"
881 fi
882
71353a5f 883 alirootInfo > ALICE_ROOT.log
fc7e1e45 884
885 #
71353a5f 886 /bin/ls
fc7e1e45 887
888 #merge calibration
71353a5f 889 chmod u+x ${mergingScript}
fc7e1e45 890 mkdir -p OCDB
444b9916 891
892 #if not provided, create the lists of files to merge
893 if [[ ! -f ${filteredFilesToMerge} ]]; then
894 echo "/bin/ls -1 ${outputDir}/*/FilterEvents_Trees.root > ${filteredFilesToMerge}"
895 /bin/ls -1 ${outputDir}/*/FilterEvents_Trees.root 2>/dev/null > ${filteredFilesToMerge}
896 fi
71353a5f 897 if [[ ! -f ${calibrationFilesToMerge} ]]; then
444b9916 898 echo "/bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root > ${calibrationFilesToMerge}"
899 /bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root 2>/dev/null > ${calibrationFilesToMerge}
900 fi
901 if [[ ! -f ${qaFilesToMerge} ]]; then
902 #find the files, but only store the directories (QAtrain_duo.C requires this)
903 echo "/bin/ls -1 ${outputDir}/*/QAresults*.root | while read x; do echo ${x%/*}; done | sort | uniq > ${qaFilesToMerge}"
904 /bin/ls -1 ${outputDir}/*/QAresults*.root | while read x; do echo ${x%/*}; done | sort | uniq > ${qaFilesToMerge}
fc7e1e45 905 fi
906
71353a5f 907 echo "${mergingScript} ${calibrationFilesToMerge} ${runNumber} local://./OCDB ${ocdbStorage}"
908 if [[ -n ${pretend} ]]; then
0eaf38c7 909 sleep ${pretendDelay}
fc7e1e45 910 touch ocdb.log
71353a5f 911 touch cpass1.localOCDB.${runNumber}.tgz
71353a5f 912 touch ${qaMergedOutputFileName}
fc7e1e45 913 touch merge.log
cb473870 914 touch trending.root
444b9916 915 touch FilterEvents_Trees.root
ab946185 916 touch CalibObjects.root
917 touch dcsTime.root
918 touch ${qaMergedOutputFileName}
617f8886 919 mkdir -p OCDB
fc7e1e45 920 else
617f8886 921 ./${mergingScript} ${calibrationFilesToMerge} ${runNumber} "local://./OCDB" ${ocdbStorage}
922
923 #merge QA (and filtered trees)
924 [[ -n ${AliAnalysisTaskFilteredTree_fLowPtTrackDownscaligF} ]] && export AliAnalysisTaskFilteredTree_fLowPtTrackDownscaligF
925 [[ -n ${AliAnalysisTaskFilteredTree_fLowPtV0DownscaligF} ]] && export AliAnalysisTaskFilteredTree_fLowPtV0DownscaligF
926
927 #echo aliroot -l -b -q "merge.C(\"${qaFilesToMerge}\",\"\",kFALSE,\"${qaMergedOutputFileName}\")"
928 echo aliroot -b -q "QAtrain_duo.C(\"_barrel\",${runNumber},\"${qaFilesToMerge}\",1,\"${ocdbStorage}\")"
71353a5f 929 #aliroot -l -b -q "merge.C(\"${qaFilesToMerge}\",\"\",kFALSE,\"${qaMergedOutputFileName}\")"
930 aliroot -b -q "QAtrain_duo.C(\"_barrel\",${runNumber},\"${qaFilesToMerge}\",1,\"${ocdbStorage}\")" > mergeQA.log
931 mv QAresults_barrel.root ${qaMergedOutputFileName}
cb473870 932 mv trending_barrel.root trending.root
444b9916 933
934 #merge filtered trees
935 echo aliroot -l -b -q "merge.C(\"${qaFilesToMerge}\",\"\",kFALSE,\"${qaMergedOutputFileName}\")"
922d0b10 936 aliroot -l -b -q "merge.C(\"${filteredFilesToMerge}\",\"\",kFALSE,\"FilterEvents_Trees.root\")" > mergeFilteredTrees.log
617f8886 937
938 #produce the calib trees for expert QA
939 echo aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")"
922d0b10 940 aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")" > calibTree.log
fc7e1e45 941 fi
617f8886 942
ef77d296 943 # Create tarball with OCDB, store on the shared directory, create signal file on batch directory
944 mkdir -p ${commonOutputPath}/meta
945 baseTar="cpass1.localOCDB.${runNumber}.tgz"
946 tar czf ${batchWorkingDirectory}/${baseTar} ./OCDB && \
947 mv ${batchWorkingDirectory}/${baseTar} ${commonOutputPath}/meta/${baseTar} && \
948 touch ${batchWorkingDirectory}/${baseTar}.done
71353a5f 949
950 /bin/ls
5ed7c82c 951
952 #copy all to output dir
0eaf38c7 953 cp -pf -R ${runpath}/* ${outputDir}
5ed7c82c 954
fc7e1e45 955 #validate merge cpass1
5ed7c82c 956 cd ${outputDir}
ef77d296 957 touch ${doneFileTmp}
958 echo "dir ${outputDir}" >> ${doneFileTmp}
959 if summarizeLogs >> ${doneFileTmp}; then
960 [[ -f CalibObjects.root ]] && echo "calibfile ${outputDir}/CalibObjects.root" >> ${doneFileTmp}
961 [[ -f ${qaMergedOutputFileName} ]] && echo "qafile ${outputDir}/${qaMergedOutputFileName}" >> ${doneFileTmp}
962 [[ -f trending.root ]] && echo "trendingfile ${outputDir}/trending.root" >> ${doneFileTmp}
963 [[ -f dcsTime.root ]] && echo "dcsTree ${outputDir}/dcsTime.root" >> ${doneFileTmp}
964 [[ -f FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFileTmp}
922d0b10 965 else
ef77d296 966 if grep "mergeQA.log.*OK" ${doneFileTmp} > /dev/null; then
967 [[ -f ${qaMergedOutputFileName} ]] && echo "qafile ${outputDir}/${qaMergedOutputFileName}" >> ${doneFileTmp}
922d0b10 968 fi
ef77d296 969 if grep "mergeFilteredTrees.log.*OK" ${doneFileTmp} > /dev/null; then
970 [[ -f FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFileTmp}
922d0b10 971 fi
ab946185 972 fi
71353a5f 973
974 [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
ef77d296 975 cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
976 [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
5de3064b 977 return 0
78d0bbfd 978)
cb473870 979
980goMerge()
78d0bbfd 981(
cb473870 982 #generic root merge using CPass1 merge.C script
71353a5f 983 inputList=${1}
984 outputFile=${2}
0eaf38c7 985 configFile=${3-"benchmark.config"}
71353a5f 986 shift 3
6fc48f9e 987 if ! parseConfig ${configFile} "$@"; then return 1; fi
39527a15 988
989 #record the working directory provided by the batch system
990 batchWorkingDirectory=${PWD}
fc7e1e45 991
71353a5f 992 [[ ! -f ${inputList} ]] && echo "inputList ${inputList} does not exist!" && return 1
993 [[ ! -f ${configFile} ]] && echo "configFile ${configFile} does not exist!" && return 1
994 umask 0002
995 [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
996 rm -f ${outputFile}
997 aliroot -b -q "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/merge.C(\"${inputList}\",\"\",kFALSE,\"${outputFile}\")" > merge_${inputList}.log
5de3064b 998 return 0
78d0bbfd 999)
fc7e1e45 1000
cb473870 1001goSubmitMakeflow()
5dfac970 1002{
cb473870 1003 #run
71353a5f 1004 productionID=${1}
1005 inputFileList=${2}
1006 configFile=${3}
1007 shift 3
444b9916 1008 extraOpts=("$@")
6fc48f9e 1009 if ! parseConfig ${configFile} "${extraOpts[@]}"; then return 1; fi
39527a15 1010
1011 #record the working directory provided by the batch system
1012 batchWorkingDirectory=${PWD}
fc7e1e45 1013
5dfac970 1014 [[ -z ${configFile} ]] && configFile="benchmark.config"
1015 [[ ! -f ${configFile} ]] && echo "no config file found (${configFile})" && return 1
fc7e1e45 1016
71353a5f 1017 if [[ ! $(which makeflow &>/dev/null) && -n ${makeflowPath} ]]; then
cb473870 1018 echo "setting the makflow path from the config: "
1019 echo " export PATH=${makeflowPath}:${PATH}"
1020 export PATH=${makeflowPath}:${PATH}
1021 fi
1022
39527a15 1023 #create the common output dir and the meta dir
1024 commonOutputPath=${baseOutputDirectory}/${productionID}
1025 if [[ -d ${commonOutputPath} ]]; then
1026 echo "output dir ${commonOutputPath} exists!"
0eaf38c7 1027 #return 1
39527a15 1028 else
1029 mkdir -p ${commonOutputPath}
1030 fi
1031 mkdir -p ${commonOutputPath}/meta
839b07ca 1032
0eaf38c7 1033 self=${0}
1034 #if which greadlink; then self=$(greadlink -f "${0}"); fi
839b07ca 1035
0eaf38c7 1036 #for reference copy the setup to the output dir
839b07ca 1037 cp ${self} ${commonOutputPath}
1038 cp ${configFile} ${commonOutputPath}
333bc229 1039 cp ${inputFileList} ${commonOutputPath}
39527a15 1040
cb473870 1041 #submit - use makeflow if available, fall back to old stuff when makeflow not there
1042 if which makeflow; then
39527a15 1043 goGenerateMakeflow ${productionID} ${inputFileList} ${configFile} "${extraOpts[@]}" commonOutputPath=${commonOutputPath} > benchmark.makeflow
1795556a 1044 cp benchmark.makeflow ${commonOutputPath}
cb473870 1045 makeflow ${makeflowOptions} benchmark.makeflow
cb473870 1046 else
1047 echo "no makeflow!"
1048 fi
5de3064b 1049 return 0
5dfac970 1050}
fc7e1e45 1051
5dfac970 1052goGenerateMakeflow()
78d0bbfd 1053(
5dfac970 1054 #generate the makeflow file
ab946185 1055 [[ $# -lt 3 ]] && echo "args: id inputFileList configFile" && return 1
71353a5f 1056 productionID=${1}
1057 inputFileList=${2}
1058 configFile=${3}
1059 shift 3
444b9916 1060 extraOpts=("$@")
0eaf38c7 1061
1062 #batch systems/makeflow sometimes handle spaces in arguments poorly, so encode them
1063 for (( i=0;i<${#extraOpts[@]};i++ )); do
1064 extraOpts[i]=$(encSpaces "${extraOpts[i]}")
1065 done
1066 extraOpts+=("encodedSpaces=1")
1067
39527a15 1068 if ! parseConfig ${configFile} "${extraOpts[@]}" &>/dev/null; then return 1; fi
1069
1a0e38e3 1070 #extra safety
1071 if [[ -z ${commonOutputPath} ]]; then
1072 commonOutputPath=${baseOutputDirectory}/${productionID}
1073 extraOpts=( "${extraOpts[@]}" "commonOutputPath=${commonOutputPath}" )
1074 fi
bff71da0 1075
39527a15 1076 #record the working directory provided by the batch system
1077 batchWorkingDirectory=${PWD}
fc7e1e45 1078
5dfac970 1079 [[ -z ${configFile} ]] && configFile="benchmark.config"
1080 [[ ! -f ${configFile} ]] && echo "no config file found (${configFile})" && return 1
fc7e1e45 1081
5dfac970 1082 #these files will be made a dependency - will be copied to the working dir of the jobs
1083 declare -a copyFiles
1084 inputFiles=(
1085 "OCDB.root"
1086 "localOCDBaccessConfig.C"
cb473870 1087 "QAtrain_duo.C"
1088 "runCPass1.sh"
1089 "recCPass1.C"
1090 "recCPass1_OuterDet.C"
1091 "runCalibTrain.C"
1092 "runCPass0.sh"
1093 "recCPass0.C"
ef77d296 1094 "runQA.sh"
5dfac970 1095 )
1096 for file in ${inputFiles[*]}; do
1097 [[ -f ${file} ]] && copyFiles+=("${file}")
1098 done
1099
1100 #create the makeflow file
cb473870 1101 [[ -n ${batchFlags} ]] && echo "BATCH_OPTIONS = ${batchFlags}"
444b9916 1102 declare -A arr_cpass0_merged arr_cpass1_merged
1103 declare -A arr_cpass0_calib_list arr_cpass1_calib_list
1104 declare -A arr_cpass1_QA_list arr_cpass1_ESD_list arr_cpass1_filtered_list
6fc48f9e 1105 declare -A arr_cpass0_profiled_outputs
444b9916 1106 declare -A listOfRuns
1107 [[ -n ${runNumber} ]] && listOfRuns[${runNumber}]=1
617f8886 1108 while read x; do tmpRun=$(guessRunNumber ${x}); [[ -n ${tmpRun} ]] && listOfRuns[${tmpRun}]=1; done < ${inputFileList}
444b9916 1109 for runNumber in "${!listOfRuns[@]}"; do
71353a5f 1110 [[ -z ${runNumber} ]] && continue
fc7e1e45 1111 [[ ! ${runNumber} =~ ^[0-9]*[0-9]$ ]] && continue
fc7e1e45 1112
444b9916 1113 unset arr_cpass0_outputs
1114 unset arr_cpass1_outputs
1115 declare -a arr_cpass0_outputs
1116 declare -a arr_cpass1_outputs
fc7e1e45 1117
ef77d296 1118 #Header
1119 echo "### Automatically generated on $(LANG=C date) ###"
1120 echo ; echo
1121
cb473870 1122 jobindex=0
6fc48f9e 1123 inputFile=""
5dfac970 1124 while read inputFile; do
1125 currentDefaultOCDB=${defaultOCDB}
cb473870 1126 [[ -z ${autoOCDB} ]] && autoOCDB=1
1127 if [[ ${autoOCDB} -ne 0 ]]; then
71353a5f 1128 currentDefaultOCDB=$(setYear ${inputFile} ${defaultOCDB})
cb473870 1129 fi
ef77d296 1130 guessRunData ${inputFile}
1131
1132 #Set variables
1133 echo "### Variables ###"
1134 echo "OUTPATH=\"${commonOutputPath}/${year}/${period}\""
1135 echo ; echo
fc7e1e45 1136
5dfac970 1137 #CPass0
ef77d296 1138 #arr_cpass0_outputs[${jobindex}]="${commonOutputPath}/meta/cpass0.job${jobindex}.run${runNumber}.done"
1139 arr_cpass0_outputs[${jobindex}]="cpass0.job${jobindex}.run${runNumber}.done"
1140 echo "### CPass0 ###"
1141 echo "${arr_cpass0_outputs[${jobindex}]}: benchmark.sh ${configFile} ${copyFiles[@]}"
1142 echo " ${alirootEnv} ./benchmark.sh CPass0 \$OUTPATH/000${runNumber}/cpass0 ${inputFile} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} ${jobindex} ${extraOpts[@]}"" "
1143 echo ; echo
5dfac970 1144
1145 #CPass1
ef77d296 1146 #arr_cpass1_outputs[${jobindex}]="${commonOutputPath}/meta/cpass1.job${jobindex}.run${runNumber}.done"
1147 arr_cpass1_outputs[${jobindex}]="cpass1.job${jobindex}.run${runNumber}.done"
1148 echo "### CPass1 ###"
1149 echo "${arr_cpass1_outputs[${jobindex}]}: benchmark.sh ${configFile} cpass0.localOCDB.${runNumber}.tgz.done ${copyFiles[@]}"
1150 echo " ${alirootEnv} ./benchmark.sh CPass1 \$OUTPATH/000${runNumber}/cpass1 ${inputFile} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} ${jobindex} ${extraOpts[@]}"" "
1151 echo ; echo
5dfac970 1152 ((jobindex++))
fc7e1e45 1153
71353a5f 1154 done< <(grep "/000${runNumber}/" ${inputFileList})
cb473870 1155
5dfac970 1156 #CPass0 list of Calib files to merge
ef77d296 1157 #arr_cpass0_calib_list[${runNumber}]="${commonOutputPath}/meta/cpass0.calib.run${runNumber}.list"
1158 arr_cpass0_calib_list[${runNumber}]="cpass0.calib.run${runNumber}.list"
1159 echo "### Produces the list of CPass0 files to merge (executes locally) ###"
1160 echo "${arr_cpass0_calib_list[${runNumber}]}: benchmark.sh ${arr_cpass0_outputs[*]}"
1161 echo " LOCAL ./benchmark.sh PrintValues calibfile ${arr_cpass0_calib_list[${runNumber}]} ${arr_cpass0_outputs[*]} && mkdir -p \$OUTPATH/meta && cp ${arr_cpass0_calib_list[${runNumber}]} \$OUTPATH/meta/${arr_cpass0_calib_list[${runNumber}]}"
1162 echo ; echo
71353a5f 1163
1164 #CPass0 merging
ef77d296 1165 echo "### Merges CPass0 files ###"
1166 #arr_cpass0_merged[${runNumber}]="${commonOutputPath}/meta/merge.cpass0.run${runNumber}.done"
1167 arr_cpass0_merged[${runNumber}]="merge.cpass0.run${runNumber}.done"
1168 echo "cpass0.localOCDB.${runNumber}.tgz.done ${arr_cpass0_merged[${runNumber}]}: benchmark.sh ${configFile} ${arr_cpass0_calib_list[${runNumber}]} ${copyFiles[@]}"
1169 echo " ${alirootEnv} ./benchmark.sh MergeCPass0 \$OUTPATH/000${runNumber}/cpass0 ${currentDefaultOCDB} ${configFile} ${runNumber} ${arr_cpass0_calib_list[${runNumber}]} ${extraOpts[@]}"" "
1170 echo ; echo
fc7e1e45 1171
ab946185 1172 #CPass1 list of Calib/QA/ESD/filtered files
1173 # the trick with QA is to have the string "Stage.txt" in the file name of the list of directories with QA output to trigger
1174 # the production of the QA trending tree (only then the task->Finish() will be called in QAtrain_duo.C, on the grid
8d6a0a43 1175 # this corresponds to the last merging stage)
ef77d296 1176 #arr_cpass1_QA_list[${runNumber}]="${commonOutputPath}/meta/cpass1.QA.run${runNumber}.lastMergingStage.txt.list"
1177 arr_cpass1_QA_list[${runNumber}]="cpass1.QA.run${runNumber}.lastMergingStage.txt.list"
1178 echo "### Lists CPass1 QA ###"
444b9916 1179 echo "${arr_cpass1_QA_list[${runNumber}]}: benchmark.sh ${arr_cpass1_outputs[*]}"
ef77d296 1180 echo " LOCAL ./benchmark.sh PrintValues dir ${arr_cpass1_QA_list[${runNumber}]} ${arr_cpass1_outputs[*]} && mkdir -p \$OUTPATH/meta && cp ${arr_cpass1_QA_list[${runNumber}]} \$OUTPATH/meta/${arr_cpass1_QA_list[${runNumber}]}"
1181 echo ; echo
1182
1183 #arr_cpass1_calib_list[${runNumber}]="${commonOutputPath}/meta/cpass1.calib.run${runNumber}.list"
1184 arr_cpass1_calib_list[${runNumber}]="cpass1.calib.run${runNumber}.list"
1185 echo "### Lists CPass1 Calib ###"
1186 echo "${arr_cpass1_calib_list[${runNumber}]}: benchmark.sh ${arr_cpass1_outputs[*]}"
1187 echo " LOCAL ./benchmark.sh PrintValues calibfile ${arr_cpass1_calib_list[${runNumber}]} ${arr_cpass1_outputs[*]} && mkdir -p \$OUTPATH/meta && cp ${arr_cpass1_calib_list[${runNumber}]} \$OUTPATH/meta/${arr_cpass1_calib_list[${runNumber}]}"
1188 echo ; echo
1189
1190 #arr_cpass1_ESD_list[${runNumber}]="${commonOutputPath}/meta/cpass1.ESD.run${runNumber}.list"
1191 arr_cpass1_ESD_list[${runNumber}]="cpass1.ESD.run${runNumber}.list"
1192 echo "### Lists CPass1 ESDs ###"
1193 echo "${arr_cpass1_ESD_list[${runNumber}]}: benchmark.sh ${arr_cpass1_outputs[*]}"
1194 echo " LOCAL ./benchmark.sh PrintValues esd ${arr_cpass1_ESD_list[${runNumber}]} ${arr_cpass1_outputs[*]} && mkdir -p \$OUTPATH/meta && cp ${arr_cpass1_ESD_list[${runNumber}]} \$OUTPATH/meta/${arr_cpass1_ESD_list[${runNumber}]}"
1195 echo ; echo
1196
1197 #arr_cpass1_filtered_list[${runNumber}]="${commonOutputPath}/meta/cpass1.filtered.run${runNumber}.list"
1198 arr_cpass1_filtered_list[${runNumber}]="cpass1.filtered.run${runNumber}.list"
1199 echo "### Lists CPass1 filtered ###"
1200 echo "${arr_cpass1_filtered_list[${runNumber}]}: benchmark.sh ${arr_cpass1_outputs[*]}"
1201 echo " LOCAL ./benchmark.sh PrintValues filteredTree ${arr_cpass1_filtered_list[${runNumber}]} ${arr_cpass1_outputs[*]} && mkdir -p \$OUTPATH/meta && cp ${arr_cpass1_filtered_list[${runNumber}]} \$OUTPATH/meta/${arr_cpass1_filtered_list[${runNumber}]}"
1202 echo ; echo
444b9916 1203
71353a5f 1204 #CPass1 merging
ef77d296 1205 #arr_cpass1_merged[${runNumber}]="${commonOutputPath}/meta/merge.cpass1.run${runNumber}.done"
1206 arr_cpass1_merged[${runNumber}]="merge.cpass1.run${runNumber}.done"
1207 echo "### Merges CPass1 files ###"
1208 echo "cpass1.localOCDB.${runNumber}.tgz.done ${arr_cpass1_merged[${runNumber}]}: benchmark.sh ${configFile} ${arr_cpass1_calib_list[${runNumber}]} ${arr_cpass1_QA_list[${runNumber}]} ${copyFiles[@]}"
1209 echo " ${alirootEnv} ./benchmark.sh MergeCPass1 \$OUTPATH/000${runNumber}/cpass1 ${currentDefaultOCDB} ${configFile} ${runNumber} ${arr_cpass1_calib_list[${runNumber}]} ${arr_cpass1_QA_list[${runNumber}]} ${arr_cpass1_filtered_list[${runNumber}]} ${extraOpts[@]}"" "
1210 echo ; echo
fc7e1e45 1211
6fc48f9e 1212 #CPass0 wrapped in a profiling tool (valgrind,....)
39527a15 1213 if [[ -n ${profilingCommand} ]]; then
0eaf38c7 1214 inputFile=$(grep -m1 "${runNumber}/" ${inputFileList})
1215 [[ -z ${nEventsProfiling} ]] && nEventsProfiling=2
1216 currentDefaultOCDB=$(setYear ${inputFile} ${defaultOCDB})
1217 jobindex="profiling"
1218
ef77d296 1219 #arr_cpass0_profiled_outputs[${runNumber}]="${commonOutputPath}/meta/profiling.cpass0.job${jobindex}.run${runNumber}.done"
1220 arr_cpass0_profiled_outputs[${runNumber}]="profiling.cpass0.job${jobindex}.run${runNumber}.done"
1221 echo "### CPass0 in a profiler ###"
1222 echo "${arr_cpass0_profiled_outputs[${runNumber}]}: benchmark.sh ${configFile} ${copyFiles[@]}"
0eaf38c7 1223 profilingCommand=$(encSpaces "${profilingCommand}")
ef77d296 1224 echo " ${alirootEnv} ./benchmark.sh CPass0 \$OUTPATH/000${runNumber}/${jobindex} ${inputFile} ${nEventsProfiling} ${currentDefaultOCDB} ${configFile} ${runNumber} ${jobindex} ${extraOpts[@]} useProfilingCommand=${profilingCommand}"
1225 echo ; echo
6fc48f9e 1226 fi
1227
444b9916 1228 done #runs
fc7e1e45 1229
5dfac970 1230 #Summary
ef77d296 1231 echo "### Summary ###"
1232 echo "summary.log: benchmark.sh ${configFile} ${arr_cpass1_merged[*]}"
1233 echo " ${alirootEnv} ./benchmark.sh MakeSummary ${configFile} ${extraOpts[@]}"
1234 echo ; echo
cb473870 1235
5de3064b 1236 return 0
78d0bbfd 1237)
cb473870 1238
1239goPrintValues()
78d0bbfd 1240(
cb473870 1241 #print the values given the key from any number of files (store in output file on second argument)
71353a5f 1242 if [[ $# -lt 3 ]]; then
1243 echo "goPrintValues key outputFile inputFiles"
ef77d296 1244 echo "if outputFile is \"-\" don't redirect to a file"
71353a5f 1245 return
1246 fi
1247 key=${1}
1248 outputFile=${2}
1249 [[ ${outputFile} =~ "-" ]] && outputFile=""
cb473870 1250 shift 2 #remove 2 first arguments from arg list to only pass the input files to awk
71353a5f 1251 awk -v key=${key} '$0 ~ key" " {print $2}' "$@" | tee ${outputFile}
5de3064b 1252 return 0
78d0bbfd 1253)
fc7e1e45 1254
5dfac970 1255goCreateQAplots()
78d0bbfd 1256(
5dfac970 1257 umask 0002
71353a5f 1258 mergedQAfileList=${1}
1259 productionID=${2}
1260 outputDir=${3}
1261 configFile=${4}
1262 shift 4
e48b4924 1263 if ! parseConfig ${configFile} "$@"; then return 1; fi
71353a5f 1264
39527a15 1265 #record the working directory provided by the batch system
1266 batchWorkingDirectory=${PWD}
1267
71353a5f 1268 [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
fc7e1e45 1269
71353a5f 1270 [[ -z ${logOutputDir} ]] && logOutputDir=${PWD}
0eaf38c7 1271 [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/makeQAplots.log
71353a5f 1272 echo "${0} $*"
1273
1274 olddir=${PWD}
1275 mkdir -p ${outputDir}
1276 cd ${outputDir}
839b07ca 1277 [[ ! "${PWD}" =~ "${outputDir}" ]] && echo "PWD is not equal to outputDir=${outputDir}" && cd ${olddir} && return 1
fc7e1e45 1278
ef77d296 1279 inputFiles=(
1280 "${batchWorkingDirectory}/runQA.sh"
1281 "${ALICE_ROOT}/PWGPP/QA/scripts/runQA.sh"
1282 )
1283 for file in ${inputFiles[*]}; do
1284 [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
1285 done
1286
1287 echo "running QA with command:"
1288 echo ./runQA.sh inputList="${mergedQAfileList}" inputListHighPtTrees="${filteringList}" ocdbStorage="${defaultOCDB}"
1289 ./runQA.sh inputList="${mergedQAfileList}" inputListHighPtTrees="${filteringList}" ocdbStorage="${defaultOCDB}"
71353a5f 1290 cd ${olddir}
5de3064b 1291 return 0
78d0bbfd 1292)
fc7e1e45 1293
5dfac970 1294goTest()
78d0bbfd 1295(
1296 echo AA
1297)
fc7e1e45 1298
5dfac970 1299alirootInfo()
78d0bbfd 1300(
5dfac970 1301 umask 0002
71353a5f 1302 # save aliroot repository info
1303 [[ -z "${ALICE_ROOT}" ]] && return 1
1304
1305 echo "\${ALICE_ROOT}=${ALICE_ROOT}"
1306 echo "\${ROOTSYS}=${ROOTSYS}"
1307 echo "\${PATH}=${PATH}"
1308 echo "\${LD_LIBRARY_PATH}=${LD_LIBRARY_PATH}"
0eaf38c7 1309 echo
71353a5f 1310
1311 pushd ${PWD}
1312 cd ${ALICE_ROOT}
71353a5f 1313
1314 currentBranch=$(git rev-parse --abbrev-ref HEAD)
1315 git status
5dfac970 1316 echo ""
1317 echo ""
71353a5f 1318 git diff ${currentBranch}
1319 popd
5de3064b 1320 return 0
78d0bbfd 1321)
fc7e1e45 1322
5dfac970 1323setYear()
78d0bbfd 1324(
5dfac970 1325 #set the year
71353a5f 1326 # ${1} - year to be set
1327 # ${2} - where to set the year
1328 year1=$(guessYear ${1})
1329 year2=$(guessYear ${2})
1330 local path=${2}
1331 [[ ${year1} -ne ${year2} && -n ${year2} && -n ${year1} ]] && path=${2/\/${year2}\//\/${year1}\/}
1332 echo ${path}
5de3064b 1333 return 0
78d0bbfd 1334)
fc7e1e45 1335
5dfac970 1336guessPeriod()
78d0bbfd 1337(
5dfac970 1338 #guess the period from the path, pick the rightmost one
1339 local IFS="/"
71353a5f 1340 declare -a path=( ${1} )
5dfac970 1341 local dirDepth=${#path[*]}
1342 for ((x=${dirDepth}-1;x>=0;x--)); do
1343 local field=${path[${x}]}
71353a5f 1344 [[ ${field} =~ ^LHC[0-9][0-9][a-z]$ ]] && period=${field} && break
5dfac970 1345 done
71353a5f 1346 echo ${period}
5de3064b 1347 return 0
78d0bbfd 1348)
fc7e1e45 1349
5dfac970 1350guessYear()
78d0bbfd 1351(
5dfac970 1352 #guess the year from the path, pick the rightmost one
1353 local IFS="/"
71353a5f 1354 declare -a path=( ${1} )
5dfac970 1355 local dirDepth=${#path[*]}
1356 for ((x=${dirDepth}-1;x>=0;x--)); do
1357 local field=${path[${x}]}
71353a5f 1358 [[ ${field} =~ ^20[0-9][0-9]$ ]] && year=${field} && break
5dfac970 1359 done
71353a5f 1360 echo ${year}
5de3064b 1361 return 0
78d0bbfd 1362)
fc7e1e45 1363
5dfac970 1364guessRunNumber()
78d0bbfd 1365(
5dfac970 1366 #guess the run number from the path, pick the rightmost one
71353a5f 1367 #works for /path/foo/000123456/bar/...
1368 #and /path/foo.run123456.bar
1369 local IFS="/."
1370 declare -a path=( ${1} )
5dfac970 1371 local dirDepth=${#path[*]}
1372 for ((x=${dirDepth}-1;x>=0;x--)); do
1373 local field=${path[${x}]}
71353a5f 1374 field=${field/run/000}
617f8886 1375 [[ ${field} =~ [0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && runNumber=${field#000} && break
5dfac970 1376 done
71353a5f 1377 echo ${runNumber}
5de3064b 1378 return 0
78d0bbfd 1379)
fc7e1e45 1380
cb473870 1381validateLog()
78d0bbfd 1382(
71353a5f 1383 log=${1}
cb473870 1384 errorConditions=(
71353a5f 1385 'There was a crash'
1386 'floating'
1387 'error while loading shared libraries'
1388 'std::bad_alloc'
1389 's_err_syswatch_'
1390 'Thread [0-9]* (Thread'
1391 'AliFatal'
1392 'core dumped'
1393 '\.C.*error:.*\.h: No such file'
839b07ca 1394 'line.*Aborted'
71353a5f 1395 )
1396
1397 warningConditions=(
1398 'This is serious !'
1399 'rocVoltage out of range:'
cb473870 1400 )
1401
71353a5f 1402 local logstatus=0
1403 local errorSummary=""
1404 local warningSummary=""
1405
1406 for ((i=0; i<${#errorConditions[@]};i++)); do
1407 local tmp=$(grep -m1 -e "${errorConditions[${i}]}" ${log})
1408 [[ -n ${tmp} ]] && tmp+=" : "
1409 errorSummary+=${tmp}
1410 done
1411
1412 for ((i=0; i<${#warningConditions[@]};i++)); do
1413 local tmp=$(grep -m1 -e "${warningConditions[${i}]}" ${log})
1414 [[ -n ${tmp} ]] && tmp+=" : "
1415 warningSummary+=${tmp}
1416 done
1417
1418 if [[ -n ${errorSummary} ]]; then
1419 echo "${errorSummary}"
1420 return 1
1421 fi
cb473870 1422
71353a5f 1423 if [[ -n ${warningSummary} ]]; then
1424 echo "${warningSummary}"
1425 return 2
1426 fi
1427
1428 return 0
78d0bbfd 1429)
cb473870 1430
5dfac970 1431summarizeLogs()
78d0bbfd 1432(
5dfac970 1433 #print a summary of logs
1434 logFiles=(
1435 "*.log"
cb473870 1436 "stdout"
1437 "stderr"
5dfac970 1438 )
fc7e1e45 1439
cb473870 1440 #check logs
71353a5f 1441 local logstatus=0
5dfac970 1442 for log in ${logFiles[*]}; do
71353a5f 1443 finallog=${PWD%/}/${log}
1444 [[ ! -f ${log} ]] && continue
1445 errorSummary=$(validateLog ${log})
1446 validationStatus=$?
1447 if [[ ${validationStatus} -eq 0 ]]; then
5dfac970 1448 #in pretend mode randomly report an error in rec.log some cases
71353a5f 1449 if [[ -n ${pretend} && "${log}" == "rec.log" ]]; then
617f8886 1450 #[[ $(( ${RANDOM}%2 )) -ge 1 ]] && echo "${finallog} BAD random error" || echo "${finallog} OK"
1451 echo "${finallog} OK"
fc7e1e45 1452 else
71353a5f 1453 echo "${finallog} OK"
fc7e1e45 1454 fi
71353a5f 1455 elif [[ ${validationStatus} -eq 1 ]]; then
1456 echo "${finallog} BAD ${errorSummary}"
ab946185 1457 logstatus=1
71353a5f 1458 elif [[ ${validationStatus} -eq 2 ]]; then
1459 echo "${finallog} OK MWAH ${errorSummary}"
5dfac970 1460 fi
1461 done
cb473870 1462
1463 #report core files
71353a5f 1464 while read x; do
1465 echo ${x}
1466 chmod 644 ${x}
1467 gdb --batch --quiet -ex "bt" -ex "quit" aliroot ${x} > stacktrace_${x//\//_}.log
1468 done < <(/bin/ls ${PWD}/*/core 2>/dev/null; /bin/ls ${PWD}/core 2>/dev/null)
cb473870 1469
71353a5f 1470 return ${logstatus}
78d0bbfd 1471)
fc7e1e45 1472
5dfac970 1473spitOutLocalOCDBaccessConfig()
1474{
1475 umask 0002
71353a5f 1476 #find ${1} -name "*root" | \
1477 /bin/ls -1 ${1}/*/*/*/*.root 2>/dev/null | \
5dfac970 1478 while read line
1479 do
71353a5f 1480 local tmp=${line#${1}}
5dfac970 1481 echo ${tmp%/*} | \
71353a5f 1482 awk -v ocdb=${1} '{print " man->SetSpecificStorage(\""$1"\",\"local://"ocdb"\");"}'
5dfac970 1483 done
5de3064b 1484 return 0
5dfac970 1485}
fc7e1e45 1486
71353a5f 1487goMakeLocalOCDBaccessConfig()
5dfac970 1488{
1489 umask 0002
1490 # make a script that sets the specific storages form all the root files produced by CPass0
71353a5f 1491 local localOCDBpathCPass0=${1}
1492 local OCDBpathPrefix=${2}
1493 [[ -z ${OCDBpathPrefix} ]] && OCDBpathPrefix="."
1494
1495 if [[ -f ${localOCDBpathCPass0} && ${localOCDBpathCPass0} =~ \.tgz$ ]]; then
1496 tar xzf ${localOCDBpathCPass0}
1497 local localOCDBpathCPass0="${OCDBpathPrefix}/OCDB"
5dfac970 1498 fi
fc7e1e45 1499
5dfac970 1500 echo
1501 echo creating the specific storage script
1502 echo localOCDBaccessConfig.C
71353a5f 1503 echo based on OCDB: ${localOCDBaccessConfig}
5dfac970 1504 echo
fc7e1e45 1505
5dfac970 1506 local tempLocalOCDB=""
1507 if [[ -f localOCDBaccessConfig.C ]]; then
0eaf38c7 1508 tempLocalOCDB=$(mktemp -t tempLocalOCDB.XXXXXX)
71353a5f 1509 echo "egrep "SetSpecificStorage" localOCDBaccessConfig.C > ${tempLocalOCDB}"
1510 egrep "SetSpecificStorage" localOCDBaccessConfig.C > ${tempLocalOCDB}
5dfac970 1511 fi
fc7e1e45 1512
5dfac970 1513 echo "localOCDBaccessConfig()" > localOCDBaccessConfig.C
1514 echo "{" >> localOCDBaccessConfig.C
1515 echo " AliCDBManager* man = AliCDBManager::Instance();" >> localOCDBaccessConfig.C
71353a5f 1516 spitOutLocalOCDBaccessConfig ${localOCDBpathCPass0}|sort|uniq >> localOCDBaccessConfig.C
1517 [[ -f "${tempLocalOCDB}" ]] && cat ${tempLocalOCDB} >> localOCDBaccessConfig.C
5dfac970 1518 echo "}" >> localOCDBaccessConfig.C
fc7e1e45 1519
71353a5f 1520 [[ -f "${tempLocalOCDB}" ]] && rm -f ${tempLocalOCDB}
fc7e1e45 1521
5dfac970 1522 if ! grep SetSpecificStorage localOCDBaccessConfig.C; then
1523 echo
1524 echo "!!!!!!! CPass0 produced no OCDB entries"
1525 return 1
1526 fi
5de3064b 1527 return 0
5dfac970 1528}
fc7e1e45 1529
cb473870 1530goMakeFilteredTrees()
78d0bbfd 1531(
71353a5f 1532 outputDir=${1}
1533 runNumber=${2}
cb473870 1534 #get path to input list
71353a5f 1535 inputListfiles=${3}
cb473870 1536 #get scale number for tracks
71353a5f 1537 filterT=${4}
cb473870 1538 #get scale number for V0s
71353a5f 1539 filterV=${5}
cb473870 1540 #get OCDB path (optional)
1541 OCDBpath=${6}
1542 #get max number of files
1543 maxFiles=${7-"1000000"}
1544 #get offset of first file
1545 offsetFile=${8-"0"}
1546 #get max number of events
1547 maxEvents=${9-"30000000"}
1548 #get offset of first event
1549 offsetEvent=${10-"0"}
1550 configFile=${11-"benchmark.config"}
1551 esdFileName=${12-"AliESDs_Barrel.root"}
71353a5f 1552 shift 12
6fc48f9e 1553 if ! parseConfig ${configFile} "$@"; then return 1; fi
39527a15 1554
1555 #record the working directory provided by the batch system
1556 batchWorkingDirectory=${PWD}
1557
78d0bbfd 1558 [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
ef77d296 1559 doneFileBase=filtering.cpass1.run${runNumber}.done
1560 doneFileTmp=${batchWorkingDirectory}/${doneFileBase}
1561 doneFile=${commonOutputPath}/meta/${doneFileBase}
71353a5f 1562
cb473870 1563 cat > filtering.log << EOF
1564 goMakeFilteredTrees config:
71353a5f 1565 runpath=${runpath}
1566 outputDir=${outputDir}
1567 commonOutputPath=${commonOutputPath}
1568 ALICE_ROOT=${ALICE_ROOT}
1569 PATH=${PATH}
1570 offsetEvent=$offsetEvent
1571 configFile=$configFile
1572 esdFileName=$esdFileName
fab31805 1573 inputListfiles=$inputListfiles
1574 doneFile=$doneFile
cb473870 1575EOF
1576
fab31805 1577 #runpath=${outputDir}
1578 #[[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
1579 #[[ ${reconstructInTemporaryDir} -eq 1 ]] && runpath=$(mktemp -d -t goMakeFilteredTrees.XXXXXX)
1580 #mkdir -p ${outputDir}
1581 #mkdir -p ${runpath}
1582 #if ! cd ${runpath}; then
1583 # echo "PWD=$PWD is not the runpath=${runpath}"
1584 # touch ${doneFile}
1585 # return 1
1586 #fi
1587
cb473870 1588 if [[ -z ${pretend} ]];then
71353a5f 1589 aliroot -l -b -q "${ALICE_ROOT}/PWGPP/macros/runFilteringTask.C(\"${inputListfiles}\",${filterT},${filterV},\"${OCDBpath}\",${maxFiles},${offsetFile},${maxEvents},${offsetEvent},\"${esdFileName}\")" &>> filtering.log
cb473870 1590 else
0eaf38c7 1591 sleep ${pretendDelay}
71353a5f 1592 touch filtering.log FilterEvents_Trees.root
cb473870 1593 fi
1594 pwd
71353a5f 1595 /bin/ls
1596 touch ${doneFile}
1597 summarizeLogs >> ${doneFile}
fab31805 1598
1599 #echo mv -f * ${outputDir}
1600 #mv -f * ${outputDir}
1601 #[[ -f ${outputDir}/FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFile}
1602 #cd ${commonOutputPath}
1603 #[[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
e344162c 1604
ef77d296 1605 cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
1606 [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
e344162c 1607
5de3064b 1608 return 0
78d0bbfd 1609)
cb473870 1610
1611submit()
1612{
1613 umask 0002
444b9916 1614 [[ $# -lt 5 ]] && echo "at least 5 args needed, you supplied $#" && return 1
71353a5f 1615 JobID=${1}
1616 startID=${2}
1617 endID=${3}
1618 waitForJOBID=${4}
1619 command=${5}
444b9916 1620 shift 5
78d0bbfd 1621 local commandArgs=("$@")
444b9916 1622
1623 #add quote strings around the extra arguments
1624 for ((i=0; i<${#commandArgs[@]}; i++)); do
1625 commandArgs[i]=\"${commandArgs[i]}\"
1626 done
cb473870 1627
71353a5f 1628 [[ -z ${waitForJOBID} ]] && waitForJOBID=0
cb473870 1629
1630 newFarm=$(which qsub|grep "^/usr/bin/qsub")
1631
1632 batchSystem="SGE"
1633
71353a5f 1634 if [[ -z "${newFarm}" ]]
cb473870 1635 then
1636 #old LSF
1637 # submit it (as job array)
71353a5f 1638 nFiles=$(( ${endID}-${startID}+1 ))
1639 while [ ${startID} -le ${nFiles} ] ; do
1640 if [ $(expr ${nFiles} - ${startID}) -gt 999 ] ; then
1641 endID=$(expr ${startID} + 999)
cb473870 1642 else
71353a5f 1643 endID=${nFiles}
cb473870 1644 fi
1645 if [[ ${waitForJOBID} -eq 0 ]]; then
71353a5f 1646 echo ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}"
1647 ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}"
cb473870 1648 else
71353a5f 1649 echo ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -w "ended(${waitForJOBID})" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}"
1650 ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -w "ended(${waitForJOBID})" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}"
cb473870 1651 fi
71353a5f 1652 startID=$(expr ${endID} + 1)
cb473870 1653 done
1654 else
1655 #new SGE farm
1656 if [[ ${waitForJOBID} =~ "000" ]]; then
444b9916 1657 echo ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}"
1658 ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}"
cb473870 1659 else
444b9916 1660 echo ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -hold_jid "${waitForJOBID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}"
1661 ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -hold_jid "${waitForJOBID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}"
cb473870 1662 fi
1663 fi
5de3064b 1664 return 0
cb473870 1665}
1666
1667goSubmitBatch()
5dfac970 1668{
71353a5f 1669 if [[ $# -lt 3 ]]; then
1670 echo "minimal use:"
1671 echo " ${0} submit fileList productionID configFile"
1672 return 0
1673 fi
1674
1675 productionID=${1}
1676 inputList=${2}
1677 configFile=${3:-"benchmark.config"}
0eaf38c7 1678 #if which greadlink; then configFile=$(greadlink -f ${configFile}); fi
71353a5f 1679 shift 3
444b9916 1680 extraOpts=("$@")
6fc48f9e 1681 if ! parseConfig ${configFile} "${extraOpts[@]}"; then return 1; fi
39527a15 1682
b0e401b4 1683 #batch systems/makeflow sometimes handle spaces in arguments poorly, so encode them
1684 for (( i=0;i<${#extraOpts[@]};i++ )); do
1685 extraOpts[i]=$(encSpaces "${extraOpts[i]}")
1686 done
1687 extraOpts+=("encodedSpaces=1")
ef77d296 1688 #this removes the copy of the done file used by makeflow (in the running dir)
1689 extraOpts+=("removeTMPdoneFile=1")
b0e401b4 1690
39527a15 1691 #record the working directory provided by the batch system
1692 batchWorkingDirectory=${PWD}
fc7e1e45 1693
5dfac970 1694 #redirect all output to submit.log
1695 echo "redirecting all output to ${PWD}/submit_${productionID//"/"/_}.log"
1696 exec 7>&1
1697 exec 1>submit_${productionID//"/"/_}.log 2>&1
fc7e1e45 1698
5dfac970 1699 umask 0002
71353a5f 1700 echo ${0}" submit $*"
1701 if [[ -z "${inputList}" || -z "${productionID}" ]]
5dfac970 1702 then
fc7e1e45 1703 echo
71353a5f 1704 echo " Usage: ${0} submit inputList productionID [configFile=benchmark.config]"
fc7e1e45 1705 echo
71353a5f 1706 return
fc7e1e45 1707 fi
fc7e1e45 1708
5dfac970 1709 # check if config file is there
71353a5f 1710 if [ ! -f ${configFile} ]; then
1711 echo "ERROR! Config File '${configFile}' not found" >&2
1712 return
5dfac970 1713 else
71353a5f 1714 echo "Using Config File: '${configFile}'"
fc7e1e45 1715 fi
1716
9510565c 1717 [[ ! -f ${alirootEnv} ]] && echo "alirootEnv script ${alirootEnv} not found!..." && return 1
fc7e1e45 1718
71353a5f 1719 #move the script, config and some other stuff to ${commonOutputPath} first, then use them from there
0eaf38c7 1720 self=${0}
1721 #if which greadlink; then self=$(greadlink -f "${0}"); fi
71353a5f 1722 configPath=$(dirname ${configFile})
5dfac970 1723 export commonOutputPath=${baseOutputDirectory}/${productionID}
71353a5f 1724
cb473870 1725 mkdir -p ${commonOutputPath}
71353a5f 1726 mkdir -p ${commonOutputPath}/logs
1727 mkdir -p ${commonOutputPath}/meta
1728
1729 cp ${self} ${commonOutputPath}
1730 cp ${configFile} ${commonOutputPath}
1731 cp ${inputList} ${commonOutputPath}
cb473870 1732 self=${commonOutputPath}/${self##*/}
71353a5f 1733 chmod u+x ${self}
cb473870 1734 configFile=${commonOutputPath}/${configFile##*/}
1735 inputList=${commonOutputPath}/${inputList##*/}
fc7e1e45 1736
5dfac970 1737 #convert to absolut pathnames
0eaf38c7 1738 #if which greadlink; then inputList=$(greadlink -f "${inputList}"); fi
5dfac970 1739 #make list of runs
71353a5f 1740 if [[ -z ${runNumber} ]]; then
1741 listOfRuns=($(while read x; do guessRunNumber ${x}; done < ${inputList} | sort | uniq))
5dfac970 1742 else
71353a5f 1743 listOfRuns=${runNumber}
5dfac970 1744 fi
fc7e1e45 1745
0eaf38c7 1746 #if which greadlink; then alirootSource=$(greadlink -f "${alirootSource}"); fi
fc7e1e45 1747
5dfac970 1748 echo ""
1749 echo "### BEGIN CONFIGURATION ###"
1750 echo ""
1751 echo "GENERAL:"
1752 echo ""
71353a5f 1753 echo " productionID: ${productionID}"
1754 echo " batchCommand: ${batchCommand}"
1755 echo " batchFlags: ${batchFlags}"
9510565c 1756 echo " alirootEnv: ${alirootEnv}"
1757 ${alirootEnv} echo ' ALICE_ROOT: ${ALICE_ROOT}'
1758 ${alirootEnv} echo ' ALIROOT_RELEASE: ${ALICE_RELEASE}'
71353a5f 1759 echo " inputList: ${inputList}"
1760 echo " configPath: ${configPath}"
1761 echo " commonOutputPath: ${commonOutputPath}"
1762 echo " defaultOCDB: ${defaultOCDB}"
1763 echo " autoOCDB: ${autoOCDB}"
1764 echo " recoTriggerOptions: ${recoTriggerOptions}"
5dfac970 1765 echo " runs:"
1766 echo " ${listOfRuns[*]}"
1767 echo ""
1768 echo "THE TRAIN WILL RUN:"
fc7e1e45 1769
71353a5f 1770 if [ ${runCPass0reco} -eq 1 ]; then
5dfac970 1771 echo " Pass0 - Recontruction"
1772 fi
fc7e1e45 1773
71353a5f 1774 if [ ${runCPass0MergeMakeOCDB} -eq 1 ]; then
5dfac970 1775 echo " Pass0 - merging and OCDB export"
1776 fi
fc7e1e45 1777
71353a5f 1778 if [ ${runCPass1reco} -eq 1 ]; then
5dfac970 1779 echo " Pass1 - Recontruction"
1780 fi
71353a5f 1781 if [ ${runCPass1MergeMakeOCDB} -eq 1 ]; then
5dfac970 1782 echo " Pass1 - merging and OCDB export"
1783 fi
fc7e1e45 1784
5dfac970 1785 echo ""
1786 echo "LIMITS:"
71353a5f 1787 echo " max. Events/Chunk: ${nEvents}"
1788 echo " max. Number of Chunks per Run: ${nMaxChunks}"
5dfac970 1789 echo ""
1790 echo "### END CONFIGURATION ###"
1791 echo ""
fc7e1e45 1792
36b5de1a 1793
5dfac970 1794 # check if input file is there
71353a5f 1795 if [ ! -f ${inputList} ]; then
1796 echo "ERROR! Input List '${inputList}' not found" >&2
1797 return
5dfac970 1798 fi
1799
1800 # define jobid (for dependent jobs)
cb473870 1801 date=$(date +%Y_%m_%d_%H%M%S)
5dfac970 1802 #for each run we submit one jobarray:
fc7e1e45 1803 for runNumber in ${listOfRuns[*]}; do
cb473870 1804
71353a5f 1805 [[ -z ${runNumber} ]] && continue
fc7e1e45 1806 [[ ! ${runNumber} =~ ^[0-9]*[0-9]$ ]] && continue
cb473870 1807
1808 JOBpostfix="${productionID//"/"/_}_${runNumber}_${date}"
1809 JOBID1="p0_${JOBpostfix}"
1810 JOBID1wait="w0_${JOBpostfix}"
1811 JOBID2="m0_${JOBpostfix}"
1812 JOBID2wait="wm0_${JOBpostfix}"
1813 JOBID3="op0_${JOBpostfix}"
1814 JOBID3wait="wop0_${JOBpostfix}"
1815 JOBID4="p1_${JOBpostfix}"
1816 JOBID4wait="w1_${JOBpostfix}"
1817 JOBID5="m1_${JOBpostfix}"
1818 JOBID5wait="wm1_${JOBpostfix}"
1819 JOBID6="s1_${JOBpostfix}"
1820 JOBID6wait="ws1_${JOBpostfix}"
1821 JOBID7="QA_${JOBpostfix}"
1822 JOBmakeESDlistCPass1="lp1_${JOBpostfix}"
1823 JOBfilterESDcpass1="fp1_${JOBpostfix}"
1824 LASTJOB="000"
1825
0eaf38c7 1826 oneInputFile=$(egrep -m1 "${runNumber}/" ${inputList})
cb473870 1827
1828 currentDefaultOCDB=${defaultOCDB}
1829 [[ -z ${autoOCDB} ]] && autoOCDB=1
1830 if [[ ${autoOCDB} -ne 0 ]]; then
71353a5f 1831 currentDefaultOCDB=$(setYear ${oneInputFile} ${defaultOCDB})
cb473870 1832 fi
ef77d296 1833 period=$(guessPeriod ${oneInputFile})
1834 year=$(guessYear ${oneInputFile})
cb473870 1835
71353a5f 1836 echo "submitting run ${runNumber} with OCDB ${currentDefaultOCDB}"
1837
1838 ###############################################################################
1839 #run one chunk with valgrind:
6fc48f9e 1840 if [[ -n ${profilingCommand} ]]; then
1841 [[ -z ${nEventsProfiling} ]] && nEventsProfiling=2
1842 [[ -z ${profilingCommand} ]] && profilingCommand="/usr/bin/valgrind --tool=callgrind --num-callers=40 -v --trace-children=yes"
ef77d296 1843 submit "profile-${JOBpostfix}" 1 1 000 "${alirootEnv} ${self}" CPass0 ${commonOutputPath}/${year}/${period}/000${runNumber}/${jobindex} ${oneInputFile} ${nEventsProfiling} ${currentDefaultOCDB} ${configFile} ${runNumber} ${jobindex} useProfilingCommand=$(encSpaces "${profilingCommand}") "${extraOpts[@]}"
71353a5f 1844 fi
fc7e1e45 1845
5dfac970 1846 ################################################################################
1847 ################################################################################
1848 # run the CPass0 if requested
fc7e1e45 1849
71353a5f 1850 if [ ${runCPass0reco} -eq 1 ]; then
fc7e1e45 1851
fc7e1e45 1852 echo
71353a5f 1853 echo "starting CPass0... for run ${runNumber}"
fc7e1e45 1854 echo
fc7e1e45 1855
5dfac970 1856 # create directory and copy all files that are needed
ef77d296 1857 targetDirectory="${commonOutputPath}/${year}/${period}/000${runNumber}/cpass0"
71353a5f 1858 mkdir -p ${targetDirectory}
1859
cb473870 1860 filesCPass0=(
71353a5f 1861 "${configPath}/runCPass0.sh"
1862 "${configPath}/recCPass0.C"
1863 "${configPath}/runCalibTrain.C"
1864 "${configPath}/localOCDBaccessConfig.C"
1865 "${configPath}/OCDB*.root"
1866 "${configPath}/sim.C"
1867 "${configPath}/rec.C"
1868 "${configPath}/Config.C"
cb473870 1869 )
1870 for file in ${filesCPass0[*]}; do
1871 [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
1872 done
fc7e1e45 1873
71353a5f 1874 localInputList=${targetDirectory}/${inputList##*/}
1875 [[ ! -f ${localInputList} ]] && egrep "\/000${runNumber}\/" ${inputList} > ${localInputList}
5dfac970 1876 # limit nFiles to nMaxChunks
71353a5f 1877 nFiles=$(wc -l < ${localInputList})
1878 [[ ${nFiles} -eq 0 ]] && echo "list contains ZERO files! exiting..." && return 1
1879 echo "raw files in list: ${nFiles}"
1880 if [[ ${nMaxChunks} -gt 0 && ${nMaxChunks} -le ${nFiles} ]]; then
1881 nFiles=${nMaxChunks}
5dfac970 1882 fi
71353a5f 1883 echo "raw files to process: ${nFiles}"
1884 [[ -z "${percentProcessedFilesToContinue}" ]] && percentProcessedFilesToContinue=100
1885 if [[ ${percentProcessedFilesToContinue} -eq 100 ]]; then
1886 nFilesToWaitFor=${nFiles}
5dfac970 1887 else
71353a5f 1888 nFilesToWaitFor=$(( ${nFiles}-${nFiles}/(100/(100-${percentProcessedFilesToContinue})) ))
5dfac970 1889 fi
71353a5f 1890 echo "requested success rate is ${percentProcessedFilesToContinue}%"
1891 echo "merging will start after ${nFilesToWaitFor} jobs are done"
fc7e1e45 1892
444b9916 1893 submit ${JOBID1} 1 ${nFiles} 000 "${alirootEnv} ${self}" CPass0 ${targetDirectory} ${localInputList} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} -1 "${extraOpts[@]}"
fc7e1e45 1894
5dfac970 1895 ## submit a monitoring job that will run until a certain number of jobs are done with reconstruction
0ff60de9 1896 submit "${JOBID1wait}" 1 1 000 "${alirootEnv} ${self}" WaitForOutput ${commonOutputPath} "meta/cpass0.job*.run${runNumber}.done" ${nFilesToWaitFor} ${maxSecondsToWait}
71353a5f 1897 LASTJOB=${JOBID1wait}
fc7e1e45 1898
5dfac970 1899 fi #end running CPass0
1900 ################################################################################
fc7e1e45 1901
fc7e1e45 1902
5dfac970 1903 ################################################################################
1904 # submit merging of CPass0, depends on the reconstruction
fc7e1e45 1905
71353a5f 1906 if [ ${runCPass0MergeMakeOCDB} -eq 1 ]; then
fc7e1e45 1907
5dfac970 1908 echo
71353a5f 1909 echo "submit CPass0 merging for run ${runNumber}"
5dfac970 1910 echo
fc7e1e45 1911
ef77d296 1912 targetDirectory="${commonOutputPath}/${year}/${period}/000${runNumber}/cpass0"
71353a5f 1913 mkdir -p ${targetDirectory}
fc7e1e45 1914
cb473870 1915 #copy the scripts
1916 filesMergeCPass0=(
71353a5f 1917 "${configPath}/OCDB.root"
1918 "${configPath}/mergeMakeOCDB.byComponent.sh"
1919 "${configPath}/mergeMakeOCDB.sh"
1920 "${configPath}/localOCDBaccessConfig.C"
1921 "${configPath}/mergeByComponent.C"
1922 "${configPath}/makeOCDB.C"
1923 "${configPath}/merge.C"
cb473870 1924 )
1925 for file in ${filesMergeCPass0[*]}; do
1926 [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
1927 done
1928
444b9916 1929 submit ${JOBID2} 1 1 "${LASTJOB}" "${alirootEnv} ${self}" MergeCPass0 ${targetDirectory} ${currentDefaultOCDB} ${configFile} ${runNumber} cpass0.calib.run${runNumber}.list "${extraOpts[@]}"
71353a5f 1930 LASTJOB=${JOBID2}
fc7e1e45 1931
71353a5f 1932 if [[ -n ${generateMC} ]]; then
444b9916 1933 submit "mrl${JOBpostfix}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" PrintValues sim ${commonOutputPath}/meta/sim.run${runNumber}.list ${commonOutputPath}/meta/cpass0.job*.run${runNumber}.done
71353a5f 1934 LASTJOB="mrl${JOBpostfix}"
1935 fi
fc7e1e45 1936
5dfac970 1937 echo
fc7e1e45 1938 fi
5dfac970 1939 # end of merging CPass0
1940 ################################################################################
fc7e1e45 1941
5dfac970 1942 ################################################################################
1943 ################################################################################
1944 # run the CPass1 if requested
fc7e1e45 1945
71353a5f 1946 if [ ${runCPass1reco} -eq 1 ]; then
fc7e1e45 1947
ef77d296 1948 targetDirectory="${commonOutputPath}/${year}/${period}/000${runNumber}/cpass1"
71353a5f 1949 rm -f ${commonOutputPath}/meta/cpass1.job*.run${runNumber}.done
fc7e1e45 1950
5dfac970 1951 # safety feature: if we are re-running for any reason we want to delete the previous output first.
71353a5f 1952 [[ -d ${targetDirectory} ]] && rm -rf ${targetDirectory}/* && echo "removed old output at ${targetDirectory}/*"
fc7e1e45 1953
5dfac970 1954 echo
71353a5f 1955 echo "starting CPass1... for run ${runNumber}"
5dfac970 1956 echo
1957
1958 # create directory and copy all files that are needed
71353a5f 1959 mkdir -p ${targetDirectory}
5dfac970 1960
cb473870 1961 filesCPass1=(
71353a5f 1962 "${configPath}/runCPass1.sh"
1963 "${configPath}/recCPass1.C"
1964 "${configPath}/recCPass1_OuterDet.C"
1965 "${configPath}/runCalibTrain.C"
1966 "${configPath}/QAtrain_duo.C"
1967 "${configPath}/localOCDBaccessConfig.C"
1968 "${configPath}/OCDB.root"
cb473870 1969 )
1970 for file in ${filesCPass1[*]}; do
1971 [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
1972 done
5dfac970 1973
71353a5f 1974 if [[ -n ${generateMC} ]]; then
1975 localInputList=${commonOutputPath}/meta/sim.run${runNumber}.list
1976 else
1977 localInputList=${targetDirectory}/${inputList##*/}
1978 [[ ! -f ${localInputList} ]] && egrep "\/000${runNumber}\/" ${inputList} > ${localInputList}
1979 fi
5dfac970 1980 # limit nFiles to nMaxChunks
71353a5f 1981 nFiles=$(wc -l < ${localInputList})
1982 [[ ${nFiles} -eq 0 ]] && echo "list contains ZERO files! continuing..." && continue
1983 echo "raw files in list: ${nFiles}"
1984 if [[ ${nMaxChunks} -gt 0 && ${nMaxChunks} -le ${nFiles} ]]; then
1985 nFiles=${nMaxChunks}
5dfac970 1986 fi
71353a5f 1987 echo "raw files to process: ${nFiles}"
1988 [[ -z "${percentProcessedFilesToContinue}" ]] && percentProcessedFilesToContinue=100
1989 if [[ ${percentProcessedFilesToContinue} -eq 100 ]]; then
1990 nFilesToWaitFor=${nFiles}
fc7e1e45 1991 else
71353a5f 1992 nFilesToWaitFor=$(( ${nFiles}-${nFiles}/(100/(100-${percentProcessedFilesToContinue})) ))
fc7e1e45 1993 fi
71353a5f 1994 echo "requested success rate is ${percentProcessedFilesToContinue}%"
1995 echo "merging will start after ${nFilesToWaitFor} jobs are done"
fc7e1e45 1996
444b9916 1997 submit ${JOBID4} 1 ${nFiles} "${LASTJOB}" "${alirootEnv} ${self}" CPass1 ${targetDirectory} ${localInputList} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} -1 "${extraOpts[@]}"
fc7e1e45 1998
5dfac970 1999 ################################################################################
2000 ## submit a monitoring job that will run until a certain number of jobs are done with reconstruction
0ff60de9 2001 submit "${JOBID4wait}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" WaitForOutput ${commonOutputPath} "meta/cpass1.job*.run${runNumber}.done" ${nFilesToWaitFor} ${maxSecondsToWait}
71353a5f 2002 LASTJOB=${JOBID4wait}
5dfac970 2003 ################################################################################
fc7e1e45 2004
5dfac970 2005 echo
2006 fi #end running CPass1
fc7e1e45 2007
5dfac970 2008 ################################################################################
2009 # submit merging of CPass1, depends on the reconstruction
71353a5f 2010 if [ ${runCPass1MergeMakeOCDB} -eq 1 ]; then
fc7e1e45 2011
5dfac970 2012 echo
71353a5f 2013 echo "submit CPass1 merging for run ${runNumber}"
5dfac970 2014 echo
fc7e1e45 2015
ef77d296 2016 targetDirectory="${commonOutputPath}/${year}/${period}/000${runNumber}/cpass1"
71353a5f 2017 rm -f ${commonOutputPath}/meta/merge.cpass1.run${runNumber}.done
2018 mkdir -p ${targetDirectory}
fc7e1e45 2019
cb473870 2020 # copy files
2021 filesMergeCPass1=(
71353a5f 2022 "${configPath}/OCDB.root"
2023 "${configPath}/localOCDBaccessConfig.C"
2024 "${configPath}/mergeMakeOCDB.byComponent.sh"
2025 "${configPath}/mergeByComponent.C"
2026 "${configPath}/makeOCDB.C"
2027 "${configPath}/merge.C"
2028 "${configPath}/mergeMakeOCDB.sh"
2029 "${configPath}/QAtrain_duo.C"
cb473870 2030 )
2031 for file in ${filesMergeCPass1[*]}; do
2032 [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
2033 done
2034
444b9916 2035 submit "${JOBID5}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" MergeCPass1 ${targetDirectory} ${currentDefaultOCDB} ${configFile} ${runNumber} cpass1.calib.run${runNumber}.list cpass1.QA.run${runNumber}.lastMergingStage.txt.list cpass1.filtered.run${runNumber}.list "${extraOpts[@]}"
71353a5f 2036 LASTJOB=${JOBID5}
5dfac970 2037 echo
2038 fi
2039
444b9916 2040 ###############################
2041 #if [ ${runESDfiltering} -eq 1 ]; then
2042 # rm -f ${commonOutputPath}/cpass1.ESD.run${runNumber}.list
2043 # rm -f ${commonOutputPath}/meta/filtering.cpass1.run*.done
2044 # echo
2045 # echo submitting filtering for run ${runNumber}
2046 # echo
2047 # submit "${JOBmakeESDlistCPass1}" 1 1 "${LASTJOB}" "${self}" PrintValues esd ${commonOutputPath}/meta/cpass1.ESD.run${runNumber}.list ${commonOutputPath}/meta/cpass1.job*.run${runNumber}.done
ef77d296 2048 # submit "${JOBfilterESDcpass1}" 1 1 "${JOBmakeESDlistCPass1}" "${alirootEnv} ${self}" MakeFilteredTrees ${commonOutputPath}/${year}/${period}/000${runNumber}/cpass1 ${runNumber} ${commonOutputPath}/meta/cpass1.ESD.run${runNumber}.list ${filteringFactorHighPt} ${filteringFactorV0s} ${currentDefaultOCDB} 1000000 0 10000000 0 ${configFile} AliESDs_Barrel.root "${extraOpts[@]}"
444b9916 2049 # LASTJOB=${JOBfilterESDcpass1}
2050 #fi
cb473870 2051
5dfac970 2052 done
2053
71353a5f 2054 #################################################################################
2055 #################################################################################
444b9916 2056 #if [ ${runESDfiltering} -eq 1 ]; then
2057 # submit "${JOBID5wait}" 1 1 "${LASTJOB}" "${self}" WaitForOutput ${commonOutputPath} "meta/filtering.cpass1.run*.done" "${#listOfRuns[@]}" ${maxSecondsToWait}
2058 #else
2059 submit "${JOBID5wait}" 1 1 "${LASTJOB}" "${self}" WaitForOutput ${commonOutputPath} "meta/merge.cpass1.run*.done" ${#listOfRuns[@]} ${maxSecondsToWait}
2060 #fi
71353a5f 2061 LASTJOB=${JOBID5wait}
cb473870 2062
5de3064b 2063 #################################################################################
2064 echo
2065 echo "submit make a summary"
2066 echo
5dfac970 2067
444b9916 2068 submit "${JOBID6}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" MakeSummary ${configFile}
5de3064b 2069 LASTJOB=${JOBID6}
71353a5f 2070 #################################################################################
5dfac970 2071
5dfac970 2072 #restore stdout
2073 exec 1>&7 7>&-
2074 echo "jobs submitted."
5de3064b 2075 return 0
fc7e1e45 2076}
2077
cb473870 2078goWaitForOutput()
78d0bbfd 2079(
cb473870 2080 umask 0002
71353a5f 2081 [[ $# -lt 3 ]] && echo "goWaitForOutput() wrong number of arguments, exiting.." && return 1
2082 echo searchPath=${1}
2083 echo fileName=${2}
2084 echo numberOfFiles=${3}
2085 echo maxSecondsToWait=${4}
2086 searchPath=${1}
2087 fileName=${2}
2088 numberOfFiles=${3}
2089 maxSecondsToWait=${4}
0ff60de9 2090 echo "command to be executed: /bin/ls -1 ${searchPath}/${fileName}"
71353a5f 2091 [[ -z "${maxSecondsToWait}" ]] && maxSecondsToWait=$(( 3600*12 ))
cb473870 2092 while true; do
cb473870 2093 n=$(/bin/ls -1 ${searchPath}/${fileName} 2>/dev/null | wc -l)
71353a5f 2094 [[ ${n} -gt 0 ]] && echo "found ${n} X ${fileName}"
2095 [[ ${n} -ge ${numberOfFiles} ]] && break
2096 [[ ${SECONDS} -gt ${maxSecondsToWait} ]] && echo "timeout of ${maxSecondsToWait}s!" && break
cb473870 2097 sleep 60
2098 done
2099 echo "DONE! exiting..."
5de3064b 2100 return 0
78d0bbfd 2101)
cb473870 2102
71353a5f 2103mergeSysLogs()
78d0bbfd 2104(
71353a5f 2105 outputFile=${1}
2106 shift
2107 inputFiles="$@"
2108 i=0
2109 if ! ls -1 ${inputFiles} &>/dev/null; then echo "the files dont exist!: ${inputFiles}"; return 1; fi
2110 while read x; do
2111 runNumber=$(guessRunNumber ${x})
2112 [[ -z ${runNumber} ]] && echo "run number cannot be guessed for ${x}" && continue
2113 awk -v run=${runNumber} -v i=${i} 'NR > 1 {print run" "$0} NR==1 && i==0 {print "run/I:"$0}' ${x}
2114 (( i++ ))
2115 done < <(ls -1 ${inputFiles}) > ${outputFile}
5de3064b 2116 return 0
78d0bbfd 2117)
71353a5f 2118
2119goMakeMergedSummaryTree()
78d0bbfd 2120(
71353a5f 2121 # create list of calibration entries
2122 # takes no arguments, just run it in the base output
2123 # directory with the following files in the working directory
2124 #
2125 # Calibration file lists:
2126 # cpass0.dcsTree.list, cpass1.dcsTree.list
2127 # QA trending root file:
2128 # trending.root
2129 #
2130 # Production infoo ascii files:
2131 # summary_pass0.tree
2132 # summary_pass1.tree
2133 #
617f8886 2134
71353a5f 2135 [[ ! -f cpass0.dcsTree.list ]] && echo "no cpass0.dcsTree.list" && return 1
2136 [[ ! -f cpass1.dcsTree.list ]] && echo "no cpass1.dcsTree.list" && return 1
2137 [[ ! -f trending.root ]] && echo "no trending.root" && return 1
2138 [[ ! -f summary_pass0.tree ]] && echo "no summary_pass0.tree" && return 1
2139 [[ ! -f summary_pass1.tree ]] && echo "no summary_pass1.tree" && return 1
2140
2141 #first, dump the C macro to file
2142 cat << EOF > mergeTree.C
2143 //
2144 // Merge summary information
2145 // Following files are expected to be in the working directory
2146 //
2147 // Calibration file lists:
2148 // cpass0.dcsTree.list, cpass1.dcsTree.list
2149 // QA trending root file:
2150 // trending.root
2151 //
2152 // Production infoo ascii files:
2153 // summary_pass0.tree
2154 // summary_pass1.tree
2155 //
2156 void mergeTree(){
2157 //
2158 //
2159 //
2160 // Calibration values dump
2161 //
839b07ca 2162 //Printf("MakeTreeFromList cpass0.dcsTree.list");
71353a5f 2163 AliXRDPROOFtoolkit::MakeTreeFromList("Calib.TPC.CPass0.root", "dcs","dcs","cpass0.dcsTree.list",1);
839b07ca 2164 //Printf("MakeTreeFromList cpass1.dcsTree.list");
71353a5f 2165 AliXRDPROOFtoolkit::MakeTreeFromList("Calib.TPC.CPass1.root", "dcs","dcs","cpass1.dcsTree.list",1);
2166 //
2167 // Calibration status dump
2168 //
2169 TFile *fprod = TFile::Open("fproduction.root","recreate");
2170 TTree tree0, tree1;
839b07ca 2171 //Printf("reading summary_pass0.tree");
71353a5f 2172 tree0.ReadFile("summary_pass0.tree");
839b07ca 2173 //Printf("reading summary_pass1.tree");
71353a5f 2174 tree1.ReadFile("summary_pass1.tree");
2175 tree0.Write("CPass0");
2176 tree1.Write("CPass1");
2177 fprod->Close();
2178 //
2179 //
2180 //
2181 TString stringSetup="";
2182 stringSetup+="1#QA.TPC#run#SummaryTPCQA/tpcQA#trending.root+"; //
2183 stringSetup+="1#Calib.TPC.CPass0#run#dcs#Calib.TPC.CPass0.root+"; //
2184 stringSetup+="1#Calib.TPC.CPass1#run#dcs#Calib.TPC.CPass1.root+"; //
2185 //
2186 stringSetup+="1#CPass0#runnumber#CPass0#fproduction.root+"; //
2187 stringSetup+="1#CPass1#runnumber#CPass1#fproduction.root+"; //
2188 //
839b07ca 2189 //Printf("stringSetup: %s", stringSetup.Data());
71353a5f 2190 AliXRDPROOFtoolkit::JoinTreesIndex("outAll.root","joinAll","run",stringSetup.Data(), 1);
2191 }
2192EOF
2193
2194 aliroot -b -q "mergeTree.C" > mergeTrees.log
2195 return $?
78d0bbfd 2196)
71353a5f 2197
177b9ebc 2198stackTraceTree()
2199(
67d69d16 2200 # make stacktrace processing in case of standard root crash log
2201 # input is a (list of) text files with the stack trace (either gdb aoutput
2202 # produced with e.g. gdb --batch --quiet -ex "bt" -ex "quit" aliroot core, or the root crash log), output is a TTree formatted table.
2203# example usage:
2204# benchmark.sh stackTraceTree /foo/*/rec.log
4a3c2422 2205 gawk '
177b9ebc 2206 BEGIN {
2207 print "frame/I:method/C:line/C:cpass/I:aliroot/I";
2208 RS="#[0-9]*";
2209 aliroot=0;
4a3c2422 2210 read=1;
177b9ebc 2211 }
4a3c2422 2212 /There was a crash/ {read=1;}
2213 /The lines below might hint at the cause of the crash/ {read=0;}
2214 read==1 {
177b9ebc 2215 if ($3 ~ /Ali*/) aliroot=1; else aliroot=0;
2216 gsub("#","",RT);
2217 if ($NF!="" && RT!="" && $3!="") print RT" "$3" "$NF" "0" "aliroot
2218 }
2219 ' "$@" 2>/dev/null
2220)
2221
71353a5f 2222goMakeSummary()
78d0bbfd 2223(
71353a5f 2224 #all the final stuff goes in here for ease of use:
2225 # summary logs
2226 # qa plot making
2227 # final file lists
ef77d296 2228 # runs in current dir - in makeflow mode it can run LOCAL, then the QA plots and summaries
2229 # will appear in the submission dir.
444b9916 2230 #some defaults:
2231 log="summary.log"
2232 productionID="qa"
71353a5f 2233
2234 configFile=${1}
2235 shift 1
444b9916 2236 extraOpts=("$@")
6fc48f9e 2237 if ! parseConfig ${configFile} "${extraOpts[@]}"; then return 1; fi
71353a5f 2238
0eaf38c7 2239 #if which greadlink; then configFile=$(greadlink -f ${configFile}); fi
839b07ca 2240
39527a15 2241 #record the working directory provided by the batch system
2242 batchWorkingDirectory=${PWD}
ef77d296 2243
2244 logTmp=${batchWorkingDirectory}/${log}
2245 logDest=${commonOutputPath}/${log}
39527a15 2246
71353a5f 2247 [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
2248
71353a5f 2249 [[ ! -f ${configFile} ]] && echo "no config file ${configFile}!" && return
2250
2251 [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
2252
39527a15 2253 #copy some useful stuff
2254 #and go to the commonOutputPath
2255 cp ${configFile} ${commonOutputPath}
39527a15 2256
ef77d296 2257 exec &> >(tee ${logTmp})
39527a15 2258
71353a5f 2259 #summarize the global stuff
2260 echo "env script: ${alirootSource} ${alirootEnv}"
2261 echo "\$ALICE_ROOT=${ALICE_ROOT}"
2262 echo "commonOutputPath=${commonOutputPath}"
2263
2264 #summarize the stacktraces
ef77d296 2265 stackTraceTree ${commonOutputPath}/*/*/000*/cpass0/*/stacktrace* > stacktrace_cpass0.tree
2266 stackTraceTree ${commonOutputPath}/*/*/000*/cpass1/*/stacktrace* > stacktrace_cpass1.tree
71353a5f 2267
2268 echo total numbers for the production:
2269 echo
2270 awk 'BEGIN {nFiles=0;nCore=0;}
2271 /^calibfile/ {nFiles++;}
2272 /core dumped/ {nCore++i;}
ef77d296 2273 END {print "cpass0 produced "nFiles" calib files, "nCore" core files";}' ${commonOutputPath}/meta/cpass0.job*done 2>/dev/null
71353a5f 2274 awk 'BEGIN {nOK=0; nBAD=0; }
2275 /\/rec.log OK/ {nOK++;}
2276 /\/rec.log BAD/ {nBAD++;}
2277 /stderr BAD/ {if ($0 ~ /rec.log/){nBAD++;}}
ef77d296 2278 END {print "cpass0 reco: OK: "nOK"\tBAD: "nBAD;}' ${commonOutputPath}/meta/cpass0.job*done 2>/dev/null
71353a5f 2279 awk 'BEGIN {nOK=0; nBAD=0; }
2280 /\/calib.log OK/ {nOK++;}
2281 /\/calib.log BAD/ {nBAD++;}
ef77d296 2282 END {print "cpass0 calib: OK: "nOK"\tBAD: "nBAD;}' ${commonOutputPath}/meta/cpass0.job*done 2>/dev/null
71353a5f 2283
2284 awk 'BEGIN {nOK=0; nBAD=0; }
2285 /merge.log OK/ {nOK++;}
2286 /merge.log BAD/ {nBAD++;}
ef77d296 2287 END {print "cpass0 merge: OK: "nOK"\tBAD: "nBAD;}' ${commonOutputPath}/meta/merge.cpass0*done 2>/dev/null
71353a5f 2288 awk 'BEGIN {nOK=0; nBAD=0; }
2289 /ocdb.log OK/ {nOK++;}
2290 /ocdb.log BAD/ {nBAD++;}
ef77d296 2291 END {print "cpass0 OCDB: OK: "nOK"\tBAD: "nBAD;}' ${commonOutputPath}/meta/merge.cpass0*done 2>/dev/null
71353a5f 2292
2293 echo
2294 awk 'BEGIN {nFiles=0;nCore=0;}
2295 /^calibfile/ {nFiles++;}
2296 /core dumped/ {nCore++;}
ef77d296 2297 END {print "cpass1 produced "nFiles" calib files, "nCore" core files";}' ${commonOutputPath}/meta/cpass1.job*done 2>/dev/null
71353a5f 2298 awk 'BEGIN {nOK=0; nBAD=0; }
2299 /\/rec.log OK/ {nOK++;}
2300 /\/rec.log BAD/ {nBAD++;}
2301 /stderr BAD/ {if ($0 ~ /rec.log/){nBAD++;}}
ef77d296 2302 END {print "cpass1 reco: OK: "nOK"\tBAD: "nBAD;}' ${commonOutputPath}/meta/cpass1.job*done 2>/dev/null
71353a5f 2303 awk 'BEGIN {nOK=0; nBAD=0; }
2304 /\/calib.log OK/ {nOK++;}
2305 /\/calib.log BAD/ {nBAD++;}
ef77d296 2306 END {print "cpass1 calib: OK: "nOK"\tBAD: "nBAD;}' ${commonOutputPath}/meta/cpass1.job*done 2>/dev/null
71353a5f 2307
2308 awk 'BEGIN {nOK=0; nBAD=0; }
2309 /merge.log OK/ {nOK++;}
2310 /merge.log BAD/ {nBAD++;}
ef77d296 2311 END {print "cpass1 merge: OK: "nOK"\tBAD: "nBAD;}' ${commonOutputPath}/meta/merge.cpass1*done 2>/dev/null
71353a5f 2312 awk 'BEGIN {nOK=0; nBAD=0; }
2313 /ocdb.log OK/ {nOK++;}
2314 /ocdb.log BAD/ {nBAD++;}
ef77d296 2315 END {print "cpass1 OCDB: OK: "nOK"\tBAD: "nBAD;}' ${commonOutputPath}/meta/merge.cpass1*done 2>/dev/null
71353a5f 2316
2317 echo
2318 echo per run stats:
ef77d296 2319 /bin/ls -1 ${commonOutputPath}/meta/merge.cpass0.run*.done | while read x
71353a5f 2320do
2321 dir=$(goPrintValues calibfile - ${x})
2322 runNumber=$(guessRunNumber ${dir})
2323 [[ -z ${runNumber} ]] && continue
2324
ef77d296 2325 if $(/bin/ls ${commonOutputPath}/meta/cpass0.job*.run${runNumber}.done &> /dev/null); then
71353a5f 2326 statusCPass0=( $(
2327 awk 'BEGIN {nOKrec=0;nBADrec=0;nOKcalib=0;nBADcalib=0;nOKstderr=0;nBADstderr=0;}
2328 /\/rec.log OK/ {nOKrec++;}
2329 /\/rec.log BAD/ {nBADrec++;}
2330 /stderr BAD/ {if ($0 ~ /rec.log/) {nBADrec++;} nBADstderr++;}
2331 /stderr OK/ {nOKstderr++;}
2332 /\/calib.log OK/ {nOKcalib++;}
2333 /\/calib.log BAD/ {nBADcalib++}
ef77d296 2334 END {print ""nOKrec" "nBADrec" "nOKstderr" "nBADstderr" "nOKcalib" "nBADcalib;}' ${commonOutputPath}/meta/cpass0.job*.run${runNumber}.done 2>/dev/null
71353a5f 2335 ) )
2336 fi
2337
ef77d296 2338 if $(/bin/ls ${commonOutputPath}/meta/cpass1.job*.run${runNumber}.done &>/dev/null); then
71353a5f 2339 statusCPass1=( $(
2340 awk 'BEGIN {nOKrec=0;nBADrec=0;nOKcalib=0;nBADcalib=0;nOKstderr=0;nBADstderr=0;nQAbarrelOK=0;nQAbarrelBAD=0;nQAouterOK=0;nQAouterBAD=0;}
2341 /\/rec.log OK/ {nOKrec++;}
2342 /\/rec.log BAD/ {nBADrec++;}
2343 /stderr BAD/ {if ($0 ~ /rec.log/) nBADrec++;nBADstderr++;}
2344 /stderr OK/ {nOKstderr++;}
2345 /\/calib.log OK/ {nOKcalib++;}
2346 /\/calib.log BAD/ {nBADcalib++}
2347 /\/qa_barrel.log OK/ {nQAbarrelOK++;}
2348 /\/qa_barrel.log BAD/ {nQAbarrelBAD++;}
2349 /\/qa_outer.log OK/ {nQAouterOK++;}
2350 /\/qa_outer.log BAD/ {nQAouterBAD++;}
ef77d296 2351 END {print ""nOKrec" "nBADrec" "nOKstderr" "nBADstderr" "nOKcalib" "nBADcalib" "nQAbarrelOK" "nQAbarrelBAD" "nQAouterOK" "nQAouterBAD;}' ${commonOutputPath}/meta/cpass1.job*.run${runNumber}.done 2>/dev/null
71353a5f 2352 ) )
2353 fi
2354
2355 statusOCDBcpass0=$(awk '/ocdb.log/ {print $2} ' ${x} 2>/dev/null)
2356 statusOCDBcpass1=$(awk '/ocdb.log/ {print $2}' ${x/cpass0/cpass1} 2>/dev/null)
2357 statusQA=$(awk '/mergeMakeOCDB.log/ {print $2}' ${x/cpass0/cpass1} 2>/dev/null)
2358
2359 printf "%s\t ocdb.log cpass0: %s\t ocdb.log cpass1: %s\tqa.log:%s\t| cpass0: rec:%s/%s stderr:%s/%s calib:%s/%s cpass1: rec:%s/%s stderr:%s/%s calib:%s/%s QAbarrel:%s/%s QAouter:%s/%s\n" ${runNumber} ${statusOCDBcpass0} ${statusOCDBcpass1} ${statusQA} ${statusCPass0[0]} ${statusCPass0[1]} ${statusCPass0[2]} ${statusCPass0[3]} ${statusCPass0[4]} ${statusCPass0[5]} ${statusCPass1[0]} ${statusCPass1[1]} ${statusCPass1[2]} ${statusCPass1[3]} ${statusCPass1[4]} ${statusCPass1[5]} ${statusCPass1[6]} ${statusCPass1[7]} ${statusCPass1[8]} ${statusCPass1[9]}
2360done
2361
2362 #make lists with output files - QA, trending, filtering and calibration
2363 ### wait for the merging of all runs to be over ###
2364 rm -f qa.list
2365 goPrintValues qafile qa.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2366 rm -f calib.list
2367 goPrintValues calibfile calib.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2368 rm -f trending.list
2369 goPrintValues trendingfile trending.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2370 rm -f filtering.list
444b9916 2371 goPrintValues filteredTree filtering.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
71353a5f 2372 rm -f cpass0.dcsTree.list
2373 goPrintValues dcsTree cpass0.dcsTree.list ${commonOutputPath}/meta/merge.cpass0.run*.done &>/dev/null
71353a5f 2374 rm -f cpass1.dcsTree.list
2375 goPrintValues dcsTree cpass1.dcsTree.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2376
2377 #merge trending
2378 rm -f ${commonOutputPath}/trending_merged.root
444b9916 2379 goMerge trending.list ${commonOutputPath}/trending.root ${configFile} "${extraOpts[@]}" &> mergeTrending.log
71353a5f 2380
2381 goMakeSummaryTree ${commonOutputPath} 0
2382 goMakeSummaryTree ${commonOutputPath} 1
2383
839b07ca 2384 goCreateQAplots "${PWD}/qa.list" "${productionID}" "QAplots" "${configFile}" "${extraOpts[@]}" filteringList="${PWD}/filtering.list" &>createQAplots.log
71353a5f 2385
2386 #make a merged summary tree out of the QA trending, dcs trees and log summary trees
2387 goMakeMergedSummaryTree
2388
2389 #if set, email the summary
ef77d296 2390 [[ -n ${MAILTO} ]] && cat ${logTmp} | mail -s "benchmark ${productionID} done" ${MAILTO}
2391
2392 # Copy log to destination (delete all on failure to signal error)
2393 cp "$logTmp" "$logDest" || rm -f "$logTmp" "$logDest"
2394 cp -r QAplots ${commonOutputPath}
2395 cp *.list ${commonOutputPath}
2396 cp *.root ${commonOutputPath}
2397 cp *.log ${commonOutputPath}
71353a5f 2398
2399 return 0
78d0bbfd 2400)
71353a5f 2401
2402goMakeSummaryTree()
78d0bbfd 2403(
71353a5f 2404 if [[ $# -lt 1 ]] ; then
2405 return
2406 fi
2407 #1. define vars/arrays
2408 DIR=${1} #use input or exec in current dir
2409 pass=${2-"0"} #pass from input
2410 outfile="summary_pass${pass}.tree"
2411 Ncolumns=0
2412 test -f ${outfile} && : >${outfile}
2413 errfile=${outfile/tree/err}
2414 test -f ${errfile} && : >${errfile}
2415
2416 declare -a counterString=(TOFevents TOFtracks TPCevents TPCtracks TRDevents TRDtracks T0events SDDevents SDDtracks MeanVertexevents)
2417 Ncounter=${#counterString[@]}
2418
2419 declare -a statusString=(TRDStatus TOFStatus TPCStatus T0Status MeanVertexStatus)
2420 Nstatus=${#statusString[@]}
2421
2422
2423 declare -a ratesString=(rec stderr calib qa_barrel qa_outer)
2424 Nrates=${#ratesString[@]}
2425
2426 runs=( $(ls -1 ${DIR}/meta/merge.cpass0* | while read x; do guessRunNumber $x; done) )
2427 Nruns=${#runs[@]}
2428
2429 echo -n runnumber/I >>${outfile}
2430 echo -n :cpass${pass}status/I >>${outfile}
2431 echo -n :cpass${pass}QAstatus/I >>${outfile}
2432 for i in ${ratesString[@]}; do
2433 echo -n :${i}OK/I >>${outfile}
2434 echo -n :${i}BAD/I >>${outfile}
2435
2436 done
2437 for i in ${counterString[@]} ${statusString[@]} ; do
2438 echo -n :${i}/I >>${outfile}
2439 done
2440 Ncolumns=$((2 + 2*Nrates + Ncounter + Nstatus))
2441 echo >> ${outfile}
2442
2443 #2. loop runs
2444
2445 for runnumber in ${runs[@]} ; do
2446
2447
2448
2449 filejob="${DIR}/meta/cpass${pass}.job*.run${runnumber}.done"
2450 filemerge="${DIR}/meta/merge.cpass${pass}.run${runnumber}.done"
2451 fileOCDB=$(grep /ocdb.log ${filemerge} | awk '{print $1}')
2452 if ! $(/bin/ls ${filemerge} &>/dev/null) ; then
2453 echo "${filemerge} does not exist!" >>${errfile}
2454 continue
2455 elif ! $(/bin/ls ${filejob} &>/dev/null) ; then
2456 echo "${filejob} does not exist!" >>${errfile}
2457 echo -n ${runnumber} >> ${outfile}
2458 for i in $(seq ${Ncolumns}) ; do
2459 echo -n "-1" >> ${outfile}
2460 done
2461 echo >> ${outfile}
2462 continue
2463 fi
2464 echo -n ${runnumber} >> ${outfile}
2465 #pass0status= grep '/ocdb.log' ${filemerge} | cut -d' ' -f2 | tr OK x1 | tr BAD xx0 | tr -d 'x'
2466 passStatus=$(grep '/ocdb.log' ${filemerge} | grep OK | wc -l)
2467 echo -n " ${passStatus}" >> ${outfile}
2468 qaStatus=$(grep '/mergeMakeOCDB.log' ${filemerge} | grep OK | wc -l)
2469 echo -n " ${qaStatus}" >> ${outfile}
2470
2471
2472 #fill OK/BAD rates
2473 for i in $(seq 0 $((${Nrates}-1))) ; do
2474 var1=$(grep "/${ratesString[${i}]}.log" ${filejob} | grep OK | wc -l)
2475 var2=$(grep "/${ratesString[${i}]}.log" ${filejob} | grep BAD | wc -l)
2476
2477 if [[ ${ratesString[${i}]} == "stderr" ]] ; then
2478 var1=$(grep "stderr" ${filejob} | grep OK | wc -l)
2479 var2=$(grep "stderr" ${filejob} | grep "rec.log" | grep BAD | wc -l)
2480 fi
2481 echo -n " ${var1}" >> ${outfile}
2482 echo -n " ${var2}" >> ${outfile}
2483 done
2484
2485 if [[ -f ${fileOCDB} ]] ; then
2486 #fill counter
2487 for i in $(seq 0 $((${Ncounter}-1))) ; do
2488 var1=$(grep Monalisa ${fileOCDB} | grep ${counterString[${i}]} | cut -f2)
2489 echo -n " ${var1:-"-1"}" >> ${outfile}
2490 done
2491
2492 #fill status
2493 for i in $(seq 0 $((${Nstatus}-1))) ; do
2494 var1=$(grep "calibration status=" ${fileOCDB} | grep ${statusString[${i}]/Status/} | cut -d'=' -f2)
2495 echo -n " ${var1:-"-1"}" >> ${outfile}
2496 done
2497 fi
2498 echo >> ${outfile}
2499 done
2500
5de3064b 2501 return 0
78d0bbfd 2502)
71353a5f 2503
2504parseConfig()
2505{
2506 configFile=${1}
2507 shift
444b9916 2508 args=("$@")
2509
0eaf38c7 2510
444b9916 2511 #some defaults
6fc48f9e 2512 #autoOCDB=0
2513 defaultOCDB="raw://"
2514 #runNumber=167123
2515 #makeflowPath="/hera/alice/aux/cctools/bin"
2516 #makeflowOptions="-T wq -N alice -d all -C ali-copilot.cern.ch:9097"
2517 #makeflowOptions="-T wq -N alice -C ali-copilot.cern.ch:9097"
2518 makeflowOptions=""
39527a15 2519 #batchCommand="/usr/bin/qsub"
ef77d296 2520 batchFlags=""
6fc48f9e 2521 baseOutputDirectory="$PWD/output"
2522 #alirootEnv="/cvmfs/alice.cern.ch/bin/alienv setenv AliRoot/v5-04-34-AN -c"
2523 #alirootEnv="/home/mkrzewic/alisoft/balice_master.sh"
2524 #trustedQAtrainMacro='/hera/alice/mkrzewic/gsisvn/Calibration/QAtrain_duo.C'
2525 reconstructInTemporaryDir=0
2526 recoTriggerOptions="\"\""
2527 percentProcessedFilesToContinue=100
2528 maxSecondsToWait=$(( 3600*24 ))
2529 nEvents=-1
2530 nMaxChunks=0
2531 postSetUpActionCPass0=""
2532 postSetUpActionCPass1=""
2533 runCPass0reco=1
2534 runCPass0MergeMakeOCDB=1
2535 runCPass1reco=1
2536 runCPass1MergeMakeOCDB=1
2537 runESDfiltering=1
444b9916 2538 filteringFactorHighPt=1e2
2539 filteringFactorV0s=1e1
6fc48f9e 2540 MAILTO=""
2541 #pretend=1
2542 #dontRedirectStdOutToLog=1
2543 logToFinalDestination=1
2544 ALIROOT_FORCE_COREDUMP=1
0eaf38c7 2545 pretendDelay=0
71353a5f 2546
2547 #first, source the config file
2548 if [ -f ${configFile} ]; then
2549 source ${configFile}
2550 else
0eaf38c7 2551 echo "config file ${configFile} not found!"
2552 return 1
71353a5f 2553 fi
2554
839b07ca 2555 unset encodedSpaces
2556 for opt in "${args[@]}"; do
0eaf38c7 2557 [[ "${opt}" =~ encodedSpaces=.* ]] && encodedSpaces=1 && echo "encodedSpaces!" && break
839b07ca 2558 done
2559
d56eeaab 2560 #then, parse the options as they override the options from file
2561 for opt in "${args[@]}"; do
0eaf38c7 2562 [[ -z ${opt} ]] && continue
2563 [[ -n ${encodedSpaces} ]] && opt="$(decSpaces ${opt})"
2564 [[ "${opt}" =~ ^[[:space:]]*$ ]] && continue
d56eeaab 2565 if [[ ! "${opt}" =~ .*=.* ]]; then
0eaf38c7 2566 echo "badly formatted option \"${opt}\" should be: option=value, stopping..."
d56eeaab 2567 return 1
2568 fi
2569 local var="${opt%%=*}"
2570 local value="${opt#*=}"
9d0c204f 2571 echo "${var}=${value}"
444b9916 2572 export ${var}="${value}"
71353a5f 2573 done
2574
6fc48f9e 2575 #do some checking
2576 [[ -z ${alirootEnv} ]] && echo "alirootEnv not defined!" && return 1
2577
71353a5f 2578 #export the aliroot function if defined to override normal behaviour
2579 [[ $(type -t aliroot) =~ "function" ]] && export -f aliroot
6fc48f9e 2580
5de3064b 2581 return 0
71353a5f 2582}
2583
2584aliroot()
2585{
e48b4924 2586 args=("$@")
6fc48f9e 2587 if [[ -n ${useProfilingCommand} ]]; then
e48b4924 2588 profilerLogFile="cpu.txt"
2589 [[ "${args}" =~ rec ]] && profilerLogFile="cpu_rec.txt"
2590 [[ "${args}}" =~ Calib ]] && profilerLogFile="cpu_calib.txt"
2591 echo running "${useProfilingCommand} aliroot ${args} &> ${profilerLogFile}"
2592 ${useProfilingCommand} aliroot "${args[@]}" &> ${profilerLogFile}
71353a5f 2593 else
2594 #to prevent an infinite recursion use "command aliroot" to disable
2595 #aliases and functions
e48b4924 2596 echo running command aliroot "${args[@]}"
2597 command aliroot "${args[@]}"
71353a5f 2598 fi
5de3064b 2599 return 0
71353a5f 2600}
2601
2602guessRunData()
2603{
2604 #guess the period from the path, pick the rightmost one
2605 period=""
2606 runNumber=""
2607 year=""
2608 pass=""
2609 legoTrainRunNumber=""
2610 dataType=""
2611
2612 local shortRunNumber=""
2613 local IFS="/"
2614 declare -a path=( $1 )
2615 local dirDepth=$(( ${#path[*]}-1 ))
2616 i=0
2617 #for ((x=${dirDepth};x>=0;x--)); do
2618 for ((x=0;x<=${dirDepth};x++)); do
2619
2620 [[ $((x-1)) -ge 0 ]] && local fieldPrev=${path[$((x-1))]}
2621 local field=${path[${x}]}
2622 local fieldNext=${path[$((x+1))]}
2623
2624 [[ ${field} =~ ^[0-9]*$ && ${fieldNext} =~ (.*\.zip$|.*\.root$) ]] && legoTrainRunNumber=${field}
2625 [[ -n ${legoTrainRunNumber} && -z ${pass} ]] && pass=${fieldPrev}
2626 [[ ${field} =~ ^LHC[0-9][0-9][a-z].*$ ]] && period=${field%_*}
2627 [[ ${field} =~ ^000[0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && runNumber=${field#000}
2628 [[ ${field} =~ ^[0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && shortRunNumber=${field}
2629 [[ ${field} =~ ^20[0-9][0-9]$ ]] && year=${field}
2630 [[ ${field} =~ ^(^sim$|^data$) ]] && dataType=${field}
2631 (( i++ ))
2632 done
2633 [[ -z ${legoTrainRunNumber} ]] && pass=${path[$((dirDepth-1))]}
2634 [[ "${dataType}" =~ ^sim$ ]] && pass="passMC" && runNumber=${shortRunNumber}
2635
2636 #if [[ -z ${dataType} || -z ${year} || -z ${period} || -z ${runNumber}} || -z ${pass} ]];
2637 if [[ -z ${runNumber}} ]];
2638 then
2639 #error condition
2640 return 1
2641 else
2642 #ALL OK
2643 return 0
2644 fi
5de3064b 2645 return 0
71353a5f 2646}
2647
839b07ca 2648#these functions encode strings to and from a space-less form
2649#use when spaces are not well handled (e.g. in arguments to
2650#commands in makeflow files, etc.
0eaf38c7 2651encSpaces()(echo "${1// /@@@@}")
2652decSpaces()(echo "${1//@@@@/ }")
839b07ca 2653
fc7e1e45 2654main "$@"