]> git.uio.no Git - u/mrichter/AliRoot.git/blame - PWGPP/benchmark/benchmark.sh
git://git.uio.no / u / mrichter / AliRoot.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
The default cuts are now for PbPb (Maximiliano)
[u/mrichter/AliRoot.git] / PWGPP / benchmark / benchmark.sh
CommitLineData
fc7e1e45 1#!/bin/bash
71353a5f 2#include benchmark.config
3
b802abe4 4# blame: Mikolaj Krzewicki, mkrzewic@cern.ch
fc7e1e45 5# this script runs the CPass0/CPass1 train
6# produced OCDB updates are local
7
8main()
9{
10 #run in proper mode depending on the selection
71353a5f 11 if [[ $# -lt 1 ]]; then
12 if [[ ! "${0}" =~ "bash" ]]; then
13 echo "uses makeflow:"
14 echo " ${0} \"run\" productionID inputList configFile [extraOpts]"
15 echo "uses a batch system (SGE):"
16 echo " ${0} \"submit\" productionID inputList configFile [extraOpts]"
17 echo "extraOpts if non-empty override the config file, e.g.:"
9510565c 18 echo " ${0} submit test1 benchmark.list benchmark.config runNumber=169123 nEvents=10"
71353a5f 19 fi
20 return
21 fi
22
23 #define some aliases - default is to call one of the functions directly
24 runMode=${1}
fc7e1e45 25 umask 0002
26 shift
71353a5f 27 case ${runMode} in
fc7e1e45 28 "CPass0") goCPass0 "$@";;
29 "CPass1") goCPass1 "$@";;
71353a5f 30 "MakeLocalOCDBaccessConfig") goMakeLocalOCDBaccessConfig "$@";;
fc7e1e45 31 "MergeCPass0") goMergeCPass0 "$@";;
32 "MergeCPass1") goMergeCPass1 "$@";;
cb473870 33 "MakeFilteredTrees") goMakeFilteredTrees "$@";;
71353a5f 34 "MakeSummary") goMakeSummary "$@";;
cb473870 35 "run") goSubmitMakeflow "$@";;
36 "submit") goSubmitBatch "$@";;
fc7e1e45 37 "test") goTest "$@";;
71353a5f 38 "GenerateMakeflow") goGenerateMakeflow "$@";;
39 "PrintValues") goPrintValues "$@";;
cb473870 40 "CreateQAplots") goCreateQAplots "$@";;
41 "WaitForOutput") goWaitForOutput "$@";;
71353a5f 42 "Merge") goMerge "$@";;
43 *)
444b9916 44 ${runMode} "$@"
fc7e1e45 45 ;;
46 esac
39527a15 47 return 0
fc7e1e45 48}
49
71353a5f 50generateMC()
51{
52 #generate one raw chunk in current directory
53 SEED=${JOB_ID}${SGE_TASK_ID}
54 export CONFIG_SEED=${SEED}
55 runNumber=${1}
56 OCDBpath=${2}
57 nEventsim=${3}
58 if [[ -n ${pretend} ]]; then
0eaf38c7 59 sleep ${pretendDelay}
71353a5f 60 touch galice.root
61 else
62 if [[ -f sim.C && -f Config.C ]] ; then
63 time aliroot -b -q -x sim.C\(${runNumber},\"${OCDBpath}\",${nEventsim}\) >sim.log 2>&1
64 mv syswatch.log simwatch.log
65 fi
66 fi
67}
68
fc7e1e45 69goCPass0()
78d0bbfd 70(
fc7e1e45 71 umask 0002
72
71353a5f 73 targetDirectory=${1}
74 inputList=${2}
75 nEvents=${3}
76 ocdbPath=${4}
77 configFile=${5}
78 runNumber=${6}
79 jobindex=${7}
80 shift 7
6fc48f9e 81 if ! parseConfig ${configFile} "$@"; then return 1; fi
71353a5f 82
39527a15 83 #record the working directory provided by the batch system
84 batchWorkingDirectory=${PWD}
85
71353a5f 86 #use the jobindex only if set and non-negative
87 if [[ -z ${jobindex} || ${jobindex} -lt 0 ]]; then
88 [[ -n "${LSB_JOBINDEX}" ]] && jobindex=${LSB_JOBINDEX}
89 [[ -n "${SGE_TASK_ID}" ]] && jobindex=${SGE_TASK_ID}
999a843d 90 if [[ -z ${jobindex} ]]; then
91 echo "no jobindex!"
92 return 1
93 fi
71353a5f 94 fi
95
96 [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
ef77d296 97
98 # This file signals that/if everything went fine
99 doneFileBase="cpass0.job${jobindex}.run${runNumber}.done"
100 [[ -n ${useProfilingCommand} ]] && doneFileBase="profiling.cpass0.job${jobindex}.run${runNumber}.done"
101
102 # We will have two copies of the file
103 mkdir -p "${commonOutputPath}/meta" || return 1
104 doneFileTmp="${batchWorkingDirectory}/${doneFileBase}"
105 doneFile="${commonOutputPath}/meta/${doneFileBase}"
71353a5f 106
107 [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
108
109 if [[ -n ${ALIROOT_FORCE_COREDUMP} ]]; then
110 ulimit -c unlimited
111 export ALIROOT_FORCE_COREDUMP
112 fi
fc7e1e45 113
cb473870 114 #the contents of this is stored in the tree and used later (e.g. AliAnalysisTaskPIDResponse)!
115 #at the QA stage the pass number is guessed from the path stored here.
116 #The Format is:
117 #Packages= ;OutputDir= ;LPMPass= ;TriggerAlias= ;LPMRunNumber= ;LPMProductionType= ;LPMInteractionType= ;LPMProductionTag= ;LPMAnchorRun= ;LPMAnchorProduction= ;LPMAnchorYear=
118 export PRODUCTION_METADATA="OutputDir=cpass0"
119
5de3064b 120 if [[ "${inputList}" =~ \.root$ ]]; then
121 infile=${inputList}
122 else
123 infile=$(sed -ne "${jobindex}p" ${inputList} | egrep '\s*\w*/\w*')
124 fi
71353a5f 125 chunkName=${infile##*/}
71353a5f 126
127 outputDir=${targetDirectory}/${jobindex}_${chunkName%.*}
128 mkdir -p ${outputDir}
1795556a 129 if [[ ! -d ${outputDir} ]]; then
ef77d296 130 touch ${doneFileTmp}
131 echo "cannot make ${outputDir}" >> ${doneFileTmp}
132 cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
133 [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
1795556a 134 return 1
135 fi
71353a5f 136
71353a5f 137 runpath=${outputDir}
0eaf38c7 138 [[ ${reconstructInTemporaryDir} -eq 1 ]] && runpath=$(mktemp -d -t cpass0.XXXXXX)
e91a9c78 139 [[ ${reconstructInTemporaryDir} -eq 2 ]] && runpath=${PWD}/rundir_cpass0_${runNumber}_${jobindex}
71353a5f 140 mkdir -p ${runpath}
1795556a 141 if [[ ! -d ${runpath} ]]; then
ef77d296 142 touch ${doneFileTmp}
143 echo "cannot make runpath ${runpath}" >> ${doneFileTmp}
144 cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
145 [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
1795556a 146 return 1
147 fi
0eaf38c7 148 if ! cd ${runpath}; then
ef77d296 149 touch ${doneFileTmp}
150 echo "PWD=$PWD is not the runpath=${runpath}" >> ${doneFileTmp}
151 cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
152 [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
0eaf38c7 153 return 1
154 fi
fc7e1e45 155
5de3064b 156 #runCPassX/C expects the raw chunk to be linked in the run dir
157 #despite it being accessed by the full path
158 ln -s ${infile} ${runpath}/${chunkName}
fc7e1e45 159
71353a5f 160 #####MC
161 if [[ -n ${generateMC} ]]; then
162 olddir=${PWD}
163 outputDirMC=${commonOutputPath}/000${runNumber}/sim/${jobindex}
164 simrunpath=${outputDirMC}
0eaf38c7 165 #[[ ${simulateInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && simrunpath=${TMPDIR}
166 [[ ${simulateInTemporaryDir} -eq 1 ]] && simrunpath=$(mktemp -d -t cpass0MC.XXXXXX)
71353a5f 167 mkdir -p ${outputDirMC}
168 mkdir -p ${simrunpath}
0eaf38c7 169 if cd ${simrunpath}; then
fc7e1e45 170
0eaf38c7 171 filesMC=(
172 "${batchWorkingDirectory}/sim.C"
173 "${batchWorkingDirectory}/rec.C"
174 "${batchWorkingDirectory}/Config.C"
175 "${batchWorkingDirectory}/OCDB_*.root"
176 )
177 for file in ${filesMC[*]}; do
178 [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
179 done
fc7e1e45 180
0eaf38c7 181 generateMC ${runNumber} ${ocdbPath} ${nEvents}
182
183 [[ ! "${simrunpath}" =~ "${outputDirMC}" ]] && mv * ${outputDirMC} #TODO check if it works
184 cd ${olddir}
185
186 ln -s ${outputDirMC}/* ${runpath}/
187
188 inputList=${outputDirMC}/galice.root #TODO not valid outside shell !!!
189 infile=""
190 fi
71353a5f 191 fi
192 ######
193
1795556a 194 if [[ ! -f ${inputList} && -z ${pretend} ]]; then
ef77d296 195 touch ${doneFileTmp}
196 echo "input file ${inputList} not found, exiting..." >> ${doneFileTmp}
197 cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
198 [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
1795556a 199 return 1
200 fi
71353a5f 201
202 logOutputDir=${runpath}
203 [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir}
0eaf38c7 204 [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/stdout
205 #[[ -z ${dontRedirectStdOutToLog} ]] && exec 2> ${logOutputDir}/stderr
71353a5f 206 echo "${0} $*"
fc7e1e45 207
208 echo "#####################"
209 echo CPass0:
210 echo JOB setup
ef77d296 211 echo nEvents ${nEvents}
212 echo runNumber ${runNumber}
213 echo ocdbPath ${ocdbPath}
214 echo infile ${infile}
215 echo chunkName ${chunkName}
216 echo jobindex ${jobindex}
217 echo recoTriggerOptions ${recoTriggerOptions}
218 echo targetDirectory ${targetDirectory}
219 echo commonOutputPath ${commonOutputPath}
220 echo doneFile ${doneFile}
221 echo batchWorkingDirectory ${batchWorkingDirectory}
222 echo runpath ${runpath}
223 echo outputDir ${outputDir}
224 echo PWD ${PWD}
225 echo ALICE_ROOT ${ALICE_ROOT}
fc7e1e45 226 echo "########## ###########"
227
71353a5f 228 alirootInfo > ALICE_ROOT.log
fc7e1e45 229
230 filesCPass0=(
39527a15 231 "${batchWorkingDirectory}/runCPass0.sh"
232 "${batchWorkingDirectory}/recCPass0.C"
233 "${batchWorkingDirectory}/runCalibTrain.C"
234 "${batchWorkingDirectory}/localOCDBaccessConfig.C"
235 "${batchWorkingDirectory}/OCDB.root"
71353a5f 236 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/runCPass0.sh"
237 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/recCPass0.C"
238 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/runCalibTrain.C"
fc7e1e45 239 )
240
241 for file in ${filesCPass0[*]}; do
242 [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
1a0e38e3 243 [[ ${file##*/} =~ .*\.sh ]] && chmod +x ${file##*/}
fc7e1e45 244 done
245
71353a5f 246 echo "this directory (${PWD}) contents:"
247 /bin/ls
fc7e1e45 248 echo
249 chmod u+x runCPass0.sh
250
71353a5f 251 sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh
252
253 if [[ -n ${postSetUpActionCPass0} ]]; then
254 echo "running ${postSetUpActionCPass0}"
255 eval ${postSetUpActionCPass0}
fc7e1e45 256 fi
257
258 #run CPass0
71353a5f 259 echo "${runpath}/runCPass0.sh ${infile} ${nEvents} ${runNumber} ${ocdbPath} ${recoTriggerOptions}"
260 if [[ -n ${pretend} ]]; then
0eaf38c7 261 sleep ${pretendDelay}
cb473870 262 touch AliESDs.root
263 touch AliESDfriends.root
fc7e1e45 264 touch AliESDfriends_v1.root
265 touch rec.log
266 touch calib.log
267 else
71353a5f 268 echo ./runCPass0.sh "${infile}" "${nEvents}" "${runNumber}" "${ocdbPath}" "${recoTriggerOptions}"
269 ./runCPass0.sh "${infile}" "${nEvents}" "${runNumber}" "${ocdbPath}" "${recoTriggerOptions}"
fc7e1e45 270 fi
271
fc7e1e45 272 #move stuff to final destination
71353a5f 273 echo "this directory (${PWD}) contents:"
274 /bin/ls
fc7e1e45 275 echo
276
ef77d296 277 # [dberzano] OK this is fine!
71353a5f 278 echo rm -f ./${chunkName}
279 rm -f ./${chunkName}
13abb339 280 echo "paranoidCp ${runpath}/* ${outputDir}"
281 paranoidCp ${runpath}/* ${outputDir}
fc7e1e45 282 echo
5ed7c82c 283
284 #validate CPass0
285 cd ${outputDir}
ef77d296 286 if summarizeLogs >> ${doneFileTmp}; then
287 [[ -f ${outputDirMC}/galice.root ]] && echo "sim ${outputDirMC}/galice.root" >> ${doneFileTmp}
288 [[ -f AliESDfriends_v1.root ]] && echo "calibfile ${outputDir}/AliESDfriends_v1.root" >> ${doneFileTmp}
289 [[ -f AliESDs.root ]] && echo "esd ${outputDir}/AliESDs.root" >> ${doneFileTmp}
ab946185 290 fi
fc7e1e45 291
0eaf38c7 292 [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath} && echo "removing ${runpath}"
ef77d296 293 cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
294 [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
5de3064b 295 return 0
78d0bbfd 296)
fc7e1e45 297
298goCPass1()
78d0bbfd 299(
fc7e1e45 300 umask 0002
301
71353a5f 302 targetDirectory=${1}
303 inputList=${2}
304 nEvents=${3}
305 ocdbPath=${4}
306 configFile=${5}
307 runNumber=${6}
308 jobindex=${7}
309 shift 7
444b9916 310 extraOpts=("$@")
6fc48f9e 311 if ! parseConfig ${configFile} "$@"; then return 1; fi
71353a5f 312
39527a15 313 #record the working directory provided by the batch system
314 batchWorkingDirectory=${PWD}
315
71353a5f 316 #use the jobindex only if set and non-negative
317 if [[ -z ${jobindex} || ${jobindex} -lt 0 ]]; then
318 [[ -n "${LSB_JOBINDEX}" ]] && jobindex=${LSB_JOBINDEX}
319 [[ -n "${SGE_TASK_ID}" ]] && jobindex=${SGE_TASK_ID}
999a843d 320 if [[ -z ${jobindex} ]]; then
321 echo "no jobindex!"
322 return 1
323 fi
71353a5f 324 fi
325
326 [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
ef77d296 327
328 # This file signals that/if everything went fine
329 doneFileBase="cpass1.job${jobindex}.run${runNumber}.done"
330 [[ -n ${useProfilingCommand} ]] && doneFileBase="profiling.cpass0.job${jobindex}.run${runNumber}.done"
331
332 # We will have two copies of the file
333 mkdir -p "${commonOutputPath}/meta" || return 1
334 doneFileTmp="${batchWorkingDirectory}/${doneFileBase}"
335 doneFile="${commonOutputPath}/meta/${doneFileBase}"
71353a5f 336
337 [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
338
339 if [[ -n ${ALIROOT_FORCE_COREDUMP} ]]; then
340 ulimit -c unlimited
341 export ALIROOT_FORCE_COREDUMP
342 fi
fc7e1e45 343
cb473870 344 #the contents of this is stored in the tree and used later (e.g. AliAnalysisTaskPIDResponse)!
345 #at the QA stage the pass number is guessed from the path stored here.
346 #The Format is:
347 #Packages= ;OutputDir= ;LPMPass= ;TriggerAlias= ;LPMRunNumber= ;LPMProductionType= ;LPMInteractionType= ;LPMProductionTag= ;LPMAnchorRun= ;LPMAnchorProduction= ;LPMAnchorYear=
348 export PRODUCTION_METADATA="OutputDir=cpass1"
349
1795556a 350 if [[ ! -f ${inputList} && -z ${pretend} ]]; then
ef77d296 351 touch ${doneFileTmp}
352 echo "input file ${inputList} not found, exiting..." >> ${doneFileTmp}
353 cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
354 [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
1795556a 355 return 1
356 fi
fc7e1e45 357 if [[ "${inputList}" =~ \.root$ ]]; then
71353a5f 358 infile=${inputList}
fc7e1e45 359 else
71353a5f 360 infile=$(sed -ne "${jobindex}p" ${inputList} | egrep '\s*\w*/\w*')
fc7e1e45 361 fi
fc7e1e45 362 chunkName=${infile##*/}
5de3064b 363
71353a5f 364 outputDir=${targetDirectory}/${jobindex}_${chunkName%.*}
365 mkdir -p ${outputDir}
1795556a 366 if [[ ! -d ${outputDir} ]];then
ef77d296 367 touch ${doneFileTmp}
368 echo "cannot make ${outputDir}" >> ${doneFileTmp}
369 cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
370 [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
1795556a 371 return 1
372 fi
fc7e1e45 373
71353a5f 374 runpath=${outputDir}
0eaf38c7 375 [[ ${reconstructInTemporaryDir} -eq 1 ]] && runpath=$(mktemp -d -t cpass1.XXXXXX)
e91a9c78 376 [[ ${reconstructInTemporaryDir} -eq 2 ]] && runpath=${PWD}/rundir_cpass1_${runNumber}_${jobindex}
71353a5f 377
378 #MC
379 if [[ "${infile}" =~ galice\.root ]]; then
380 ln -s ${inputList%/*}/* ${runpath}
381 infile=""
382 fi
fc7e1e45 383
384 #init the running path
71353a5f 385 mkdir -p ${runpath}
1795556a 386 if [[ ! -d ${runpath} ]]; then
ef77d296 387 touch ${doneFileTmp}
388 echo "cannot make runpath ${runpath}" >> ${doneFileTmp}
389 cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
390 [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
1795556a 391 return 1
392 fi
0eaf38c7 393 if ! cd ${runpath}; then
ef77d296 394 touch ${doneFileTmp}
395 echo "PWD=$PWD is not the runpath=${runpath}" >> ${doneFileTmp}
396 cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
397 [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
0eaf38c7 398 return 1
399 fi
fc7e1e45 400
71353a5f 401 #this is needed for runCPass1.sh
402 ln -s ${infile} ${runpath}/${chunkName}
fc7e1e45 403
71353a5f 404 logOutputDir=${runpath}
405 [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir}
0eaf38c7 406 [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/stdout
407 #[[ -z ${dontRedirectStdOutToLog} ]] && exec 2> ${logOutputDir}/stderr
71353a5f 408 echo "${0} $*"
fc7e1e45 409
410 echo "#####################"
411 echo CPass1:
412 echo JOB setup
ef77d296 413 echo nEvents ${nEvents}
414 echo runNumber ${runNumber}
415 echo ocdbPath ${ocdbPath}
416 echo infile ${infile}
417 echo chunkName ${chunkName}
418 echo jobindex ${jobindex}
419 echo recoTriggerOptions ${recoTriggerOptions}
420 echo targetDirectory ${targetDirectory}
421 echo commonOutputPath ${commonOutputPath}
422 echo doneFile ${doneFile}
423 echo runpath ${runpath}
424 echo outputDir ${outputDir}
1795556a 425 echo batchWorkingDirectory ${batchWorkingDirectory}
ef77d296 426 echo ALICE_ROOT ${ALICE_ROOT}
427 echo PWD ${PWD}
428 echo "#####################"
fc7e1e45 429
71353a5f 430 alirootInfo > ALICE_ROOT.log
fc7e1e45 431
432 filesCPass1=(
39527a15 433 "${batchWorkingDirectory}/runCPass1.sh"
434 "${batchWorkingDirectory}/recCPass1.C"
435 "${batchWorkingDirectory}/recCPass1_OuterDet.C"
436 "${batchWorkingDirectory}/runCalibTrain.C"
437 "${batchWorkingDirectory}/QAtrain_duo.C"
438 "${batchWorkingDirectory}/localOCDBaccessConfig.C"
0eaf38c7 439 "${batchWorkingDirectory}/${configFile}"
617f8886 440 "${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz"
39527a15 441 "${batchWorkingDirectory}/OCDB.root"
71353a5f 442 "${trustedQAtrainMacro}"
443 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/runCPass1.sh"
444 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/recCPass1.C"
445 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/recCPass1_OuterDet.C"
446 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/runCalibTrain.C"
447 "${ALICE_ROOT}/ANALYSIS/macros/QAtrain_duo.C"
fc7e1e45 448 )
449
78d0bbfd 450 for file in "${filesCPass1[@]}"; do
fc7e1e45 451 [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
78d0bbfd 452 [[ ${file##*/} =~ .*\.sh ]] && echo "making ${file##*/} executable" && chmod +x ${file##*/}
fc7e1e45 453 done
454
71353a5f 455 echo "this directory (${PWD}) contents:"
456 /bin/ls
fc7e1e45 457 echo
458
78d0bbfd 459 #remove spaces around commas from calls to root
71353a5f 460 sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh
461
462 if [[ -n ${postSetUpActionCPass1} ]]; then
463 echo "running ${postSetUpActionCPass1}"
464 eval ${postSetUpActionCPass1}
fc7e1e45 465 echo
466 fi
467
468 #configure local OCDB storage from CPass0 (creates the localOCDBaccessConfig.C script)
469 if [[ -f cpass0.localOCDB.${runNumber}.tgz ]]; then
71353a5f 470 echo goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz"
471 goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz"
cb473870 472 else
473 echo "WARNING: file cpass0.localOCDB.${runNumber}.tgz not found!"
474 fi
475
71353a5f 476 if [[ ! $(/bin/ls -1 OCDB/*/*/*/*.root 2>/dev/null) ]]; then
ef77d296 477 touch ${doneFileTmp}
478 echo "cpass0 produced no calibration! exiting..." >> ${doneFileTmp}
479 cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
480 [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
71353a5f 481 return 1
cb473870 482 fi
fc7e1e45 483
71353a5f 484 #create the Barrel and OuterDet directories for CPass1 and link the local OCDB directory
485 #there to make the localOCDBaccessConfig.C file work, since it may point to the OCDB
486 #entries using a relative path, e.g. local://./OCDB
78d0bbfd 487 echo "linking the OCDB/ for Barrel and OuterDet directories"
71353a5f 488 mkdir Barrel OuterDet
78d0bbfd 489 ls -l
71353a5f 490 ln -s ../OCDB Barrel/OCDB
491 ln -s ../OCDB OuterDet/OCDB
492
444b9916 493 #setup the filtering
494 #the following option enables the filtering task inside the QAtrain_duo.C
495 [[ -n $runESDfiltering ]] && export QA_TaskFilteredTree=1
496 #set the downscaling factors during the filtering fro expert QA (overrides the previous values)
497 if [[ -n ${filteringFactorHighPt} ]]; then
498 export AliAnalysisTaskFilteredTree_fLowPtTrackDownscaligF=${filteringFactorHighPt}
499 fi
500 if [[ -n ${filteringFactorV0s} ]]; then
501 export AliAnalysisTaskFilteredTree_fLowPtV0DownscaligF=${filteringFactorV0s}
502 fi
503
fc7e1e45 504 #run CPass1
505 chmod u+x runCPass1.sh
71353a5f 506 echo "${runpath}/runCPass1.sh ${infile} ${nEvents} ${runNumber} ${ocdbPath} ${recoTriggerOptions}"
507 if [[ -n ${pretend} ]]; then
0eaf38c7 508 sleep ${pretendDelay}
cb473870 509 touch AliESDs_Barrel.root
510 touch AliESDfriends_Barrel.root
fc7e1e45 511 touch AliESDfriends_v1.root
cb473870 512 touch QAresults_barrel.root
513 touch EventStat_temp_barrel.root
514 touch AODtpITS.root
515 touch AliESDs_Outer.root
516 touch AliESDfriends_Outer.root
517 touch QAresults_outer.root
518 touch EventStat_temp_outer.root
fc7e1e45 519 touch rec.log
520 touch calib.log
521 touch qa.log
444b9916 522 touch filtering.log FilterEvents_Trees.root
fc7e1e45 523 else
71353a5f 524 ./runCPass1.sh "${infile}" "${nEvents}" "${runNumber}" "${ocdbPath}" "${recoTriggerOptions}"
444b9916 525
526 [[ ! -f AliESDs_Barrel.root && -f Barrel/AliESDs.root ]] && mv Barrel/AliESDs.root AliESDs_Barrel.root
527 [[ ! -f AliESDfriends_Barrel.root && -f Barrel/AliESDfriends.root ]] && mv Barrel/AliESDfriends.root AliESDfriends_Barrel.root
528 [[ ! -f AliESDfriends_v1.root && -f Barrel/AliESDfriends_v1.root ]] && mv Barrel/AliESDfriends_v1.root .
529 [[ ! -f QAresults_barrel.root && -f Barrel/QAresults_barrel.root ]] && mv Barrel/QAresults_barrel.root .
530 [[ ! -f AliESDs_Outer.root && -f OuterDet/AliESDs.root ]] && mv OuterDet/AliESDs.root AliESDs_Outer.root
531 [[ ! -f AliESDfriends_Outer.root && -f OuterDet/AliESDfriends.root ]] && mv OuterDet/AliESDfriends.root AliESDfriends_Outer.root
532 [[ ! -f QAresults_outer.root && -f OuterDet/QAresults_outer.root ]] && mv OuterDet/QAresults_outer.root .
533 [[ ! -f FilterEvents_Trees.root && -f Barrel/FilterEvents_Trees.root ]] && mv Barrel/FilterEvents_Trees.root .
534
535 #make the filtered tree (if requested and not already produced by QA
536 [[ -f AliESDs_Barrel.root ]] && echo "AliESDs_Barrel.root" > filtered.list
fab31805 537 if [[ -n ${runESDfiltering} && ! -f FilterEvents_Trees.root && -f filtered.list ]]; then
538 goMakeFilteredTrees ${PWD} ${runNumber} "${PWD}/filtered.list" ${filteringFactorHighPt} ${filteringFactorV0s} ${ocdbPath} 1000000 0 10000000 0 ${configFile} AliESDs_Barrel.root "${extraOpts[@]}" >filtering.log
539 else
540 echo ""
444b9916 541 fi
542
fc7e1e45 543 fi
cb473870 544
545 ##handle possible crashes in QA (happens often in trunk)
546 ##rerun QA with a trusted aliroot version
547 #if [[ $(validateLog qa_barrel.log) ]]; then
548 # echo "qa_barrel.log not validated!"
549 #fi
550 #if [[ ! -f QAresults_barrel.root && -f ${setupTrustedAliROOTenvInCurrentShell} || $(validateLog qa_barrel.log) ]]; then
71353a5f 551 # echo "WARNING: using trusted QA aliroot ${ALICE_ROOT}"
cb473870 552 # source ${setupTrustedAliROOTenvInCurrentShell}
553 # cd Barrel
554 # rm QAresults_barrel.root
555 # rm EventStat_temp_barrel.root
556 # rm AODtpITS.root
557 # [[ ! -f AliESDs.root ]] && ln -s ../AliESDs_Barrel.root AliESDs.root
558 # [[ ! -f AliESDfriends.root ]] && ln -s ../AliESDfriends_Barrel.root AliESDfriends.root
71353a5f 559 # if [[ -n ${trustedQAtrainMacro} ]]; then
560 # eval "cp ${trustedQAtrainMacro} QAtrain_duo_trusted.C"
cb473870 561 # fi
71353a5f 562 # echo executing aliroot -b -q "QAtrain_duo_trusted.C(\"_barrel\",${runNumber},\"wn.xml\",0,\"${ocdbPath}\")"
563 # time aliroot -b -q "QAtrain_duo.C(\"_barrel\",${runNumber},\"wn.xml\",0,\"${ocdbPath}\")" &> ../qa_barrel_trusted.log
cb473870 564 # cd ../
565 #fi
cb473870 566
fc7e1e45 567 #move stuff to final destination
71353a5f 568 echo "this directory (${PWD}) contents:"
569 /bin/ls
570 echo rm -f ./${chunkName}
571 rm -f ./${chunkName}
13abb339 572 echo "paranoidCp ${runpath}/* ${outputDir}"
573 paranoidCp ${runpath}/* ${outputDir}
fc7e1e45 574 echo
575
5ed7c82c 576 #validate CPass1
577 cd ${outputDir}
ef77d296 578 if summarizeLogs >> ${doneFileTmp}; then
579 [[ -f AliESDs_Barrel.root ]] && echo "esd ${outputDir}/AliESDs_Barrel.root" >> ${doneFileTmp}
580 [[ -f AliESDfriends_v1.root ]] && echo "calibfile ${outputDir}/AliESDfriends_v1.root" >> ${doneFileTmp}
581 [[ -f QAresults_Barrel.root ]] && echo "qafile ${outputDir}/QAresults_Barrel.root" >> ${doneFileTmp}
582 [[ -f QAresults_Outer.root ]] && echo "qafile ${outputDir}/QAresults_Outer.root" >> ${doneFileTmp}
583 [[ -f QAresults_barrel.root ]] && echo "qafile ${outputDir}/QAresults_barrel.root" >> ${doneFileTmp}
584 [[ -f QAresults_outer.root ]] && echo "qafile ${outputDir}/QAresults_outer.root" >> ${doneFileTmp}
585 [[ -f FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFileTmp}
922d0b10 586 else
ef77d296 587 if grep "qa_outer.log.*OK" ${doneFileTmp} > /dev/null; then
588 [[ -f QAresults_Outer.root ]] && echo "qafile ${outputDir}/QAresults_Outer.root" >> ${doneFileTmp}
589 [[ -f QAresults_outer.root ]] && echo "qafile ${outputDir}/QAresults_outer.root" >> ${doneFileTmp}
922d0b10 590 fi
ef77d296 591 if grep "qa_barrel.log.*OK" ${doneFileTmp} > /dev/null; then
592 [[ -f QAresults_Barrel.root ]] && echo "qafile ${outputDir}/QAresults_Barrel.root" >> ${doneFileTmp}
593 [[ -f QAresults_barrel.root ]] && echo "qafile ${outputDir}/QAresults_barrel.root" >> ${doneFileTmp}
922d0b10 594 fi
ef77d296 595 if grep "filtering.log.*OK" ${doneFileTmp} > /dev/null; then
596 [[ -f FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFileTmp}
922d0b10 597 fi
ab946185 598 fi
599
71353a5f 600 [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
ef77d296 601 cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
602 [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
5de3064b 603 return 0
78d0bbfd 604)
fc7e1e45 605
606
607goMergeCPass0()
78d0bbfd 608(
fc7e1e45 609 #
610 # find the output files and merge them
611 #
612
71353a5f 613 outputDir=${1}
614 ocdbStorage=${2}
615 configFile=${3}
616 runNumber=${4}
617 calibrationFilesToMerge=${5} #can be a non-existent file, will then be produced on the fly
618 shift 5
6fc48f9e 619 if ! parseConfig ${configFile} "$@"; then return 1; fi
71353a5f 620
39527a15 621 #record the working directory provided by the batch system
622 batchWorkingDirectory=${PWD}
623
71353a5f 624 [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
ef77d296 625
626 # This file signals that everything went fine
627 doneFileBase="merge.cpass0.run${runNumber}.done"
628
629 # We will have two copies of the file
630 mkdir -p "${commonOutputPath}/meta" || return 1
631 doneFileTmp="${batchWorkingDirectory}/${doneFileBase}"
632 doneFile="${commonOutputPath}/meta/${doneFileBase}"
71353a5f 633
634 umask 0002
635 ulimit -c unlimited
fc7e1e45 636
71353a5f 637 [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
fc7e1e45 638
71353a5f 639 runpath=${outputDir}
0eaf38c7 640 [[ ${reconstructInTemporaryDir} -eq 1 ]] && runpath=$(mktemp -d -t mergeCPass0.XXXXXX)
e91a9c78 641 [[ ${reconstructInTemporaryDir} -eq 2 ]] && runpath=${PWD}/rundir_mergeCPass0_${runNumber}
fc7e1e45 642
71353a5f 643 mkdir -p ${runpath}
1795556a 644 if [[ ! -d ${runpath} ]]; then
ef77d296 645 touch ${doneFileTmp}
646 echo "not able to make the runpath ${runpath}" >> ${doneFileTmp}
647 cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
648 [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
1795556a 649 return 1
650 fi
0eaf38c7 651 if ! cd ${runpath}; then
ef77d296 652 touch ${doneFileTmp}
653 echo "PWD=$PWD is not the runpath=${runpath}" >> ${doneFileTmp}
654 cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
655 [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
0eaf38c7 656 return 1
657 fi
fc7e1e45 658
71353a5f 659 logOutputDir=${runpath}
660 [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir}
1795556a 661 [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/stdout
71353a5f 662 echo "${0} $*"
fc7e1e45 663
fc7e1e45 664 mergingScript="mergeMakeOCDB.byComponent.sh"
fc7e1e45 665
666 echo goMergeCPass0 SETUP:
71353a5f 667 echo runNumber=${runNumber}
668 echo outputDir=${outputDir}
669 echo ocdbStorage=${ocdbStorage}
670 echo calibrationFilesToMerge=${calibrationFilesToMerge}
671 echo mergingScript=${mergingScript}
672 echo commonOutputPath=${commonOutputPath}
673 echo runpath=${runpath}
fc7e1e45 674
675 # copy files in case they are not already there
676 filesMergeCPass0=(
39527a15 677 "${batchWorkingDirectory}/${calibrationFilesToMerge}"
678 "${batchWorkingDirectory}/OCDB.root"
679 "${batchWorkingDirectory}/localOCDBaccessConfig.C"
71353a5f 680 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/mergeMakeOCDB.byComponent.sh"
681 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/mergeByComponent.C"
682 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/makeOCDB.C"
683 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/merge.C"
684 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/mergeMakeOCDB.sh"
fc7e1e45 685 )
686 for file in ${filesMergeCPass0[*]}; do
687 [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
1a0e38e3 688 [[ ${file##*/} =~ .*\.sh ]] && chmod +x ${file##*/}
fc7e1e45 689 done
690
71353a5f 691 sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh
692
693 alirootInfo > ALICE_ROOT.log
fc7e1e45 694
695 #
cb473870 696 echo "PWD"
71353a5f 697 /bin/ls
cb473870 698 echo "PWD/.."
71353a5f 699 /bin/ls ../
cb473870 700
fc7e1e45 701
702 #merge calibration
71353a5f 703 chmod u+x ${mergingScript}
fc7e1e45 704 mkdir -p ./OCDB
71353a5f 705 if [[ ! -f ${calibrationFilesToMerge} ]]; then
706 echo "/bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root > ${calibrationFilesToMerge}"
707 /bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root 2>/dev/null > ${calibrationFilesToMerge}
fc7e1e45 708 fi
709
71353a5f 710 echo "${mergingScript} ${calibrationFilesToMerge} ${runNumber} local://./OCDB ${ocdbStorage}"
711 if [[ -n ${pretend} ]]; then
0eaf38c7 712 sleep ${pretendDelay}
fc7e1e45 713 touch CalibObjects.root
714 touch ocdb.log
715 touch merge.log
617f8886 716 touch dcsTime.root
71353a5f 717 mkdir -p ./OCDB/TPC/Calib/TimeGain/
718 mkdir -p ./OCDB/TPC/Calib/TimeDrift/
1795556a 719 echo "some calibration" >> ./OCDB/TPC/Calib/TimeGain/someCalibObject_0-999999_cpass0.root
720 echo "some calibration" >> ./OCDB/TPC/Calib/TimeDrift/otherCalibObject_0-999999_cpass0.root
fc7e1e45 721 else
1795556a 722 ./${mergingScript} ${calibrationFilesToMerge} ${runNumber} "local://./OCDB" ${ocdbStorage} >> "mergeMakeOCDB.log"
fc7e1e45 723
617f8886 724 #produce the calib trees for expert QA (dcsTime.root)
725 goMakeLocalOCDBaccessConfig ./OCDB
726 echo aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")"
727 aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")"
728 fi
729
fc7e1e45 730 ### produce the output
731 #tar the produced OCDB for reuse
ef77d296 732 #tar czf ${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz ./OCDB
733
734 # Create tarball with OCDB, store on the shared directory, create signal file on batch directory
735 mkdir -p ${commonOutputPath}/meta
736 baseTar="cpass0.localOCDB.${runNumber}.tgz"
737 tar czf ${batchWorkingDirectory}/${baseTar} ./OCDB && \
738 mv ${batchWorkingDirectory}/${baseTar} ${commonOutputPath}/meta/${baseTar} && \
739 touch ${batchWorkingDirectory}/${baseTar}.done
fc7e1e45 740
71353a5f 741 /bin/ls
5ed7c82c 742
743 #copy all to output dir
13abb339 744 echo "paranoidCp ${runpath}/* ${outputDir}"
745 paranoidCp ${runpath}/* ${outputDir}
617f8886 746
71353a5f 747 if [[ -n ${generateMC} ]]; then
748 goPrintValues sim ${commonOutputPath}/meta/sim.run${runNumber}.list ${commonOutputPath}/meta/cpass0.job*.run${runNumber}.done
749 fi
750
fc7e1e45 751 #validate merging cpass0
5ed7c82c 752 cd ${outputDir}
ef77d296 753 if summarizeLogs >> ${doneFileTmp}; then
754 [[ -f CalibObjects.root ]] && echo "calibfile ${outputDir}/CalibObjects.root" >> ${doneFileTmp}
755 [[ -f dcsTime.root ]] && echo "dcsTree ${outputDir}/dcsTime.root" >> ${doneFileTmp}
ab946185 756 fi
fc7e1e45 757
71353a5f 758 [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
ef77d296 759 cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
760 [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
5de3064b 761 return 0
78d0bbfd 762)
fc7e1e45 763
764goMergeCPass1()
78d0bbfd 765(
fc7e1e45 766 #
767 # find the output files and merge them
768 #
769
71353a5f 770 outputDir=${1}
771 ocdbStorage=${2}
772 configFile=${3}
773 runNumber=${4}
774 calibrationFilesToMerge=${5}
775 qaFilesToMerge=${6}
444b9916 776 filteredFilesToMerge=${7}
777 shift 7
6fc48f9e 778 if ! parseConfig ${configFile} "$@"; then return 1; fi
71353a5f 779
39527a15 780 #record the working directory provided by the batch system
781 batchWorkingDirectory=${PWD}
782
71353a5f 783 [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
ef77d296 784
785 # This file signals that everything went fine
786 doneFileBase="merge.cpass1.run${runNumber}.done"
787
788 # We will have two copies of the file
789 mkdir -p "${commonOutputPath}/meta" || return 1
790 doneFileTmp="${batchWorkingDirectory}/${doneFileBase}"
791 doneFile="${commonOutputPath}/meta/${doneFileBase}"
71353a5f 792
793 umask 0002
794 ulimit -c unlimited
fc7e1e45 795
796 #clean up first:
71353a5f 797 rm -f ${outputDir}/*.log
798 rm -f ${outputDir}/*.root
799 rm -f ${outputDir}/*done
800
801 [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
802
71353a5f 803 runpath=${outputDir}
0eaf38c7 804 [[ ${reconstructInTemporaryDir} -eq 1 ]] && runpath=$(mktemp -d -t mergeCPass1.XXXXXX)
e91a9c78 805 [[ ${reconstructInTemporaryDir} -eq 2 ]] && runpath=${PWD}/rundir_mergeCPass1_${runNumber}
71353a5f 806
807 mkdir -p ${runpath}
1795556a 808 if [[ ! -d ${runpath} ]]; then
ef77d296 809 touch ${doneFileTmp}
810 echo "not able to make the runpath ${runpath}" >> ${doneFileTmp}
811 cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
812 [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
1795556a 813 return 1
814 fi
0eaf38c7 815 if ! cd ${runpath}; then
ef77d296 816 touch ${doneFileTmp}
817 echo "PWD=$PWD is not the runpath=${runpath}" >> ${doneFileTmp}
818 cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
819 [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
0eaf38c7 820 return 1
821 fi
71353a5f 822
823 logOutputDir=${runpath}
824 [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir}
825 [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/mergeMakeOCDB.log
826 echo "${0} $*"
827
828 calibrationOutputFileName='AliESDfriends_v1.root'
829 qaOutputFileName='QAresults*.root'
fc7e1e45 830 mergingScript="mergeMakeOCDB.byComponent.sh"
cb473870 831 #important to have the string "Stage.txt" in the filename to trigger the merging
832 #it has to be a list of directories containing the files
fc7e1e45 833 qaMergedOutputFileName="QAresults_merged.root"
834
835 echo goMergeCPass1 SETUP:
71353a5f 836 echo runNumber=${runNumber}
837 echo outputDir=${outputDir}
838 echo ocdbStorage=${ocdbStorage}
839 echo calibrationFilesToMerge=$calibrationFilesToMerge
840 echo qaFilesToMerge=$qaFilesToMerge
841 echo calibrationOutputFileName=${calibrationOutputFileName}
842 echo mergingScript=${mergingScript}
fc7e1e45 843
844 # copy files in case they are not already there
845 filesMergeCPass1=(
39527a15 846 "${batchWorkingDirectory}/${calibrationFilesToMerge}"
847 "${batchWorkingDirectory}/${qaFilesToMerge}"
848 "${batchWorkingDirectory}/OCDB.root"
849 "${batchWorkingDirectory}/localOCDBaccessConfig.C"
3fbef3b9 850 "${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz"
39527a15 851 "${batchWorkingDirectory}/QAtrain_duo.C"
71353a5f 852 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/mergeMakeOCDB.byComponent.sh"
853 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/mergeByComponent.C"
854 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/makeOCDB.C"
855 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/merge.C"
856 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/mergeMakeOCDB.sh"
857 "${trustedQAtrainMacro}"
858 "${ALICE_ROOT}/ANALYSIS/macros/QAtrain_duo.C"
fc7e1e45 859 )
860 for file in ${filesMergeCPass1[*]}; do
861 [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
1a0e38e3 862 [[ ${file##*/} =~ .*\.sh ]] && chmod +x ${file##*/}
fc7e1e45 863 done
864
71353a5f 865 sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh
866
cb473870 867 #configure local OCDB storage from CPass0 (creates the localOCDBaccessConfig.C script)
868 if [[ -f cpass0.localOCDB.${runNumber}.tgz ]]; then
71353a5f 869 echo goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz"
870 goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz"
cb473870 871 else
872 echo "WARNING: file cpass0.localOCDB.${runNumber}.tgz not found!"
873 fi
874
71353a5f 875 alirootInfo > ALICE_ROOT.log
fc7e1e45 876
877 #
71353a5f 878 /bin/ls
fc7e1e45 879
880 #merge calibration
71353a5f 881 chmod u+x ${mergingScript}
fc7e1e45 882 mkdir -p OCDB
444b9916 883
884 #if not provided, create the lists of files to merge
885 if [[ ! -f ${filteredFilesToMerge} ]]; then
886 echo "/bin/ls -1 ${outputDir}/*/FilterEvents_Trees.root > ${filteredFilesToMerge}"
887 /bin/ls -1 ${outputDir}/*/FilterEvents_Trees.root 2>/dev/null > ${filteredFilesToMerge}
888 fi
71353a5f 889 if [[ ! -f ${calibrationFilesToMerge} ]]; then
444b9916 890 echo "/bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root > ${calibrationFilesToMerge}"
891 /bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root 2>/dev/null > ${calibrationFilesToMerge}
892 fi
893 if [[ ! -f ${qaFilesToMerge} ]]; then
894 #find the files, but only store the directories (QAtrain_duo.C requires this)
895 echo "/bin/ls -1 ${outputDir}/*/QAresults*.root | while read x; do echo ${x%/*}; done | sort | uniq > ${qaFilesToMerge}"
896 /bin/ls -1 ${outputDir}/*/QAresults*.root | while read x; do echo ${x%/*}; done | sort | uniq > ${qaFilesToMerge}
fc7e1e45 897 fi
898
71353a5f 899 echo "${mergingScript} ${calibrationFilesToMerge} ${runNumber} local://./OCDB ${ocdbStorage}"
900 if [[ -n ${pretend} ]]; then
0eaf38c7 901 sleep ${pretendDelay}
fc7e1e45 902 touch ocdb.log
71353a5f 903 touch cpass1.localOCDB.${runNumber}.tgz
71353a5f 904 touch ${qaMergedOutputFileName}
fc7e1e45 905 touch merge.log
cb473870 906 touch trending.root
444b9916 907 touch FilterEvents_Trees.root
ab946185 908 touch CalibObjects.root
909 touch dcsTime.root
910 touch ${qaMergedOutputFileName}
617f8886 911 mkdir -p OCDB
fc7e1e45 912 else
617f8886 913 ./${mergingScript} ${calibrationFilesToMerge} ${runNumber} "local://./OCDB" ${ocdbStorage}
914
915 #merge QA (and filtered trees)
916 [[ -n ${AliAnalysisTaskFilteredTree_fLowPtTrackDownscaligF} ]] && export AliAnalysisTaskFilteredTree_fLowPtTrackDownscaligF
917 [[ -n ${AliAnalysisTaskFilteredTree_fLowPtV0DownscaligF} ]] && export AliAnalysisTaskFilteredTree_fLowPtV0DownscaligF
918
919 #echo aliroot -l -b -q "merge.C(\"${qaFilesToMerge}\",\"\",kFALSE,\"${qaMergedOutputFileName}\")"
920 echo aliroot -b -q "QAtrain_duo.C(\"_barrel\",${runNumber},\"${qaFilesToMerge}\",1,\"${ocdbStorage}\")"
71353a5f 921 #aliroot -l -b -q "merge.C(\"${qaFilesToMerge}\",\"\",kFALSE,\"${qaMergedOutputFileName}\")"
922 aliroot -b -q "QAtrain_duo.C(\"_barrel\",${runNumber},\"${qaFilesToMerge}\",1,\"${ocdbStorage}\")" > mergeQA.log
923 mv QAresults_barrel.root ${qaMergedOutputFileName}
cb473870 924 mv trending_barrel.root trending.root
444b9916 925
926 #merge filtered trees
927 echo aliroot -l -b -q "merge.C(\"${qaFilesToMerge}\",\"\",kFALSE,\"${qaMergedOutputFileName}\")"
922d0b10 928 aliroot -l -b -q "merge.C(\"${filteredFilesToMerge}\",\"\",kFALSE,\"FilterEvents_Trees.root\")" > mergeFilteredTrees.log
617f8886 929
930 #produce the calib trees for expert QA
931 echo aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")"
922d0b10 932 aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")" > calibTree.log
fc7e1e45 933 fi
617f8886 934
ef77d296 935 # Create tarball with OCDB, store on the shared directory, create signal file on batch directory
936 mkdir -p ${commonOutputPath}/meta
937 baseTar="cpass1.localOCDB.${runNumber}.tgz"
938 tar czf ${batchWorkingDirectory}/${baseTar} ./OCDB && \
939 mv ${batchWorkingDirectory}/${baseTar} ${commonOutputPath}/meta/${baseTar} && \
940 touch ${batchWorkingDirectory}/${baseTar}.done
71353a5f 941
942 /bin/ls
5ed7c82c 943
944 #copy all to output dir
13abb339 945 echo "paranoidCp ${runpath}/* ${outputDir}"
946 paranoidCp ${runpath}/* ${outputDir}
5ed7c82c 947
fc7e1e45 948 #validate merge cpass1
5ed7c82c 949 cd ${outputDir}
ef77d296 950 if summarizeLogs >> ${doneFileTmp}; then
951 [[ -f CalibObjects.root ]] && echo "calibfile ${outputDir}/CalibObjects.root" >> ${doneFileTmp}
952 [[ -f ${qaMergedOutputFileName} ]] && echo "qafile ${outputDir}/${qaMergedOutputFileName}" >> ${doneFileTmp}
953 [[ -f trending.root ]] && echo "trendingfile ${outputDir}/trending.root" >> ${doneFileTmp}
954 [[ -f dcsTime.root ]] && echo "dcsTree ${outputDir}/dcsTime.root" >> ${doneFileTmp}
955 [[ -f FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFileTmp}
922d0b10 956 else
ef77d296 957 if grep "mergeQA.log.*OK" ${doneFileTmp} > /dev/null; then
958 [[ -f ${qaMergedOutputFileName} ]] && echo "qafile ${outputDir}/${qaMergedOutputFileName}" >> ${doneFileTmp}
922d0b10 959 fi
ef77d296 960 if grep "mergeFilteredTrees.log.*OK" ${doneFileTmp} > /dev/null; then
961 [[ -f FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFileTmp}
922d0b10 962 fi
ab946185 963 fi
71353a5f 964
965 [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
ef77d296 966 cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
967 [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
5de3064b 968 return 0
78d0bbfd 969)
cb473870 970
971goMerge()
78d0bbfd 972(
cb473870 973 #generic root merge using CPass1 merge.C script
71353a5f 974 inputList=${1}
975 outputFile=${2}
0eaf38c7 976 configFile=${3-"benchmark.config"}
71353a5f 977 shift 3
6fc48f9e 978 if ! parseConfig ${configFile} "$@"; then return 1; fi
39527a15 979
980 #record the working directory provided by the batch system
981 batchWorkingDirectory=${PWD}
fc7e1e45 982
71353a5f 983 [[ ! -f ${inputList} ]] && echo "inputList ${inputList} does not exist!" && return 1
984 [[ ! -f ${configFile} ]] && echo "configFile ${configFile} does not exist!" && return 1
985 umask 0002
986 [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
987 rm -f ${outputFile}
988 aliroot -b -q "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/merge.C(\"${inputList}\",\"\",kFALSE,\"${outputFile}\")" > merge_${inputList}.log
5de3064b 989 return 0
78d0bbfd 990)
fc7e1e45 991
cb473870 992goSubmitMakeflow()
5dfac970 993{
cb473870 994 #run
71353a5f 995 productionID=${1}
996 inputFileList=${2}
997 configFile=${3}
998 shift 3
444b9916 999 extraOpts=("$@")
6fc48f9e 1000 if ! parseConfig ${configFile} "${extraOpts[@]}"; then return 1; fi
39527a15 1001
1002 #record the working directory provided by the batch system
1003 batchWorkingDirectory=${PWD}
fc7e1e45 1004
5dfac970 1005 [[ -z ${configFile} ]] && configFile="benchmark.config"
1006 [[ ! -f ${configFile} ]] && echo "no config file found (${configFile})" && return 1
fc7e1e45 1007
71353a5f 1008 if [[ ! $(which makeflow &>/dev/null) && -n ${makeflowPath} ]]; then
cb473870 1009 echo "setting the makflow path from the config: "
1010 echo " export PATH=${makeflowPath}:${PATH}"
1011 export PATH=${makeflowPath}:${PATH}
1012 fi
1013
39527a15 1014 #create the common output dir and the meta dir
1015 commonOutputPath=${baseOutputDirectory}/${productionID}
1016 if [[ -d ${commonOutputPath} ]]; then
1017 echo "output dir ${commonOutputPath} exists!"
0eaf38c7 1018 #return 1
39527a15 1019 else
1020 mkdir -p ${commonOutputPath}
1021 fi
1022 mkdir -p ${commonOutputPath}/meta
839b07ca 1023
0eaf38c7 1024 self=${0}
1025 #if which greadlink; then self=$(greadlink -f "${0}"); fi
839b07ca 1026
0eaf38c7 1027 #for reference copy the setup to the output dir
839b07ca 1028 cp ${self} ${commonOutputPath}
1029 cp ${configFile} ${commonOutputPath}
333bc229 1030 cp ${inputFileList} ${commonOutputPath}
39527a15 1031
cb473870 1032 #submit - use makeflow if available, fall back to old stuff when makeflow not there
1033 if which makeflow; then
39527a15 1034 goGenerateMakeflow ${productionID} ${inputFileList} ${configFile} "${extraOpts[@]}" commonOutputPath=${commonOutputPath} > benchmark.makeflow
1795556a 1035 cp benchmark.makeflow ${commonOutputPath}
cb473870 1036 makeflow ${makeflowOptions} benchmark.makeflow
cb473870 1037 else
1038 echo "no makeflow!"
1039 fi
054fdec5 1040
1041 #summarize the run based on the makeflow log
1042 #and add it to the end of summary log
1043 awk '/STARTED/ {startTime=$3}
1044 /COMPLETED/ {endTime=$3}
1045 END {print "makeflow running time: "(endTime-startTime)/1000000/3600" hours"}' \
1046 benchmark.makeflow.makeflowlog | tee -a summary.log
1047 paranoidCp summary.log ${commonOutputPath}
1048
5de3064b 1049 return 0
5dfac970 1050}
fc7e1e45 1051
5dfac970 1052goGenerateMakeflow()
78d0bbfd 1053(
5dfac970 1054 #generate the makeflow file
ab946185 1055 [[ $# -lt 3 ]] && echo "args: id inputFileList configFile" && return 1
71353a5f 1056 productionID=${1}
1057 inputFileList=${2}
1058 configFile=${3}
1059 shift 3
444b9916 1060 extraOpts=("$@")
0eaf38c7 1061
1062 #batch systems/makeflow sometimes handle spaces in arguments poorly, so encode them
1063 for (( i=0;i<${#extraOpts[@]};i++ )); do
1064 extraOpts[i]=$(encSpaces "${extraOpts[i]}")
1065 done
1066 extraOpts+=("encodedSpaces=1")
1067
39527a15 1068 if ! parseConfig ${configFile} "${extraOpts[@]}" &>/dev/null; then return 1; fi
1069
1a0e38e3 1070 #extra safety
1071 if [[ -z ${commonOutputPath} ]]; then
1072 commonOutputPath=${baseOutputDirectory}/${productionID}
1073 extraOpts=( "${extraOpts[@]}" "commonOutputPath=${commonOutputPath}" )
1074 fi
bff71da0 1075
39527a15 1076 #record the working directory provided by the batch system
1077 batchWorkingDirectory=${PWD}
fc7e1e45 1078
5dfac970 1079 [[ -z ${configFile} ]] && configFile="benchmark.config"
1080 [[ ! -f ${configFile} ]] && echo "no config file found (${configFile})" && return 1
fc7e1e45 1081
5dfac970 1082 #these files will be made a dependency - will be copied to the working dir of the jobs
1083 declare -a copyFiles
1084 inputFiles=(
1085 "OCDB.root"
1086 "localOCDBaccessConfig.C"
cb473870 1087 "QAtrain_duo.C"
1088 "runCPass1.sh"
1089 "recCPass1.C"
1090 "recCPass1_OuterDet.C"
1091 "runCalibTrain.C"
1092 "runCPass0.sh"
1093 "recCPass0.C"
ef77d296 1094 "runQA.sh"
5dfac970 1095 )
1096 for file in ${inputFiles[*]}; do
1097 [[ -f ${file} ]] && copyFiles+=("${file}")
1098 done
1099
1100 #create the makeflow file
cb473870 1101 [[ -n ${batchFlags} ]] && echo "BATCH_OPTIONS = ${batchFlags}"
444b9916 1102 declare -A arr_cpass0_merged arr_cpass1_merged
1103 declare -A arr_cpass0_calib_list arr_cpass1_calib_list
1104 declare -A arr_cpass1_QA_list arr_cpass1_ESD_list arr_cpass1_filtered_list
6fc48f9e 1105 declare -A arr_cpass0_profiled_outputs
444b9916 1106 declare -A listOfRuns
1107 [[ -n ${runNumber} ]] && listOfRuns[${runNumber}]=1
617f8886 1108 while read x; do tmpRun=$(guessRunNumber ${x}); [[ -n ${tmpRun} ]] && listOfRuns[${tmpRun}]=1; done < ${inputFileList}
444b9916 1109 for runNumber in "${!listOfRuns[@]}"; do
71353a5f 1110 [[ -z ${runNumber} ]] && continue
fc7e1e45 1111 [[ ! ${runNumber} =~ ^[0-9]*[0-9]$ ]] && continue
fc7e1e45 1112
444b9916 1113 unset arr_cpass0_outputs
1114 unset arr_cpass1_outputs
1115 declare -a arr_cpass0_outputs
1116 declare -a arr_cpass1_outputs
fc7e1e45 1117
ef77d296 1118 #Header
1119 echo "### Automatically generated on $(LANG=C date) ###"
1120 echo ; echo
1121
cb473870 1122 jobindex=0
6fc48f9e 1123 inputFile=""
5dfac970 1124 while read inputFile; do
1125 currentDefaultOCDB=${defaultOCDB}
cb473870 1126 [[ -z ${autoOCDB} ]] && autoOCDB=1
1127 if [[ ${autoOCDB} -ne 0 ]]; then
71353a5f 1128 currentDefaultOCDB=$(setYear ${inputFile} ${defaultOCDB})
cb473870 1129 fi
ef77d296 1130 guessRunData ${inputFile}
1131
1132 #Set variables
1133 echo "### Variables ###"
1134 echo "OUTPATH=\"${commonOutputPath}/${year}/${period}\""
1135 echo ; echo
fc7e1e45 1136
5dfac970 1137 #CPass0
ef77d296 1138 #arr_cpass0_outputs[${jobindex}]="${commonOutputPath}/meta/cpass0.job${jobindex}.run${runNumber}.done"
1139 arr_cpass0_outputs[${jobindex}]="cpass0.job${jobindex}.run${runNumber}.done"
1140 echo "### CPass0 ###"
1141 echo "${arr_cpass0_outputs[${jobindex}]}: benchmark.sh ${configFile} ${copyFiles[@]}"
1142 echo " ${alirootEnv} ./benchmark.sh CPass0 \$OUTPATH/000${runNumber}/cpass0 ${inputFile} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} ${jobindex} ${extraOpts[@]}"" "
1143 echo ; echo
5dfac970 1144
1145 #CPass1
ef77d296 1146 #arr_cpass1_outputs[${jobindex}]="${commonOutputPath}/meta/cpass1.job${jobindex}.run${runNumber}.done"
1147 arr_cpass1_outputs[${jobindex}]="cpass1.job${jobindex}.run${runNumber}.done"
1148 echo "### CPass1 ###"
1149 echo "${arr_cpass1_outputs[${jobindex}]}: benchmark.sh ${configFile} cpass0.localOCDB.${runNumber}.tgz.done ${copyFiles[@]}"
1150 echo " ${alirootEnv} ./benchmark.sh CPass1 \$OUTPATH/000${runNumber}/cpass1 ${inputFile} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} ${jobindex} ${extraOpts[@]}"" "
1151 echo ; echo
5dfac970 1152 ((jobindex++))
fc7e1e45 1153
71353a5f 1154 done< <(grep "/000${runNumber}/" ${inputFileList})
cb473870 1155
5dfac970 1156 #CPass0 list of Calib files to merge
ef77d296 1157 #arr_cpass0_calib_list[${runNumber}]="${commonOutputPath}/meta/cpass0.calib.run${runNumber}.list"
1158 arr_cpass0_calib_list[${runNumber}]="cpass0.calib.run${runNumber}.list"
1159 echo "### Produces the list of CPass0 files to merge (executes locally) ###"
1160 echo "${arr_cpass0_calib_list[${runNumber}]}: benchmark.sh ${arr_cpass0_outputs[*]}"
1161 echo " LOCAL ./benchmark.sh PrintValues calibfile ${arr_cpass0_calib_list[${runNumber}]} ${arr_cpass0_outputs[*]} && mkdir -p \$OUTPATH/meta && cp ${arr_cpass0_calib_list[${runNumber}]} \$OUTPATH/meta/${arr_cpass0_calib_list[${runNumber}]}"
1162 echo ; echo
71353a5f 1163
1164 #CPass0 merging
ef77d296 1165 echo "### Merges CPass0 files ###"
1166 #arr_cpass0_merged[${runNumber}]="${commonOutputPath}/meta/merge.cpass0.run${runNumber}.done"
1167 arr_cpass0_merged[${runNumber}]="merge.cpass0.run${runNumber}.done"
1168 echo "cpass0.localOCDB.${runNumber}.tgz.done ${arr_cpass0_merged[${runNumber}]}: benchmark.sh ${configFile} ${arr_cpass0_calib_list[${runNumber}]} ${copyFiles[@]}"
1169 echo " ${alirootEnv} ./benchmark.sh MergeCPass0 \$OUTPATH/000${runNumber}/cpass0 ${currentDefaultOCDB} ${configFile} ${runNumber} ${arr_cpass0_calib_list[${runNumber}]} ${extraOpts[@]}"" "
1170 echo ; echo
fc7e1e45 1171
ab946185 1172 #CPass1 list of Calib/QA/ESD/filtered files
1173 # the trick with QA is to have the string "Stage.txt" in the file name of the list of directories with QA output to trigger
1174 # the production of the QA trending tree (only then the task->Finish() will be called in QAtrain_duo.C, on the grid
8d6a0a43 1175 # this corresponds to the last merging stage)
ef77d296 1176 #arr_cpass1_QA_list[${runNumber}]="${commonOutputPath}/meta/cpass1.QA.run${runNumber}.lastMergingStage.txt.list"
1177 arr_cpass1_QA_list[${runNumber}]="cpass1.QA.run${runNumber}.lastMergingStage.txt.list"
1178 echo "### Lists CPass1 QA ###"
444b9916 1179 echo "${arr_cpass1_QA_list[${runNumber}]}: benchmark.sh ${arr_cpass1_outputs[*]}"
ef77d296 1180 echo " LOCAL ./benchmark.sh PrintValues dir ${arr_cpass1_QA_list[${runNumber}]} ${arr_cpass1_outputs[*]} && mkdir -p \$OUTPATH/meta && cp ${arr_cpass1_QA_list[${runNumber}]} \$OUTPATH/meta/${arr_cpass1_QA_list[${runNumber}]}"
1181 echo ; echo
1182
1183 #arr_cpass1_calib_list[${runNumber}]="${commonOutputPath}/meta/cpass1.calib.run${runNumber}.list"
1184 arr_cpass1_calib_list[${runNumber}]="cpass1.calib.run${runNumber}.list"
1185 echo "### Lists CPass1 Calib ###"
1186 echo "${arr_cpass1_calib_list[${runNumber}]}: benchmark.sh ${arr_cpass1_outputs[*]}"
1187 echo " LOCAL ./benchmark.sh PrintValues calibfile ${arr_cpass1_calib_list[${runNumber}]} ${arr_cpass1_outputs[*]} && mkdir -p \$OUTPATH/meta && cp ${arr_cpass1_calib_list[${runNumber}]} \$OUTPATH/meta/${arr_cpass1_calib_list[${runNumber}]}"
1188 echo ; echo
1189
1190 #arr_cpass1_ESD_list[${runNumber}]="${commonOutputPath}/meta/cpass1.ESD.run${runNumber}.list"
1191 arr_cpass1_ESD_list[${runNumber}]="cpass1.ESD.run${runNumber}.list"
1192 echo "### Lists CPass1 ESDs ###"
1193 echo "${arr_cpass1_ESD_list[${runNumber}]}: benchmark.sh ${arr_cpass1_outputs[*]}"
1194 echo " LOCAL ./benchmark.sh PrintValues esd ${arr_cpass1_ESD_list[${runNumber}]} ${arr_cpass1_outputs[*]} && mkdir -p \$OUTPATH/meta && cp ${arr_cpass1_ESD_list[${runNumber}]} \$OUTPATH/meta/${arr_cpass1_ESD_list[${runNumber}]}"
1195 echo ; echo
1196
1197 #arr_cpass1_filtered_list[${runNumber}]="${commonOutputPath}/meta/cpass1.filtered.run${runNumber}.list"
1198 arr_cpass1_filtered_list[${runNumber}]="cpass1.filtered.run${runNumber}.list"
1199 echo "### Lists CPass1 filtered ###"
1200 echo "${arr_cpass1_filtered_list[${runNumber}]}: benchmark.sh ${arr_cpass1_outputs[*]}"
1201 echo " LOCAL ./benchmark.sh PrintValues filteredTree ${arr_cpass1_filtered_list[${runNumber}]} ${arr_cpass1_outputs[*]} && mkdir -p \$OUTPATH/meta && cp ${arr_cpass1_filtered_list[${runNumber}]} \$OUTPATH/meta/${arr_cpass1_filtered_list[${runNumber}]}"
1202 echo ; echo
444b9916 1203
71353a5f 1204 #CPass1 merging
ef77d296 1205 #arr_cpass1_merged[${runNumber}]="${commonOutputPath}/meta/merge.cpass1.run${runNumber}.done"
1206 arr_cpass1_merged[${runNumber}]="merge.cpass1.run${runNumber}.done"
1207 echo "### Merges CPass1 files ###"
1208 echo "cpass1.localOCDB.${runNumber}.tgz.done ${arr_cpass1_merged[${runNumber}]}: benchmark.sh ${configFile} ${arr_cpass1_calib_list[${runNumber}]} ${arr_cpass1_QA_list[${runNumber}]} ${copyFiles[@]}"
1209 echo " ${alirootEnv} ./benchmark.sh MergeCPass1 \$OUTPATH/000${runNumber}/cpass1 ${currentDefaultOCDB} ${configFile} ${runNumber} ${arr_cpass1_calib_list[${runNumber}]} ${arr_cpass1_QA_list[${runNumber}]} ${arr_cpass1_filtered_list[${runNumber}]} ${extraOpts[@]}"" "
1210 echo ; echo
fc7e1e45 1211
6fc48f9e 1212 #CPass0 wrapped in a profiling tool (valgrind,....)
39527a15 1213 if [[ -n ${profilingCommand} ]]; then
0eaf38c7 1214 inputFile=$(grep -m1 "${runNumber}/" ${inputFileList})
1215 [[ -z ${nEventsProfiling} ]] && nEventsProfiling=2
1216 currentDefaultOCDB=$(setYear ${inputFile} ${defaultOCDB})
1217 jobindex="profiling"
1218
ef77d296 1219 #arr_cpass0_profiled_outputs[${runNumber}]="${commonOutputPath}/meta/profiling.cpass0.job${jobindex}.run${runNumber}.done"
1220 arr_cpass0_profiled_outputs[${runNumber}]="profiling.cpass0.job${jobindex}.run${runNumber}.done"
1221 echo "### CPass0 in a profiler ###"
1222 echo "${arr_cpass0_profiled_outputs[${runNumber}]}: benchmark.sh ${configFile} ${copyFiles[@]}"
0eaf38c7 1223 profilingCommand=$(encSpaces "${profilingCommand}")
ef77d296 1224 echo " ${alirootEnv} ./benchmark.sh CPass0 \$OUTPATH/000${runNumber}/${jobindex} ${inputFile} ${nEventsProfiling} ${currentDefaultOCDB} ${configFile} ${runNumber} ${jobindex} ${extraOpts[@]} useProfilingCommand=${profilingCommand}"
1225 echo ; echo
6fc48f9e 1226 fi
1227
444b9916 1228 done #runs
fc7e1e45 1229
5dfac970 1230 #Summary
ef77d296 1231 echo "### Summary ###"
1232 echo "summary.log: benchmark.sh ${configFile} ${arr_cpass1_merged[*]}"
1233 echo " ${alirootEnv} ./benchmark.sh MakeSummary ${configFile} ${extraOpts[@]}"
1234 echo ; echo
cb473870 1235
5de3064b 1236 return 0
78d0bbfd 1237)
cb473870 1238
1239goPrintValues()
78d0bbfd 1240(
cb473870 1241 #print the values given the key from any number of files (store in output file on second argument)
71353a5f 1242 if [[ $# -lt 3 ]]; then
1243 echo "goPrintValues key outputFile inputFiles"
ef77d296 1244 echo "if outputFile is \"-\" don't redirect to a file"
71353a5f 1245 return
1246 fi
1247 key=${1}
1248 outputFile=${2}
1249 [[ ${outputFile} =~ "-" ]] && outputFile=""
cb473870 1250 shift 2 #remove 2 first arguments from arg list to only pass the input files to awk
71353a5f 1251 awk -v key=${key} '$0 ~ key" " {print $2}' "$@" | tee ${outputFile}
5de3064b 1252 return 0
78d0bbfd 1253)
fc7e1e45 1254
5dfac970 1255goCreateQAplots()
78d0bbfd 1256(
5dfac970 1257 umask 0002
71353a5f 1258 mergedQAfileList=${1}
1259 productionID=${2}
1260 outputDir=${3}
1261 configFile=${4}
1262 shift 4
e48b4924 1263 if ! parseConfig ${configFile} "$@"; then return 1; fi
71353a5f 1264
39527a15 1265 #record the working directory provided by the batch system
1266 batchWorkingDirectory=${PWD}
1267
71353a5f 1268 [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
fc7e1e45 1269
71353a5f 1270 [[ -z ${logOutputDir} ]] && logOutputDir=${PWD}
0eaf38c7 1271 [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/makeQAplots.log
71353a5f 1272 echo "${0} $*"
1273
1274 olddir=${PWD}
1275 mkdir -p ${outputDir}
1276 cd ${outputDir}
839b07ca 1277 [[ ! "${PWD}" =~ "${outputDir}" ]] && echo "PWD is not equal to outputDir=${outputDir}" && cd ${olddir} && return 1
fc7e1e45 1278
ef77d296 1279 inputFiles=(
1280 "${batchWorkingDirectory}/runQA.sh"
1281 "${ALICE_ROOT}/PWGPP/QA/scripts/runQA.sh"
1282 )
1283 for file in ${inputFiles[*]}; do
1284 [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
1285 done
1286
1287 echo "running QA with command:"
1288 echo ./runQA.sh inputList="${mergedQAfileList}" inputListHighPtTrees="${filteringList}" ocdbStorage="${defaultOCDB}"
1289 ./runQA.sh inputList="${mergedQAfileList}" inputListHighPtTrees="${filteringList}" ocdbStorage="${defaultOCDB}"
71353a5f 1290 cd ${olddir}
5de3064b 1291 return 0
78d0bbfd 1292)
fc7e1e45 1293
5dfac970 1294goTest()
78d0bbfd 1295(
1296 echo AA
1297)
fc7e1e45 1298
5dfac970 1299alirootInfo()
78d0bbfd 1300(
5dfac970 1301 umask 0002
71353a5f 1302 # save aliroot repository info
1303 [[ -z "${ALICE_ROOT}" ]] && return 1
1304
1305 echo "\${ALICE_ROOT}=${ALICE_ROOT}"
1306 echo "\${ROOTSYS}=${ROOTSYS}"
1307 echo "\${PATH}=${PATH}"
1308 echo "\${LD_LIBRARY_PATH}=${LD_LIBRARY_PATH}"
0eaf38c7 1309 echo
71353a5f 1310
1311 pushd ${PWD}
1312 cd ${ALICE_ROOT}
71353a5f 1313
1314 currentBranch=$(git rev-parse --abbrev-ref HEAD)
1315 git status
5dfac970 1316 echo ""
1317 echo ""
71353a5f 1318 git diff ${currentBranch}
1319 popd
5de3064b 1320 return 0
78d0bbfd 1321)
fc7e1e45 1322
5dfac970 1323setYear()
78d0bbfd 1324(
5dfac970 1325 #set the year
71353a5f 1326 # ${1} - year to be set
1327 # ${2} - where to set the year
1328 year1=$(guessYear ${1})
1329 year2=$(guessYear ${2})
1330 local path=${2}
1331 [[ ${year1} -ne ${year2} && -n ${year2} && -n ${year1} ]] && path=${2/\/${year2}\//\/${year1}\/}
1332 echo ${path}
5de3064b 1333 return 0
78d0bbfd 1334)
fc7e1e45 1335
5dfac970 1336guessPeriod()
78d0bbfd 1337(
5dfac970 1338 #guess the period from the path, pick the rightmost one
1339 local IFS="/"
71353a5f 1340 declare -a path=( ${1} )
5dfac970 1341 local dirDepth=${#path[*]}
1342 for ((x=${dirDepth}-1;x>=0;x--)); do
1343 local field=${path[${x}]}
71353a5f 1344 [[ ${field} =~ ^LHC[0-9][0-9][a-z]$ ]] && period=${field} && break
5dfac970 1345 done
71353a5f 1346 echo ${period}
5de3064b 1347 return 0
78d0bbfd 1348)
fc7e1e45 1349
5dfac970 1350guessYear()
78d0bbfd 1351(
5dfac970 1352 #guess the year from the path, pick the rightmost one
1353 local IFS="/"
71353a5f 1354 declare -a path=( ${1} )
5dfac970 1355 local dirDepth=${#path[*]}
1356 for ((x=${dirDepth}-1;x>=0;x--)); do
1357 local field=${path[${x}]}
71353a5f 1358 [[ ${field} =~ ^20[0-9][0-9]$ ]] && year=${field} && break
5dfac970 1359 done
71353a5f 1360 echo ${year}
5de3064b 1361 return 0
78d0bbfd 1362)
fc7e1e45 1363
5dfac970 1364guessRunNumber()
78d0bbfd 1365(
5dfac970 1366 #guess the run number from the path, pick the rightmost one
71353a5f 1367 #works for /path/foo/000123456/bar/...
1368 #and /path/foo.run123456.bar
1369 local IFS="/."
1370 declare -a path=( ${1} )
5dfac970 1371 local dirDepth=${#path[*]}
1372 for ((x=${dirDepth}-1;x>=0;x--)); do
1373 local field=${path[${x}]}
71353a5f 1374 field=${field/run/000}
617f8886 1375 [[ ${field} =~ [0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && runNumber=${field#000} && break
5dfac970 1376 done
71353a5f 1377 echo ${runNumber}
5de3064b 1378 return 0
78d0bbfd 1379)
fc7e1e45 1380
cb473870 1381validateLog()
78d0bbfd 1382(
71353a5f 1383 log=${1}
cb473870 1384 errorConditions=(
71353a5f 1385 'There was a crash'
1386 'floating'
1387 'error while loading shared libraries'
1388 'std::bad_alloc'
1389 's_err_syswatch_'
1390 'Thread [0-9]* (Thread'
1391 'AliFatal'
1392 'core dumped'
1393 '\.C.*error:.*\.h: No such file'
839b07ca 1394 'line.*Aborted'
71353a5f 1395 )
1396
1397 warningConditions=(
1398 'This is serious !'
1399 'rocVoltage out of range:'
cb473870 1400 )
1401
71353a5f 1402 local logstatus=0
1403 local errorSummary=""
1404 local warningSummary=""
1405
1406 for ((i=0; i<${#errorConditions[@]};i++)); do
1407 local tmp=$(grep -m1 -e "${errorConditions[${i}]}" ${log})
1408 [[ -n ${tmp} ]] && tmp+=" : "
1409 errorSummary+=${tmp}
1410 done
1411
1412 for ((i=0; i<${#warningConditions[@]};i++)); do
1413 local tmp=$(grep -m1 -e "${warningConditions[${i}]}" ${log})
1414 [[ -n ${tmp} ]] && tmp+=" : "
1415 warningSummary+=${tmp}
1416 done
1417
1418 if [[ -n ${errorSummary} ]]; then
1419 echo "${errorSummary}"
1420 return 1
1421 fi
cb473870 1422
71353a5f 1423 if [[ -n ${warningSummary} ]]; then
1424 echo "${warningSummary}"
1425 return 2
1426 fi
1427
1428 return 0
78d0bbfd 1429)
cb473870 1430
5dfac970 1431summarizeLogs()
78d0bbfd 1432(
5dfac970 1433 #print a summary of logs
1434 logFiles=(
1435 "*.log"
cb473870 1436 "stdout"
1437 "stderr"
5dfac970 1438 )
fc7e1e45 1439
9c124791 1440 #put dir information in the output
1441 echo "dir $PWD"
1442
cb473870 1443 #check logs
71353a5f 1444 local logstatus=0
5dfac970 1445 for log in ${logFiles[*]}; do
71353a5f 1446 finallog=${PWD%/}/${log}
1447 [[ ! -f ${log} ]] && continue
1448 errorSummary=$(validateLog ${log})
1449 validationStatus=$?
1450 if [[ ${validationStatus} -eq 0 ]]; then
5dfac970 1451 #in pretend mode randomly report an error in rec.log some cases
71353a5f 1452 if [[ -n ${pretend} && "${log}" == "rec.log" ]]; then
617f8886 1453 #[[ $(( ${RANDOM}%2 )) -ge 1 ]] && echo "${finallog} BAD random error" || echo "${finallog} OK"
1454 echo "${finallog} OK"
fc7e1e45 1455 else
71353a5f 1456 echo "${finallog} OK"
fc7e1e45 1457 fi
71353a5f 1458 elif [[ ${validationStatus} -eq 1 ]]; then
1459 echo "${finallog} BAD ${errorSummary}"
ab946185 1460 logstatus=1
71353a5f 1461 elif [[ ${validationStatus} -eq 2 ]]; then
1462 echo "${finallog} OK MWAH ${errorSummary}"
5dfac970 1463 fi
1464 done
cb473870 1465
1466 #report core files
71353a5f 1467 while read x; do
1468 echo ${x}
1469 chmod 644 ${x}
1470 gdb --batch --quiet -ex "bt" -ex "quit" aliroot ${x} > stacktrace_${x//\//_}.log
1471 done < <(/bin/ls ${PWD}/*/core 2>/dev/null; /bin/ls ${PWD}/core 2>/dev/null)
cb473870 1472
71353a5f 1473 return ${logstatus}
78d0bbfd 1474)
fc7e1e45 1475
5dfac970 1476spitOutLocalOCDBaccessConfig()
1477{
1478 umask 0002
71353a5f 1479 #find ${1} -name "*root" | \
1480 /bin/ls -1 ${1}/*/*/*/*.root 2>/dev/null | \
5dfac970 1481 while read line
1482 do
71353a5f 1483 local tmp=${line#${1}}
5dfac970 1484 echo ${tmp%/*} | \
71353a5f 1485 awk -v ocdb=${1} '{print " man->SetSpecificStorage(\""$1"\",\"local://"ocdb"\");"}'
5dfac970 1486 done
5de3064b 1487 return 0
5dfac970 1488}
fc7e1e45 1489
71353a5f 1490goMakeLocalOCDBaccessConfig()
5dfac970 1491{
1492 umask 0002
1493 # make a script that sets the specific storages form all the root files produced by CPass0
71353a5f 1494 local localOCDBpathCPass0=${1}
1495 local OCDBpathPrefix=${2}
1496 [[ -z ${OCDBpathPrefix} ]] && OCDBpathPrefix="."
1497
1498 if [[ -f ${localOCDBpathCPass0} && ${localOCDBpathCPass0} =~ \.tgz$ ]]; then
1499 tar xzf ${localOCDBpathCPass0}
1500 local localOCDBpathCPass0="${OCDBpathPrefix}/OCDB"
5dfac970 1501 fi
fc7e1e45 1502
5dfac970 1503 echo
1504 echo creating the specific storage script
1505 echo localOCDBaccessConfig.C
71353a5f 1506 echo based on OCDB: ${localOCDBaccessConfig}
5dfac970 1507 echo
fc7e1e45 1508
5dfac970 1509 local tempLocalOCDB=""
1510 if [[ -f localOCDBaccessConfig.C ]]; then
0eaf38c7 1511 tempLocalOCDB=$(mktemp -t tempLocalOCDB.XXXXXX)
71353a5f 1512 echo "egrep "SetSpecificStorage" localOCDBaccessConfig.C > ${tempLocalOCDB}"
1513 egrep "SetSpecificStorage" localOCDBaccessConfig.C > ${tempLocalOCDB}
5dfac970 1514 fi
fc7e1e45 1515
5dfac970 1516 echo "localOCDBaccessConfig()" > localOCDBaccessConfig.C
1517 echo "{" >> localOCDBaccessConfig.C
1518 echo " AliCDBManager* man = AliCDBManager::Instance();" >> localOCDBaccessConfig.C
71353a5f 1519 spitOutLocalOCDBaccessConfig ${localOCDBpathCPass0}|sort|uniq >> localOCDBaccessConfig.C
1520 [[ -f "${tempLocalOCDB}" ]] && cat ${tempLocalOCDB} >> localOCDBaccessConfig.C
5dfac970 1521 echo "}" >> localOCDBaccessConfig.C
fc7e1e45 1522
71353a5f 1523 [[ -f "${tempLocalOCDB}" ]] && rm -f ${tempLocalOCDB}
fc7e1e45 1524
5dfac970 1525 if ! grep SetSpecificStorage localOCDBaccessConfig.C; then
1526 echo
1527 echo "!!!!!!! CPass0 produced no OCDB entries"
1528 return 1
1529 fi
5de3064b 1530 return 0
5dfac970 1531}
fc7e1e45 1532
cb473870 1533goMakeFilteredTrees()
78d0bbfd 1534(
71353a5f 1535 outputDir=${1}
1536 runNumber=${2}
cb473870 1537 #get path to input list
71353a5f 1538 inputListfiles=${3}
cb473870 1539 #get scale number for tracks
71353a5f 1540 filterT=${4}
cb473870 1541 #get scale number for V0s
71353a5f 1542 filterV=${5}
cb473870 1543 #get OCDB path (optional)
1544 OCDBpath=${6}
1545 #get max number of files
1546 maxFiles=${7-"1000000"}
1547 #get offset of first file
1548 offsetFile=${8-"0"}
1549 #get max number of events
1550 maxEvents=${9-"30000000"}
1551 #get offset of first event
1552 offsetEvent=${10-"0"}
1553 configFile=${11-"benchmark.config"}
1554 esdFileName=${12-"AliESDs_Barrel.root"}
71353a5f 1555 shift 12
6fc48f9e 1556 if ! parseConfig ${configFile} "$@"; then return 1; fi
39527a15 1557
1558 #record the working directory provided by the batch system
1559 batchWorkingDirectory=${PWD}
1560
78d0bbfd 1561 [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
ef77d296 1562 doneFileBase=filtering.cpass1.run${runNumber}.done
1563 doneFileTmp=${batchWorkingDirectory}/${doneFileBase}
1564 doneFile=${commonOutputPath}/meta/${doneFileBase}
71353a5f 1565
cb473870 1566 cat > filtering.log << EOF
1567 goMakeFilteredTrees config:
71353a5f 1568 runpath=${runpath}
1569 outputDir=${outputDir}
1570 commonOutputPath=${commonOutputPath}
1571 ALICE_ROOT=${ALICE_ROOT}
1572 PATH=${PATH}
1573 offsetEvent=$offsetEvent
1574 configFile=$configFile
1575 esdFileName=$esdFileName
fab31805 1576 inputListfiles=$inputListfiles
1577 doneFile=$doneFile
cb473870 1578EOF
1579
fab31805 1580 #runpath=${outputDir}
1581 #[[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
1582 #[[ ${reconstructInTemporaryDir} -eq 1 ]] && runpath=$(mktemp -d -t goMakeFilteredTrees.XXXXXX)
1583 #mkdir -p ${outputDir}
1584 #mkdir -p ${runpath}
1585 #if ! cd ${runpath}; then
1586 # echo "PWD=$PWD is not the runpath=${runpath}"
1587 # touch ${doneFile}
1588 # return 1
1589 #fi
1590
cb473870 1591 if [[ -z ${pretend} ]];then
71353a5f 1592 aliroot -l -b -q "${ALICE_ROOT}/PWGPP/macros/runFilteringTask.C(\"${inputListfiles}\",${filterT},${filterV},\"${OCDBpath}\",${maxFiles},${offsetFile},${maxEvents},${offsetEvent},\"${esdFileName}\")" &>> filtering.log
cb473870 1593 else
0eaf38c7 1594 sleep ${pretendDelay}
71353a5f 1595 touch filtering.log FilterEvents_Trees.root
cb473870 1596 fi
1597 pwd
71353a5f 1598 /bin/ls
71353a5f 1599 summarizeLogs >> ${doneFile}
fab31805 1600
1601 #echo mv -f * ${outputDir}
1602 #mv -f * ${outputDir}
1603 #[[ -f ${outputDir}/FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFile}
1604 #cd ${commonOutputPath}
1605 #[[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
e344162c 1606
ef77d296 1607 cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
1608 [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
e344162c 1609
5de3064b 1610 return 0
78d0bbfd 1611)
cb473870 1612
1613submit()
1614{
1615 umask 0002
444b9916 1616 [[ $# -lt 5 ]] && echo "at least 5 args needed, you supplied $#" && return 1
71353a5f 1617 JobID=${1}
1618 startID=${2}
1619 endID=${3}
1620 waitForJOBID=${4}
1621 command=${5}
444b9916 1622 shift 5
78d0bbfd 1623 local commandArgs=("$@")
444b9916 1624
1625 #add quote strings around the extra arguments
1626 for ((i=0; i<${#commandArgs[@]}; i++)); do
1627 commandArgs[i]=\"${commandArgs[i]}\"
1628 done
cb473870 1629
71353a5f 1630 [[ -z ${waitForJOBID} ]] && waitForJOBID=0
cb473870 1631
1632 newFarm=$(which qsub|grep "^/usr/bin/qsub")
1633
1634 batchSystem="SGE"
1635
71353a5f 1636 if [[ -z "${newFarm}" ]]
cb473870 1637 then
1638 #old LSF
1639 # submit it (as job array)
71353a5f 1640 nFiles=$(( ${endID}-${startID}+1 ))
1641 while [ ${startID} -le ${nFiles} ] ; do
1642 if [ $(expr ${nFiles} - ${startID}) -gt 999 ] ; then
1643 endID=$(expr ${startID} + 999)
cb473870 1644 else
71353a5f 1645 endID=${nFiles}
cb473870 1646 fi
1647 if [[ ${waitForJOBID} -eq 0 ]]; then
71353a5f 1648 echo ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}"
1649 ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}"
cb473870 1650 else
71353a5f 1651 echo ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -w "ended(${waitForJOBID})" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}"
1652 ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -w "ended(${waitForJOBID})" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}"
cb473870 1653 fi
71353a5f 1654 startID=$(expr ${endID} + 1)
cb473870 1655 done
1656 else
1657 #new SGE farm
1658 if [[ ${waitForJOBID} =~ "000" ]]; then
444b9916 1659 echo ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}"
1660 ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}"
cb473870 1661 else
444b9916 1662 echo ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -hold_jid "${waitForJOBID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}"
1663 ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -hold_jid "${waitForJOBID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}"
cb473870 1664 fi
1665 fi
5de3064b 1666 return 0
cb473870 1667}
1668
1669goSubmitBatch()
5dfac970 1670{
71353a5f 1671 if [[ $# -lt 3 ]]; then
1672 echo "minimal use:"
1673 echo " ${0} submit fileList productionID configFile"
1674 return 0
1675 fi
1676
1677 productionID=${1}
1678 inputList=${2}
1679 configFile=${3:-"benchmark.config"}
0eaf38c7 1680 #if which greadlink; then configFile=$(greadlink -f ${configFile}); fi
71353a5f 1681 shift 3
444b9916 1682 extraOpts=("$@")
6fc48f9e 1683 if ! parseConfig ${configFile} "${extraOpts[@]}"; then return 1; fi
39527a15 1684
b0e401b4 1685 #batch systems/makeflow sometimes handle spaces in arguments poorly, so encode them
1686 for (( i=0;i<${#extraOpts[@]};i++ )); do
1687 extraOpts[i]=$(encSpaces "${extraOpts[i]}")
1688 done
1689 extraOpts+=("encodedSpaces=1")
ef77d296 1690 #this removes the copy of the done file used by makeflow (in the running dir)
1691 extraOpts+=("removeTMPdoneFile=1")
b0e401b4 1692
39527a15 1693 #record the working directory provided by the batch system
1694 batchWorkingDirectory=${PWD}
fc7e1e45 1695
5dfac970 1696 #redirect all output to submit.log
1697 echo "redirecting all output to ${PWD}/submit_${productionID//"/"/_}.log"
1698 exec 7>&1
1699 exec 1>submit_${productionID//"/"/_}.log 2>&1
fc7e1e45 1700
5dfac970 1701 umask 0002
71353a5f 1702 echo ${0}" submit $*"
1703 if [[ -z "${inputList}" || -z "${productionID}" ]]
5dfac970 1704 then
fc7e1e45 1705 echo
71353a5f 1706 echo " Usage: ${0} submit inputList productionID [configFile=benchmark.config]"
fc7e1e45 1707 echo
71353a5f 1708 return
fc7e1e45 1709 fi
fc7e1e45 1710
5dfac970 1711 # check if config file is there
71353a5f 1712 if [ ! -f ${configFile} ]; then
1713 echo "ERROR! Config File '${configFile}' not found" >&2
1714 return
5dfac970 1715 else
71353a5f 1716 echo "Using Config File: '${configFile}'"
fc7e1e45 1717 fi
1718
9510565c 1719 [[ ! -f ${alirootEnv} ]] && echo "alirootEnv script ${alirootEnv} not found!..." && return 1
fc7e1e45 1720
71353a5f 1721 #move the script, config and some other stuff to ${commonOutputPath} first, then use them from there
0eaf38c7 1722 self=${0}
1723 #if which greadlink; then self=$(greadlink -f "${0}"); fi
71353a5f 1724 configPath=$(dirname ${configFile})
5dfac970 1725 export commonOutputPath=${baseOutputDirectory}/${productionID}
71353a5f 1726
cb473870 1727 mkdir -p ${commonOutputPath}
71353a5f 1728 mkdir -p ${commonOutputPath}/logs
1729 mkdir -p ${commonOutputPath}/meta
1730
1731 cp ${self} ${commonOutputPath}
1732 cp ${configFile} ${commonOutputPath}
1733 cp ${inputList} ${commonOutputPath}
cb473870 1734 self=${commonOutputPath}/${self##*/}
71353a5f 1735 chmod u+x ${self}
cb473870 1736 configFile=${commonOutputPath}/${configFile##*/}
1737 inputList=${commonOutputPath}/${inputList##*/}
fc7e1e45 1738
5dfac970 1739 #convert to absolut pathnames
0eaf38c7 1740 #if which greadlink; then inputList=$(greadlink -f "${inputList}"); fi
5dfac970 1741 #make list of runs
71353a5f 1742 if [[ -z ${runNumber} ]]; then
1743 listOfRuns=($(while read x; do guessRunNumber ${x}; done < ${inputList} | sort | uniq))
5dfac970 1744 else
71353a5f 1745 listOfRuns=${runNumber}
5dfac970 1746 fi
fc7e1e45 1747
0eaf38c7 1748 #if which greadlink; then alirootSource=$(greadlink -f "${alirootSource}"); fi
fc7e1e45 1749
5dfac970 1750 echo ""
1751 echo "### BEGIN CONFIGURATION ###"
1752 echo ""
1753 echo "GENERAL:"
1754 echo ""
71353a5f 1755 echo " productionID: ${productionID}"
1756 echo " batchCommand: ${batchCommand}"
1757 echo " batchFlags: ${batchFlags}"
9510565c 1758 echo " alirootEnv: ${alirootEnv}"
1759 ${alirootEnv} echo ' ALICE_ROOT: ${ALICE_ROOT}'
1760 ${alirootEnv} echo ' ALIROOT_RELEASE: ${ALICE_RELEASE}'
71353a5f 1761 echo " inputList: ${inputList}"
1762 echo " configPath: ${configPath}"
1763 echo " commonOutputPath: ${commonOutputPath}"
1764 echo " defaultOCDB: ${defaultOCDB}"
1765 echo " autoOCDB: ${autoOCDB}"
1766 echo " recoTriggerOptions: ${recoTriggerOptions}"
5dfac970 1767 echo " runs:"
1768 echo " ${listOfRuns[*]}"
1769 echo ""
1770 echo "THE TRAIN WILL RUN:"
fc7e1e45 1771
71353a5f 1772 if [ ${runCPass0reco} -eq 1 ]; then
5dfac970 1773 echo " Pass0 - Recontruction"
1774 fi
fc7e1e45 1775
71353a5f 1776 if [ ${runCPass0MergeMakeOCDB} -eq 1 ]; then
5dfac970 1777 echo " Pass0 - merging and OCDB export"
1778 fi
fc7e1e45 1779
71353a5f 1780 if [ ${runCPass1reco} -eq 1 ]; then
5dfac970 1781 echo " Pass1 - Recontruction"
1782 fi
71353a5f 1783 if [ ${runCPass1MergeMakeOCDB} -eq 1 ]; then
5dfac970 1784 echo " Pass1 - merging and OCDB export"
1785 fi
fc7e1e45 1786
5dfac970 1787 echo ""
1788 echo "LIMITS:"
71353a5f 1789 echo " max. Events/Chunk: ${nEvents}"
1790 echo " max. Number of Chunks per Run: ${nMaxChunks}"
5dfac970 1791 echo ""
1792 echo "### END CONFIGURATION ###"
1793 echo ""
fc7e1e45 1794
36b5de1a 1795
5dfac970 1796 # check if input file is there
71353a5f 1797 if [ ! -f ${inputList} ]; then
1798 echo "ERROR! Input List '${inputList}' not found" >&2
1799 return
5dfac970 1800 fi
1801
1802 # define jobid (for dependent jobs)
cb473870 1803 date=$(date +%Y_%m_%d_%H%M%S)
5dfac970 1804 #for each run we submit one jobarray:
fc7e1e45 1805 for runNumber in ${listOfRuns[*]}; do
cb473870 1806
71353a5f 1807 [[ -z ${runNumber} ]] && continue
fc7e1e45 1808 [[ ! ${runNumber} =~ ^[0-9]*[0-9]$ ]] && continue
cb473870 1809
1810 JOBpostfix="${productionID//"/"/_}_${runNumber}_${date}"
1811 JOBID1="p0_${JOBpostfix}"
1812 JOBID1wait="w0_${JOBpostfix}"
1813 JOBID2="m0_${JOBpostfix}"
1814 JOBID2wait="wm0_${JOBpostfix}"
1815 JOBID3="op0_${JOBpostfix}"
1816 JOBID3wait="wop0_${JOBpostfix}"
1817 JOBID4="p1_${JOBpostfix}"
1818 JOBID4wait="w1_${JOBpostfix}"
1819 JOBID5="m1_${JOBpostfix}"
1820 JOBID5wait="wm1_${JOBpostfix}"
1821 JOBID6="s1_${JOBpostfix}"
1822 JOBID6wait="ws1_${JOBpostfix}"
1823 JOBID7="QA_${JOBpostfix}"
1824 JOBmakeESDlistCPass1="lp1_${JOBpostfix}"
1825 JOBfilterESDcpass1="fp1_${JOBpostfix}"
1826 LASTJOB="000"
1827
0eaf38c7 1828 oneInputFile=$(egrep -m1 "${runNumber}/" ${inputList})
cb473870 1829
1830 currentDefaultOCDB=${defaultOCDB}
1831 [[ -z ${autoOCDB} ]] && autoOCDB=1
1832 if [[ ${autoOCDB} -ne 0 ]]; then
71353a5f 1833 currentDefaultOCDB=$(setYear ${oneInputFile} ${defaultOCDB})
cb473870 1834 fi
ef77d296 1835 period=$(guessPeriod ${oneInputFile})
1836 year=$(guessYear ${oneInputFile})
cb473870 1837
71353a5f 1838 echo "submitting run ${runNumber} with OCDB ${currentDefaultOCDB}"
1839
1840 ###############################################################################
1841 #run one chunk with valgrind:
6fc48f9e 1842 if [[ -n ${profilingCommand} ]]; then
1843 [[ -z ${nEventsProfiling} ]] && nEventsProfiling=2
1844 [[ -z ${profilingCommand} ]] && profilingCommand="/usr/bin/valgrind --tool=callgrind --num-callers=40 -v --trace-children=yes"
ef77d296 1845 submit "profile-${JOBpostfix}" 1 1 000 "${alirootEnv} ${self}" CPass0 ${commonOutputPath}/${year}/${period}/000${runNumber}/${jobindex} ${oneInputFile} ${nEventsProfiling} ${currentDefaultOCDB} ${configFile} ${runNumber} ${jobindex} useProfilingCommand=$(encSpaces "${profilingCommand}") "${extraOpts[@]}"
71353a5f 1846 fi
fc7e1e45 1847
5dfac970 1848 ################################################################################
1849 ################################################################################
1850 # run the CPass0 if requested
fc7e1e45 1851
71353a5f 1852 if [ ${runCPass0reco} -eq 1 ]; then
fc7e1e45 1853
fc7e1e45 1854 echo
71353a5f 1855 echo "starting CPass0... for run ${runNumber}"
fc7e1e45 1856 echo
fc7e1e45 1857
5dfac970 1858 # create directory and copy all files that are needed
ef77d296 1859 targetDirectory="${commonOutputPath}/${year}/${period}/000${runNumber}/cpass0"
71353a5f 1860 mkdir -p ${targetDirectory}
1861
cb473870 1862 filesCPass0=(
71353a5f 1863 "${configPath}/runCPass0.sh"
1864 "${configPath}/recCPass0.C"
1865 "${configPath}/runCalibTrain.C"
1866 "${configPath}/localOCDBaccessConfig.C"
1867 "${configPath}/OCDB*.root"
1868 "${configPath}/sim.C"
1869 "${configPath}/rec.C"
1870 "${configPath}/Config.C"
cb473870 1871 )
1872 for file in ${filesCPass0[*]}; do
1873 [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
1874 done
fc7e1e45 1875
71353a5f 1876 localInputList=${targetDirectory}/${inputList##*/}
1877 [[ ! -f ${localInputList} ]] && egrep "\/000${runNumber}\/" ${inputList} > ${localInputList}
5dfac970 1878 # limit nFiles to nMaxChunks
71353a5f 1879 nFiles=$(wc -l < ${localInputList})
1880 [[ ${nFiles} -eq 0 ]] && echo "list contains ZERO files! exiting..." && return 1
1881 echo "raw files in list: ${nFiles}"
1882 if [[ ${nMaxChunks} -gt 0 && ${nMaxChunks} -le ${nFiles} ]]; then
1883 nFiles=${nMaxChunks}
5dfac970 1884 fi
71353a5f 1885 echo "raw files to process: ${nFiles}"
1886 [[ -z "${percentProcessedFilesToContinue}" ]] && percentProcessedFilesToContinue=100
1887 if [[ ${percentProcessedFilesToContinue} -eq 100 ]]; then
1888 nFilesToWaitFor=${nFiles}
5dfac970 1889 else
71353a5f 1890 nFilesToWaitFor=$(( ${nFiles}-${nFiles}/(100/(100-${percentProcessedFilesToContinue})) ))
5dfac970 1891 fi
71353a5f 1892 echo "requested success rate is ${percentProcessedFilesToContinue}%"
1893 echo "merging will start after ${nFilesToWaitFor} jobs are done"
fc7e1e45 1894
444b9916 1895 submit ${JOBID1} 1 ${nFiles} 000 "${alirootEnv} ${self}" CPass0 ${targetDirectory} ${localInputList} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} -1 "${extraOpts[@]}"
fc7e1e45 1896
5dfac970 1897 ## submit a monitoring job that will run until a certain number of jobs are done with reconstruction
0ff60de9 1898 submit "${JOBID1wait}" 1 1 000 "${alirootEnv} ${self}" WaitForOutput ${commonOutputPath} "meta/cpass0.job*.run${runNumber}.done" ${nFilesToWaitFor} ${maxSecondsToWait}
71353a5f 1899 LASTJOB=${JOBID1wait}
fc7e1e45 1900
5dfac970 1901 fi #end running CPass0
1902 ################################################################################
fc7e1e45 1903
fc7e1e45 1904
5dfac970 1905 ################################################################################
1906 # submit merging of CPass0, depends on the reconstruction
fc7e1e45 1907
71353a5f 1908 if [ ${runCPass0MergeMakeOCDB} -eq 1 ]; then
fc7e1e45 1909
5dfac970 1910 echo
71353a5f 1911 echo "submit CPass0 merging for run ${runNumber}"
5dfac970 1912 echo
fc7e1e45 1913
ef77d296 1914 targetDirectory="${commonOutputPath}/${year}/${period}/000${runNumber}/cpass0"
71353a5f 1915 mkdir -p ${targetDirectory}
fc7e1e45 1916
cb473870 1917 #copy the scripts
1918 filesMergeCPass0=(
71353a5f 1919 "${configPath}/OCDB.root"
1920 "${configPath}/mergeMakeOCDB.byComponent.sh"
1921 "${configPath}/mergeMakeOCDB.sh"
1922 "${configPath}/localOCDBaccessConfig.C"
1923 "${configPath}/mergeByComponent.C"
1924 "${configPath}/makeOCDB.C"
1925 "${configPath}/merge.C"
cb473870 1926 )
1927 for file in ${filesMergeCPass0[*]}; do
1928 [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
1929 done
1930
444b9916 1931 submit ${JOBID2} 1 1 "${LASTJOB}" "${alirootEnv} ${self}" MergeCPass0 ${targetDirectory} ${currentDefaultOCDB} ${configFile} ${runNumber} cpass0.calib.run${runNumber}.list "${extraOpts[@]}"
71353a5f 1932 LASTJOB=${JOBID2}
fc7e1e45 1933
71353a5f 1934 if [[ -n ${generateMC} ]]; then
444b9916 1935 submit "mrl${JOBpostfix}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" PrintValues sim ${commonOutputPath}/meta/sim.run${runNumber}.list ${commonOutputPath}/meta/cpass0.job*.run${runNumber}.done
71353a5f 1936 LASTJOB="mrl${JOBpostfix}"
1937 fi
fc7e1e45 1938
5dfac970 1939 echo
fc7e1e45 1940 fi
5dfac970 1941 # end of merging CPass0
1942 ################################################################################
fc7e1e45 1943
5dfac970 1944 ################################################################################
1945 ################################################################################
1946 # run the CPass1 if requested
fc7e1e45 1947
71353a5f 1948 if [ ${runCPass1reco} -eq 1 ]; then
fc7e1e45 1949
ef77d296 1950 targetDirectory="${commonOutputPath}/${year}/${period}/000${runNumber}/cpass1"
71353a5f 1951 rm -f ${commonOutputPath}/meta/cpass1.job*.run${runNumber}.done
fc7e1e45 1952
5dfac970 1953 # safety feature: if we are re-running for any reason we want to delete the previous output first.
71353a5f 1954 [[ -d ${targetDirectory} ]] && rm -rf ${targetDirectory}/* && echo "removed old output at ${targetDirectory}/*"
fc7e1e45 1955
5dfac970 1956 echo
71353a5f 1957 echo "starting CPass1... for run ${runNumber}"
5dfac970 1958 echo
1959
1960 # create directory and copy all files that are needed
71353a5f 1961 mkdir -p ${targetDirectory}
5dfac970 1962
cb473870 1963 filesCPass1=(
71353a5f 1964 "${configPath}/runCPass1.sh"
1965 "${configPath}/recCPass1.C"
1966 "${configPath}/recCPass1_OuterDet.C"
1967 "${configPath}/runCalibTrain.C"
1968 "${configPath}/QAtrain_duo.C"
1969 "${configPath}/localOCDBaccessConfig.C"
1970 "${configPath}/OCDB.root"
cb473870 1971 )
1972 for file in ${filesCPass1[*]}; do
1973 [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
1974 done
5dfac970 1975
71353a5f 1976 if [[ -n ${generateMC} ]]; then
1977 localInputList=${commonOutputPath}/meta/sim.run${runNumber}.list
1978 else
1979 localInputList=${targetDirectory}/${inputList##*/}
1980 [[ ! -f ${localInputList} ]] && egrep "\/000${runNumber}\/" ${inputList} > ${localInputList}
1981 fi
5dfac970 1982 # limit nFiles to nMaxChunks
71353a5f 1983 nFiles=$(wc -l < ${localInputList})
1984 [[ ${nFiles} -eq 0 ]] && echo "list contains ZERO files! continuing..." && continue
1985 echo "raw files in list: ${nFiles}"
1986 if [[ ${nMaxChunks} -gt 0 && ${nMaxChunks} -le ${nFiles} ]]; then
1987 nFiles=${nMaxChunks}
5dfac970 1988 fi
71353a5f 1989 echo "raw files to process: ${nFiles}"
1990 [[ -z "${percentProcessedFilesToContinue}" ]] && percentProcessedFilesToContinue=100
1991 if [[ ${percentProcessedFilesToContinue} -eq 100 ]]; then
1992 nFilesToWaitFor=${nFiles}
fc7e1e45 1993 else
71353a5f 1994 nFilesToWaitFor=$(( ${nFiles}-${nFiles}/(100/(100-${percentProcessedFilesToContinue})) ))
fc7e1e45 1995 fi
71353a5f 1996 echo "requested success rate is ${percentProcessedFilesToContinue}%"
1997 echo "merging will start after ${nFilesToWaitFor} jobs are done"
fc7e1e45 1998
444b9916 1999 submit ${JOBID4} 1 ${nFiles} "${LASTJOB}" "${alirootEnv} ${self}" CPass1 ${targetDirectory} ${localInputList} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} -1 "${extraOpts[@]}"
fc7e1e45 2000
5dfac970 2001 ################################################################################
2002 ## submit a monitoring job that will run until a certain number of jobs are done with reconstruction
0ff60de9 2003 submit "${JOBID4wait}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" WaitForOutput ${commonOutputPath} "meta/cpass1.job*.run${runNumber}.done" ${nFilesToWaitFor} ${maxSecondsToWait}
71353a5f 2004 LASTJOB=${JOBID4wait}
5dfac970 2005 ################################################################################
fc7e1e45 2006
5dfac970 2007 echo
2008 fi #end running CPass1
fc7e1e45 2009
5dfac970 2010 ################################################################################
2011 # submit merging of CPass1, depends on the reconstruction
71353a5f 2012 if [ ${runCPass1MergeMakeOCDB} -eq 1 ]; then
fc7e1e45 2013
5dfac970 2014 echo
71353a5f 2015 echo "submit CPass1 merging for run ${runNumber}"
5dfac970 2016 echo
fc7e1e45 2017
ef77d296 2018 targetDirectory="${commonOutputPath}/${year}/${period}/000${runNumber}/cpass1"
71353a5f 2019 rm -f ${commonOutputPath}/meta/merge.cpass1.run${runNumber}.done
2020 mkdir -p ${targetDirectory}
fc7e1e45 2021
cb473870 2022 # copy files
2023 filesMergeCPass1=(
71353a5f 2024 "${configPath}/OCDB.root"
2025 "${configPath}/localOCDBaccessConfig.C"
2026 "${configPath}/mergeMakeOCDB.byComponent.sh"
2027 "${configPath}/mergeByComponent.C"
2028 "${configPath}/makeOCDB.C"
2029 "${configPath}/merge.C"
2030 "${configPath}/mergeMakeOCDB.sh"
2031 "${configPath}/QAtrain_duo.C"
cb473870 2032 )
2033 for file in ${filesMergeCPass1[*]}; do
2034 [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
2035 done
2036
444b9916 2037 submit "${JOBID5}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" MergeCPass1 ${targetDirectory} ${currentDefaultOCDB} ${configFile} ${runNumber} cpass1.calib.run${runNumber}.list cpass1.QA.run${runNumber}.lastMergingStage.txt.list cpass1.filtered.run${runNumber}.list "${extraOpts[@]}"
71353a5f 2038 LASTJOB=${JOBID5}
5dfac970 2039 echo
2040 fi
2041
444b9916 2042 ###############################
2043 #if [ ${runESDfiltering} -eq 1 ]; then
2044 # rm -f ${commonOutputPath}/cpass1.ESD.run${runNumber}.list
2045 # rm -f ${commonOutputPath}/meta/filtering.cpass1.run*.done
2046 # echo
2047 # echo submitting filtering for run ${runNumber}
2048 # echo
2049 # submit "${JOBmakeESDlistCPass1}" 1 1 "${LASTJOB}" "${self}" PrintValues esd ${commonOutputPath}/meta/cpass1.ESD.run${runNumber}.list ${commonOutputPath}/meta/cpass1.job*.run${runNumber}.done
ef77d296 2050 # submit "${JOBfilterESDcpass1}" 1 1 "${JOBmakeESDlistCPass1}" "${alirootEnv} ${self}" MakeFilteredTrees ${commonOutputPath}/${year}/${period}/000${runNumber}/cpass1 ${runNumber} ${commonOutputPath}/meta/cpass1.ESD.run${runNumber}.list ${filteringFactorHighPt} ${filteringFactorV0s} ${currentDefaultOCDB} 1000000 0 10000000 0 ${configFile} AliESDs_Barrel.root "${extraOpts[@]}"
444b9916 2051 # LASTJOB=${JOBfilterESDcpass1}
2052 #fi
cb473870 2053
5dfac970 2054 done
2055
71353a5f 2056 #################################################################################
2057 #################################################################################
444b9916 2058 #if [ ${runESDfiltering} -eq 1 ]; then
2059 # submit "${JOBID5wait}" 1 1 "${LASTJOB}" "${self}" WaitForOutput ${commonOutputPath} "meta/filtering.cpass1.run*.done" "${#listOfRuns[@]}" ${maxSecondsToWait}
2060 #else
2061 submit "${JOBID5wait}" 1 1 "${LASTJOB}" "${self}" WaitForOutput ${commonOutputPath} "meta/merge.cpass1.run*.done" ${#listOfRuns[@]} ${maxSecondsToWait}
2062 #fi
71353a5f 2063 LASTJOB=${JOBID5wait}
cb473870 2064
5de3064b 2065 #################################################################################
2066 echo
2067 echo "submit make a summary"
2068 echo
5dfac970 2069
3d530ff9 2070 [[ -z ${alirootEnvQA} ]] && alirootEnvQA=$(encSpaces "${alirootEnv}")
4f8ecc60 2071 submit "${JOBID6}" 1 1 "${LASTJOB}" "${alirootEnvQA} ${self}" MakeSummary ${configFile} "commonOutputPath=${commonOutputPath}"
5de3064b 2072 LASTJOB=${JOBID6}
71353a5f 2073 #################################################################################
5dfac970 2074
5dfac970 2075 #restore stdout
2076 exec 1>&7 7>&-
2077 echo "jobs submitted."
5de3064b 2078 return 0
fc7e1e45 2079}
2080
cb473870 2081goWaitForOutput()
78d0bbfd 2082(
cb473870 2083 umask 0002
71353a5f 2084 [[ $# -lt 3 ]] && echo "goWaitForOutput() wrong number of arguments, exiting.." && return 1
2085 echo searchPath=${1}
2086 echo fileName=${2}
2087 echo numberOfFiles=${3}
2088 echo maxSecondsToWait=${4}
2089 searchPath=${1}
2090 fileName=${2}
2091 numberOfFiles=${3}
2092 maxSecondsToWait=${4}
0ff60de9 2093 echo "command to be executed: /bin/ls -1 ${searchPath}/${fileName}"
71353a5f 2094 [[ -z "${maxSecondsToWait}" ]] && maxSecondsToWait=$(( 3600*12 ))
cb473870 2095 while true; do
cb473870 2096 n=$(/bin/ls -1 ${searchPath}/${fileName} 2>/dev/null | wc -l)
71353a5f 2097 [[ ${n} -gt 0 ]] && echo "found ${n} X ${fileName}"
2098 [[ ${n} -ge ${numberOfFiles} ]] && break
2099 [[ ${SECONDS} -gt ${maxSecondsToWait} ]] && echo "timeout of ${maxSecondsToWait}s!" && break
cb473870 2100 sleep 60
2101 done
2102 echo "DONE! exiting..."
5de3064b 2103 return 0
78d0bbfd 2104)
cb473870 2105
71353a5f 2106mergeSysLogs()
78d0bbfd 2107(
71353a5f 2108 outputFile=${1}
2109 shift
2110 inputFiles="$@"
2111 i=0
2112 if ! ls -1 ${inputFiles} &>/dev/null; then echo "the files dont exist!: ${inputFiles}"; return 1; fi
2113 while read x; do
2114 runNumber=$(guessRunNumber ${x})
2115 [[ -z ${runNumber} ]] && echo "run number cannot be guessed for ${x}" && continue
2116 awk -v run=${runNumber} -v i=${i} 'NR > 1 {print run" "$0} NR==1 && i==0 {print "run/I:"$0}' ${x}
2117 (( i++ ))
2118 done < <(ls -1 ${inputFiles}) > ${outputFile}
5de3064b 2119 return 0
78d0bbfd 2120)
71353a5f 2121
2122goMakeMergedSummaryTree()
78d0bbfd 2123(
71353a5f 2124 # create list of calibration entries
2125 # takes no arguments, just run it in the base output
2126 # directory with the following files in the working directory
2127 #
2128 # Calibration file lists:
2129 # cpass0.dcsTree.list, cpass1.dcsTree.list
2130 # QA trending root file:
2131 # trending.root
2132 #
2133 # Production infoo ascii files:
2134 # summary_pass0.tree
2135 # summary_pass1.tree
2136 #
617f8886 2137
71353a5f 2138 [[ ! -f cpass0.dcsTree.list ]] && echo "no cpass0.dcsTree.list" && return 1
2139 [[ ! -f cpass1.dcsTree.list ]] && echo "no cpass1.dcsTree.list" && return 1
2140 [[ ! -f trending.root ]] && echo "no trending.root" && return 1
2141 [[ ! -f summary_pass0.tree ]] && echo "no summary_pass0.tree" && return 1
2142 [[ ! -f summary_pass1.tree ]] && echo "no summary_pass1.tree" && return 1
2143
2144 #first, dump the C macro to file
2145 cat << EOF > mergeTree.C
2146 //
2147 // Merge summary information
2148 // Following files are expected to be in the working directory
2149 //
2150 // Calibration file lists:
2151 // cpass0.dcsTree.list, cpass1.dcsTree.list
2152 // QA trending root file:
2153 // trending.root
2154 //
2155 // Production infoo ascii files:
2156 // summary_pass0.tree
2157 // summary_pass1.tree
2158 //
2159 void mergeTree(){
2160 //
2161 //
2162 //
2163 // Calibration values dump
2164 //
839b07ca 2165 //Printf("MakeTreeFromList cpass0.dcsTree.list");
71353a5f 2166 AliXRDPROOFtoolkit::MakeTreeFromList("Calib.TPC.CPass0.root", "dcs","dcs","cpass0.dcsTree.list",1);
839b07ca 2167 //Printf("MakeTreeFromList cpass1.dcsTree.list");
71353a5f 2168 AliXRDPROOFtoolkit::MakeTreeFromList("Calib.TPC.CPass1.root", "dcs","dcs","cpass1.dcsTree.list",1);
2169 //
2170 // Calibration status dump
2171 //
2172 TFile *fprod = TFile::Open("fproduction.root","recreate");
2173 TTree tree0, tree1;
839b07ca 2174 //Printf("reading summary_pass0.tree");
71353a5f 2175 tree0.ReadFile("summary_pass0.tree");
839b07ca 2176 //Printf("reading summary_pass1.tree");
71353a5f 2177 tree1.ReadFile("summary_pass1.tree");
2178 tree0.Write("CPass0");
2179 tree1.Write("CPass1");
2180 fprod->Close();
2181 //
2182 //
2183 //
2184 TString stringSetup="";
2185 stringSetup+="1#QA.TPC#run#SummaryTPCQA/tpcQA#trending.root+"; //
2186 stringSetup+="1#Calib.TPC.CPass0#run#dcs#Calib.TPC.CPass0.root+"; //
2187 stringSetup+="1#Calib.TPC.CPass1#run#dcs#Calib.TPC.CPass1.root+"; //
2188 //
2189 stringSetup+="1#CPass0#runnumber#CPass0#fproduction.root+"; //
2190 stringSetup+="1#CPass1#runnumber#CPass1#fproduction.root+"; //
2191 //
839b07ca 2192 //Printf("stringSetup: %s", stringSetup.Data());
71353a5f 2193 AliXRDPROOFtoolkit::JoinTreesIndex("outAll.root","joinAll","run",stringSetup.Data(), 1);
2194 }
2195EOF
2196
2197 aliroot -b -q "mergeTree.C" > mergeTrees.log
2198 return $?
78d0bbfd 2199)
71353a5f 2200
177b9ebc 2201stackTraceTree()
2202(
e00eb304 2203 if [[ $# -lt 1 ]]; then
2204 echo 'make stacktrace processing in case of standard root crash log'
2205 echo 'input is a (list of) text files with the stack trace (either gdb aoutput'
2206 echo 'produced with e.g. gdb --batch --quiet -ex "bt" -ex "quit" aliroot core,'
2207 echo 'or the root crash log), output is a TTree formatted table.'
2208 echo 'example usage:'
2209 echo 'benchmark.sh stackTraceTree /foo/*/rec.log'
2210 echo 'benchmark.sh stackTraceTree $(cat file.list)'
0685f982 2211 echo 'benchmark.sh stackTraceTree `cat file.list`'
e00eb304 2212 return 0
2213 fi
4a3c2422 2214 gawk '
177b9ebc 2215 BEGIN {
80c75839 2216 print "frame/I:method/C:line/C:cpass/I:aliroot/I:file/C";
177b9ebc 2217 RS="#[0-9]*";
2218 aliroot=0;
4a3c2422 2219 read=1;
177b9ebc 2220 }
4a3c2422 2221 /There was a crash/ {read=1;}
2222 /The lines below might hint at the cause of the crash/ {read=0;}
2223 read==1 {
177b9ebc 2224 if ($3 ~ /Ali*/) aliroot=1; else aliroot=0;
2225 gsub("#","",RT);
80c75839 2226 if ($NF!="" && RT!="" && $3!="") print RT" "$3" "$NF" "0" "aliroot" "FILENAME
177b9ebc 2227 }
2228 ' "$@" 2>/dev/null
2229)
2230
71353a5f 2231goMakeSummary()
78d0bbfd 2232(
71353a5f 2233 #all the final stuff goes in here for ease of use:
2234 # summary logs
2235 # qa plot making
2236 # final file lists
ef77d296 2237 # runs in current dir - in makeflow mode it can run LOCAL, then the QA plots and summaries
2238 # will appear in the submission dir.
444b9916 2239 #some defaults:
2240 log="summary.log"
2241 productionID="qa"
71353a5f 2242
2243 configFile=${1}
2244 shift 1
444b9916 2245 extraOpts=("$@")
6fc48f9e 2246 if ! parseConfig ${configFile} "${extraOpts[@]}"; then return 1; fi
71353a5f 2247
0eaf38c7 2248 #if which greadlink; then configFile=$(greadlink -f ${configFile}); fi
839b07ca 2249
39527a15 2250 #record the working directory provided by the batch system
2251 batchWorkingDirectory=${PWD}
ef77d296 2252
2253 logTmp=${batchWorkingDirectory}/${log}
2254 logDest=${commonOutputPath}/${log}
39527a15 2255
71353a5f 2256 [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
2257
71353a5f 2258 [[ ! -f ${configFile} ]] && echo "no config file ${configFile}!" && return
2259
2260 [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
2261
39527a15 2262 #copy some useful stuff
2263 #and go to the commonOutputPath
2264 cp ${configFile} ${commonOutputPath}
39527a15 2265
ef77d296 2266 exec &> >(tee ${logTmp})
39527a15 2267
71353a5f 2268 #summarize the global stuff
2269 echo "env script: ${alirootSource} ${alirootEnv}"
2270 echo "\$ALICE_ROOT=${ALICE_ROOT}"
2271 echo "commonOutputPath=${commonOutputPath}"
2272
2273 #summarize the stacktraces
ef77d296 2274 stackTraceTree ${commonOutputPath}/*/*/000*/cpass0/*/stacktrace* > stacktrace_cpass0.tree
2275 stackTraceTree ${commonOutputPath}/*/*/000*/cpass1/*/stacktrace* > stacktrace_cpass1.tree
71353a5f 2276
2277 echo total numbers for the production:
2278 echo
2279 awk 'BEGIN {nFiles=0;nCore=0;}
2280 /^calibfile/ {nFiles++;}
2281 /core dumped/ {nCore++i;}
ef77d296 2282 END {print "cpass0 produced "nFiles" calib files, "nCore" core files";}' ${commonOutputPath}/meta/cpass0.job*done 2>/dev/null
71353a5f 2283 awk 'BEGIN {nOK=0; nBAD=0; }
2284 /\/rec.log OK/ {nOK++;}
2285 /\/rec.log BAD/ {nBAD++;}
2286 /stderr BAD/ {if ($0 ~ /rec.log/){nBAD++;}}
ef77d296 2287 END {print "cpass0 reco: OK: "nOK"\tBAD: "nBAD;}' ${commonOutputPath}/meta/cpass0.job*done 2>/dev/null
71353a5f 2288 awk 'BEGIN {nOK=0; nBAD=0; }
2289 /\/calib.log OK/ {nOK++;}
2290 /\/calib.log BAD/ {nBAD++;}
ef77d296 2291 END {print "cpass0 calib: OK: "nOK"\tBAD: "nBAD;}' ${commonOutputPath}/meta/cpass0.job*done 2>/dev/null
71353a5f 2292
2293 awk 'BEGIN {nOK=0; nBAD=0; }
2294 /merge.log OK/ {nOK++;}
2295 /merge.log BAD/ {nBAD++;}
ef77d296 2296 END {print "cpass0 merge: OK: "nOK"\tBAD: "nBAD;}' ${commonOutputPath}/meta/merge.cpass0*done 2>/dev/null
71353a5f 2297 awk 'BEGIN {nOK=0; nBAD=0; }
2298 /ocdb.log OK/ {nOK++;}
2299 /ocdb.log BAD/ {nBAD++;}
ef77d296 2300 END {print "cpass0 OCDB: OK: "nOK"\tBAD: "nBAD;}' ${commonOutputPath}/meta/merge.cpass0*done 2>/dev/null
71353a5f 2301
2302 echo
2303 awk 'BEGIN {nFiles=0;nCore=0;}
2304 /^calibfile/ {nFiles++;}
2305 /core dumped/ {nCore++;}
ef77d296 2306 END {print "cpass1 produced "nFiles" calib files, "nCore" core files";}' ${commonOutputPath}/meta/cpass1.job*done 2>/dev/null
71353a5f 2307 awk 'BEGIN {nOK=0; nBAD=0; }
2308 /\/rec.log OK/ {nOK++;}
2309 /\/rec.log BAD/ {nBAD++;}
2310 /stderr BAD/ {if ($0 ~ /rec.log/){nBAD++;}}
ef77d296 2311 END {print "cpass1 reco: OK: "nOK"\tBAD: "nBAD;}' ${commonOutputPath}/meta/cpass1.job*done 2>/dev/null
71353a5f 2312 awk 'BEGIN {nOK=0; nBAD=0; }
2313 /\/calib.log OK/ {nOK++;}
2314 /\/calib.log BAD/ {nBAD++;}
ef77d296 2315 END {print "cpass1 calib: OK: "nOK"\tBAD: "nBAD;}' ${commonOutputPath}/meta/cpass1.job*done 2>/dev/null
71353a5f 2316
2317 awk 'BEGIN {nOK=0; nBAD=0; }
2318 /merge.log OK/ {nOK++;}
2319 /merge.log BAD/ {nBAD++;}
ef77d296 2320 END {print "cpass1 merge: OK: "nOK"\tBAD: "nBAD;}' ${commonOutputPath}/meta/merge.cpass1*done 2>/dev/null
71353a5f 2321 awk 'BEGIN {nOK=0; nBAD=0; }
2322 /ocdb.log OK/ {nOK++;}
2323 /ocdb.log BAD/ {nBAD++;}
ef77d296 2324 END {print "cpass1 OCDB: OK: "nOK"\tBAD: "nBAD;}' ${commonOutputPath}/meta/merge.cpass1*done 2>/dev/null
71353a5f 2325
2326 echo
2327 echo per run stats:
ef77d296 2328 /bin/ls -1 ${commonOutputPath}/meta/merge.cpass0.run*.done | while read x
71353a5f 2329do
5a2ee6ff 2330 dir=$(goPrintValues dir - ${x})
71353a5f 2331 runNumber=$(guessRunNumber ${dir})
2332 [[ -z ${runNumber} ]] && continue
2333
ef77d296 2334 if $(/bin/ls ${commonOutputPath}/meta/cpass0.job*.run${runNumber}.done &> /dev/null); then
71353a5f 2335 statusCPass0=( $(
2336 awk 'BEGIN {nOKrec=0;nBADrec=0;nOKcalib=0;nBADcalib=0;nOKstderr=0;nBADstderr=0;}
2337 /\/rec.log OK/ {nOKrec++;}
2338 /\/rec.log BAD/ {nBADrec++;}
2339 /stderr BAD/ {if ($0 ~ /rec.log/) {nBADrec++;} nBADstderr++;}
2340 /stderr OK/ {nOKstderr++;}
2341 /\/calib.log OK/ {nOKcalib++;}
2342 /\/calib.log BAD/ {nBADcalib++}
ef77d296 2343 END {print ""nOKrec" "nBADrec" "nOKstderr" "nBADstderr" "nOKcalib" "nBADcalib;}' ${commonOutputPath}/meta/cpass0.job*.run${runNumber}.done 2>/dev/null
71353a5f 2344 ) )
2345 fi
2346
ef77d296 2347 if $(/bin/ls ${commonOutputPath}/meta/cpass1.job*.run${runNumber}.done &>/dev/null); then
71353a5f 2348 statusCPass1=( $(
2349 awk 'BEGIN {nOKrec=0;nBADrec=0;nOKcalib=0;nBADcalib=0;nOKstderr=0;nBADstderr=0;nQAbarrelOK=0;nQAbarrelBAD=0;nQAouterOK=0;nQAouterBAD=0;}
2350 /\/rec.log OK/ {nOKrec++;}
2351 /\/rec.log BAD/ {nBADrec++;}
2352 /stderr BAD/ {if ($0 ~ /rec.log/) nBADrec++;nBADstderr++;}
2353 /stderr OK/ {nOKstderr++;}
2354 /\/calib.log OK/ {nOKcalib++;}
2355 /\/calib.log BAD/ {nBADcalib++}
2356 /\/qa_barrel.log OK/ {nQAbarrelOK++;}
2357 /\/qa_barrel.log BAD/ {nQAbarrelBAD++;}
2358 /\/qa_outer.log OK/ {nQAouterOK++;}
2359 /\/qa_outer.log BAD/ {nQAouterBAD++;}
ef77d296 2360 END {print ""nOKrec" "nBADrec" "nOKstderr" "nBADstderr" "nOKcalib" "nBADcalib" "nQAbarrelOK" "nQAbarrelBAD" "nQAouterOK" "nQAouterBAD;}' ${commonOutputPath}/meta/cpass1.job*.run${runNumber}.done 2>/dev/null
71353a5f 2361 ) )
2362 fi
2363
2364 statusOCDBcpass0=$(awk '/ocdb.log/ {print $2} ' ${x} 2>/dev/null)
2365 statusOCDBcpass1=$(awk '/ocdb.log/ {print $2}' ${x/cpass0/cpass1} 2>/dev/null)
2366 statusQA=$(awk '/mergeMakeOCDB.log/ {print $2}' ${x/cpass0/cpass1} 2>/dev/null)
2367
2368 printf "%s\t ocdb.log cpass0: %s\t ocdb.log cpass1: %s\tqa.log:%s\t| cpass0: rec:%s/%s stderr:%s/%s calib:%s/%s cpass1: rec:%s/%s stderr:%s/%s calib:%s/%s QAbarrel:%s/%s QAouter:%s/%s\n" ${runNumber} ${statusOCDBcpass0} ${statusOCDBcpass1} ${statusQA} ${statusCPass0[0]} ${statusCPass0[1]} ${statusCPass0[2]} ${statusCPass0[3]} ${statusCPass0[4]} ${statusCPass0[5]} ${statusCPass1[0]} ${statusCPass1[1]} ${statusCPass1[2]} ${statusCPass1[3]} ${statusCPass1[4]} ${statusCPass1[5]} ${statusCPass1[6]} ${statusCPass1[7]} ${statusCPass1[8]} ${statusCPass1[9]}
2369done
2370
2371 #make lists with output files - QA, trending, filtering and calibration
2372 ### wait for the merging of all runs to be over ###
2373 rm -f qa.list
2374 goPrintValues qafile qa.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2375 rm -f calib.list
2376 goPrintValues calibfile calib.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2377 rm -f trending.list
2378 goPrintValues trendingfile trending.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2379 rm -f filtering.list
444b9916 2380 goPrintValues filteredTree filtering.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
71353a5f 2381 rm -f cpass0.dcsTree.list
2382 goPrintValues dcsTree cpass0.dcsTree.list ${commonOutputPath}/meta/merge.cpass0.run*.done &>/dev/null
71353a5f 2383 rm -f cpass1.dcsTree.list
2384 goPrintValues dcsTree cpass1.dcsTree.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2385
2386 #merge trending
fecdbe75 2387 rm -f trending.root
2388 goMerge trending.list trending.root ${configFile} "${extraOpts[@]}" &> mergeTrending.log
71353a5f 2389
2390 goMakeSummaryTree ${commonOutputPath} 0
2391 goMakeSummaryTree ${commonOutputPath} 1
2392
839b07ca 2393 goCreateQAplots "${PWD}/qa.list" "${productionID}" "QAplots" "${configFile}" "${extraOpts[@]}" filteringList="${PWD}/filtering.list" &>createQAplots.log
71353a5f 2394
2395 #make a merged summary tree out of the QA trending, dcs trees and log summary trees
2396 goMakeMergedSummaryTree
2397
2398 #if set, email the summary
ef77d296 2399 [[ -n ${MAILTO} ]] && cat ${logTmp} | mail -s "benchmark ${productionID} done" ${MAILTO}
2400
2401 # Copy log to destination (delete all on failure to signal error)
2402 cp "$logTmp" "$logDest" || rm -f "$logTmp" "$logDest"
fecdbe75 2403
2404 #copy output files
054fdec5 2405 exec &> >(tee fileCopy.log)
13abb339 2406 paranoidCp QAplots ${commonOutputPath}
2407 paranoidCp *.list ${commonOutputPath}
2408 paranoidCp *.root ${commonOutputPath}
2409 paranoidCp *.log ${commonOutputPath}
054fdec5 2410 paranoidCp fileCopy.log ${commonOutputPath}
71353a5f 2411
2412 return 0
78d0bbfd 2413)
71353a5f 2414
2415goMakeSummaryTree()
78d0bbfd 2416(
71353a5f 2417 if [[ $# -lt 1 ]] ; then
2418 return
2419 fi
2420 #1. define vars/arrays
2421 DIR=${1} #use input or exec in current dir
2422 pass=${2-"0"} #pass from input
2423 outfile="summary_pass${pass}.tree"
2424 Ncolumns=0
2425 test -f ${outfile} && : >${outfile}
2426 errfile=${outfile/tree/err}
2427 test -f ${errfile} && : >${errfile}
2428
2429 declare -a counterString=(TOFevents TOFtracks TPCevents TPCtracks TRDevents TRDtracks T0events SDDevents SDDtracks MeanVertexevents)
2430 Ncounter=${#counterString[@]}
2431
2432 declare -a statusString=(TRDStatus TOFStatus TPCStatus T0Status MeanVertexStatus)
2433 Nstatus=${#statusString[@]}
2434
2435
2436 declare -a ratesString=(rec stderr calib qa_barrel qa_outer)
2437 Nrates=${#ratesString[@]}
2438
2439 runs=( $(ls -1 ${DIR}/meta/merge.cpass0* | while read x; do guessRunNumber $x; done) )
2440 Nruns=${#runs[@]}
2441
2442 echo -n runnumber/I >>${outfile}
2443 echo -n :cpass${pass}status/I >>${outfile}
2444 echo -n :cpass${pass}QAstatus/I >>${outfile}
2445 for i in ${ratesString[@]}; do
2446 echo -n :${i}OK/I >>${outfile}
2447 echo -n :${i}BAD/I >>${outfile}
2448
2449 done
2450 for i in ${counterString[@]} ${statusString[@]} ; do
2451 echo -n :${i}/I >>${outfile}
2452 done
2453 Ncolumns=$((2 + 2*Nrates + Ncounter + Nstatus))
2454 echo >> ${outfile}
2455
2456 #2. loop runs
2457
2458 for runnumber in ${runs[@]} ; do
2459
2460
2461
2462 filejob="${DIR}/meta/cpass${pass}.job*.run${runnumber}.done"
2463 filemerge="${DIR}/meta/merge.cpass${pass}.run${runnumber}.done"
2464 fileOCDB=$(grep /ocdb.log ${filemerge} | awk '{print $1}')
2465 if ! $(/bin/ls ${filemerge} &>/dev/null) ; then
2466 echo "${filemerge} does not exist!" >>${errfile}
2467 continue
2468 elif ! $(/bin/ls ${filejob} &>/dev/null) ; then
2469 echo "${filejob} does not exist!" >>${errfile}
2470 echo -n ${runnumber} >> ${outfile}
2471 for i in $(seq ${Ncolumns}) ; do
2472 echo -n "-1" >> ${outfile}
2473 done
2474 echo >> ${outfile}
2475 continue
2476 fi
2477 echo -n ${runnumber} >> ${outfile}
2478 #pass0status= grep '/ocdb.log' ${filemerge} | cut -d' ' -f2 | tr OK x1 | tr BAD xx0 | tr -d 'x'
2479 passStatus=$(grep '/ocdb.log' ${filemerge} | grep OK | wc -l)
2480 echo -n " ${passStatus}" >> ${outfile}
2481 qaStatus=$(grep '/mergeMakeOCDB.log' ${filemerge} | grep OK | wc -l)
2482 echo -n " ${qaStatus}" >> ${outfile}
2483
2484
2485 #fill OK/BAD rates
2486 for i in $(seq 0 $((${Nrates}-1))) ; do
2487 var1=$(grep "/${ratesString[${i}]}.log" ${filejob} | grep OK | wc -l)
2488 var2=$(grep "/${ratesString[${i}]}.log" ${filejob} | grep BAD | wc -l)
2489
2490 if [[ ${ratesString[${i}]} == "stderr" ]] ; then
2491 var1=$(grep "stderr" ${filejob} | grep OK | wc -l)
2492 var2=$(grep "stderr" ${filejob} | grep "rec.log" | grep BAD | wc -l)
2493 fi
2494 echo -n " ${var1}" >> ${outfile}
2495 echo -n " ${var2}" >> ${outfile}
2496 done
2497
2498 if [[ -f ${fileOCDB} ]] ; then
2499 #fill counter
2500 for i in $(seq 0 $((${Ncounter}-1))) ; do
2501 var1=$(grep Monalisa ${fileOCDB} | grep ${counterString[${i}]} | cut -f2)
2502 echo -n " ${var1:-"-1"}" >> ${outfile}
2503 done
2504
2505 #fill status
2506 for i in $(seq 0 $((${Nstatus}-1))) ; do
2507 var1=$(grep "calibration status=" ${fileOCDB} | grep ${statusString[${i}]/Status/} | cut -d'=' -f2)
2508 echo -n " ${var1:-"-1"}" >> ${outfile}
2509 done
2510 fi
2511 echo >> ${outfile}
2512 done
2513
5de3064b 2514 return 0
78d0bbfd 2515)
71353a5f 2516
2517parseConfig()
2518{
2519 configFile=${1}
2520 shift
444b9916 2521 args=("$@")
2522
0eaf38c7 2523
444b9916 2524 #some defaults
6fc48f9e 2525 #autoOCDB=0
2526 defaultOCDB="raw://"
2527 #runNumber=167123
2528 #makeflowPath="/hera/alice/aux/cctools/bin"
2529 #makeflowOptions="-T wq -N alice -d all -C ali-copilot.cern.ch:9097"
2530 #makeflowOptions="-T wq -N alice -C ali-copilot.cern.ch:9097"
2531 makeflowOptions=""
39527a15 2532 #batchCommand="/usr/bin/qsub"
ef77d296 2533 batchFlags=""
6fc48f9e 2534 baseOutputDirectory="$PWD/output"
2535 #alirootEnv="/cvmfs/alice.cern.ch/bin/alienv setenv AliRoot/v5-04-34-AN -c"
2536 #alirootEnv="/home/mkrzewic/alisoft/balice_master.sh"
2537 #trustedQAtrainMacro='/hera/alice/mkrzewic/gsisvn/Calibration/QAtrain_duo.C'
2538 reconstructInTemporaryDir=0
2539 recoTriggerOptions="\"\""
2540 percentProcessedFilesToContinue=100
2541 maxSecondsToWait=$(( 3600*24 ))
2542 nEvents=-1
2543 nMaxChunks=0
2544 postSetUpActionCPass0=""
2545 postSetUpActionCPass1=""
2546 runCPass0reco=1
2547 runCPass0MergeMakeOCDB=1
2548 runCPass1reco=1
2549 runCPass1MergeMakeOCDB=1
2550 runESDfiltering=1
444b9916 2551 filteringFactorHighPt=1e2
2552 filteringFactorV0s=1e1
6fc48f9e 2553 MAILTO=""
2554 #pretend=1
2555 #dontRedirectStdOutToLog=1
2556 logToFinalDestination=1
2557 ALIROOT_FORCE_COREDUMP=1
0eaf38c7 2558 pretendDelay=0
71353a5f 2559
2560 #first, source the config file
2561 if [ -f ${configFile} ]; then
2562 source ${configFile}
2563 else
0eaf38c7 2564 echo "config file ${configFile} not found!"
2565 return 1
71353a5f 2566 fi
2567
839b07ca 2568 unset encodedSpaces
2569 for opt in "${args[@]}"; do
0eaf38c7 2570 [[ "${opt}" =~ encodedSpaces=.* ]] && encodedSpaces=1 && echo "encodedSpaces!" && break
839b07ca 2571 done
2572
d56eeaab 2573 #then, parse the options as they override the options from file
2574 for opt in "${args[@]}"; do
0eaf38c7 2575 [[ -z ${opt} ]] && continue
2576 [[ -n ${encodedSpaces} ]] && opt="$(decSpaces ${opt})"
2577 [[ "${opt}" =~ ^[[:space:]]*$ ]] && continue
d56eeaab 2578 if [[ ! "${opt}" =~ .*=.* ]]; then
0eaf38c7 2579 echo "badly formatted option \"${opt}\" should be: option=value, stopping..."
d56eeaab 2580 return 1
2581 fi
2582 local var="${opt%%=*}"
2583 local value="${opt#*=}"
9d0c204f 2584 echo "${var}=${value}"
444b9916 2585 export ${var}="${value}"
71353a5f 2586 done
2587
6fc48f9e 2588 #do some checking
2589 [[ -z ${alirootEnv} ]] && echo "alirootEnv not defined!" && return 1
2590
71353a5f 2591 #export the aliroot function if defined to override normal behaviour
fecdbe75 2592 [[ $(type -t aliroot) =~ "function" ]] && export -f aliroot && echo "exporting aliroot() function..."
6fc48f9e 2593
5de3064b 2594 return 0
71353a5f 2595}
2596
2597aliroot()
2598{
e48b4924 2599 args=("$@")
6fc48f9e 2600 if [[ -n ${useProfilingCommand} ]]; then
e48b4924 2601 profilerLogFile="cpu.txt"
1a0fcd62 2602 [[ "${args[@]}" =~ rec ]] && profilerLogFile="cpu_rec.txt"
2603 [[ "${args[@]}" =~ Calib ]] && profilerLogFile="cpu_calib.txt"
2f2d99b8 2604 echo running "${useProfilingCommand} aliroot ${args[@]} &> ${profilerLogFile}"
e48b4924 2605 ${useProfilingCommand} aliroot "${args[@]}" &> ${profilerLogFile}
71353a5f 2606 else
2607 #to prevent an infinite recursion use "command aliroot" to disable
2608 #aliases and functions
e48b4924 2609 echo running command aliroot "${args[@]}"
2610 command aliroot "${args[@]}"
71353a5f 2611 fi
5de3064b 2612 return 0
71353a5f 2613}
2614
13abb339 2615paranoidCp()
2616(
2617 #recursively copy files and directories
2618 #to avoid using find and the like as they kill
2619 #the performance on some cluster file systems
2620 #does not copy links to avoid problems
2621 sourceFiles=("${@}")
2622 destination="${sourceFiles[@]:(-1)}" #last element
2623 unset sourceFiles[${#sourceFiles[@]}-1] #remove last element (dst)
2624 for src in "${sourceFiles[@]}"; do
2625 if [[ -f "${src}" && ! -h "${src}" ]]; then
2626 paranoidCopyFile "${src}" "${destination}"
2627 elif [[ -d "${src}" && ! -h "${src}" ]]; then
2628 src="${src%/}"
2629 dst="${destination}/${src##*/}"
2630 mkdir -p "${dst}"
2631 paranoidCp "${src}"/* "${dst}"
2632 fi
2633 done
2634)
2635
2636paranoidCopyFile()
2637(
2638 #copy a single file to a target in an existing dir
2639 #repeat a few times if copy fails
2640 src="${1}"
2641 dst="${2}"
2642 [[ -d "${dst}" ]] && dst="${dst}/${src##*/}"
2643 [[ -z "${maxCopyTries}" ]] && maxCopyTries=5
054fdec5 2644 #echo "maxCopyTries=${maxCopyTries}"
13abb339 2645 echo "cp ${src} ${dst}"
2646 cp "${src}" "${dst}"
2647 i=0
2648 until cmp -s "${src}" "${dst}"; do
2649 echo "try: ${i}"
2650 [[ -f "${dst}" ]] && rm "${dst}"
2651 cp "${src}" "${dst}"
2652 [[ ${i} -gt ${maxCopyTries} ]] && ret=1 && return 1
2653 (( i++ ))
2654 done
2655 return 0
2656)
2657
71353a5f 2658guessRunData()
2659{
2660 #guess the period from the path, pick the rightmost one
2661 period=""
2662 runNumber=""
2663 year=""
2664 pass=""
2665 legoTrainRunNumber=""
2666 dataType=""
2667
2668 local shortRunNumber=""
2669 local IFS="/"
2670 declare -a path=( $1 )
2671 local dirDepth=$(( ${#path[*]}-1 ))
2672 i=0
2673 #for ((x=${dirDepth};x>=0;x--)); do
2674 for ((x=0;x<=${dirDepth};x++)); do
2675
2676 [[ $((x-1)) -ge 0 ]] && local fieldPrev=${path[$((x-1))]}
2677 local field=${path[${x}]}
2678 local fieldNext=${path[$((x+1))]}
2679
2680 [[ ${field} =~ ^[0-9]*$ && ${fieldNext} =~ (.*\.zip$|.*\.root$) ]] && legoTrainRunNumber=${field}
2681 [[ -n ${legoTrainRunNumber} && -z ${pass} ]] && pass=${fieldPrev}
2682 [[ ${field} =~ ^LHC[0-9][0-9][a-z].*$ ]] && period=${field%_*}
2683 [[ ${field} =~ ^000[0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && runNumber=${field#000}
2684 [[ ${field} =~ ^[0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && shortRunNumber=${field}
2685 [[ ${field} =~ ^20[0-9][0-9]$ ]] && year=${field}
2686 [[ ${field} =~ ^(^sim$|^data$) ]] && dataType=${field}
2687 (( i++ ))
2688 done
2689 [[ -z ${legoTrainRunNumber} ]] && pass=${path[$((dirDepth-1))]}
2690 [[ "${dataType}" =~ ^sim$ ]] && pass="passMC" && runNumber=${shortRunNumber}
2691
2692 #if [[ -z ${dataType} || -z ${year} || -z ${period} || -z ${runNumber}} || -z ${pass} ]];
2693 if [[ -z ${runNumber}} ]];
2694 then
2695 #error condition
2696 return 1
2697 else
2698 #ALL OK
2699 return 0
2700 fi
5de3064b 2701 return 0
71353a5f 2702}
2703
839b07ca 2704#these functions encode strings to and from a space-less form
2705#use when spaces are not well handled (e.g. in arguments to
2706#commands in makeflow files, etc.
0eaf38c7 2707encSpaces()(echo "${1// /@@@@}")
2708decSpaces()(echo "${1//@@@@/ }")
839b07ca 2709
fc7e1e45 2710main "$@"