]>
Commit | Line | Data |
---|---|---|
fc7e1e45 | 1 | #!/bin/bash |
71353a5f | 2 | #include benchmark.config |
3 | ||
b802abe4 | 4 | # blame: Mikolaj Krzewicki, mkrzewic@cern.ch |
fc7e1e45 | 5 | # this script runs the CPass0/CPass1 train |
6 | # produced OCDB updates are local | |
7 | ||
8 | main() | |
9 | { | |
10 | #run in proper mode depending on the selection | |
71353a5f | 11 | if [[ $# -lt 1 ]]; then |
12 | if [[ ! "${0}" =~ "bash" ]]; then | |
13 | echo "uses makeflow:" | |
14 | echo " ${0} \"run\" productionID inputList configFile [extraOpts]" | |
15 | echo "uses a batch system (SGE):" | |
16 | echo " ${0} \"submit\" productionID inputList configFile [extraOpts]" | |
17 | echo "extraOpts if non-empty override the config file, e.g.:" | |
9510565c | 18 | echo " ${0} submit test1 benchmark.list benchmark.config runNumber=169123 nEvents=10" |
71353a5f | 19 | fi |
20 | return | |
21 | fi | |
22 | ||
23 | #define some aliases - default is to call one of the functions directly | |
24 | runMode=${1} | |
fc7e1e45 | 25 | umask 0002 |
26 | shift | |
71353a5f | 27 | case ${runMode} in |
fc7e1e45 | 28 | "CPass0") goCPass0 "$@";; |
29 | "CPass1") goCPass1 "$@";; | |
71353a5f | 30 | "MakeLocalOCDBaccessConfig") goMakeLocalOCDBaccessConfig "$@";; |
fc7e1e45 | 31 | "MergeCPass0") goMergeCPass0 "$@";; |
32 | "MergeCPass1") goMergeCPass1 "$@";; | |
cb473870 | 33 | "MakeFilteredTrees") goMakeFilteredTrees "$@";; |
71353a5f | 34 | "MakeSummary") goMakeSummary "$@";; |
cb473870 | 35 | "run") goSubmitMakeflow "$@";; |
36 | "submit") goSubmitBatch "$@";; | |
fc7e1e45 | 37 | "test") goTest "$@";; |
71353a5f | 38 | "GenerateMakeflow") goGenerateMakeflow "$@";; |
39 | "PrintValues") goPrintValues "$@";; | |
cb473870 | 40 | "CreateQAplots") goCreateQAplots "$@";; |
41 | "WaitForOutput") goWaitForOutput "$@";; | |
71353a5f | 42 | "Merge") goMerge "$@";; |
43 | *) | |
444b9916 | 44 | ${runMode} "$@" |
fc7e1e45 | 45 | ;; |
46 | esac | |
3ce223dd | 47 | return 0 |
fc7e1e45 | 48 | } |
49 | ||
71353a5f | 50 | generateMC() |
51 | { | |
52 | #generate one raw chunk in current directory | |
53 | SEED=${JOB_ID}${SGE_TASK_ID} | |
54 | export CONFIG_SEED=${SEED} | |
55 | runNumber=${1} | |
56 | OCDBpath=${2} | |
57 | nEventsim=${3} | |
58 | if [[ -n ${pretend} ]]; then | |
0eaf38c7 | 59 | sleep ${pretendDelay} |
71353a5f | 60 | touch galice.root |
61 | else | |
62 | if [[ -f sim.C && -f Config.C ]] ; then | |
63 | time aliroot -b -q -x sim.C\(${runNumber},\"${OCDBpath}\",${nEventsim}\) >sim.log 2>&1 | |
64 | mv syswatch.log simwatch.log | |
65 | fi | |
66 | fi | |
67 | } | |
68 | ||
fc7e1e45 | 69 | goCPass0() |
78d0bbfd | 70 | ( |
fc7e1e45 | 71 | umask 0002 |
72 | ||
71353a5f | 73 | targetDirectory=${1} |
74 | inputList=${2} | |
75 | nEvents=${3} | |
76 | ocdbPath=${4} | |
77 | configFile=${5} | |
78 | runNumber=${6} | |
79 | jobindex=${7} | |
80 | shift 7 | |
6fc48f9e | 81 | if ! parseConfig ${configFile} "$@"; then return 1; fi |
71353a5f | 82 | |
39527a15 | 83 | #record the working directory provided by the batch system |
84 | batchWorkingDirectory=${PWD} | |
85 | ||
71353a5f | 86 | #use the jobindex only if set and non-negative |
87 | if [[ -z ${jobindex} || ${jobindex} -lt 0 ]]; then | |
88 | [[ -n "${LSB_JOBINDEX}" ]] && jobindex=${LSB_JOBINDEX} | |
89 | [[ -n "${SGE_TASK_ID}" ]] && jobindex=${SGE_TASK_ID} | |
999a843d | 90 | if [[ -z ${jobindex} ]]; then |
91 | echo "no jobindex!" | |
92 | return 1 | |
93 | fi | |
71353a5f | 94 | fi |
95 | ||
96 | [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD} | |
ef77d296 | 97 | |
98 | # This file signals that/if everything went fine | |
99 | doneFileBase="cpass0.job${jobindex}.run${runNumber}.done" | |
100 | [[ -n ${useProfilingCommand} ]] && doneFileBase="profiling.cpass0.job${jobindex}.run${runNumber}.done" | |
101 | ||
102 | # We will have two copies of the file | |
103 | mkdir -p "${commonOutputPath}/meta" || return 1 | |
104 | doneFileTmp="${batchWorkingDirectory}/${doneFileBase}" | |
105 | doneFile="${commonOutputPath}/meta/${doneFileBase}" | |
71353a5f | 106 | |
107 | [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource} | |
108 | ||
109 | if [[ -n ${ALIROOT_FORCE_COREDUMP} ]]; then | |
110 | ulimit -c unlimited | |
111 | export ALIROOT_FORCE_COREDUMP | |
112 | fi | |
fc7e1e45 | 113 | |
cb473870 | 114 | #the contents of this is stored in the tree and used later (e.g. AliAnalysisTaskPIDResponse)! |
115 | #at the QA stage the pass number is guessed from the path stored here. | |
116 | #The Format is: | |
117 | #Packages= ;OutputDir= ;LPMPass= ;TriggerAlias= ;LPMRunNumber= ;LPMProductionType= ;LPMInteractionType= ;LPMProductionTag= ;LPMAnchorRun= ;LPMAnchorProduction= ;LPMAnchorYear= | |
118 | export PRODUCTION_METADATA="OutputDir=cpass0" | |
119 | ||
5de3064b | 120 | if [[ "${inputList}" =~ \.root$ ]]; then |
121 | infile=${inputList} | |
122 | else | |
123 | infile=$(sed -ne "${jobindex}p" ${inputList} | egrep '\s*\w*/\w*') | |
124 | fi | |
71353a5f | 125 | chunkName=${infile##*/} |
71353a5f | 126 | |
127 | outputDir=${targetDirectory}/${jobindex}_${chunkName%.*} | |
128 | mkdir -p ${outputDir} | |
1795556a | 129 | if [[ ! -d ${outputDir} ]]; then |
ef77d296 | 130 | touch ${doneFileTmp} |
131 | echo "cannot make ${outputDir}" >> ${doneFileTmp} | |
132 | cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile" | |
133 | [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp} | |
1795556a | 134 | return 1 |
135 | fi | |
71353a5f | 136 | |
71353a5f | 137 | runpath=${outputDir} |
0eaf38c7 | 138 | [[ ${reconstructInTemporaryDir} -eq 1 ]] && runpath=$(mktemp -d -t cpass0.XXXXXX) |
e91a9c78 | 139 | [[ ${reconstructInTemporaryDir} -eq 2 ]] && runpath=${PWD}/rundir_cpass0_${runNumber}_${jobindex} |
71353a5f | 140 | mkdir -p ${runpath} |
1795556a | 141 | if [[ ! -d ${runpath} ]]; then |
ef77d296 | 142 | touch ${doneFileTmp} |
143 | echo "cannot make runpath ${runpath}" >> ${doneFileTmp} | |
144 | cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile" | |
145 | [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp} | |
1795556a | 146 | return 1 |
147 | fi | |
0eaf38c7 | 148 | if ! cd ${runpath}; then |
ef77d296 | 149 | touch ${doneFileTmp} |
150 | echo "PWD=$PWD is not the runpath=${runpath}" >> ${doneFileTmp} | |
151 | cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile" | |
152 | [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp} | |
0eaf38c7 | 153 | return 1 |
154 | fi | |
fc7e1e45 | 155 | |
5de3064b | 156 | #runCPassX/C expects the raw chunk to be linked in the run dir |
157 | #despite it being accessed by the full path | |
158 | ln -s ${infile} ${runpath}/${chunkName} | |
fc7e1e45 | 159 | |
71353a5f | 160 | #####MC |
161 | if [[ -n ${generateMC} ]]; then | |
162 | olddir=${PWD} | |
163 | outputDirMC=${commonOutputPath}/000${runNumber}/sim/${jobindex} | |
164 | simrunpath=${outputDirMC} | |
0eaf38c7 | 165 | #[[ ${simulateInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && simrunpath=${TMPDIR} |
166 | [[ ${simulateInTemporaryDir} -eq 1 ]] && simrunpath=$(mktemp -d -t cpass0MC.XXXXXX) | |
71353a5f | 167 | mkdir -p ${outputDirMC} |
168 | mkdir -p ${simrunpath} | |
0eaf38c7 | 169 | if cd ${simrunpath}; then |
fc7e1e45 | 170 | |
0eaf38c7 | 171 | filesMC=( |
172 | "${batchWorkingDirectory}/sim.C" | |
173 | "${batchWorkingDirectory}/rec.C" | |
174 | "${batchWorkingDirectory}/Config.C" | |
175 | "${batchWorkingDirectory}/OCDB_*.root" | |
176 | ) | |
177 | for file in ${filesMC[*]}; do | |
178 | [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} . | |
179 | done | |
fc7e1e45 | 180 | |
0eaf38c7 | 181 | generateMC ${runNumber} ${ocdbPath} ${nEvents} |
182 | ||
183 | [[ ! "${simrunpath}" =~ "${outputDirMC}" ]] && mv * ${outputDirMC} #TODO check if it works | |
184 | cd ${olddir} | |
185 | ||
186 | ln -s ${outputDirMC}/* ${runpath}/ | |
187 | ||
188 | inputList=${outputDirMC}/galice.root #TODO not valid outside shell !!! | |
189 | infile="" | |
190 | fi | |
71353a5f | 191 | fi |
192 | ###### | |
193 | ||
1795556a | 194 | if [[ ! -f ${inputList} && -z ${pretend} ]]; then |
ef77d296 | 195 | touch ${doneFileTmp} |
196 | echo "input file ${inputList} not found, exiting..." >> ${doneFileTmp} | |
197 | cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile" | |
198 | [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp} | |
1795556a | 199 | return 1 |
200 | fi | |
71353a5f | 201 | |
202 | logOutputDir=${runpath} | |
203 | [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir} | |
0eaf38c7 | 204 | [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/stdout |
205 | #[[ -z ${dontRedirectStdOutToLog} ]] && exec 2> ${logOutputDir}/stderr | |
71353a5f | 206 | echo "${0} $*" |
fc7e1e45 | 207 | |
208 | echo "#####################" | |
209 | echo CPass0: | |
210 | echo JOB setup | |
ef77d296 | 211 | echo nEvents ${nEvents} |
212 | echo runNumber ${runNumber} | |
213 | echo ocdbPath ${ocdbPath} | |
214 | echo infile ${infile} | |
215 | echo chunkName ${chunkName} | |
216 | echo jobindex ${jobindex} | |
217 | echo recoTriggerOptions ${recoTriggerOptions} | |
218 | echo targetDirectory ${targetDirectory} | |
219 | echo commonOutputPath ${commonOutputPath} | |
220 | echo doneFile ${doneFile} | |
221 | echo batchWorkingDirectory ${batchWorkingDirectory} | |
222 | echo runpath ${runpath} | |
223 | echo outputDir ${outputDir} | |
224 | echo PWD ${PWD} | |
225 | echo ALICE_ROOT ${ALICE_ROOT} | |
fc7e1e45 | 226 | echo "########## ###########" |
227 | ||
71353a5f | 228 | alirootInfo > ALICE_ROOT.log |
fc7e1e45 | 229 | |
230 | filesCPass0=( | |
39527a15 | 231 | "${batchWorkingDirectory}/runCPass0.sh" |
232 | "${batchWorkingDirectory}/recCPass0.C" | |
233 | "${batchWorkingDirectory}/runCalibTrain.C" | |
234 | "${batchWorkingDirectory}/localOCDBaccessConfig.C" | |
235 | "${batchWorkingDirectory}/OCDB.root" | |
71353a5f | 236 | "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/runCPass0.sh" |
237 | "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/recCPass0.C" | |
238 | "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/runCalibTrain.C" | |
fc7e1e45 | 239 | ) |
240 | ||
241 | for file in ${filesCPass0[*]}; do | |
242 | [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} . | |
1a0e38e3 | 243 | [[ ${file##*/} =~ .*\.sh ]] && chmod +x ${file##*/} |
fc7e1e45 | 244 | done |
245 | ||
71353a5f | 246 | echo "this directory (${PWD}) contents:" |
247 | /bin/ls | |
fc7e1e45 | 248 | echo |
249 | chmod u+x runCPass0.sh | |
250 | ||
71353a5f | 251 | sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh |
252 | ||
253 | if [[ -n ${postSetUpActionCPass0} ]]; then | |
254 | echo "running ${postSetUpActionCPass0}" | |
255 | eval ${postSetUpActionCPass0} | |
fc7e1e45 | 256 | fi |
257 | ||
258 | #run CPass0 | |
71353a5f | 259 | echo "${runpath}/runCPass0.sh ${infile} ${nEvents} ${runNumber} ${ocdbPath} ${recoTriggerOptions}" |
260 | if [[ -n ${pretend} ]]; then | |
0eaf38c7 | 261 | sleep ${pretendDelay} |
cb473870 | 262 | touch AliESDs.root |
263 | touch AliESDfriends.root | |
fc7e1e45 | 264 | touch AliESDfriends_v1.root |
265 | touch rec.log | |
266 | touch calib.log | |
267 | else | |
71353a5f | 268 | echo ./runCPass0.sh "${infile}" "${nEvents}" "${runNumber}" "${ocdbPath}" "${recoTriggerOptions}" |
269 | ./runCPass0.sh "${infile}" "${nEvents}" "${runNumber}" "${ocdbPath}" "${recoTriggerOptions}" | |
fc7e1e45 | 270 | fi |
271 | ||
fc7e1e45 | 272 | #move stuff to final destination |
71353a5f | 273 | echo "this directory (${PWD}) contents:" |
274 | /bin/ls | |
fc7e1e45 | 275 | echo |
276 | ||
ef77d296 | 277 | # [dberzano] OK this is fine! |
71353a5f | 278 | echo rm -f ./${chunkName} |
279 | rm -f ./${chunkName} | |
13abb339 | 280 | echo "paranoidCp ${runpath}/* ${outputDir}" |
281 | paranoidCp ${runpath}/* ${outputDir} | |
fc7e1e45 | 282 | echo |
5ed7c82c | 283 | |
284 | #validate CPass0 | |
285 | cd ${outputDir} | |
ef77d296 | 286 | if summarizeLogs >> ${doneFileTmp}; then |
287 | [[ -f ${outputDirMC}/galice.root ]] && echo "sim ${outputDirMC}/galice.root" >> ${doneFileTmp} | |
288 | [[ -f AliESDfriends_v1.root ]] && echo "calibfile ${outputDir}/AliESDfriends_v1.root" >> ${doneFileTmp} | |
289 | [[ -f AliESDs.root ]] && echo "esd ${outputDir}/AliESDs.root" >> ${doneFileTmp} | |
ab946185 | 290 | fi |
fc7e1e45 | 291 | |
0eaf38c7 | 292 | [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath} && echo "removing ${runpath}" |
ef77d296 | 293 | cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile" |
294 | [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp} | |
5de3064b | 295 | return 0 |
78d0bbfd | 296 | ) |
fc7e1e45 | 297 | |
298 | goCPass1() | |
78d0bbfd | 299 | ( |
fc7e1e45 | 300 | umask 0002 |
301 | ||
71353a5f | 302 | targetDirectory=${1} |
303 | inputList=${2} | |
304 | nEvents=${3} | |
305 | ocdbPath=${4} | |
306 | configFile=${5} | |
307 | runNumber=${6} | |
308 | jobindex=${7} | |
309 | shift 7 | |
444b9916 | 310 | extraOpts=("$@") |
6fc48f9e | 311 | if ! parseConfig ${configFile} "$@"; then return 1; fi |
71353a5f | 312 | |
39527a15 | 313 | #record the working directory provided by the batch system |
314 | batchWorkingDirectory=${PWD} | |
315 | ||
71353a5f | 316 | #use the jobindex only if set and non-negative |
317 | if [[ -z ${jobindex} || ${jobindex} -lt 0 ]]; then | |
318 | [[ -n "${LSB_JOBINDEX}" ]] && jobindex=${LSB_JOBINDEX} | |
319 | [[ -n "${SGE_TASK_ID}" ]] && jobindex=${SGE_TASK_ID} | |
999a843d | 320 | if [[ -z ${jobindex} ]]; then |
321 | echo "no jobindex!" | |
322 | return 1 | |
323 | fi | |
71353a5f | 324 | fi |
325 | ||
326 | [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD} | |
ef77d296 | 327 | |
328 | # This file signals that/if everything went fine | |
329 | doneFileBase="cpass1.job${jobindex}.run${runNumber}.done" | |
330 | [[ -n ${useProfilingCommand} ]] && doneFileBase="profiling.cpass0.job${jobindex}.run${runNumber}.done" | |
331 | ||
332 | # We will have two copies of the file | |
333 | mkdir -p "${commonOutputPath}/meta" || return 1 | |
334 | doneFileTmp="${batchWorkingDirectory}/${doneFileBase}" | |
335 | doneFile="${commonOutputPath}/meta/${doneFileBase}" | |
71353a5f | 336 | |
337 | [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource} | |
338 | ||
339 | if [[ -n ${ALIROOT_FORCE_COREDUMP} ]]; then | |
340 | ulimit -c unlimited | |
341 | export ALIROOT_FORCE_COREDUMP | |
342 | fi | |
fc7e1e45 | 343 | |
cb473870 | 344 | #the contents of this is stored in the tree and used later (e.g. AliAnalysisTaskPIDResponse)! |
345 | #at the QA stage the pass number is guessed from the path stored here. | |
346 | #The Format is: | |
347 | #Packages= ;OutputDir= ;LPMPass= ;TriggerAlias= ;LPMRunNumber= ;LPMProductionType= ;LPMInteractionType= ;LPMProductionTag= ;LPMAnchorRun= ;LPMAnchorProduction= ;LPMAnchorYear= | |
348 | export PRODUCTION_METADATA="OutputDir=cpass1" | |
349 | ||
1795556a | 350 | if [[ ! -f ${inputList} && -z ${pretend} ]]; then |
ef77d296 | 351 | touch ${doneFileTmp} |
352 | echo "input file ${inputList} not found, exiting..." >> ${doneFileTmp} | |
353 | cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile" | |
354 | [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp} | |
1795556a | 355 | return 1 |
356 | fi | |
fc7e1e45 | 357 | if [[ "${inputList}" =~ \.root$ ]]; then |
71353a5f | 358 | infile=${inputList} |
fc7e1e45 | 359 | else |
71353a5f | 360 | infile=$(sed -ne "${jobindex}p" ${inputList} | egrep '\s*\w*/\w*') |
fc7e1e45 | 361 | fi |
fc7e1e45 | 362 | chunkName=${infile##*/} |
5de3064b | 363 | |
71353a5f | 364 | outputDir=${targetDirectory}/${jobindex}_${chunkName%.*} |
365 | mkdir -p ${outputDir} | |
1795556a | 366 | if [[ ! -d ${outputDir} ]];then |
ef77d296 | 367 | touch ${doneFileTmp} |
368 | echo "cannot make ${outputDir}" >> ${doneFileTmp} | |
369 | cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile" | |
370 | [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp} | |
1795556a | 371 | return 1 |
372 | fi | |
fc7e1e45 | 373 | |
71353a5f | 374 | runpath=${outputDir} |
0eaf38c7 | 375 | [[ ${reconstructInTemporaryDir} -eq 1 ]] && runpath=$(mktemp -d -t cpass1.XXXXXX) |
e91a9c78 | 376 | [[ ${reconstructInTemporaryDir} -eq 2 ]] && runpath=${PWD}/rundir_cpass1_${runNumber}_${jobindex} |
71353a5f | 377 | |
378 | #MC | |
379 | if [[ "${infile}" =~ galice\.root ]]; then | |
380 | ln -s ${inputList%/*}/* ${runpath} | |
381 | infile="" | |
382 | fi | |
fc7e1e45 | 383 | |
384 | #init the running path | |
71353a5f | 385 | mkdir -p ${runpath} |
1795556a | 386 | if [[ ! -d ${runpath} ]]; then |
ef77d296 | 387 | touch ${doneFileTmp} |
388 | echo "cannot make runpath ${runpath}" >> ${doneFileTmp} | |
389 | cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile" | |
390 | [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp} | |
1795556a | 391 | return 1 |
392 | fi | |
0eaf38c7 | 393 | if ! cd ${runpath}; then |
ef77d296 | 394 | touch ${doneFileTmp} |
395 | echo "PWD=$PWD is not the runpath=${runpath}" >> ${doneFileTmp} | |
396 | cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile" | |
397 | [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp} | |
0eaf38c7 | 398 | return 1 |
399 | fi | |
fc7e1e45 | 400 | |
71353a5f | 401 | #this is needed for runCPass1.sh |
402 | ln -s ${infile} ${runpath}/${chunkName} | |
fc7e1e45 | 403 | |
71353a5f | 404 | logOutputDir=${runpath} |
405 | [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir} | |
0eaf38c7 | 406 | [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/stdout |
407 | #[[ -z ${dontRedirectStdOutToLog} ]] && exec 2> ${logOutputDir}/stderr | |
71353a5f | 408 | echo "${0} $*" |
fc7e1e45 | 409 | |
410 | echo "#####################" | |
411 | echo CPass1: | |
412 | echo JOB setup | |
ef77d296 | 413 | echo nEvents ${nEvents} |
414 | echo runNumber ${runNumber} | |
415 | echo ocdbPath ${ocdbPath} | |
416 | echo infile ${infile} | |
417 | echo chunkName ${chunkName} | |
418 | echo jobindex ${jobindex} | |
419 | echo recoTriggerOptions ${recoTriggerOptions} | |
420 | echo targetDirectory ${targetDirectory} | |
421 | echo commonOutputPath ${commonOutputPath} | |
422 | echo doneFile ${doneFile} | |
423 | echo runpath ${runpath} | |
424 | echo outputDir ${outputDir} | |
1795556a | 425 | echo batchWorkingDirectory ${batchWorkingDirectory} |
ef77d296 | 426 | echo ALICE_ROOT ${ALICE_ROOT} |
427 | echo PWD ${PWD} | |
428 | echo "#####################" | |
fc7e1e45 | 429 | |
71353a5f | 430 | alirootInfo > ALICE_ROOT.log |
fc7e1e45 | 431 | |
432 | filesCPass1=( | |
39527a15 | 433 | "${batchWorkingDirectory}/runCPass1.sh" |
434 | "${batchWorkingDirectory}/recCPass1.C" | |
435 | "${batchWorkingDirectory}/recCPass1_OuterDet.C" | |
436 | "${batchWorkingDirectory}/runCalibTrain.C" | |
437 | "${batchWorkingDirectory}/QAtrain_duo.C" | |
438 | "${batchWorkingDirectory}/localOCDBaccessConfig.C" | |
0eaf38c7 | 439 | "${batchWorkingDirectory}/${configFile}" |
617f8886 | 440 | "${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz" |
39527a15 | 441 | "${batchWorkingDirectory}/OCDB.root" |
71353a5f | 442 | "${trustedQAtrainMacro}" |
443 | "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/runCPass1.sh" | |
444 | "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/recCPass1.C" | |
445 | "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/recCPass1_OuterDet.C" | |
446 | "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/runCalibTrain.C" | |
447 | "${ALICE_ROOT}/ANALYSIS/macros/QAtrain_duo.C" | |
fc7e1e45 | 448 | ) |
449 | ||
78d0bbfd | 450 | for file in "${filesCPass1[@]}"; do |
fc7e1e45 | 451 | [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} . |
78d0bbfd | 452 | [[ ${file##*/} =~ .*\.sh ]] && echo "making ${file##*/} executable" && chmod +x ${file##*/} |
fc7e1e45 | 453 | done |
454 | ||
71353a5f | 455 | echo "this directory (${PWD}) contents:" |
456 | /bin/ls | |
fc7e1e45 | 457 | echo |
458 | ||
78d0bbfd | 459 | #remove spaces around commas from calls to root |
71353a5f | 460 | sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh |
461 | ||
462 | if [[ -n ${postSetUpActionCPass1} ]]; then | |
463 | echo "running ${postSetUpActionCPass1}" | |
464 | eval ${postSetUpActionCPass1} | |
fc7e1e45 | 465 | echo |
466 | fi | |
467 | ||
468 | #configure local OCDB storage from CPass0 (creates the localOCDBaccessConfig.C script) | |
469 | if [[ -f cpass0.localOCDB.${runNumber}.tgz ]]; then | |
71353a5f | 470 | echo goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz" |
471 | goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz" | |
cb473870 | 472 | else |
473 | echo "WARNING: file cpass0.localOCDB.${runNumber}.tgz not found!" | |
474 | fi | |
475 | ||
71353a5f | 476 | if [[ ! $(/bin/ls -1 OCDB/*/*/*/*.root 2>/dev/null) ]]; then |
ef77d296 | 477 | touch ${doneFileTmp} |
478 | echo "cpass0 produced no calibration! exiting..." >> ${doneFileTmp} | |
479 | cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile" | |
480 | [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp} | |
71353a5f | 481 | return 1 |
cb473870 | 482 | fi |
fc7e1e45 | 483 | |
71353a5f | 484 | #create the Barrel and OuterDet directories for CPass1 and link the local OCDB directory |
485 | #there to make the localOCDBaccessConfig.C file work, since it may point to the OCDB | |
486 | #entries using a relative path, e.g. local://./OCDB | |
78d0bbfd | 487 | echo "linking the OCDB/ for Barrel and OuterDet directories" |
71353a5f | 488 | mkdir Barrel OuterDet |
78d0bbfd | 489 | ls -l |
71353a5f | 490 | ln -s ../OCDB Barrel/OCDB |
491 | ln -s ../OCDB OuterDet/OCDB | |
492 | ||
444b9916 | 493 | #setup the filtering |
494 | #the following option enables the filtering task inside the QAtrain_duo.C | |
495 | [[ -n $runESDfiltering ]] && export QA_TaskFilteredTree=1 | |
496 | #set the downscaling factors during the filtering fro expert QA (overrides the previous values) | |
497 | if [[ -n ${filteringFactorHighPt} ]]; then | |
498 | export AliAnalysisTaskFilteredTree_fLowPtTrackDownscaligF=${filteringFactorHighPt} | |
499 | fi | |
500 | if [[ -n ${filteringFactorV0s} ]]; then | |
501 | export AliAnalysisTaskFilteredTree_fLowPtV0DownscaligF=${filteringFactorV0s} | |
502 | fi | |
503 | ||
fc7e1e45 | 504 | #run CPass1 |
505 | chmod u+x runCPass1.sh | |
71353a5f | 506 | echo "${runpath}/runCPass1.sh ${infile} ${nEvents} ${runNumber} ${ocdbPath} ${recoTriggerOptions}" |
507 | if [[ -n ${pretend} ]]; then | |
0eaf38c7 | 508 | sleep ${pretendDelay} |
cb473870 | 509 | touch AliESDs_Barrel.root |
510 | touch AliESDfriends_Barrel.root | |
fc7e1e45 | 511 | touch AliESDfriends_v1.root |
cb473870 | 512 | touch QAresults_barrel.root |
513 | touch EventStat_temp_barrel.root | |
514 | touch AODtpITS.root | |
515 | touch AliESDs_Outer.root | |
516 | touch AliESDfriends_Outer.root | |
517 | touch QAresults_outer.root | |
518 | touch EventStat_temp_outer.root | |
fc7e1e45 | 519 | touch rec.log |
520 | touch calib.log | |
521 | touch qa.log | |
444b9916 | 522 | touch filtering.log FilterEvents_Trees.root |
fc7e1e45 | 523 | else |
71353a5f | 524 | ./runCPass1.sh "${infile}" "${nEvents}" "${runNumber}" "${ocdbPath}" "${recoTriggerOptions}" |
444b9916 | 525 | |
526 | [[ ! -f AliESDs_Barrel.root && -f Barrel/AliESDs.root ]] && mv Barrel/AliESDs.root AliESDs_Barrel.root | |
527 | [[ ! -f AliESDfriends_Barrel.root && -f Barrel/AliESDfriends.root ]] && mv Barrel/AliESDfriends.root AliESDfriends_Barrel.root | |
528 | [[ ! -f AliESDfriends_v1.root && -f Barrel/AliESDfriends_v1.root ]] && mv Barrel/AliESDfriends_v1.root . | |
529 | [[ ! -f QAresults_barrel.root && -f Barrel/QAresults_barrel.root ]] && mv Barrel/QAresults_barrel.root . | |
530 | [[ ! -f AliESDs_Outer.root && -f OuterDet/AliESDs.root ]] && mv OuterDet/AliESDs.root AliESDs_Outer.root | |
531 | [[ ! -f AliESDfriends_Outer.root && -f OuterDet/AliESDfriends.root ]] && mv OuterDet/AliESDfriends.root AliESDfriends_Outer.root | |
532 | [[ ! -f QAresults_outer.root && -f OuterDet/QAresults_outer.root ]] && mv OuterDet/QAresults_outer.root . | |
533 | [[ ! -f FilterEvents_Trees.root && -f Barrel/FilterEvents_Trees.root ]] && mv Barrel/FilterEvents_Trees.root . | |
534 | ||
535 | #make the filtered tree (if requested and not already produced by QA | |
536 | [[ -f AliESDs_Barrel.root ]] && echo "AliESDs_Barrel.root" > filtered.list | |
fab31805 | 537 | if [[ -n ${runESDfiltering} && ! -f FilterEvents_Trees.root && -f filtered.list ]]; then |
538 | goMakeFilteredTrees ${PWD} ${runNumber} "${PWD}/filtered.list" ${filteringFactorHighPt} ${filteringFactorV0s} ${ocdbPath} 1000000 0 10000000 0 ${configFile} AliESDs_Barrel.root "${extraOpts[@]}" >filtering.log | |
539 | else | |
540 | echo "" | |
444b9916 | 541 | fi |
542 | ||
fc7e1e45 | 543 | fi |
cb473870 | 544 | |
545 | ##handle possible crashes in QA (happens often in trunk) | |
546 | ##rerun QA with a trusted aliroot version | |
547 | #if [[ $(validateLog qa_barrel.log) ]]; then | |
548 | # echo "qa_barrel.log not validated!" | |
549 | #fi | |
550 | #if [[ ! -f QAresults_barrel.root && -f ${setupTrustedAliROOTenvInCurrentShell} || $(validateLog qa_barrel.log) ]]; then | |
71353a5f | 551 | # echo "WARNING: using trusted QA aliroot ${ALICE_ROOT}" |
cb473870 | 552 | # source ${setupTrustedAliROOTenvInCurrentShell} |
553 | # cd Barrel | |
554 | # rm QAresults_barrel.root | |
555 | # rm EventStat_temp_barrel.root | |
556 | # rm AODtpITS.root | |
557 | # [[ ! -f AliESDs.root ]] && ln -s ../AliESDs_Barrel.root AliESDs.root | |
558 | # [[ ! -f AliESDfriends.root ]] && ln -s ../AliESDfriends_Barrel.root AliESDfriends.root | |
71353a5f | 559 | # if [[ -n ${trustedQAtrainMacro} ]]; then |
560 | # eval "cp ${trustedQAtrainMacro} QAtrain_duo_trusted.C" | |
cb473870 | 561 | # fi |
71353a5f | 562 | # echo executing aliroot -b -q "QAtrain_duo_trusted.C(\"_barrel\",${runNumber},\"wn.xml\",0,\"${ocdbPath}\")" |
563 | # time aliroot -b -q "QAtrain_duo.C(\"_barrel\",${runNumber},\"wn.xml\",0,\"${ocdbPath}\")" &> ../qa_barrel_trusted.log | |
cb473870 | 564 | # cd ../ |
565 | #fi | |
cb473870 | 566 | |
fc7e1e45 | 567 | #move stuff to final destination |
71353a5f | 568 | echo "this directory (${PWD}) contents:" |
569 | /bin/ls | |
570 | echo rm -f ./${chunkName} | |
571 | rm -f ./${chunkName} | |
13abb339 | 572 | echo "paranoidCp ${runpath}/* ${outputDir}" |
573 | paranoidCp ${runpath}/* ${outputDir} | |
fc7e1e45 | 574 | echo |
575 | ||
5ed7c82c | 576 | #validate CPass1 |
577 | cd ${outputDir} | |
ef77d296 | 578 | if summarizeLogs >> ${doneFileTmp}; then |
579 | [[ -f AliESDs_Barrel.root ]] && echo "esd ${outputDir}/AliESDs_Barrel.root" >> ${doneFileTmp} | |
580 | [[ -f AliESDfriends_v1.root ]] && echo "calibfile ${outputDir}/AliESDfriends_v1.root" >> ${doneFileTmp} | |
581 | [[ -f QAresults_Barrel.root ]] && echo "qafile ${outputDir}/QAresults_Barrel.root" >> ${doneFileTmp} | |
582 | [[ -f QAresults_Outer.root ]] && echo "qafile ${outputDir}/QAresults_Outer.root" >> ${doneFileTmp} | |
583 | [[ -f QAresults_barrel.root ]] && echo "qafile ${outputDir}/QAresults_barrel.root" >> ${doneFileTmp} | |
584 | [[ -f QAresults_outer.root ]] && echo "qafile ${outputDir}/QAresults_outer.root" >> ${doneFileTmp} | |
585 | [[ -f FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFileTmp} | |
922d0b10 | 586 | else |
ef77d296 | 587 | if grep "qa_outer.log.*OK" ${doneFileTmp} > /dev/null; then |
588 | [[ -f QAresults_Outer.root ]] && echo "qafile ${outputDir}/QAresults_Outer.root" >> ${doneFileTmp} | |
589 | [[ -f QAresults_outer.root ]] && echo "qafile ${outputDir}/QAresults_outer.root" >> ${doneFileTmp} | |
922d0b10 | 590 | fi |
ef77d296 | 591 | if grep "qa_barrel.log.*OK" ${doneFileTmp} > /dev/null; then |
592 | [[ -f QAresults_Barrel.root ]] && echo "qafile ${outputDir}/QAresults_Barrel.root" >> ${doneFileTmp} | |
593 | [[ -f QAresults_barrel.root ]] && echo "qafile ${outputDir}/QAresults_barrel.root" >> ${doneFileTmp} | |
922d0b10 | 594 | fi |
ef77d296 | 595 | if grep "filtering.log.*OK" ${doneFileTmp} > /dev/null; then |
596 | [[ -f FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFileTmp} | |
922d0b10 | 597 | fi |
ab946185 | 598 | fi |
599 | ||
71353a5f | 600 | [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath} |
ef77d296 | 601 | cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile" |
602 | [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp} | |
5de3064b | 603 | return 0 |
78d0bbfd | 604 | ) |
fc7e1e45 | 605 | |
606 | ||
607 | goMergeCPass0() | |
78d0bbfd | 608 | ( |
fc7e1e45 | 609 | # |
610 | # find the output files and merge them | |
611 | # | |
612 | ||
71353a5f | 613 | outputDir=${1} |
614 | ocdbStorage=${2} | |
615 | configFile=${3} | |
616 | runNumber=${4} | |
617 | calibrationFilesToMerge=${5} #can be a non-existent file, will then be produced on the fly | |
618 | shift 5 | |
6fc48f9e | 619 | if ! parseConfig ${configFile} "$@"; then return 1; fi |
71353a5f | 620 | |
39527a15 | 621 | #record the working directory provided by the batch system |
622 | batchWorkingDirectory=${PWD} | |
623 | ||
71353a5f | 624 | [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD} |
ef77d296 | 625 | |
626 | # This file signals that everything went fine | |
627 | doneFileBase="merge.cpass0.run${runNumber}.done" | |
628 | ||
629 | # We will have two copies of the file | |
630 | mkdir -p "${commonOutputPath}/meta" || return 1 | |
631 | doneFileTmp="${batchWorkingDirectory}/${doneFileBase}" | |
632 | doneFile="${commonOutputPath}/meta/${doneFileBase}" | |
71353a5f | 633 | |
634 | umask 0002 | |
635 | ulimit -c unlimited | |
fc7e1e45 | 636 | |
71353a5f | 637 | [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource} |
fc7e1e45 | 638 | |
71353a5f | 639 | runpath=${outputDir} |
0eaf38c7 | 640 | [[ ${reconstructInTemporaryDir} -eq 1 ]] && runpath=$(mktemp -d -t mergeCPass0.XXXXXX) |
e91a9c78 | 641 | [[ ${reconstructInTemporaryDir} -eq 2 ]] && runpath=${PWD}/rundir_mergeCPass0_${runNumber} |
fc7e1e45 | 642 | |
71353a5f | 643 | mkdir -p ${runpath} |
1795556a | 644 | if [[ ! -d ${runpath} ]]; then |
ef77d296 | 645 | touch ${doneFileTmp} |
646 | echo "not able to make the runpath ${runpath}" >> ${doneFileTmp} | |
647 | cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile" | |
648 | [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp} | |
1795556a | 649 | return 1 |
650 | fi | |
0eaf38c7 | 651 | if ! cd ${runpath}; then |
ef77d296 | 652 | touch ${doneFileTmp} |
653 | echo "PWD=$PWD is not the runpath=${runpath}" >> ${doneFileTmp} | |
654 | cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile" | |
655 | [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp} | |
0eaf38c7 | 656 | return 1 |
657 | fi | |
fc7e1e45 | 658 | |
71353a5f | 659 | logOutputDir=${runpath} |
660 | [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir} | |
1795556a | 661 | [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/stdout |
71353a5f | 662 | echo "${0} $*" |
fc7e1e45 | 663 | |
fc7e1e45 | 664 | mergingScript="mergeMakeOCDB.byComponent.sh" |
fc7e1e45 | 665 | |
666 | echo goMergeCPass0 SETUP: | |
71353a5f | 667 | echo runNumber=${runNumber} |
668 | echo outputDir=${outputDir} | |
669 | echo ocdbStorage=${ocdbStorage} | |
670 | echo calibrationFilesToMerge=${calibrationFilesToMerge} | |
671 | echo mergingScript=${mergingScript} | |
672 | echo commonOutputPath=${commonOutputPath} | |
673 | echo runpath=${runpath} | |
fc7e1e45 | 674 | |
675 | # copy files in case they are not already there | |
676 | filesMergeCPass0=( | |
39527a15 | 677 | "${batchWorkingDirectory}/${calibrationFilesToMerge}" |
678 | "${batchWorkingDirectory}/OCDB.root" | |
679 | "${batchWorkingDirectory}/localOCDBaccessConfig.C" | |
71353a5f | 680 | "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/mergeMakeOCDB.byComponent.sh" |
681 | "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/mergeByComponent.C" | |
682 | "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/makeOCDB.C" | |
683 | "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/merge.C" | |
684 | "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/mergeMakeOCDB.sh" | |
fc7e1e45 | 685 | ) |
686 | for file in ${filesMergeCPass0[*]}; do | |
687 | [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} . | |
1a0e38e3 | 688 | [[ ${file##*/} =~ .*\.sh ]] && chmod +x ${file##*/} |
fc7e1e45 | 689 | done |
690 | ||
71353a5f | 691 | sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh |
692 | ||
693 | alirootInfo > ALICE_ROOT.log | |
fc7e1e45 | 694 | |
695 | # | |
cb473870 | 696 | echo "PWD" |
71353a5f | 697 | /bin/ls |
cb473870 | 698 | echo "PWD/.." |
71353a5f | 699 | /bin/ls ../ |
cb473870 | 700 | |
fc7e1e45 | 701 | |
702 | #merge calibration | |
71353a5f | 703 | chmod u+x ${mergingScript} |
fc7e1e45 | 704 | mkdir -p ./OCDB |
71353a5f | 705 | if [[ ! -f ${calibrationFilesToMerge} ]]; then |
706 | echo "/bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root > ${calibrationFilesToMerge}" | |
707 | /bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root 2>/dev/null > ${calibrationFilesToMerge} | |
fc7e1e45 | 708 | fi |
709 | ||
71353a5f | 710 | echo "${mergingScript} ${calibrationFilesToMerge} ${runNumber} local://./OCDB ${ocdbStorage}" |
711 | if [[ -n ${pretend} ]]; then | |
0eaf38c7 | 712 | sleep ${pretendDelay} |
fc7e1e45 | 713 | touch CalibObjects.root |
714 | touch ocdb.log | |
715 | touch merge.log | |
617f8886 | 716 | touch dcsTime.root |
71353a5f | 717 | mkdir -p ./OCDB/TPC/Calib/TimeGain/ |
718 | mkdir -p ./OCDB/TPC/Calib/TimeDrift/ | |
1795556a | 719 | echo "some calibration" >> ./OCDB/TPC/Calib/TimeGain/someCalibObject_0-999999_cpass0.root |
720 | echo "some calibration" >> ./OCDB/TPC/Calib/TimeDrift/otherCalibObject_0-999999_cpass0.root | |
fc7e1e45 | 721 | else |
1795556a | 722 | ./${mergingScript} ${calibrationFilesToMerge} ${runNumber} "local://./OCDB" ${ocdbStorage} >> "mergeMakeOCDB.log" |
fc7e1e45 | 723 | |
617f8886 | 724 | #produce the calib trees for expert QA (dcsTime.root) |
725 | goMakeLocalOCDBaccessConfig ./OCDB | |
726 | echo aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")" | |
727 | aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")" | |
728 | fi | |
729 | ||
fc7e1e45 | 730 | ### produce the output |
731 | #tar the produced OCDB for reuse | |
ef77d296 | 732 | #tar czf ${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz ./OCDB |
733 | ||
734 | # Create tarball with OCDB, store on the shared directory, create signal file on batch directory | |
735 | mkdir -p ${commonOutputPath}/meta | |
736 | baseTar="cpass0.localOCDB.${runNumber}.tgz" | |
737 | tar czf ${batchWorkingDirectory}/${baseTar} ./OCDB && \ | |
738 | mv ${batchWorkingDirectory}/${baseTar} ${commonOutputPath}/meta/${baseTar} && \ | |
739 | touch ${batchWorkingDirectory}/${baseTar}.done | |
fc7e1e45 | 740 | |
71353a5f | 741 | /bin/ls |
5ed7c82c | 742 | |
743 | #copy all to output dir | |
13abb339 | 744 | echo "paranoidCp ${runpath}/* ${outputDir}" |
745 | paranoidCp ${runpath}/* ${outputDir} | |
617f8886 | 746 | |
71353a5f | 747 | if [[ -n ${generateMC} ]]; then |
748 | goPrintValues sim ${commonOutputPath}/meta/sim.run${runNumber}.list ${commonOutputPath}/meta/cpass0.job*.run${runNumber}.done | |
749 | fi | |
750 | ||
fc7e1e45 | 751 | #validate merging cpass0 |
5ed7c82c | 752 | cd ${outputDir} |
ef77d296 | 753 | if summarizeLogs >> ${doneFileTmp}; then |
754 | [[ -f CalibObjects.root ]] && echo "calibfile ${outputDir}/CalibObjects.root" >> ${doneFileTmp} | |
755 | [[ -f dcsTime.root ]] && echo "dcsTree ${outputDir}/dcsTime.root" >> ${doneFileTmp} | |
ab946185 | 756 | fi |
fc7e1e45 | 757 | |
71353a5f | 758 | [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath} |
ef77d296 | 759 | cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile" |
760 | [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp} | |
5de3064b | 761 | return 0 |
78d0bbfd | 762 | ) |
fc7e1e45 | 763 | |
764 | goMergeCPass1() | |
78d0bbfd | 765 | ( |
fc7e1e45 | 766 | # |
767 | # find the output files and merge them | |
768 | # | |
769 | ||
71353a5f | 770 | outputDir=${1} |
771 | ocdbStorage=${2} | |
772 | configFile=${3} | |
773 | runNumber=${4} | |
774 | calibrationFilesToMerge=${5} | |
775 | qaFilesToMerge=${6} | |
444b9916 | 776 | filteredFilesToMerge=${7} |
777 | shift 7 | |
6fc48f9e | 778 | if ! parseConfig ${configFile} "$@"; then return 1; fi |
71353a5f | 779 | |
39527a15 | 780 | #record the working directory provided by the batch system |
781 | batchWorkingDirectory=${PWD} | |
782 | ||
71353a5f | 783 | [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD} |
ef77d296 | 784 | |
785 | # This file signals that everything went fine | |
786 | doneFileBase="merge.cpass1.run${runNumber}.done" | |
787 | ||
788 | # We will have two copies of the file | |
789 | mkdir -p "${commonOutputPath}/meta" || return 1 | |
790 | doneFileTmp="${batchWorkingDirectory}/${doneFileBase}" | |
791 | doneFile="${commonOutputPath}/meta/${doneFileBase}" | |
71353a5f | 792 | |
793 | umask 0002 | |
794 | ulimit -c unlimited | |
fc7e1e45 | 795 | |
796 | #clean up first: | |
71353a5f | 797 | rm -f ${outputDir}/*.log |
798 | rm -f ${outputDir}/*.root | |
799 | rm -f ${outputDir}/*done | |
800 | ||
801 | [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource} | |
802 | ||
71353a5f | 803 | runpath=${outputDir} |
0eaf38c7 | 804 | [[ ${reconstructInTemporaryDir} -eq 1 ]] && runpath=$(mktemp -d -t mergeCPass1.XXXXXX) |
e91a9c78 | 805 | [[ ${reconstructInTemporaryDir} -eq 2 ]] && runpath=${PWD}/rundir_mergeCPass1_${runNumber} |
71353a5f | 806 | |
807 | mkdir -p ${runpath} | |
1795556a | 808 | if [[ ! -d ${runpath} ]]; then |
ef77d296 | 809 | touch ${doneFileTmp} |
810 | echo "not able to make the runpath ${runpath}" >> ${doneFileTmp} | |
811 | cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile" | |
812 | [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp} | |
1795556a | 813 | return 1 |
814 | fi | |
0eaf38c7 | 815 | if ! cd ${runpath}; then |
ef77d296 | 816 | touch ${doneFileTmp} |
817 | echo "PWD=$PWD is not the runpath=${runpath}" >> ${doneFileTmp} | |
818 | cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile" | |
819 | [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp} | |
0eaf38c7 | 820 | return 1 |
821 | fi | |
71353a5f | 822 | |
823 | logOutputDir=${runpath} | |
824 | [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir} | |
825 | [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/mergeMakeOCDB.log | |
826 | echo "${0} $*" | |
827 | ||
828 | calibrationOutputFileName='AliESDfriends_v1.root' | |
829 | qaOutputFileName='QAresults*.root' | |
fc7e1e45 | 830 | mergingScript="mergeMakeOCDB.byComponent.sh" |
cb473870 | 831 | #important to have the string "Stage.txt" in the filename to trigger the merging |
832 | #it has to be a list of directories containing the files | |
fc7e1e45 | 833 | qaMergedOutputFileName="QAresults_merged.root" |
834 | ||
835 | echo goMergeCPass1 SETUP: | |
71353a5f | 836 | echo runNumber=${runNumber} |
837 | echo outputDir=${outputDir} | |
838 | echo ocdbStorage=${ocdbStorage} | |
839 | echo calibrationFilesToMerge=$calibrationFilesToMerge | |
840 | echo qaFilesToMerge=$qaFilesToMerge | |
841 | echo calibrationOutputFileName=${calibrationOutputFileName} | |
842 | echo mergingScript=${mergingScript} | |
fc7e1e45 | 843 | |
844 | # copy files in case they are not already there | |
845 | filesMergeCPass1=( | |
39527a15 | 846 | "${batchWorkingDirectory}/${calibrationFilesToMerge}" |
847 | "${batchWorkingDirectory}/${qaFilesToMerge}" | |
848 | "${batchWorkingDirectory}/OCDB.root" | |
849 | "${batchWorkingDirectory}/localOCDBaccessConfig.C" | |
3fbef3b9 | 850 | "${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz" |
39527a15 | 851 | "${batchWorkingDirectory}/QAtrain_duo.C" |
71353a5f | 852 | "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/mergeMakeOCDB.byComponent.sh" |
853 | "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/mergeByComponent.C" | |
854 | "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/makeOCDB.C" | |
855 | "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/merge.C" | |
856 | "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/mergeMakeOCDB.sh" | |
857 | "${trustedQAtrainMacro}" | |
858 | "${ALICE_ROOT}/ANALYSIS/macros/QAtrain_duo.C" | |
fc7e1e45 | 859 | ) |
860 | for file in ${filesMergeCPass1[*]}; do | |
861 | [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} . | |
1a0e38e3 | 862 | [[ ${file##*/} =~ .*\.sh ]] && chmod +x ${file##*/} |
fc7e1e45 | 863 | done |
864 | ||
71353a5f | 865 | sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh |
866 | ||
cb473870 | 867 | #configure local OCDB storage from CPass0 (creates the localOCDBaccessConfig.C script) |
868 | if [[ -f cpass0.localOCDB.${runNumber}.tgz ]]; then | |
71353a5f | 869 | echo goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz" |
870 | goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz" | |
cb473870 | 871 | else |
872 | echo "WARNING: file cpass0.localOCDB.${runNumber}.tgz not found!" | |
873 | fi | |
874 | ||
71353a5f | 875 | alirootInfo > ALICE_ROOT.log |
fc7e1e45 | 876 | |
877 | # | |
71353a5f | 878 | /bin/ls |
fc7e1e45 | 879 | |
880 | #merge calibration | |
71353a5f | 881 | chmod u+x ${mergingScript} |
fc7e1e45 | 882 | mkdir -p OCDB |
444b9916 | 883 | |
884 | #if not provided, create the lists of files to merge | |
885 | if [[ ! -f ${filteredFilesToMerge} ]]; then | |
886 | echo "/bin/ls -1 ${outputDir}/*/FilterEvents_Trees.root > ${filteredFilesToMerge}" | |
887 | /bin/ls -1 ${outputDir}/*/FilterEvents_Trees.root 2>/dev/null > ${filteredFilesToMerge} | |
888 | fi | |
71353a5f | 889 | if [[ ! -f ${calibrationFilesToMerge} ]]; then |
444b9916 | 890 | echo "/bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root > ${calibrationFilesToMerge}" |
891 | /bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root 2>/dev/null > ${calibrationFilesToMerge} | |
892 | fi | |
893 | if [[ ! -f ${qaFilesToMerge} ]]; then | |
894 | #find the files, but only store the directories (QAtrain_duo.C requires this) | |
895 | echo "/bin/ls -1 ${outputDir}/*/QAresults*.root | while read x; do echo ${x%/*}; done | sort | uniq > ${qaFilesToMerge}" | |
896 | /bin/ls -1 ${outputDir}/*/QAresults*.root | while read x; do echo ${x%/*}; done | sort | uniq > ${qaFilesToMerge} | |
fc7e1e45 | 897 | fi |
898 | ||
71353a5f | 899 | echo "${mergingScript} ${calibrationFilesToMerge} ${runNumber} local://./OCDB ${ocdbStorage}" |
900 | if [[ -n ${pretend} ]]; then | |
0eaf38c7 | 901 | sleep ${pretendDelay} |
fc7e1e45 | 902 | touch ocdb.log |
71353a5f | 903 | touch cpass1.localOCDB.${runNumber}.tgz |
71353a5f | 904 | touch ${qaMergedOutputFileName} |
fc7e1e45 | 905 | touch merge.log |
cb473870 | 906 | touch trending.root |
444b9916 | 907 | touch FilterEvents_Trees.root |
ab946185 | 908 | touch CalibObjects.root |
909 | touch dcsTime.root | |
910 | touch ${qaMergedOutputFileName} | |
617f8886 | 911 | mkdir -p OCDB |
fc7e1e45 | 912 | else |
617f8886 | 913 | ./${mergingScript} ${calibrationFilesToMerge} ${runNumber} "local://./OCDB" ${ocdbStorage} |
914 | ||
915 | #merge QA (and filtered trees) | |
916 | [[ -n ${AliAnalysisTaskFilteredTree_fLowPtTrackDownscaligF} ]] && export AliAnalysisTaskFilteredTree_fLowPtTrackDownscaligF | |
917 | [[ -n ${AliAnalysisTaskFilteredTree_fLowPtV0DownscaligF} ]] && export AliAnalysisTaskFilteredTree_fLowPtV0DownscaligF | |
918 | ||
919 | #echo aliroot -l -b -q "merge.C(\"${qaFilesToMerge}\",\"\",kFALSE,\"${qaMergedOutputFileName}\")" | |
920 | echo aliroot -b -q "QAtrain_duo.C(\"_barrel\",${runNumber},\"${qaFilesToMerge}\",1,\"${ocdbStorage}\")" | |
71353a5f | 921 | #aliroot -l -b -q "merge.C(\"${qaFilesToMerge}\",\"\",kFALSE,\"${qaMergedOutputFileName}\")" |
922 | aliroot -b -q "QAtrain_duo.C(\"_barrel\",${runNumber},\"${qaFilesToMerge}\",1,\"${ocdbStorage}\")" > mergeQA.log | |
923 | mv QAresults_barrel.root ${qaMergedOutputFileName} | |
cb473870 | 924 | mv trending_barrel.root trending.root |
444b9916 | 925 | |
926 | #merge filtered trees | |
927 | echo aliroot -l -b -q "merge.C(\"${qaFilesToMerge}\",\"\",kFALSE,\"${qaMergedOutputFileName}\")" | |
922d0b10 | 928 | aliroot -l -b -q "merge.C(\"${filteredFilesToMerge}\",\"\",kFALSE,\"FilterEvents_Trees.root\")" > mergeFilteredTrees.log |
617f8886 | 929 | |
930 | #produce the calib trees for expert QA | |
931 | echo aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")" | |
922d0b10 | 932 | aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")" > calibTree.log |
fc7e1e45 | 933 | fi |
617f8886 | 934 | |
ef77d296 | 935 | # Create tarball with OCDB, store on the shared directory, create signal file on batch directory |
936 | mkdir -p ${commonOutputPath}/meta | |
937 | baseTar="cpass1.localOCDB.${runNumber}.tgz" | |
938 | tar czf ${batchWorkingDirectory}/${baseTar} ./OCDB && \ | |
939 | mv ${batchWorkingDirectory}/${baseTar} ${commonOutputPath}/meta/${baseTar} && \ | |
940 | touch ${batchWorkingDirectory}/${baseTar}.done | |
71353a5f | 941 | |
942 | /bin/ls | |
5ed7c82c | 943 | |
944 | #copy all to output dir | |
13abb339 | 945 | echo "paranoidCp ${runpath}/* ${outputDir}" |
946 | paranoidCp ${runpath}/* ${outputDir} | |
5ed7c82c | 947 | |
fc7e1e45 | 948 | #validate merge cpass1 |
5ed7c82c | 949 | cd ${outputDir} |
ef77d296 | 950 | if summarizeLogs >> ${doneFileTmp}; then |
951 | [[ -f CalibObjects.root ]] && echo "calibfile ${outputDir}/CalibObjects.root" >> ${doneFileTmp} | |
952 | [[ -f ${qaMergedOutputFileName} ]] && echo "qafile ${outputDir}/${qaMergedOutputFileName}" >> ${doneFileTmp} | |
953 | [[ -f trending.root ]] && echo "trendingfile ${outputDir}/trending.root" >> ${doneFileTmp} | |
954 | [[ -f dcsTime.root ]] && echo "dcsTree ${outputDir}/dcsTime.root" >> ${doneFileTmp} | |
955 | [[ -f FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFileTmp} | |
922d0b10 | 956 | else |
ef77d296 | 957 | if grep "mergeQA.log.*OK" ${doneFileTmp} > /dev/null; then |
958 | [[ -f ${qaMergedOutputFileName} ]] && echo "qafile ${outputDir}/${qaMergedOutputFileName}" >> ${doneFileTmp} | |
922d0b10 | 959 | fi |
ef77d296 | 960 | if grep "mergeFilteredTrees.log.*OK" ${doneFileTmp} > /dev/null; then |
961 | [[ -f FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFileTmp} | |
922d0b10 | 962 | fi |
ab946185 | 963 | fi |
71353a5f | 964 | |
965 | [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath} | |
ef77d296 | 966 | cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile" |
967 | [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp} | |
5de3064b | 968 | return 0 |
78d0bbfd | 969 | ) |
cb473870 | 970 | |
971 | goMerge() | |
78d0bbfd | 972 | ( |
cb473870 | 973 | #generic root merge using CPass1 merge.C script |
71353a5f | 974 | inputList=${1} |
975 | outputFile=${2} | |
0eaf38c7 | 976 | configFile=${3-"benchmark.config"} |
71353a5f | 977 | shift 3 |
6fc48f9e | 978 | if ! parseConfig ${configFile} "$@"; then return 1; fi |
39527a15 | 979 | |
980 | #record the working directory provided by the batch system | |
981 | batchWorkingDirectory=${PWD} | |
fc7e1e45 | 982 | |
71353a5f | 983 | [[ ! -f ${inputList} ]] && echo "inputList ${inputList} does not exist!" && return 1 |
984 | [[ ! -f ${configFile} ]] && echo "configFile ${configFile} does not exist!" && return 1 | |
985 | umask 0002 | |
986 | [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource} | |
987 | rm -f ${outputFile} | |
988 | aliroot -b -q "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/merge.C(\"${inputList}\",\"\",kFALSE,\"${outputFile}\")" > merge_${inputList}.log | |
5de3064b | 989 | return 0 |
78d0bbfd | 990 | ) |
fc7e1e45 | 991 | |
cb473870 | 992 | goSubmitMakeflow() |
5dfac970 | 993 | { |
cb473870 | 994 | #run |
71353a5f | 995 | productionID=${1} |
996 | inputFileList=${2} | |
997 | configFile=${3} | |
998 | shift 3 | |
444b9916 | 999 | extraOpts=("$@") |
6fc48f9e | 1000 | if ! parseConfig ${configFile} "${extraOpts[@]}"; then return 1; fi |
39527a15 | 1001 | |
1002 | #record the working directory provided by the batch system | |
1003 | batchWorkingDirectory=${PWD} | |
fc7e1e45 | 1004 | |
5dfac970 | 1005 | [[ -z ${configFile} ]] && configFile="benchmark.config" |
1006 | [[ ! -f ${configFile} ]] && echo "no config file found (${configFile})" && return 1 | |
fc7e1e45 | 1007 | |
71353a5f | 1008 | if [[ ! $(which makeflow &>/dev/null) && -n ${makeflowPath} ]]; then |
cb473870 | 1009 | echo "setting the makflow path from the config: " |
1010 | echo " export PATH=${makeflowPath}:${PATH}" | |
1011 | export PATH=${makeflowPath}:${PATH} | |
1012 | fi | |
1013 | ||
39527a15 | 1014 | #create the common output dir and the meta dir |
1015 | commonOutputPath=${baseOutputDirectory}/${productionID} | |
1016 | if [[ -d ${commonOutputPath} ]]; then | |
1017 | echo "output dir ${commonOutputPath} exists!" | |
0eaf38c7 | 1018 | #return 1 |
39527a15 | 1019 | else |
1020 | mkdir -p ${commonOutputPath} | |
1021 | fi | |
1022 | mkdir -p ${commonOutputPath}/meta | |
839b07ca | 1023 | |
0eaf38c7 | 1024 | self=${0} |
1025 | #if which greadlink; then self=$(greadlink -f "${0}"); fi | |
839b07ca | 1026 | |
0eaf38c7 | 1027 | #for reference copy the setup to the output dir |
839b07ca | 1028 | cp ${self} ${commonOutputPath} |
1029 | cp ${configFile} ${commonOutputPath} | |
333bc229 | 1030 | cp ${inputFileList} ${commonOutputPath} |
39527a15 | 1031 | |
cb473870 | 1032 | #submit - use makeflow if available, fall back to old stuff when makeflow not there |
1033 | if which makeflow; then | |
39527a15 | 1034 | goGenerateMakeflow ${productionID} ${inputFileList} ${configFile} "${extraOpts[@]}" commonOutputPath=${commonOutputPath} > benchmark.makeflow |
1795556a | 1035 | cp benchmark.makeflow ${commonOutputPath} |
cb473870 | 1036 | makeflow ${makeflowOptions} benchmark.makeflow |
cb473870 | 1037 | else |
1038 | echo "no makeflow!" | |
1039 | fi | |
054fdec5 | 1040 | |
1041 | #summarize the run based on the makeflow log | |
1042 | #and add it to the end of summary log | |
1043 | awk '/STARTED/ {startTime=$3} | |
1044 | /COMPLETED/ {endTime=$3} | |
1045 | END {print "makeflow running time: "(endTime-startTime)/1000000/3600" hours"}' \ | |
1046 | benchmark.makeflow.makeflowlog | tee -a summary.log | |
1047 | paranoidCp summary.log ${commonOutputPath} | |
1048 | ||
3ce223dd | 1049 | return 0 |
5dfac970 | 1050 | } |
fc7e1e45 | 1051 | |
5dfac970 | 1052 | goGenerateMakeflow() |
78d0bbfd | 1053 | ( |
5dfac970 | 1054 | #generate the makeflow file |
ab946185 | 1055 | [[ $# -lt 3 ]] && echo "args: id inputFileList configFile" && return 1 |
71353a5f | 1056 | productionID=${1} |
1057 | inputFileList=${2} | |
1058 | configFile=${3} | |
1059 | shift 3 | |
444b9916 | 1060 | extraOpts=("$@") |
0eaf38c7 | 1061 | |
1062 | #batch systems/makeflow sometimes handle spaces in arguments poorly, so encode them | |
1063 | for (( i=0;i<${#extraOpts[@]};i++ )); do | |
1064 | extraOpts[i]=$(encSpaces "${extraOpts[i]}") | |
1065 | done | |
1066 | extraOpts+=("encodedSpaces=1") | |
1067 | ||
39527a15 | 1068 | if ! parseConfig ${configFile} "${extraOpts[@]}" &>/dev/null; then return 1; fi |
1069 | ||
1a0e38e3 | 1070 | #extra safety |
1071 | if [[ -z ${commonOutputPath} ]]; then | |
1072 | commonOutputPath=${baseOutputDirectory}/${productionID} | |
1073 | extraOpts=( "${extraOpts[@]}" "commonOutputPath=${commonOutputPath}" ) | |
1074 | fi | |
bff71da0 | 1075 | |
39527a15 | 1076 | #record the working directory provided by the batch system |
1077 | batchWorkingDirectory=${PWD} | |
fc7e1e45 | 1078 | |
5dfac970 | 1079 | [[ -z ${configFile} ]] && configFile="benchmark.config" |
1080 | [[ ! -f ${configFile} ]] && echo "no config file found (${configFile})" && return 1 | |
fc7e1e45 | 1081 | |
5dfac970 | 1082 | #these files will be made a dependency - will be copied to the working dir of the jobs |
1083 | declare -a copyFiles | |
1084 | inputFiles=( | |
1085 | "OCDB.root" | |
1086 | "localOCDBaccessConfig.C" | |
cb473870 | 1087 | "QAtrain_duo.C" |
1088 | "runCPass1.sh" | |
1089 | "recCPass1.C" | |
1090 | "recCPass1_OuterDet.C" | |
1091 | "runCalibTrain.C" | |
1092 | "runCPass0.sh" | |
1093 | "recCPass0.C" | |
ef77d296 | 1094 | "runQA.sh" |
5dfac970 | 1095 | ) |
1096 | for file in ${inputFiles[*]}; do | |
1097 | [[ -f ${file} ]] && copyFiles+=("${file}") | |
1098 | done | |
1099 | ||
1100 | #create the makeflow file | |
cb473870 | 1101 | [[ -n ${batchFlags} ]] && echo "BATCH_OPTIONS = ${batchFlags}" |
444b9916 | 1102 | declare -A arr_cpass0_merged arr_cpass1_merged |
1103 | declare -A arr_cpass0_calib_list arr_cpass1_calib_list | |
1104 | declare -A arr_cpass1_QA_list arr_cpass1_ESD_list arr_cpass1_filtered_list | |
6fc48f9e | 1105 | declare -A arr_cpass0_profiled_outputs |
444b9916 | 1106 | declare -A listOfRuns |
1107 | [[ -n ${runNumber} ]] && listOfRuns[${runNumber}]=1 | |
617f8886 | 1108 | while read x; do tmpRun=$(guessRunNumber ${x}); [[ -n ${tmpRun} ]] && listOfRuns[${tmpRun}]=1; done < ${inputFileList} |
444b9916 | 1109 | for runNumber in "${!listOfRuns[@]}"; do |
71353a5f | 1110 | [[ -z ${runNumber} ]] && continue |
fc7e1e45 | 1111 | [[ ! ${runNumber} =~ ^[0-9]*[0-9]$ ]] && continue |
fc7e1e45 | 1112 | |
444b9916 | 1113 | unset arr_cpass0_outputs |
1114 | unset arr_cpass1_outputs | |
1115 | declare -a arr_cpass0_outputs | |
1116 | declare -a arr_cpass1_outputs | |
fc7e1e45 | 1117 | |
ef77d296 | 1118 | #Header |
1119 | echo "### Automatically generated on $(LANG=C date) ###" | |
1120 | echo ; echo | |
1121 | ||
cb473870 | 1122 | jobindex=0 |
6fc48f9e | 1123 | inputFile="" |
5dfac970 | 1124 | while read inputFile; do |
1125 | currentDefaultOCDB=${defaultOCDB} | |
cb473870 | 1126 | [[ -z ${autoOCDB} ]] && autoOCDB=1 |
1127 | if [[ ${autoOCDB} -ne 0 ]]; then | |
71353a5f | 1128 | currentDefaultOCDB=$(setYear ${inputFile} ${defaultOCDB}) |
cb473870 | 1129 | fi |
ef77d296 | 1130 | guessRunData ${inputFile} |
1131 | ||
1132 | #Set variables | |
1133 | echo "### Variables ###" | |
1134 | echo "OUTPATH=\"${commonOutputPath}/${year}/${period}\"" | |
1135 | echo ; echo | |
fc7e1e45 | 1136 | |
5dfac970 | 1137 | #CPass0 |
ef77d296 | 1138 | #arr_cpass0_outputs[${jobindex}]="${commonOutputPath}/meta/cpass0.job${jobindex}.run${runNumber}.done" |
1139 | arr_cpass0_outputs[${jobindex}]="cpass0.job${jobindex}.run${runNumber}.done" | |
1140 | echo "### CPass0 ###" | |
1141 | echo "${arr_cpass0_outputs[${jobindex}]}: benchmark.sh ${configFile} ${copyFiles[@]}" | |
1142 | echo " ${alirootEnv} ./benchmark.sh CPass0 \$OUTPATH/000${runNumber}/cpass0 ${inputFile} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} ${jobindex} ${extraOpts[@]}"" " | |
1143 | echo ; echo | |
5dfac970 | 1144 | |
1145 | #CPass1 | |
ef77d296 | 1146 | #arr_cpass1_outputs[${jobindex}]="${commonOutputPath}/meta/cpass1.job${jobindex}.run${runNumber}.done" |
1147 | arr_cpass1_outputs[${jobindex}]="cpass1.job${jobindex}.run${runNumber}.done" | |
1148 | echo "### CPass1 ###" | |
1149 | echo "${arr_cpass1_outputs[${jobindex}]}: benchmark.sh ${configFile} cpass0.localOCDB.${runNumber}.tgz.done ${copyFiles[@]}" | |
1150 | echo " ${alirootEnv} ./benchmark.sh CPass1 \$OUTPATH/000${runNumber}/cpass1 ${inputFile} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} ${jobindex} ${extraOpts[@]}"" " | |
1151 | echo ; echo | |
5dfac970 | 1152 | ((jobindex++)) |
fc7e1e45 | 1153 | |
71353a5f | 1154 | done< <(grep "/000${runNumber}/" ${inputFileList}) |
cb473870 | 1155 | |
5dfac970 | 1156 | #CPass0 list of Calib files to merge |
ef77d296 | 1157 | #arr_cpass0_calib_list[${runNumber}]="${commonOutputPath}/meta/cpass0.calib.run${runNumber}.list" |
1158 | arr_cpass0_calib_list[${runNumber}]="cpass0.calib.run${runNumber}.list" | |
1159 | echo "### Produces the list of CPass0 files to merge (executes locally) ###" | |
1160 | echo "${arr_cpass0_calib_list[${runNumber}]}: benchmark.sh ${arr_cpass0_outputs[*]}" | |
1161 | echo " LOCAL ./benchmark.sh PrintValues calibfile ${arr_cpass0_calib_list[${runNumber}]} ${arr_cpass0_outputs[*]} && mkdir -p \$OUTPATH/meta && cp ${arr_cpass0_calib_list[${runNumber}]} \$OUTPATH/meta/${arr_cpass0_calib_list[${runNumber}]}" | |
1162 | echo ; echo | |
71353a5f | 1163 | |
1164 | #CPass0 merging | |
ef77d296 | 1165 | echo "### Merges CPass0 files ###" |
1166 | #arr_cpass0_merged[${runNumber}]="${commonOutputPath}/meta/merge.cpass0.run${runNumber}.done" | |
1167 | arr_cpass0_merged[${runNumber}]="merge.cpass0.run${runNumber}.done" | |
1168 | echo "cpass0.localOCDB.${runNumber}.tgz.done ${arr_cpass0_merged[${runNumber}]}: benchmark.sh ${configFile} ${arr_cpass0_calib_list[${runNumber}]} ${copyFiles[@]}" | |
1169 | echo " ${alirootEnv} ./benchmark.sh MergeCPass0 \$OUTPATH/000${runNumber}/cpass0 ${currentDefaultOCDB} ${configFile} ${runNumber} ${arr_cpass0_calib_list[${runNumber}]} ${extraOpts[@]}"" " | |
1170 | echo ; echo | |
fc7e1e45 | 1171 | |
ab946185 | 1172 | #CPass1 list of Calib/QA/ESD/filtered files |
1173 | # the trick with QA is to have the string "Stage.txt" in the file name of the list of directories with QA output to trigger | |
1174 | # the production of the QA trending tree (only then the task->Finish() will be called in QAtrain_duo.C, on the grid | |
8d6a0a43 | 1175 | # this corresponds to the last merging stage) |
ef77d296 | 1176 | #arr_cpass1_QA_list[${runNumber}]="${commonOutputPath}/meta/cpass1.QA.run${runNumber}.lastMergingStage.txt.list" |
1177 | arr_cpass1_QA_list[${runNumber}]="cpass1.QA.run${runNumber}.lastMergingStage.txt.list" | |
1178 | echo "### Lists CPass1 QA ###" | |
444b9916 | 1179 | echo "${arr_cpass1_QA_list[${runNumber}]}: benchmark.sh ${arr_cpass1_outputs[*]}" |
ef77d296 | 1180 | echo " LOCAL ./benchmark.sh PrintValues dir ${arr_cpass1_QA_list[${runNumber}]} ${arr_cpass1_outputs[*]} && mkdir -p \$OUTPATH/meta && cp ${arr_cpass1_QA_list[${runNumber}]} \$OUTPATH/meta/${arr_cpass1_QA_list[${runNumber}]}" |
1181 | echo ; echo | |
1182 | ||
1183 | #arr_cpass1_calib_list[${runNumber}]="${commonOutputPath}/meta/cpass1.calib.run${runNumber}.list" | |
1184 | arr_cpass1_calib_list[${runNumber}]="cpass1.calib.run${runNumber}.list" | |
1185 | echo "### Lists CPass1 Calib ###" | |
1186 | echo "${arr_cpass1_calib_list[${runNumber}]}: benchmark.sh ${arr_cpass1_outputs[*]}" | |
1187 | echo " LOCAL ./benchmark.sh PrintValues calibfile ${arr_cpass1_calib_list[${runNumber}]} ${arr_cpass1_outputs[*]} && mkdir -p \$OUTPATH/meta && cp ${arr_cpass1_calib_list[${runNumber}]} \$OUTPATH/meta/${arr_cpass1_calib_list[${runNumber}]}" | |
1188 | echo ; echo | |
1189 | ||
1190 | #arr_cpass1_ESD_list[${runNumber}]="${commonOutputPath}/meta/cpass1.ESD.run${runNumber}.list" | |
1191 | arr_cpass1_ESD_list[${runNumber}]="cpass1.ESD.run${runNumber}.list" | |
1192 | echo "### Lists CPass1 ESDs ###" | |
1193 | echo "${arr_cpass1_ESD_list[${runNumber}]}: benchmark.sh ${arr_cpass1_outputs[*]}" | |
1194 | echo " LOCAL ./benchmark.sh PrintValues esd ${arr_cpass1_ESD_list[${runNumber}]} ${arr_cpass1_outputs[*]} && mkdir -p \$OUTPATH/meta && cp ${arr_cpass1_ESD_list[${runNumber}]} \$OUTPATH/meta/${arr_cpass1_ESD_list[${runNumber}]}" | |
1195 | echo ; echo | |
1196 | ||
1197 | #arr_cpass1_filtered_list[${runNumber}]="${commonOutputPath}/meta/cpass1.filtered.run${runNumber}.list" | |
1198 | arr_cpass1_filtered_list[${runNumber}]="cpass1.filtered.run${runNumber}.list" | |
1199 | echo "### Lists CPass1 filtered ###" | |
1200 | echo "${arr_cpass1_filtered_list[${runNumber}]}: benchmark.sh ${arr_cpass1_outputs[*]}" | |
1201 | echo " LOCAL ./benchmark.sh PrintValues filteredTree ${arr_cpass1_filtered_list[${runNumber}]} ${arr_cpass1_outputs[*]} && mkdir -p \$OUTPATH/meta && cp ${arr_cpass1_filtered_list[${runNumber}]} \$OUTPATH/meta/${arr_cpass1_filtered_list[${runNumber}]}" | |
1202 | echo ; echo | |
444b9916 | 1203 | |
71353a5f | 1204 | #CPass1 merging |
ef77d296 | 1205 | #arr_cpass1_merged[${runNumber}]="${commonOutputPath}/meta/merge.cpass1.run${runNumber}.done" |
1206 | arr_cpass1_merged[${runNumber}]="merge.cpass1.run${runNumber}.done" | |
1207 | echo "### Merges CPass1 files ###" | |
1208 | echo "cpass1.localOCDB.${runNumber}.tgz.done ${arr_cpass1_merged[${runNumber}]}: benchmark.sh ${configFile} ${arr_cpass1_calib_list[${runNumber}]} ${arr_cpass1_QA_list[${runNumber}]} ${copyFiles[@]}" | |
1209 | echo " ${alirootEnv} ./benchmark.sh MergeCPass1 \$OUTPATH/000${runNumber}/cpass1 ${currentDefaultOCDB} ${configFile} ${runNumber} ${arr_cpass1_calib_list[${runNumber}]} ${arr_cpass1_QA_list[${runNumber}]} ${arr_cpass1_filtered_list[${runNumber}]} ${extraOpts[@]}"" " | |
1210 | echo ; echo | |
fc7e1e45 | 1211 | |
6fc48f9e | 1212 | #CPass0 wrapped in a profiling tool (valgrind,....) |
39527a15 | 1213 | if [[ -n ${profilingCommand} ]]; then |
0eaf38c7 | 1214 | inputFile=$(grep -m1 "${runNumber}/" ${inputFileList}) |
1215 | [[ -z ${nEventsProfiling} ]] && nEventsProfiling=2 | |
1216 | currentDefaultOCDB=$(setYear ${inputFile} ${defaultOCDB}) | |
1217 | jobindex="profiling" | |
1218 | ||
ef77d296 | 1219 | #arr_cpass0_profiled_outputs[${runNumber}]="${commonOutputPath}/meta/profiling.cpass0.job${jobindex}.run${runNumber}.done" |
1220 | arr_cpass0_profiled_outputs[${runNumber}]="profiling.cpass0.job${jobindex}.run${runNumber}.done" | |
1221 | echo "### CPass0 in a profiler ###" | |
1222 | echo "${arr_cpass0_profiled_outputs[${runNumber}]}: benchmark.sh ${configFile} ${copyFiles[@]}" | |
0eaf38c7 | 1223 | profilingCommand=$(encSpaces "${profilingCommand}") |
ef77d296 | 1224 | echo " ${alirootEnv} ./benchmark.sh CPass0 \$OUTPATH/000${runNumber}/${jobindex} ${inputFile} ${nEventsProfiling} ${currentDefaultOCDB} ${configFile} ${runNumber} ${jobindex} ${extraOpts[@]} useProfilingCommand=${profilingCommand}" |
1225 | echo ; echo | |
6fc48f9e | 1226 | fi |
1227 | ||
444b9916 | 1228 | done #runs |
fc7e1e45 | 1229 | |
5dfac970 | 1230 | #Summary |
ef77d296 | 1231 | echo "### Summary ###" |
1232 | echo "summary.log: benchmark.sh ${configFile} ${arr_cpass1_merged[*]}" | |
1233 | echo " ${alirootEnv} ./benchmark.sh MakeSummary ${configFile} ${extraOpts[@]}" | |
1234 | echo ; echo | |
cb473870 | 1235 | |
5de3064b | 1236 | return 0 |
78d0bbfd | 1237 | ) |
cb473870 | 1238 | |
1239 | goPrintValues() | |
78d0bbfd | 1240 | ( |
cb473870 | 1241 | #print the values given the key from any number of files (store in output file on second argument) |
71353a5f | 1242 | if [[ $# -lt 3 ]]; then |
1243 | echo "goPrintValues key outputFile inputFiles" | |
ef77d296 | 1244 | echo "if outputFile is \"-\" don't redirect to a file" |
71353a5f | 1245 | return |
1246 | fi | |
1247 | key=${1} | |
1248 | outputFile=${2} | |
1249 | [[ ${outputFile} =~ "-" ]] && outputFile="" | |
cb473870 | 1250 | shift 2 #remove 2 first arguments from arg list to only pass the input files to awk |
71353a5f | 1251 | awk -v key=${key} '$0 ~ key" " {print $2}' "$@" | tee ${outputFile} |
5de3064b | 1252 | return 0 |
78d0bbfd | 1253 | ) |
fc7e1e45 | 1254 | |
5dfac970 | 1255 | goCreateQAplots() |
78d0bbfd | 1256 | ( |
5dfac970 | 1257 | umask 0002 |
71353a5f | 1258 | mergedQAfileList=${1} |
1259 | productionID=${2} | |
1260 | outputDir=${3} | |
1261 | configFile=${4} | |
1262 | shift 4 | |
e48b4924 | 1263 | if ! parseConfig ${configFile} "$@"; then return 1; fi |
71353a5f | 1264 | |
39527a15 | 1265 | #record the working directory provided by the batch system |
1266 | batchWorkingDirectory=${PWD} | |
1267 | ||
71353a5f | 1268 | [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource} |
fc7e1e45 | 1269 | |
71353a5f | 1270 | [[ -z ${logOutputDir} ]] && logOutputDir=${PWD} |
0eaf38c7 | 1271 | [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/makeQAplots.log |
71353a5f | 1272 | echo "${0} $*" |
1273 | ||
1274 | olddir=${PWD} | |
1275 | mkdir -p ${outputDir} | |
1276 | cd ${outputDir} | |
839b07ca | 1277 | [[ ! "${PWD}" =~ "${outputDir}" ]] && echo "PWD is not equal to outputDir=${outputDir}" && cd ${olddir} && return 1 |
fc7e1e45 | 1278 | |
ef77d296 | 1279 | inputFiles=( |
1280 | "${batchWorkingDirectory}/runQA.sh" | |
1281 | "${ALICE_ROOT}/PWGPP/QA/scripts/runQA.sh" | |
1282 | ) | |
1283 | for file in ${inputFiles[*]}; do | |
1284 | [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} . | |
1285 | done | |
1286 | ||
1287 | echo "running QA with command:" | |
1288 | echo ./runQA.sh inputList="${mergedQAfileList}" inputListHighPtTrees="${filteringList}" ocdbStorage="${defaultOCDB}" | |
1289 | ./runQA.sh inputList="${mergedQAfileList}" inputListHighPtTrees="${filteringList}" ocdbStorage="${defaultOCDB}" | |
71353a5f | 1290 | cd ${olddir} |
5de3064b | 1291 | return 0 |
78d0bbfd | 1292 | ) |
fc7e1e45 | 1293 | |
5dfac970 | 1294 | goTest() |
78d0bbfd | 1295 | ( |
1296 | echo AA | |
1297 | ) | |
fc7e1e45 | 1298 | |
5dfac970 | 1299 | alirootInfo() |
78d0bbfd | 1300 | ( |
5dfac970 | 1301 | umask 0002 |
71353a5f | 1302 | # save aliroot repository info |
1303 | [[ -z "${ALICE_ROOT}" ]] && return 1 | |
1304 | ||
1305 | echo "\${ALICE_ROOT}=${ALICE_ROOT}" | |
1306 | echo "\${ROOTSYS}=${ROOTSYS}" | |
1307 | echo "\${PATH}=${PATH}" | |
1308 | echo "\${LD_LIBRARY_PATH}=${LD_LIBRARY_PATH}" | |
0eaf38c7 | 1309 | echo |
71353a5f | 1310 | |
1311 | pushd ${PWD} | |
1312 | cd ${ALICE_ROOT} | |
71353a5f | 1313 | |
1314 | currentBranch=$(git rev-parse --abbrev-ref HEAD) | |
1315 | git status | |
5dfac970 | 1316 | echo "" |
1317 | echo "" | |
71353a5f | 1318 | git diff ${currentBranch} |
1319 | popd | |
5de3064b | 1320 | return 0 |
78d0bbfd | 1321 | ) |
fc7e1e45 | 1322 | |
5dfac970 | 1323 | setYear() |
78d0bbfd | 1324 | ( |
5dfac970 | 1325 | #set the year |
71353a5f | 1326 | # ${1} - year to be set |
1327 | # ${2} - where to set the year | |
1328 | year1=$(guessYear ${1}) | |
1329 | year2=$(guessYear ${2}) | |
1330 | local path=${2} | |
1331 | [[ ${year1} -ne ${year2} && -n ${year2} && -n ${year1} ]] && path=${2/\/${year2}\//\/${year1}\/} | |
1332 | echo ${path} | |
5de3064b | 1333 | return 0 |
78d0bbfd | 1334 | ) |
fc7e1e45 | 1335 | |
5dfac970 | 1336 | guessPeriod() |
78d0bbfd | 1337 | ( |
5dfac970 | 1338 | #guess the period from the path, pick the rightmost one |
1339 | local IFS="/" | |
71353a5f | 1340 | declare -a path=( ${1} ) |
5dfac970 | 1341 | local dirDepth=${#path[*]} |
1342 | for ((x=${dirDepth}-1;x>=0;x--)); do | |
1343 | local field=${path[${x}]} | |
71353a5f | 1344 | [[ ${field} =~ ^LHC[0-9][0-9][a-z]$ ]] && period=${field} && break |
5dfac970 | 1345 | done |
71353a5f | 1346 | echo ${period} |
5de3064b | 1347 | return 0 |
78d0bbfd | 1348 | ) |
fc7e1e45 | 1349 | |
5dfac970 | 1350 | guessYear() |
78d0bbfd | 1351 | ( |
5dfac970 | 1352 | #guess the year from the path, pick the rightmost one |
1353 | local IFS="/" | |
71353a5f | 1354 | declare -a path=( ${1} ) |
5dfac970 | 1355 | local dirDepth=${#path[*]} |
1356 | for ((x=${dirDepth}-1;x>=0;x--)); do | |
1357 | local field=${path[${x}]} | |
71353a5f | 1358 | [[ ${field} =~ ^20[0-9][0-9]$ ]] && year=${field} && break |
5dfac970 | 1359 | done |
71353a5f | 1360 | echo ${year} |
5de3064b | 1361 | return 0 |
78d0bbfd | 1362 | ) |
fc7e1e45 | 1363 | |
5dfac970 | 1364 | guessRunNumber() |
78d0bbfd | 1365 | ( |
5dfac970 | 1366 | #guess the run number from the path, pick the rightmost one |
71353a5f | 1367 | #works for /path/foo/000123456/bar/... |
1368 | #and /path/foo.run123456.bar | |
1369 | local IFS="/." | |
1370 | declare -a path=( ${1} ) | |
5dfac970 | 1371 | local dirDepth=${#path[*]} |
1372 | for ((x=${dirDepth}-1;x>=0;x--)); do | |
1373 | local field=${path[${x}]} | |
71353a5f | 1374 | field=${field/run/000} |
617f8886 | 1375 | [[ ${field} =~ [0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && runNumber=${field#000} && break |
5dfac970 | 1376 | done |
71353a5f | 1377 | echo ${runNumber} |
5de3064b | 1378 | return 0 |
78d0bbfd | 1379 | ) |
fc7e1e45 | 1380 | |
cb473870 | 1381 | validateLog() |
78d0bbfd | 1382 | ( |
71353a5f | 1383 | log=${1} |
cb473870 | 1384 | errorConditions=( |
71353a5f | 1385 | 'There was a crash' |
1386 | 'floating' | |
1387 | 'error while loading shared libraries' | |
1388 | 'std::bad_alloc' | |
1389 | 's_err_syswatch_' | |
1390 | 'Thread [0-9]* (Thread' | |
1391 | 'AliFatal' | |
1392 | 'core dumped' | |
1393 | '\.C.*error:.*\.h: No such file' | |
839b07ca | 1394 | 'line.*Aborted' |
71353a5f | 1395 | ) |
1396 | ||
1397 | warningConditions=( | |
1398 | 'This is serious !' | |
1399 | 'rocVoltage out of range:' | |
cb473870 | 1400 | ) |
1401 | ||
71353a5f | 1402 | local logstatus=0 |
1403 | local errorSummary="" | |
1404 | local warningSummary="" | |
1405 | ||
1406 | for ((i=0; i<${#errorConditions[@]};i++)); do | |
1407 | local tmp=$(grep -m1 -e "${errorConditions[${i}]}" ${log}) | |
1408 | [[ -n ${tmp} ]] && tmp+=" : " | |
1409 | errorSummary+=${tmp} | |
1410 | done | |
1411 | ||
1412 | for ((i=0; i<${#warningConditions[@]};i++)); do | |
1413 | local tmp=$(grep -m1 -e "${warningConditions[${i}]}" ${log}) | |
1414 | [[ -n ${tmp} ]] && tmp+=" : " | |
1415 | warningSummary+=${tmp} | |
1416 | done | |
1417 | ||
1418 | if [[ -n ${errorSummary} ]]; then | |
1419 | echo "${errorSummary}" | |
1420 | return 1 | |
1421 | fi | |
cb473870 | 1422 | |
71353a5f | 1423 | if [[ -n ${warningSummary} ]]; then |
1424 | echo "${warningSummary}" | |
1425 | return 2 | |
1426 | fi | |
1427 | ||
1428 | return 0 | |
78d0bbfd | 1429 | ) |
cb473870 | 1430 | |
5dfac970 | 1431 | summarizeLogs() |
78d0bbfd | 1432 | ( |
5dfac970 | 1433 | #print a summary of logs |
1434 | logFiles=( | |
1435 | "*.log" | |
cb473870 | 1436 | "stdout" |
1437 | "stderr" | |
5dfac970 | 1438 | ) |
fc7e1e45 | 1439 | |
9c124791 | 1440 | #put dir information in the output |
1441 | echo "dir $PWD" | |
1442 | ||
cb473870 | 1443 | #check logs |
71353a5f | 1444 | local logstatus=0 |
5dfac970 | 1445 | for log in ${logFiles[*]}; do |
71353a5f | 1446 | finallog=${PWD%/}/${log} |
1447 | [[ ! -f ${log} ]] && continue | |
1448 | errorSummary=$(validateLog ${log}) | |
1449 | validationStatus=$? | |
1450 | if [[ ${validationStatus} -eq 0 ]]; then | |
5dfac970 | 1451 | #in pretend mode randomly report an error in rec.log some cases |
71353a5f | 1452 | if [[ -n ${pretend} && "${log}" == "rec.log" ]]; then |
617f8886 | 1453 | #[[ $(( ${RANDOM}%2 )) -ge 1 ]] && echo "${finallog} BAD random error" || echo "${finallog} OK" |
1454 | echo "${finallog} OK" | |
fc7e1e45 | 1455 | else |
71353a5f | 1456 | echo "${finallog} OK" |
fc7e1e45 | 1457 | fi |
71353a5f | 1458 | elif [[ ${validationStatus} -eq 1 ]]; then |
1459 | echo "${finallog} BAD ${errorSummary}" | |
ab946185 | 1460 | logstatus=1 |
71353a5f | 1461 | elif [[ ${validationStatus} -eq 2 ]]; then |
1462 | echo "${finallog} OK MWAH ${errorSummary}" | |
5dfac970 | 1463 | fi |
1464 | done | |
cb473870 | 1465 | |
1466 | #report core files | |
71353a5f | 1467 | while read x; do |
1468 | echo ${x} | |
1469 | chmod 644 ${x} | |
1470 | gdb --batch --quiet -ex "bt" -ex "quit" aliroot ${x} > stacktrace_${x//\//_}.log | |
1471 | done < <(/bin/ls ${PWD}/*/core 2>/dev/null; /bin/ls ${PWD}/core 2>/dev/null) | |
cb473870 | 1472 | |
71353a5f | 1473 | return ${logstatus} |
78d0bbfd | 1474 | ) |
fc7e1e45 | 1475 | |
5dfac970 | 1476 | spitOutLocalOCDBaccessConfig() |
1477 | { | |
1478 | umask 0002 | |
71353a5f | 1479 | #find ${1} -name "*root" | \ |
1480 | /bin/ls -1 ${1}/*/*/*/*.root 2>/dev/null | \ | |
5dfac970 | 1481 | while read line |
1482 | do | |
71353a5f | 1483 | local tmp=${line#${1}} |
5dfac970 | 1484 | echo ${tmp%/*} | \ |
71353a5f | 1485 | awk -v ocdb=${1} '{print " man->SetSpecificStorage(\""$1"\",\"local://"ocdb"\");"}' |
5dfac970 | 1486 | done |
5de3064b | 1487 | return 0 |
5dfac970 | 1488 | } |
fc7e1e45 | 1489 | |
71353a5f | 1490 | goMakeLocalOCDBaccessConfig() |
5dfac970 | 1491 | { |
1492 | umask 0002 | |
1493 | # make a script that sets the specific storages form all the root files produced by CPass0 | |
71353a5f | 1494 | local localOCDBpathCPass0=${1} |
1495 | local OCDBpathPrefix=${2} | |
1496 | [[ -z ${OCDBpathPrefix} ]] && OCDBpathPrefix="." | |
1497 | ||
1498 | if [[ -f ${localOCDBpathCPass0} && ${localOCDBpathCPass0} =~ \.tgz$ ]]; then | |
1499 | tar xzf ${localOCDBpathCPass0} | |
1500 | local localOCDBpathCPass0="${OCDBpathPrefix}/OCDB" | |
5dfac970 | 1501 | fi |
fc7e1e45 | 1502 | |
5dfac970 | 1503 | echo |
1504 | echo creating the specific storage script | |
1505 | echo localOCDBaccessConfig.C | |
71353a5f | 1506 | echo based on OCDB: ${localOCDBaccessConfig} |
5dfac970 | 1507 | echo |
fc7e1e45 | 1508 | |
5dfac970 | 1509 | local tempLocalOCDB="" |
1510 | if [[ -f localOCDBaccessConfig.C ]]; then | |
0eaf38c7 | 1511 | tempLocalOCDB=$(mktemp -t tempLocalOCDB.XXXXXX) |
71353a5f | 1512 | echo "egrep "SetSpecificStorage" localOCDBaccessConfig.C > ${tempLocalOCDB}" |
1513 | egrep "SetSpecificStorage" localOCDBaccessConfig.C > ${tempLocalOCDB} | |
5dfac970 | 1514 | fi |
fc7e1e45 | 1515 | |
5dfac970 | 1516 | echo "localOCDBaccessConfig()" > localOCDBaccessConfig.C |
1517 | echo "{" >> localOCDBaccessConfig.C | |
1518 | echo " AliCDBManager* man = AliCDBManager::Instance();" >> localOCDBaccessConfig.C | |
71353a5f | 1519 | spitOutLocalOCDBaccessConfig ${localOCDBpathCPass0}|sort|uniq >> localOCDBaccessConfig.C |
1520 | [[ -f "${tempLocalOCDB}" ]] && cat ${tempLocalOCDB} >> localOCDBaccessConfig.C | |
5dfac970 | 1521 | echo "}" >> localOCDBaccessConfig.C |
fc7e1e45 | 1522 | |
71353a5f | 1523 | [[ -f "${tempLocalOCDB}" ]] && rm -f ${tempLocalOCDB} |
fc7e1e45 | 1524 | |
5dfac970 | 1525 | if ! grep SetSpecificStorage localOCDBaccessConfig.C; then |
1526 | echo | |
1527 | echo "!!!!!!! CPass0 produced no OCDB entries" | |
1528 | return 1 | |
1529 | fi | |
5de3064b | 1530 | return 0 |
5dfac970 | 1531 | } |
fc7e1e45 | 1532 | |
cb473870 | 1533 | goMakeFilteredTrees() |
78d0bbfd | 1534 | ( |
71353a5f | 1535 | outputDir=${1} |
1536 | runNumber=${2} | |
cb473870 | 1537 | #get path to input list |
71353a5f | 1538 | inputListfiles=${3} |
cb473870 | 1539 | #get scale number for tracks |
71353a5f | 1540 | filterT=${4} |
cb473870 | 1541 | #get scale number for V0s |
71353a5f | 1542 | filterV=${5} |
cb473870 | 1543 | #get OCDB path (optional) |
1544 | OCDBpath=${6} | |
1545 | #get max number of files | |
1546 | maxFiles=${7-"1000000"} | |
1547 | #get offset of first file | |
1548 | offsetFile=${8-"0"} | |
1549 | #get max number of events | |
1550 | maxEvents=${9-"30000000"} | |
1551 | #get offset of first event | |
1552 | offsetEvent=${10-"0"} | |
1553 | configFile=${11-"benchmark.config"} | |
1554 | esdFileName=${12-"AliESDs_Barrel.root"} | |
71353a5f | 1555 | shift 12 |
6fc48f9e | 1556 | if ! parseConfig ${configFile} "$@"; then return 1; fi |
39527a15 | 1557 | |
1558 | #record the working directory provided by the batch system | |
1559 | batchWorkingDirectory=${PWD} | |
1560 | ||
78d0bbfd | 1561 | [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD} |
ef77d296 | 1562 | doneFileBase=filtering.cpass1.run${runNumber}.done |
1563 | doneFileTmp=${batchWorkingDirectory}/${doneFileBase} | |
1564 | doneFile=${commonOutputPath}/meta/${doneFileBase} | |
71353a5f | 1565 | |
cb473870 | 1566 | cat > filtering.log << EOF |
1567 | goMakeFilteredTrees config: | |
71353a5f | 1568 | runpath=${runpath} |
1569 | outputDir=${outputDir} | |
1570 | commonOutputPath=${commonOutputPath} | |
1571 | ALICE_ROOT=${ALICE_ROOT} | |
1572 | PATH=${PATH} | |
1573 | offsetEvent=$offsetEvent | |
1574 | configFile=$configFile | |
1575 | esdFileName=$esdFileName | |
fab31805 | 1576 | inputListfiles=$inputListfiles |
1577 | doneFile=$doneFile | |
cb473870 | 1578 | EOF |
1579 | ||
fab31805 | 1580 | #runpath=${outputDir} |
1581 | #[[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR} | |
1582 | #[[ ${reconstructInTemporaryDir} -eq 1 ]] && runpath=$(mktemp -d -t goMakeFilteredTrees.XXXXXX) | |
1583 | #mkdir -p ${outputDir} | |
1584 | #mkdir -p ${runpath} | |
1585 | #if ! cd ${runpath}; then | |
1586 | # echo "PWD=$PWD is not the runpath=${runpath}" | |
1587 | # touch ${doneFile} | |
1588 | # return 1 | |
1589 | #fi | |
1590 | ||
cb473870 | 1591 | if [[ -z ${pretend} ]];then |
71353a5f | 1592 | aliroot -l -b -q "${ALICE_ROOT}/PWGPP/macros/runFilteringTask.C(\"${inputListfiles}\",${filterT},${filterV},\"${OCDBpath}\",${maxFiles},${offsetFile},${maxEvents},${offsetEvent},\"${esdFileName}\")" &>> filtering.log |
cb473870 | 1593 | else |
0eaf38c7 | 1594 | sleep ${pretendDelay} |
71353a5f | 1595 | touch filtering.log FilterEvents_Trees.root |
cb473870 | 1596 | fi |
1597 | pwd | |
71353a5f | 1598 | /bin/ls |
71353a5f | 1599 | summarizeLogs >> ${doneFile} |
fab31805 | 1600 | |
1601 | #echo mv -f * ${outputDir} | |
1602 | #mv -f * ${outputDir} | |
1603 | #[[ -f ${outputDir}/FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFile} | |
1604 | #cd ${commonOutputPath} | |
1605 | #[[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath} | |
e344162c | 1606 | |
ef77d296 | 1607 | cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile" |
1608 | [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp} | |
e344162c | 1609 | |
5de3064b | 1610 | return 0 |
78d0bbfd | 1611 | ) |
cb473870 | 1612 | |
1613 | submit() | |
1614 | { | |
1615 | umask 0002 | |
444b9916 | 1616 | [[ $# -lt 5 ]] && echo "at least 5 args needed, you supplied $#" && return 1 |
71353a5f | 1617 | JobID=${1} |
1618 | startID=${2} | |
1619 | endID=${3} | |
1620 | waitForJOBID=${4} | |
1621 | command=${5} | |
444b9916 | 1622 | shift 5 |
78d0bbfd | 1623 | local commandArgs=("$@") |
444b9916 | 1624 | |
1625 | #add quote strings around the extra arguments | |
1626 | for ((i=0; i<${#commandArgs[@]}; i++)); do | |
1627 | commandArgs[i]=\"${commandArgs[i]}\" | |
1628 | done | |
cb473870 | 1629 | |
71353a5f | 1630 | [[ -z ${waitForJOBID} ]] && waitForJOBID=0 |
cb473870 | 1631 | |
1632 | newFarm=$(which qsub|grep "^/usr/bin/qsub") | |
1633 | ||
1634 | batchSystem="SGE" | |
1635 | ||
71353a5f | 1636 | if [[ -z "${newFarm}" ]] |
cb473870 | 1637 | then |
1638 | #old LSF | |
1639 | # submit it (as job array) | |
71353a5f | 1640 | nFiles=$(( ${endID}-${startID}+1 )) |
1641 | while [ ${startID} -le ${nFiles} ] ; do | |
1642 | if [ $(expr ${nFiles} - ${startID}) -gt 999 ] ; then | |
1643 | endID=$(expr ${startID} + 999) | |
cb473870 | 1644 | else |
71353a5f | 1645 | endID=${nFiles} |
cb473870 | 1646 | fi |
1647 | if [[ ${waitForJOBID} -eq 0 ]]; then | |
71353a5f | 1648 | echo ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}" |
1649 | ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}" | |
cb473870 | 1650 | else |
71353a5f | 1651 | echo ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -w "ended(${waitForJOBID})" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}" |
1652 | ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -w "ended(${waitForJOBID})" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}" | |
cb473870 | 1653 | fi |
71353a5f | 1654 | startID=$(expr ${endID} + 1) |
cb473870 | 1655 | done |
1656 | else | |
1657 | #new SGE farm | |
1658 | if [[ ${waitForJOBID} =~ "000" ]]; then | |
444b9916 | 1659 | echo ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}" |
1660 | ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}" | |
cb473870 | 1661 | else |
444b9916 | 1662 | echo ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -hold_jid "${waitForJOBID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}" |
1663 | ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -hold_jid "${waitForJOBID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}" | |
cb473870 | 1664 | fi |
1665 | fi | |
5de3064b | 1666 | return 0 |
cb473870 | 1667 | } |
1668 | ||
1669 | goSubmitBatch() | |
5dfac970 | 1670 | { |
71353a5f | 1671 | if [[ $# -lt 3 ]]; then |
1672 | echo "minimal use:" | |
1673 | echo " ${0} submit fileList productionID configFile" | |
1674 | return 0 | |
1675 | fi | |
1676 | ||
1677 | productionID=${1} | |
1678 | inputList=${2} | |
1679 | configFile=${3:-"benchmark.config"} | |
0eaf38c7 | 1680 | #if which greadlink; then configFile=$(greadlink -f ${configFile}); fi |
71353a5f | 1681 | shift 3 |
444b9916 | 1682 | extraOpts=("$@") |
6fc48f9e | 1683 | if ! parseConfig ${configFile} "${extraOpts[@]}"; then return 1; fi |
39527a15 | 1684 | |
b0e401b4 | 1685 | #batch systems/makeflow sometimes handle spaces in arguments poorly, so encode them |
1686 | for (( i=0;i<${#extraOpts[@]};i++ )); do | |
1687 | extraOpts[i]=$(encSpaces "${extraOpts[i]}") | |
1688 | done | |
1689 | extraOpts+=("encodedSpaces=1") | |
ef77d296 | 1690 | #this removes the copy of the done file used by makeflow (in the running dir) |
1691 | extraOpts+=("removeTMPdoneFile=1") | |
b0e401b4 | 1692 | |
39527a15 | 1693 | #record the working directory provided by the batch system |
1694 | batchWorkingDirectory=${PWD} | |
fc7e1e45 | 1695 | |
5dfac970 | 1696 | #redirect all output to submit.log |
1697 | echo "redirecting all output to ${PWD}/submit_${productionID//"/"/_}.log" | |
1698 | exec 7>&1 | |
1699 | exec 1>submit_${productionID//"/"/_}.log 2>&1 | |
fc7e1e45 | 1700 | |
5dfac970 | 1701 | umask 0002 |
71353a5f | 1702 | echo ${0}" submit $*" |
1703 | if [[ -z "${inputList}" || -z "${productionID}" ]] | |
5dfac970 | 1704 | then |
fc7e1e45 | 1705 | echo |
71353a5f | 1706 | echo " Usage: ${0} submit inputList productionID [configFile=benchmark.config]" |
fc7e1e45 | 1707 | echo |
71353a5f | 1708 | return |
fc7e1e45 | 1709 | fi |
fc7e1e45 | 1710 | |
5dfac970 | 1711 | # check if config file is there |
71353a5f | 1712 | if [ ! -f ${configFile} ]; then |
1713 | echo "ERROR! Config File '${configFile}' not found" >&2 | |
1714 | return | |
5dfac970 | 1715 | else |
71353a5f | 1716 | echo "Using Config File: '${configFile}'" |
fc7e1e45 | 1717 | fi |
1718 | ||
9510565c | 1719 | [[ ! -f ${alirootEnv} ]] && echo "alirootEnv script ${alirootEnv} not found!..." && return 1 |
fc7e1e45 | 1720 | |
71353a5f | 1721 | #move the script, config and some other stuff to ${commonOutputPath} first, then use them from there |
0eaf38c7 | 1722 | self=${0} |
1723 | #if which greadlink; then self=$(greadlink -f "${0}"); fi | |
71353a5f | 1724 | configPath=$(dirname ${configFile}) |
5dfac970 | 1725 | export commonOutputPath=${baseOutputDirectory}/${productionID} |
71353a5f | 1726 | |
cb473870 | 1727 | mkdir -p ${commonOutputPath} |
71353a5f | 1728 | mkdir -p ${commonOutputPath}/logs |
1729 | mkdir -p ${commonOutputPath}/meta | |
1730 | ||
1731 | cp ${self} ${commonOutputPath} | |
1732 | cp ${configFile} ${commonOutputPath} | |
1733 | cp ${inputList} ${commonOutputPath} | |
cb473870 | 1734 | self=${commonOutputPath}/${self##*/} |
71353a5f | 1735 | chmod u+x ${self} |
cb473870 | 1736 | configFile=${commonOutputPath}/${configFile##*/} |
1737 | inputList=${commonOutputPath}/${inputList##*/} | |
fc7e1e45 | 1738 | |
5dfac970 | 1739 | #convert to absolut pathnames |
0eaf38c7 | 1740 | #if which greadlink; then inputList=$(greadlink -f "${inputList}"); fi |
5dfac970 | 1741 | #make list of runs |
71353a5f | 1742 | if [[ -z ${runNumber} ]]; then |
1743 | listOfRuns=($(while read x; do guessRunNumber ${x}; done < ${inputList} | sort | uniq)) | |
5dfac970 | 1744 | else |
71353a5f | 1745 | listOfRuns=${runNumber} |
5dfac970 | 1746 | fi |
fc7e1e45 | 1747 | |
0eaf38c7 | 1748 | #if which greadlink; then alirootSource=$(greadlink -f "${alirootSource}"); fi |
fc7e1e45 | 1749 | |
5dfac970 | 1750 | echo "" |
1751 | echo "### BEGIN CONFIGURATION ###" | |
1752 | echo "" | |
1753 | echo "GENERAL:" | |
1754 | echo "" | |
71353a5f | 1755 | echo " productionID: ${productionID}" |
1756 | echo " batchCommand: ${batchCommand}" | |
1757 | echo " batchFlags: ${batchFlags}" | |
9510565c | 1758 | echo " alirootEnv: ${alirootEnv}" |
1759 | ${alirootEnv} echo ' ALICE_ROOT: ${ALICE_ROOT}' | |
1760 | ${alirootEnv} echo ' ALIROOT_RELEASE: ${ALICE_RELEASE}' | |
71353a5f | 1761 | echo " inputList: ${inputList}" |
1762 | echo " configPath: ${configPath}" | |
1763 | echo " commonOutputPath: ${commonOutputPath}" | |
1764 | echo " defaultOCDB: ${defaultOCDB}" | |
1765 | echo " autoOCDB: ${autoOCDB}" | |
1766 | echo " recoTriggerOptions: ${recoTriggerOptions}" | |
5dfac970 | 1767 | echo " runs:" |
1768 | echo " ${listOfRuns[*]}" | |
1769 | echo "" | |
1770 | echo "THE TRAIN WILL RUN:" | |
fc7e1e45 | 1771 | |
71353a5f | 1772 | if [ ${runCPass0reco} -eq 1 ]; then |
5dfac970 | 1773 | echo " Pass0 - Recontruction" |
1774 | fi | |
fc7e1e45 | 1775 | |
71353a5f | 1776 | if [ ${runCPass0MergeMakeOCDB} -eq 1 ]; then |
5dfac970 | 1777 | echo " Pass0 - merging and OCDB export" |
1778 | fi | |
fc7e1e45 | 1779 | |
71353a5f | 1780 | if [ ${runCPass1reco} -eq 1 ]; then |
5dfac970 | 1781 | echo " Pass1 - Recontruction" |
1782 | fi | |
71353a5f | 1783 | if [ ${runCPass1MergeMakeOCDB} -eq 1 ]; then |
5dfac970 | 1784 | echo " Pass1 - merging and OCDB export" |
1785 | fi | |
fc7e1e45 | 1786 | |
5dfac970 | 1787 | echo "" |
1788 | echo "LIMITS:" | |
71353a5f | 1789 | echo " max. Events/Chunk: ${nEvents}" |
1790 | echo " max. Number of Chunks per Run: ${nMaxChunks}" | |
5dfac970 | 1791 | echo "" |
1792 | echo "### END CONFIGURATION ###" | |
1793 | echo "" | |
fc7e1e45 | 1794 | |
36b5de1a | 1795 | |
5dfac970 | 1796 | # check if input file is there |
71353a5f | 1797 | if [ ! -f ${inputList} ]; then |
1798 | echo "ERROR! Input List '${inputList}' not found" >&2 | |
1799 | return | |
5dfac970 | 1800 | fi |
1801 | ||
1802 | # define jobid (for dependent jobs) | |
cb473870 | 1803 | date=$(date +%Y_%m_%d_%H%M%S) |
5dfac970 | 1804 | #for each run we submit one jobarray: |
fc7e1e45 | 1805 | for runNumber in ${listOfRuns[*]}; do |
cb473870 | 1806 | |
71353a5f | 1807 | [[ -z ${runNumber} ]] && continue |
fc7e1e45 | 1808 | [[ ! ${runNumber} =~ ^[0-9]*[0-9]$ ]] && continue |
cb473870 | 1809 | |
1810 | JOBpostfix="${productionID//"/"/_}_${runNumber}_${date}" | |
1811 | JOBID1="p0_${JOBpostfix}" | |
1812 | JOBID1wait="w0_${JOBpostfix}" | |
1813 | JOBID2="m0_${JOBpostfix}" | |
1814 | JOBID2wait="wm0_${JOBpostfix}" | |
1815 | JOBID3="op0_${JOBpostfix}" | |
1816 | JOBID3wait="wop0_${JOBpostfix}" | |
1817 | JOBID4="p1_${JOBpostfix}" | |
1818 | JOBID4wait="w1_${JOBpostfix}" | |
1819 | JOBID5="m1_${JOBpostfix}" | |
1820 | JOBID5wait="wm1_${JOBpostfix}" | |
1821 | JOBID6="s1_${JOBpostfix}" | |
1822 | JOBID6wait="ws1_${JOBpostfix}" | |
1823 | JOBID7="QA_${JOBpostfix}" | |
1824 | JOBmakeESDlistCPass1="lp1_${JOBpostfix}" | |
1825 | JOBfilterESDcpass1="fp1_${JOBpostfix}" | |
1826 | LASTJOB="000" | |
1827 | ||
0eaf38c7 | 1828 | oneInputFile=$(egrep -m1 "${runNumber}/" ${inputList}) |
cb473870 | 1829 | |
1830 | currentDefaultOCDB=${defaultOCDB} | |
1831 | [[ -z ${autoOCDB} ]] && autoOCDB=1 | |
1832 | if [[ ${autoOCDB} -ne 0 ]]; then | |
71353a5f | 1833 | currentDefaultOCDB=$(setYear ${oneInputFile} ${defaultOCDB}) |
cb473870 | 1834 | fi |
ef77d296 | 1835 | period=$(guessPeriod ${oneInputFile}) |
1836 | year=$(guessYear ${oneInputFile}) | |
cb473870 | 1837 | |
71353a5f | 1838 | echo "submitting run ${runNumber} with OCDB ${currentDefaultOCDB}" |
1839 | ||
1840 | ############################################################################### | |
1841 | #run one chunk with valgrind: | |
6fc48f9e | 1842 | if [[ -n ${profilingCommand} ]]; then |
1843 | [[ -z ${nEventsProfiling} ]] && nEventsProfiling=2 | |
1844 | [[ -z ${profilingCommand} ]] && profilingCommand="/usr/bin/valgrind --tool=callgrind --num-callers=40 -v --trace-children=yes" | |
ef77d296 | 1845 | submit "profile-${JOBpostfix}" 1 1 000 "${alirootEnv} ${self}" CPass0 ${commonOutputPath}/${year}/${period}/000${runNumber}/${jobindex} ${oneInputFile} ${nEventsProfiling} ${currentDefaultOCDB} ${configFile} ${runNumber} ${jobindex} useProfilingCommand=$(encSpaces "${profilingCommand}") "${extraOpts[@]}" |
71353a5f | 1846 | fi |
fc7e1e45 | 1847 | |
5dfac970 | 1848 | ################################################################################ |
1849 | ################################################################################ | |
1850 | # run the CPass0 if requested | |
fc7e1e45 | 1851 | |
71353a5f | 1852 | if [ ${runCPass0reco} -eq 1 ]; then |
fc7e1e45 | 1853 | |
fc7e1e45 | 1854 | echo |
71353a5f | 1855 | echo "starting CPass0... for run ${runNumber}" |
fc7e1e45 | 1856 | echo |
fc7e1e45 | 1857 | |
5dfac970 | 1858 | # create directory and copy all files that are needed |
ef77d296 | 1859 | targetDirectory="${commonOutputPath}/${year}/${period}/000${runNumber}/cpass0" |
71353a5f | 1860 | mkdir -p ${targetDirectory} |
1861 | ||
cb473870 | 1862 | filesCPass0=( |
71353a5f | 1863 | "${configPath}/runCPass0.sh" |
1864 | "${configPath}/recCPass0.C" | |
1865 | "${configPath}/runCalibTrain.C" | |
1866 | "${configPath}/localOCDBaccessConfig.C" | |
1867 | "${configPath}/OCDB*.root" | |
1868 | "${configPath}/sim.C" | |
1869 | "${configPath}/rec.C" | |
1870 | "${configPath}/Config.C" | |
cb473870 | 1871 | ) |
1872 | for file in ${filesCPass0[*]}; do | |
1873 | [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath} | |
1874 | done | |
fc7e1e45 | 1875 | |
71353a5f | 1876 | localInputList=${targetDirectory}/${inputList##*/} |
1877 | [[ ! -f ${localInputList} ]] && egrep "\/000${runNumber}\/" ${inputList} > ${localInputList} | |
5dfac970 | 1878 | # limit nFiles to nMaxChunks |
71353a5f | 1879 | nFiles=$(wc -l < ${localInputList}) |
1880 | [[ ${nFiles} -eq 0 ]] && echo "list contains ZERO files! exiting..." && return 1 | |
1881 | echo "raw files in list: ${nFiles}" | |
1882 | if [[ ${nMaxChunks} -gt 0 && ${nMaxChunks} -le ${nFiles} ]]; then | |
1883 | nFiles=${nMaxChunks} | |
5dfac970 | 1884 | fi |
71353a5f | 1885 | echo "raw files to process: ${nFiles}" |
1886 | [[ -z "${percentProcessedFilesToContinue}" ]] && percentProcessedFilesToContinue=100 | |
1887 | if [[ ${percentProcessedFilesToContinue} -eq 100 ]]; then | |
1888 | nFilesToWaitFor=${nFiles} | |
5dfac970 | 1889 | else |
71353a5f | 1890 | nFilesToWaitFor=$(( ${nFiles}-${nFiles}/(100/(100-${percentProcessedFilesToContinue})) )) |
5dfac970 | 1891 | fi |
71353a5f | 1892 | echo "requested success rate is ${percentProcessedFilesToContinue}%" |
1893 | echo "merging will start after ${nFilesToWaitFor} jobs are done" | |
fc7e1e45 | 1894 | |
444b9916 | 1895 | submit ${JOBID1} 1 ${nFiles} 000 "${alirootEnv} ${self}" CPass0 ${targetDirectory} ${localInputList} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} -1 "${extraOpts[@]}" |
fc7e1e45 | 1896 | |
5dfac970 | 1897 | ## submit a monitoring job that will run until a certain number of jobs are done with reconstruction |
0ff60de9 | 1898 | submit "${JOBID1wait}" 1 1 000 "${alirootEnv} ${self}" WaitForOutput ${commonOutputPath} "meta/cpass0.job*.run${runNumber}.done" ${nFilesToWaitFor} ${maxSecondsToWait} |
71353a5f | 1899 | LASTJOB=${JOBID1wait} |
fc7e1e45 | 1900 | |
5dfac970 | 1901 | fi #end running CPass0 |
1902 | ################################################################################ | |
fc7e1e45 | 1903 | |
fc7e1e45 | 1904 | |
5dfac970 | 1905 | ################################################################################ |
1906 | # submit merging of CPass0, depends on the reconstruction | |
fc7e1e45 | 1907 | |
71353a5f | 1908 | if [ ${runCPass0MergeMakeOCDB} -eq 1 ]; then |
fc7e1e45 | 1909 | |
5dfac970 | 1910 | echo |
71353a5f | 1911 | echo "submit CPass0 merging for run ${runNumber}" |
5dfac970 | 1912 | echo |
fc7e1e45 | 1913 | |
ef77d296 | 1914 | targetDirectory="${commonOutputPath}/${year}/${period}/000${runNumber}/cpass0" |
71353a5f | 1915 | mkdir -p ${targetDirectory} |
fc7e1e45 | 1916 | |
cb473870 | 1917 | #copy the scripts |
1918 | filesMergeCPass0=( | |
71353a5f | 1919 | "${configPath}/OCDB.root" |
1920 | "${configPath}/mergeMakeOCDB.byComponent.sh" | |
1921 | "${configPath}/mergeMakeOCDB.sh" | |
1922 | "${configPath}/localOCDBaccessConfig.C" | |
1923 | "${configPath}/mergeByComponent.C" | |
1924 | "${configPath}/makeOCDB.C" | |
1925 | "${configPath}/merge.C" | |
cb473870 | 1926 | ) |
1927 | for file in ${filesMergeCPass0[*]}; do | |
1928 | [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath} | |
1929 | done | |
1930 | ||
444b9916 | 1931 | submit ${JOBID2} 1 1 "${LASTJOB}" "${alirootEnv} ${self}" MergeCPass0 ${targetDirectory} ${currentDefaultOCDB} ${configFile} ${runNumber} cpass0.calib.run${runNumber}.list "${extraOpts[@]}" |
71353a5f | 1932 | LASTJOB=${JOBID2} |
fc7e1e45 | 1933 | |
71353a5f | 1934 | if [[ -n ${generateMC} ]]; then |
444b9916 | 1935 | submit "mrl${JOBpostfix}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" PrintValues sim ${commonOutputPath}/meta/sim.run${runNumber}.list ${commonOutputPath}/meta/cpass0.job*.run${runNumber}.done |
71353a5f | 1936 | LASTJOB="mrl${JOBpostfix}" |
1937 | fi | |
fc7e1e45 | 1938 | |
5dfac970 | 1939 | echo |
fc7e1e45 | 1940 | fi |
5dfac970 | 1941 | # end of merging CPass0 |
1942 | ################################################################################ | |
fc7e1e45 | 1943 | |
5dfac970 | 1944 | ################################################################################ |
1945 | ################################################################################ | |
1946 | # run the CPass1 if requested | |
fc7e1e45 | 1947 | |
71353a5f | 1948 | if [ ${runCPass1reco} -eq 1 ]; then |
fc7e1e45 | 1949 | |
ef77d296 | 1950 | targetDirectory="${commonOutputPath}/${year}/${period}/000${runNumber}/cpass1" |
71353a5f | 1951 | rm -f ${commonOutputPath}/meta/cpass1.job*.run${runNumber}.done |
fc7e1e45 | 1952 | |
5dfac970 | 1953 | # safety feature: if we are re-running for any reason we want to delete the previous output first. |
71353a5f | 1954 | [[ -d ${targetDirectory} ]] && rm -rf ${targetDirectory}/* && echo "removed old output at ${targetDirectory}/*" |
fc7e1e45 | 1955 | |
5dfac970 | 1956 | echo |
71353a5f | 1957 | echo "starting CPass1... for run ${runNumber}" |
5dfac970 | 1958 | echo |
1959 | ||
1960 | # create directory and copy all files that are needed | |
71353a5f | 1961 | mkdir -p ${targetDirectory} |
5dfac970 | 1962 | |
cb473870 | 1963 | filesCPass1=( |
71353a5f | 1964 | "${configPath}/runCPass1.sh" |
1965 | "${configPath}/recCPass1.C" | |
1966 | "${configPath}/recCPass1_OuterDet.C" | |
1967 | "${configPath}/runCalibTrain.C" | |
1968 | "${configPath}/QAtrain_duo.C" | |
1969 | "${configPath}/localOCDBaccessConfig.C" | |
1970 | "${configPath}/OCDB.root" | |
cb473870 | 1971 | ) |
1972 | for file in ${filesCPass1[*]}; do | |
1973 | [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath} | |
1974 | done | |
5dfac970 | 1975 | |
71353a5f | 1976 | if [[ -n ${generateMC} ]]; then |
1977 | localInputList=${commonOutputPath}/meta/sim.run${runNumber}.list | |
1978 | else | |
1979 | localInputList=${targetDirectory}/${inputList##*/} | |
1980 | [[ ! -f ${localInputList} ]] && egrep "\/000${runNumber}\/" ${inputList} > ${localInputList} | |
1981 | fi | |
5dfac970 | 1982 | # limit nFiles to nMaxChunks |
71353a5f | 1983 | nFiles=$(wc -l < ${localInputList}) |
1984 | [[ ${nFiles} -eq 0 ]] && echo "list contains ZERO files! continuing..." && continue | |
1985 | echo "raw files in list: ${nFiles}" | |
1986 | if [[ ${nMaxChunks} -gt 0 && ${nMaxChunks} -le ${nFiles} ]]; then | |
1987 | nFiles=${nMaxChunks} | |
5dfac970 | 1988 | fi |
71353a5f | 1989 | echo "raw files to process: ${nFiles}" |
1990 | [[ -z "${percentProcessedFilesToContinue}" ]] && percentProcessedFilesToContinue=100 | |
1991 | if [[ ${percentProcessedFilesToContinue} -eq 100 ]]; then | |
1992 | nFilesToWaitFor=${nFiles} | |
fc7e1e45 | 1993 | else |
71353a5f | 1994 | nFilesToWaitFor=$(( ${nFiles}-${nFiles}/(100/(100-${percentProcessedFilesToContinue})) )) |
fc7e1e45 | 1995 | fi |
71353a5f | 1996 | echo "requested success rate is ${percentProcessedFilesToContinue}%" |
1997 | echo "merging will start after ${nFilesToWaitFor} jobs are done" | |
fc7e1e45 | 1998 | |
444b9916 | 1999 | submit ${JOBID4} 1 ${nFiles} "${LASTJOB}" "${alirootEnv} ${self}" CPass1 ${targetDirectory} ${localInputList} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} -1 "${extraOpts[@]}" |
fc7e1e45 | 2000 | |
5dfac970 | 2001 | ################################################################################ |
2002 | ## submit a monitoring job that will run until a certain number of jobs are done with reconstruction | |
0ff60de9 | 2003 | submit "${JOBID4wait}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" WaitForOutput ${commonOutputPath} "meta/cpass1.job*.run${runNumber}.done" ${nFilesToWaitFor} ${maxSecondsToWait} |
71353a5f | 2004 | LASTJOB=${JOBID4wait} |
5dfac970 | 2005 | ################################################################################ |
fc7e1e45 | 2006 | |
5dfac970 | 2007 | echo |
2008 | fi #end running CPass1 | |
fc7e1e45 | 2009 | |
5dfac970 | 2010 | ################################################################################ |
2011 | # submit merging of CPass1, depends on the reconstruction | |
71353a5f | 2012 | if [ ${runCPass1MergeMakeOCDB} -eq 1 ]; then |
fc7e1e45 | 2013 | |
5dfac970 | 2014 | echo |
71353a5f | 2015 | echo "submit CPass1 merging for run ${runNumber}" |
5dfac970 | 2016 | echo |
fc7e1e45 | 2017 | |
ef77d296 | 2018 | targetDirectory="${commonOutputPath}/${year}/${period}/000${runNumber}/cpass1" |
71353a5f | 2019 | rm -f ${commonOutputPath}/meta/merge.cpass1.run${runNumber}.done |
2020 | mkdir -p ${targetDirectory} | |
fc7e1e45 | 2021 | |
cb473870 | 2022 | # copy files |
2023 | filesMergeCPass1=( | |
71353a5f | 2024 | "${configPath}/OCDB.root" |
2025 | "${configPath}/localOCDBaccessConfig.C" | |
2026 | "${configPath}/mergeMakeOCDB.byComponent.sh" | |
2027 | "${configPath}/mergeByComponent.C" | |
2028 | "${configPath}/makeOCDB.C" | |
2029 | "${configPath}/merge.C" | |
2030 | "${configPath}/mergeMakeOCDB.sh" | |
2031 | "${configPath}/QAtrain_duo.C" | |
cb473870 | 2032 | ) |
2033 | for file in ${filesMergeCPass1[*]}; do | |
2034 | [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath} | |
2035 | done | |
2036 | ||
444b9916 | 2037 | submit "${JOBID5}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" MergeCPass1 ${targetDirectory} ${currentDefaultOCDB} ${configFile} ${runNumber} cpass1.calib.run${runNumber}.list cpass1.QA.run${runNumber}.lastMergingStage.txt.list cpass1.filtered.run${runNumber}.list "${extraOpts[@]}" |
71353a5f | 2038 | LASTJOB=${JOBID5} |
5dfac970 | 2039 | echo |
2040 | fi | |
2041 | ||
444b9916 | 2042 | ############################### |
2043 | #if [ ${runESDfiltering} -eq 1 ]; then | |
2044 | # rm -f ${commonOutputPath}/cpass1.ESD.run${runNumber}.list | |
2045 | # rm -f ${commonOutputPath}/meta/filtering.cpass1.run*.done | |
2046 | # echo | |
2047 | # echo submitting filtering for run ${runNumber} | |
2048 | # echo | |
2049 | # submit "${JOBmakeESDlistCPass1}" 1 1 "${LASTJOB}" "${self}" PrintValues esd ${commonOutputPath}/meta/cpass1.ESD.run${runNumber}.list ${commonOutputPath}/meta/cpass1.job*.run${runNumber}.done | |
ef77d296 | 2050 | # submit "${JOBfilterESDcpass1}" 1 1 "${JOBmakeESDlistCPass1}" "${alirootEnv} ${self}" MakeFilteredTrees ${commonOutputPath}/${year}/${period}/000${runNumber}/cpass1 ${runNumber} ${commonOutputPath}/meta/cpass1.ESD.run${runNumber}.list ${filteringFactorHighPt} ${filteringFactorV0s} ${currentDefaultOCDB} 1000000 0 10000000 0 ${configFile} AliESDs_Barrel.root "${extraOpts[@]}" |
444b9916 | 2051 | # LASTJOB=${JOBfilterESDcpass1} |
2052 | #fi | |
cb473870 | 2053 | |
5dfac970 | 2054 | done |
2055 | ||
71353a5f | 2056 | ################################################################################# |
2057 | ################################################################################# | |
444b9916 | 2058 | #if [ ${runESDfiltering} -eq 1 ]; then |
2059 | # submit "${JOBID5wait}" 1 1 "${LASTJOB}" "${self}" WaitForOutput ${commonOutputPath} "meta/filtering.cpass1.run*.done" "${#listOfRuns[@]}" ${maxSecondsToWait} | |
2060 | #else | |
2061 | submit "${JOBID5wait}" 1 1 "${LASTJOB}" "${self}" WaitForOutput ${commonOutputPath} "meta/merge.cpass1.run*.done" ${#listOfRuns[@]} ${maxSecondsToWait} | |
2062 | #fi | |
71353a5f | 2063 | LASTJOB=${JOBID5wait} |
cb473870 | 2064 | |
5de3064b | 2065 | ################################################################################# |
2066 | echo | |
2067 | echo "submit make a summary" | |
2068 | echo | |
5dfac970 | 2069 | |
3d530ff9 | 2070 | [[ -z ${alirootEnvQA} ]] && alirootEnvQA=$(encSpaces "${alirootEnv}") |
4f8ecc60 | 2071 | submit "${JOBID6}" 1 1 "${LASTJOB}" "${alirootEnvQA} ${self}" MakeSummary ${configFile} "commonOutputPath=${commonOutputPath}" |
5de3064b | 2072 | LASTJOB=${JOBID6} |
71353a5f | 2073 | ################################################################################# |
5dfac970 | 2074 | |
5dfac970 | 2075 | #restore stdout |
2076 | exec 1>&7 7>&- | |
2077 | echo "jobs submitted." | |
5de3064b | 2078 | return 0 |
fc7e1e45 | 2079 | } |
2080 | ||
cb473870 | 2081 | goWaitForOutput() |
78d0bbfd | 2082 | ( |
cb473870 | 2083 | umask 0002 |
71353a5f | 2084 | [[ $# -lt 3 ]] && echo "goWaitForOutput() wrong number of arguments, exiting.." && return 1 |
2085 | echo searchPath=${1} | |
2086 | echo fileName=${2} | |
2087 | echo numberOfFiles=${3} | |
2088 | echo maxSecondsToWait=${4} | |
2089 | searchPath=${1} | |
2090 | fileName=${2} | |
2091 | numberOfFiles=${3} | |
2092 | maxSecondsToWait=${4} | |
0ff60de9 | 2093 | echo "command to be executed: /bin/ls -1 ${searchPath}/${fileName}" |
71353a5f | 2094 | [[ -z "${maxSecondsToWait}" ]] && maxSecondsToWait=$(( 3600*12 )) |
cb473870 | 2095 | while true; do |
cb473870 | 2096 | n=$(/bin/ls -1 ${searchPath}/${fileName} 2>/dev/null | wc -l) |
71353a5f | 2097 | [[ ${n} -gt 0 ]] && echo "found ${n} X ${fileName}" |
2098 | [[ ${n} -ge ${numberOfFiles} ]] && break | |
2099 | [[ ${SECONDS} -gt ${maxSecondsToWait} ]] && echo "timeout of ${maxSecondsToWait}s!" && break | |
cb473870 | 2100 | sleep 60 |
2101 | done | |
2102 | echo "DONE! exiting..." | |
5de3064b | 2103 | return 0 |
78d0bbfd | 2104 | ) |
cb473870 | 2105 | |
71353a5f | 2106 | mergeSysLogs() |
78d0bbfd | 2107 | ( |
71353a5f | 2108 | outputFile=${1} |
2109 | shift | |
2110 | inputFiles="$@" | |
2111 | i=0 | |
2112 | if ! ls -1 ${inputFiles} &>/dev/null; then echo "the files dont exist!: ${inputFiles}"; return 1; fi | |
2113 | while read x; do | |
2114 | runNumber=$(guessRunNumber ${x}) | |
2115 | [[ -z ${runNumber} ]] && echo "run number cannot be guessed for ${x}" && continue | |
2116 | awk -v run=${runNumber} -v i=${i} 'NR > 1 {print run" "$0} NR==1 && i==0 {print "run/I:"$0}' ${x} | |
2117 | (( i++ )) | |
2118 | done < <(ls -1 ${inputFiles}) > ${outputFile} | |
5de3064b | 2119 | return 0 |
78d0bbfd | 2120 | ) |
71353a5f | 2121 | |
2122 | goMakeMergedSummaryTree() | |
78d0bbfd | 2123 | ( |
71353a5f | 2124 | # create list of calibration entries |
2125 | # takes no arguments, just run it in the base output | |
2126 | # directory with the following files in the working directory | |
2127 | # | |
2128 | # Calibration file lists: | |
2129 | # cpass0.dcsTree.list, cpass1.dcsTree.list | |
2130 | # QA trending root file: | |
2131 | # trending.root | |
2132 | # | |
2133 | # Production infoo ascii files: | |
2134 | # summary_pass0.tree | |
2135 | # summary_pass1.tree | |
2136 | # | |
617f8886 | 2137 | |
71353a5f | 2138 | [[ ! -f cpass0.dcsTree.list ]] && echo "no cpass0.dcsTree.list" && return 1 |
2139 | [[ ! -f cpass1.dcsTree.list ]] && echo "no cpass1.dcsTree.list" && return 1 | |
2140 | [[ ! -f trending.root ]] && echo "no trending.root" && return 1 | |
2141 | [[ ! -f summary_pass0.tree ]] && echo "no summary_pass0.tree" && return 1 | |
2142 | [[ ! -f summary_pass1.tree ]] && echo "no summary_pass1.tree" && return 1 | |
2143 | ||
2144 | #first, dump the C macro to file | |
2145 | cat << EOF > mergeTree.C | |
2146 | // | |
2147 | // Merge summary information | |
2148 | // Following files are expected to be in the working directory | |
2149 | // | |
2150 | // Calibration file lists: | |
2151 | // cpass0.dcsTree.list, cpass1.dcsTree.list | |
2152 | // QA trending root file: | |
2153 | // trending.root | |
2154 | // | |
2155 | // Production infoo ascii files: | |
2156 | // summary_pass0.tree | |
2157 | // summary_pass1.tree | |
2158 | // | |
2159 | void mergeTree(){ | |
2160 | // | |
2161 | // | |
2162 | // | |
2163 | // Calibration values dump | |
2164 | // | |
839b07ca | 2165 | //Printf("MakeTreeFromList cpass0.dcsTree.list"); |
71353a5f | 2166 | AliXRDPROOFtoolkit::MakeTreeFromList("Calib.TPC.CPass0.root", "dcs","dcs","cpass0.dcsTree.list",1); |
839b07ca | 2167 | //Printf("MakeTreeFromList cpass1.dcsTree.list"); |
71353a5f | 2168 | AliXRDPROOFtoolkit::MakeTreeFromList("Calib.TPC.CPass1.root", "dcs","dcs","cpass1.dcsTree.list",1); |
2169 | // | |
2170 | // Calibration status dump | |
2171 | // | |
2172 | TFile *fprod = TFile::Open("fproduction.root","recreate"); | |
2173 | TTree tree0, tree1; | |
839b07ca | 2174 | //Printf("reading summary_pass0.tree"); |
71353a5f | 2175 | tree0.ReadFile("summary_pass0.tree"); |
839b07ca | 2176 | //Printf("reading summary_pass1.tree"); |
71353a5f | 2177 | tree1.ReadFile("summary_pass1.tree"); |
2178 | tree0.Write("CPass0"); | |
2179 | tree1.Write("CPass1"); | |
2180 | fprod->Close(); | |
2181 | // | |
2182 | // | |
2183 | // | |
2184 | TString stringSetup=""; | |
2185 | stringSetup+="1#QA.TPC#run#SummaryTPCQA/tpcQA#trending.root+"; // | |
2186 | stringSetup+="1#Calib.TPC.CPass0#run#dcs#Calib.TPC.CPass0.root+"; // | |
2187 | stringSetup+="1#Calib.TPC.CPass1#run#dcs#Calib.TPC.CPass1.root+"; // | |
2188 | // | |
2189 | stringSetup+="1#CPass0#runnumber#CPass0#fproduction.root+"; // | |
2190 | stringSetup+="1#CPass1#runnumber#CPass1#fproduction.root+"; // | |
2191 | // | |
839b07ca | 2192 | //Printf("stringSetup: %s", stringSetup.Data()); |
71353a5f | 2193 | AliXRDPROOFtoolkit::JoinTreesIndex("outAll.root","joinAll","run",stringSetup.Data(), 1); |
2194 | } | |
2195 | EOF | |
2196 | ||
2197 | aliroot -b -q "mergeTree.C" > mergeTrees.log | |
2198 | return $? | |
78d0bbfd | 2199 | ) |
71353a5f | 2200 | |
177b9ebc | 2201 | stackTraceTree() |
2202 | ( | |
e00eb304 | 2203 | if [[ $# -lt 1 ]]; then |
2204 | echo 'make stacktrace processing in case of standard root crash log' | |
2205 | echo 'input is a (list of) text files with the stack trace (either gdb aoutput' | |
2206 | echo 'produced with e.g. gdb --batch --quiet -ex "bt" -ex "quit" aliroot core,' | |
2207 | echo 'or the root crash log), output is a TTree formatted table.' | |
2208 | echo 'example usage:' | |
2209 | echo 'benchmark.sh stackTraceTree /foo/*/rec.log' | |
2210 | echo 'benchmark.sh stackTraceTree $(cat file.list)' | |
0685f982 | 2211 | echo 'benchmark.sh stackTraceTree `cat file.list`' |
e00eb304 | 2212 | return 0 |
2213 | fi | |
4a3c2422 | 2214 | gawk ' |
177b9ebc | 2215 | BEGIN { |
80c75839 | 2216 | print "frame/I:method/C:line/C:cpass/I:aliroot/I:file/C"; |
177b9ebc | 2217 | RS="#[0-9]*"; |
2218 | aliroot=0; | |
4a3c2422 | 2219 | read=1; |
177b9ebc | 2220 | } |
4a3c2422 | 2221 | /There was a crash/ {read=1;} |
2222 | /The lines below might hint at the cause of the crash/ {read=0;} | |
2223 | read==1 { | |
177b9ebc | 2224 | if ($3 ~ /Ali*/) aliroot=1; else aliroot=0; |
2225 | gsub("#","",RT); | |
80c75839 | 2226 | if ($NF!="" && RT!="" && $3!="") print RT" "$3" "$NF" "0" "aliroot" "FILENAME |
177b9ebc | 2227 | } |
2228 | ' "$@" 2>/dev/null | |
2229 | ) | |
2230 | ||
71353a5f | 2231 | goMakeSummary() |
78d0bbfd | 2232 | ( |
71353a5f | 2233 | #all the final stuff goes in here for ease of use: |
2234 | # summary logs | |
2235 | # qa plot making | |
2236 | # final file lists | |
ef77d296 | 2237 | # runs in current dir - in makeflow mode it can run LOCAL, then the QA plots and summaries |
2238 | # will appear in the submission dir. | |
444b9916 | 2239 | #some defaults: |
2240 | log="summary.log" | |
2241 | productionID="qa" | |
71353a5f | 2242 | |
2243 | configFile=${1} | |
2244 | shift 1 | |
444b9916 | 2245 | extraOpts=("$@") |
6fc48f9e | 2246 | if ! parseConfig ${configFile} "${extraOpts[@]}"; then return 1; fi |
71353a5f | 2247 | |
0eaf38c7 | 2248 | #if which greadlink; then configFile=$(greadlink -f ${configFile}); fi |
839b07ca | 2249 | |
39527a15 | 2250 | #record the working directory provided by the batch system |
2251 | batchWorkingDirectory=${PWD} | |
ef77d296 | 2252 | |
2253 | logTmp=${batchWorkingDirectory}/${log} | |
2254 | logDest=${commonOutputPath}/${log} | |
39527a15 | 2255 | |
71353a5f | 2256 | [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource} |
2257 | ||
71353a5f | 2258 | [[ ! -f ${configFile} ]] && echo "no config file ${configFile}!" && return |
2259 | ||
2260 | [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD} | |
2261 | ||
39527a15 | 2262 | #copy some useful stuff |
2263 | #and go to the commonOutputPath | |
2264 | cp ${configFile} ${commonOutputPath} | |
39527a15 | 2265 | |
ef77d296 | 2266 | exec &> >(tee ${logTmp}) |
39527a15 | 2267 | |
71353a5f | 2268 | #summarize the global stuff |
2269 | echo "env script: ${alirootSource} ${alirootEnv}" | |
2270 | echo "\$ALICE_ROOT=${ALICE_ROOT}" | |
2271 | echo "commonOutputPath=${commonOutputPath}" | |
2272 | ||
2273 | #summarize the stacktraces | |
ef77d296 | 2274 | stackTraceTree ${commonOutputPath}/*/*/000*/cpass0/*/stacktrace* > stacktrace_cpass0.tree |
2275 | stackTraceTree ${commonOutputPath}/*/*/000*/cpass1/*/stacktrace* > stacktrace_cpass1.tree | |
71353a5f | 2276 | |
2277 | echo total numbers for the production: | |
2278 | echo | |
2279 | awk 'BEGIN {nFiles=0;nCore=0;} | |
2280 | /^calibfile/ {nFiles++;} | |
2281 | /core dumped/ {nCore++i;} | |
ef77d296 | 2282 | END {print "cpass0 produced "nFiles" calib files, "nCore" core files";}' ${commonOutputPath}/meta/cpass0.job*done 2>/dev/null |
71353a5f | 2283 | awk 'BEGIN {nOK=0; nBAD=0; } |
2284 | /\/rec.log OK/ {nOK++;} | |
2285 | /\/rec.log BAD/ {nBAD++;} | |
2286 | /stderr BAD/ {if ($0 ~ /rec.log/){nBAD++;}} | |
ef77d296 | 2287 | END {print "cpass0 reco: OK: "nOK"\tBAD: "nBAD;}' ${commonOutputPath}/meta/cpass0.job*done 2>/dev/null |
71353a5f | 2288 | awk 'BEGIN {nOK=0; nBAD=0; } |
2289 | /\/calib.log OK/ {nOK++;} | |
2290 | /\/calib.log BAD/ {nBAD++;} | |
ef77d296 | 2291 | END {print "cpass0 calib: OK: "nOK"\tBAD: "nBAD;}' ${commonOutputPath}/meta/cpass0.job*done 2>/dev/null |
71353a5f | 2292 | |
2293 | awk 'BEGIN {nOK=0; nBAD=0; } | |
2294 | /merge.log OK/ {nOK++;} | |
2295 | /merge.log BAD/ {nBAD++;} | |
ef77d296 | 2296 | END {print "cpass0 merge: OK: "nOK"\tBAD: "nBAD;}' ${commonOutputPath}/meta/merge.cpass0*done 2>/dev/null |
71353a5f | 2297 | awk 'BEGIN {nOK=0; nBAD=0; } |
2298 | /ocdb.log OK/ {nOK++;} | |
2299 | /ocdb.log BAD/ {nBAD++;} | |
ef77d296 | 2300 | END {print "cpass0 OCDB: OK: "nOK"\tBAD: "nBAD;}' ${commonOutputPath}/meta/merge.cpass0*done 2>/dev/null |
71353a5f | 2301 | |
2302 | echo | |
2303 | awk 'BEGIN {nFiles=0;nCore=0;} | |
2304 | /^calibfile/ {nFiles++;} | |
2305 | /core dumped/ {nCore++;} | |
ef77d296 | 2306 | END {print "cpass1 produced "nFiles" calib files, "nCore" core files";}' ${commonOutputPath}/meta/cpass1.job*done 2>/dev/null |
71353a5f | 2307 | awk 'BEGIN {nOK=0; nBAD=0; } |
2308 | /\/rec.log OK/ {nOK++;} | |
2309 | /\/rec.log BAD/ {nBAD++;} | |
2310 | /stderr BAD/ {if ($0 ~ /rec.log/){nBAD++;}} | |
ef77d296 | 2311 | END {print "cpass1 reco: OK: "nOK"\tBAD: "nBAD;}' ${commonOutputPath}/meta/cpass1.job*done 2>/dev/null |
71353a5f | 2312 | awk 'BEGIN {nOK=0; nBAD=0; } |
2313 | /\/calib.log OK/ {nOK++;} | |
2314 | /\/calib.log BAD/ {nBAD++;} | |
ef77d296 | 2315 | END {print "cpass1 calib: OK: "nOK"\tBAD: "nBAD;}' ${commonOutputPath}/meta/cpass1.job*done 2>/dev/null |
71353a5f | 2316 | |
2317 | awk 'BEGIN {nOK=0; nBAD=0; } | |
2318 | /merge.log OK/ {nOK++;} | |
2319 | /merge.log BAD/ {nBAD++;} | |
ef77d296 | 2320 | END {print "cpass1 merge: OK: "nOK"\tBAD: "nBAD;}' ${commonOutputPath}/meta/merge.cpass1*done 2>/dev/null |
71353a5f | 2321 | awk 'BEGIN {nOK=0; nBAD=0; } |
2322 | /ocdb.log OK/ {nOK++;} | |
2323 | /ocdb.log BAD/ {nBAD++;} | |
ef77d296 | 2324 | END {print "cpass1 OCDB: OK: "nOK"\tBAD: "nBAD;}' ${commonOutputPath}/meta/merge.cpass1*done 2>/dev/null |
71353a5f | 2325 | |
2326 | echo | |
2327 | echo per run stats: | |
ef77d296 | 2328 | /bin/ls -1 ${commonOutputPath}/meta/merge.cpass0.run*.done | while read x |
71353a5f | 2329 | do |
5a2ee6ff | 2330 | dir=$(goPrintValues dir - ${x}) |
71353a5f | 2331 | runNumber=$(guessRunNumber ${dir}) |
2332 | [[ -z ${runNumber} ]] && continue | |
2333 | ||
ef77d296 | 2334 | if $(/bin/ls ${commonOutputPath}/meta/cpass0.job*.run${runNumber}.done &> /dev/null); then |
71353a5f | 2335 | statusCPass0=( $( |
2336 | awk 'BEGIN {nOKrec=0;nBADrec=0;nOKcalib=0;nBADcalib=0;nOKstderr=0;nBADstderr=0;} | |
2337 | /\/rec.log OK/ {nOKrec++;} | |
2338 | /\/rec.log BAD/ {nBADrec++;} | |
2339 | /stderr BAD/ {if ($0 ~ /rec.log/) {nBADrec++;} nBADstderr++;} | |
2340 | /stderr OK/ {nOKstderr++;} | |
2341 | /\/calib.log OK/ {nOKcalib++;} | |
2342 | /\/calib.log BAD/ {nBADcalib++} | |
ef77d296 | 2343 | END {print ""nOKrec" "nBADrec" "nOKstderr" "nBADstderr" "nOKcalib" "nBADcalib;}' ${commonOutputPath}/meta/cpass0.job*.run${runNumber}.done 2>/dev/null |
71353a5f | 2344 | ) ) |
2345 | fi | |
2346 | ||
ef77d296 | 2347 | if $(/bin/ls ${commonOutputPath}/meta/cpass1.job*.run${runNumber}.done &>/dev/null); then |
71353a5f | 2348 | statusCPass1=( $( |
2349 | awk 'BEGIN {nOKrec=0;nBADrec=0;nOKcalib=0;nBADcalib=0;nOKstderr=0;nBADstderr=0;nQAbarrelOK=0;nQAbarrelBAD=0;nQAouterOK=0;nQAouterBAD=0;} | |
2350 | /\/rec.log OK/ {nOKrec++;} | |
2351 | /\/rec.log BAD/ {nBADrec++;} | |
2352 | /stderr BAD/ {if ($0 ~ /rec.log/) nBADrec++;nBADstderr++;} | |
2353 | /stderr OK/ {nOKstderr++;} | |
2354 | /\/calib.log OK/ {nOKcalib++;} | |
2355 | /\/calib.log BAD/ {nBADcalib++} | |
2356 | /\/qa_barrel.log OK/ {nQAbarrelOK++;} | |
2357 | /\/qa_barrel.log BAD/ {nQAbarrelBAD++;} | |
2358 | /\/qa_outer.log OK/ {nQAouterOK++;} | |
2359 | /\/qa_outer.log BAD/ {nQAouterBAD++;} | |
ef77d296 | 2360 | END {print ""nOKrec" "nBADrec" "nOKstderr" "nBADstderr" "nOKcalib" "nBADcalib" "nQAbarrelOK" "nQAbarrelBAD" "nQAouterOK" "nQAouterBAD;}' ${commonOutputPath}/meta/cpass1.job*.run${runNumber}.done 2>/dev/null |
71353a5f | 2361 | ) ) |
2362 | fi | |
2363 | ||
2364 | statusOCDBcpass0=$(awk '/ocdb.log/ {print $2} ' ${x} 2>/dev/null) | |
2365 | statusOCDBcpass1=$(awk '/ocdb.log/ {print $2}' ${x/cpass0/cpass1} 2>/dev/null) | |
2366 | statusQA=$(awk '/mergeMakeOCDB.log/ {print $2}' ${x/cpass0/cpass1} 2>/dev/null) | |
2367 | ||
2368 | printf "%s\t ocdb.log cpass0: %s\t ocdb.log cpass1: %s\tqa.log:%s\t| cpass0: rec:%s/%s stderr:%s/%s calib:%s/%s cpass1: rec:%s/%s stderr:%s/%s calib:%s/%s QAbarrel:%s/%s QAouter:%s/%s\n" ${runNumber} ${statusOCDBcpass0} ${statusOCDBcpass1} ${statusQA} ${statusCPass0[0]} ${statusCPass0[1]} ${statusCPass0[2]} ${statusCPass0[3]} ${statusCPass0[4]} ${statusCPass0[5]} ${statusCPass1[0]} ${statusCPass1[1]} ${statusCPass1[2]} ${statusCPass1[3]} ${statusCPass1[4]} ${statusCPass1[5]} ${statusCPass1[6]} ${statusCPass1[7]} ${statusCPass1[8]} ${statusCPass1[9]} | |
2369 | done | |
2370 | ||
2371 | #make lists with output files - QA, trending, filtering and calibration | |
2372 | ### wait for the merging of all runs to be over ### | |
2373 | rm -f qa.list | |
2374 | goPrintValues qafile qa.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null | |
2375 | rm -f calib.list | |
2376 | goPrintValues calibfile calib.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null | |
2377 | rm -f trending.list | |
2378 | goPrintValues trendingfile trending.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null | |
2379 | rm -f filtering.list | |
444b9916 | 2380 | goPrintValues filteredTree filtering.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null |
71353a5f | 2381 | rm -f cpass0.dcsTree.list |
2382 | goPrintValues dcsTree cpass0.dcsTree.list ${commonOutputPath}/meta/merge.cpass0.run*.done &>/dev/null | |
71353a5f | 2383 | rm -f cpass1.dcsTree.list |
2384 | goPrintValues dcsTree cpass1.dcsTree.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null | |
2385 | ||
2386 | #merge trending | |
fecdbe75 | 2387 | rm -f trending.root |
2388 | goMerge trending.list trending.root ${configFile} "${extraOpts[@]}" &> mergeTrending.log | |
71353a5f | 2389 | |
2390 | goMakeSummaryTree ${commonOutputPath} 0 | |
2391 | goMakeSummaryTree ${commonOutputPath} 1 | |
2392 | ||
839b07ca | 2393 | goCreateQAplots "${PWD}/qa.list" "${productionID}" "QAplots" "${configFile}" "${extraOpts[@]}" filteringList="${PWD}/filtering.list" &>createQAplots.log |
71353a5f | 2394 | |
2395 | #make a merged summary tree out of the QA trending, dcs trees and log summary trees | |
2396 | goMakeMergedSummaryTree | |
2397 | ||
2398 | #if set, email the summary | |
ef77d296 | 2399 | [[ -n ${MAILTO} ]] && cat ${logTmp} | mail -s "benchmark ${productionID} done" ${MAILTO} |
2400 | ||
2401 | # Copy log to destination (delete all on failure to signal error) | |
2402 | cp "$logTmp" "$logDest" || rm -f "$logTmp" "$logDest" | |
fecdbe75 | 2403 | |
2404 | #copy output files | |
054fdec5 | 2405 | exec &> >(tee fileCopy.log) |
13abb339 | 2406 | paranoidCp QAplots ${commonOutputPath} |
2407 | paranoidCp *.list ${commonOutputPath} | |
2408 | paranoidCp *.root ${commonOutputPath} | |
2409 | paranoidCp *.log ${commonOutputPath} | |
054fdec5 | 2410 | paranoidCp fileCopy.log ${commonOutputPath} |
71353a5f | 2411 | |
2412 | return 0 | |
78d0bbfd | 2413 | ) |
71353a5f | 2414 | |
2415 | goMakeSummaryTree() | |
78d0bbfd | 2416 | ( |
71353a5f | 2417 | if [[ $# -lt 1 ]] ; then |
2418 | return | |
2419 | fi | |
2420 | #1. define vars/arrays | |
2421 | DIR=${1} #use input or exec in current dir | |
2422 | pass=${2-"0"} #pass from input | |
2423 | outfile="summary_pass${pass}.tree" | |
2424 | Ncolumns=0 | |
2425 | test -f ${outfile} && : >${outfile} | |
2426 | errfile=${outfile/tree/err} | |
2427 | test -f ${errfile} && : >${errfile} | |
2428 | ||
2429 | declare -a counterString=(TOFevents TOFtracks TPCevents TPCtracks TRDevents TRDtracks T0events SDDevents SDDtracks MeanVertexevents) | |
2430 | Ncounter=${#counterString[@]} | |
2431 | ||
2432 | declare -a statusString=(TRDStatus TOFStatus TPCStatus T0Status MeanVertexStatus) | |
2433 | Nstatus=${#statusString[@]} | |
2434 | ||
2435 | ||
2436 | declare -a ratesString=(rec stderr calib qa_barrel qa_outer) | |
2437 | Nrates=${#ratesString[@]} | |
2438 | ||
2439 | runs=( $(ls -1 ${DIR}/meta/merge.cpass0* | while read x; do guessRunNumber $x; done) ) | |
2440 | Nruns=${#runs[@]} | |
2441 | ||
2442 | echo -n runnumber/I >>${outfile} | |
2443 | echo -n :cpass${pass}status/I >>${outfile} | |
2444 | echo -n :cpass${pass}QAstatus/I >>${outfile} | |
2445 | for i in ${ratesString[@]}; do | |
2446 | echo -n :${i}OK/I >>${outfile} | |
2447 | echo -n :${i}BAD/I >>${outfile} | |
2448 | ||
2449 | done | |
2450 | for i in ${counterString[@]} ${statusString[@]} ; do | |
2451 | echo -n :${i}/I >>${outfile} | |
2452 | done | |
2453 | Ncolumns=$((2 + 2*Nrates + Ncounter + Nstatus)) | |
2454 | echo >> ${outfile} | |
2455 | ||
2456 | #2. loop runs | |
2457 | ||
2458 | for runnumber in ${runs[@]} ; do | |
2459 | ||
2460 | ||
2461 | ||
2462 | filejob="${DIR}/meta/cpass${pass}.job*.run${runnumber}.done" | |
2463 | filemerge="${DIR}/meta/merge.cpass${pass}.run${runnumber}.done" | |
2464 | fileOCDB=$(grep /ocdb.log ${filemerge} | awk '{print $1}') | |
2465 | if ! $(/bin/ls ${filemerge} &>/dev/null) ; then | |
2466 | echo "${filemerge} does not exist!" >>${errfile} | |
2467 | continue | |
2468 | elif ! $(/bin/ls ${filejob} &>/dev/null) ; then | |
2469 | echo "${filejob} does not exist!" >>${errfile} | |
2470 | echo -n ${runnumber} >> ${outfile} | |
2471 | for i in $(seq ${Ncolumns}) ; do | |
2472 | echo -n "-1" >> ${outfile} | |
2473 | done | |
2474 | echo >> ${outfile} | |
2475 | continue | |
2476 | fi | |
2477 | echo -n ${runnumber} >> ${outfile} | |
2478 | #pass0status= grep '/ocdb.log' ${filemerge} | cut -d' ' -f2 | tr OK x1 | tr BAD xx0 | tr -d 'x' | |
2479 | passStatus=$(grep '/ocdb.log' ${filemerge} | grep OK | wc -l) | |
2480 | echo -n " ${passStatus}" >> ${outfile} | |
2481 | qaStatus=$(grep '/mergeMakeOCDB.log' ${filemerge} | grep OK | wc -l) | |
2482 | echo -n " ${qaStatus}" >> ${outfile} | |
2483 | ||
2484 | ||
2485 | #fill OK/BAD rates | |
2486 | for i in $(seq 0 $((${Nrates}-1))) ; do | |
2487 | var1=$(grep "/${ratesString[${i}]}.log" ${filejob} | grep OK | wc -l) | |
2488 | var2=$(grep "/${ratesString[${i}]}.log" ${filejob} | grep BAD | wc -l) | |
2489 | ||
2490 | if [[ ${ratesString[${i}]} == "stderr" ]] ; then | |
2491 | var1=$(grep "stderr" ${filejob} | grep OK | wc -l) | |
2492 | var2=$(grep "stderr" ${filejob} | grep "rec.log" | grep BAD | wc -l) | |
2493 | fi | |
2494 | echo -n " ${var1}" >> ${outfile} | |
2495 | echo -n " ${var2}" >> ${outfile} | |
2496 | done | |
2497 | ||
2498 | if [[ -f ${fileOCDB} ]] ; then | |
2499 | #fill counter | |
2500 | for i in $(seq 0 $((${Ncounter}-1))) ; do | |
2501 | var1=$(grep Monalisa ${fileOCDB} | grep ${counterString[${i}]} | cut -f2) | |
2502 | echo -n " ${var1:-"-1"}" >> ${outfile} | |
2503 | done | |
2504 | ||
2505 | #fill status | |
2506 | for i in $(seq 0 $((${Nstatus}-1))) ; do | |
2507 | var1=$(grep "calibration status=" ${fileOCDB} | grep ${statusString[${i}]/Status/} | cut -d'=' -f2) | |
2508 | echo -n " ${var1:-"-1"}" >> ${outfile} | |
2509 | done | |
2510 | fi | |
2511 | echo >> ${outfile} | |
2512 | done | |
2513 | ||
5de3064b | 2514 | return 0 |
78d0bbfd | 2515 | ) |
71353a5f | 2516 | |
2517 | parseConfig() | |
2518 | { | |
2519 | configFile=${1} | |
2520 | shift | |
444b9916 | 2521 | args=("$@") |
2522 | ||
0eaf38c7 | 2523 | |
444b9916 | 2524 | #some defaults |
6fc48f9e | 2525 | #autoOCDB=0 |
2526 | defaultOCDB="raw://" | |
2527 | #runNumber=167123 | |
2528 | #makeflowPath="/hera/alice/aux/cctools/bin" | |
2529 | #makeflowOptions="-T wq -N alice -d all -C ali-copilot.cern.ch:9097" | |
2530 | #makeflowOptions="-T wq -N alice -C ali-copilot.cern.ch:9097" | |
2531 | makeflowOptions="" | |
39527a15 | 2532 | #batchCommand="/usr/bin/qsub" |
ef77d296 | 2533 | batchFlags="" |
6fc48f9e | 2534 | baseOutputDirectory="$PWD/output" |
2535 | #alirootEnv="/cvmfs/alice.cern.ch/bin/alienv setenv AliRoot/v5-04-34-AN -c" | |
2536 | #alirootEnv="/home/mkrzewic/alisoft/balice_master.sh" | |
2537 | #trustedQAtrainMacro='/hera/alice/mkrzewic/gsisvn/Calibration/QAtrain_duo.C' | |
2538 | reconstructInTemporaryDir=0 | |
2539 | recoTriggerOptions="\"\"" | |
2540 | percentProcessedFilesToContinue=100 | |
2541 | maxSecondsToWait=$(( 3600*24 )) | |
2542 | nEvents=-1 | |
2543 | nMaxChunks=0 | |
2544 | postSetUpActionCPass0="" | |
2545 | postSetUpActionCPass1="" | |
2546 | runCPass0reco=1 | |
2547 | runCPass0MergeMakeOCDB=1 | |
2548 | runCPass1reco=1 | |
2549 | runCPass1MergeMakeOCDB=1 | |
2550 | runESDfiltering=1 | |
444b9916 | 2551 | filteringFactorHighPt=1e2 |
2552 | filteringFactorV0s=1e1 | |
6fc48f9e | 2553 | MAILTO="" |
2554 | #pretend=1 | |
2555 | #dontRedirectStdOutToLog=1 | |
2556 | logToFinalDestination=1 | |
2557 | ALIROOT_FORCE_COREDUMP=1 | |
0eaf38c7 | 2558 | pretendDelay=0 |
71353a5f | 2559 | |
2560 | #first, source the config file | |
2561 | if [ -f ${configFile} ]; then | |
2562 | source ${configFile} | |
2563 | else | |
0eaf38c7 | 2564 | echo "config file ${configFile} not found!" |
2565 | return 1 | |
71353a5f | 2566 | fi |
2567 | ||
839b07ca | 2568 | unset encodedSpaces |
2569 | for opt in "${args[@]}"; do | |
0eaf38c7 | 2570 | [[ "${opt}" =~ encodedSpaces=.* ]] && encodedSpaces=1 && echo "encodedSpaces!" && break |
839b07ca | 2571 | done |
2572 | ||
d56eeaab | 2573 | #then, parse the options as they override the options from file |
2574 | for opt in "${args[@]}"; do | |
0eaf38c7 | 2575 | [[ -z ${opt} ]] && continue |
2576 | [[ -n ${encodedSpaces} ]] && opt="$(decSpaces ${opt})" | |
2577 | [[ "${opt}" =~ ^[[:space:]]*$ ]] && continue | |
d56eeaab | 2578 | if [[ ! "${opt}" =~ .*=.* ]]; then |
0eaf38c7 | 2579 | echo "badly formatted option \"${opt}\" should be: option=value, stopping..." |
d56eeaab | 2580 | return 1 |
2581 | fi | |
2582 | local var="${opt%%=*}" | |
2583 | local value="${opt#*=}" | |
9d0c204f | 2584 | echo "${var}=${value}" |
444b9916 | 2585 | export ${var}="${value}" |
71353a5f | 2586 | done |
2587 | ||
6fc48f9e | 2588 | #do some checking |
2589 | [[ -z ${alirootEnv} ]] && echo "alirootEnv not defined!" && return 1 | |
2590 | ||
71353a5f | 2591 | #export the aliroot function if defined to override normal behaviour |
fecdbe75 | 2592 | [[ $(type -t aliroot) =~ "function" ]] && export -f aliroot && echo "exporting aliroot() function..." |
6fc48f9e | 2593 | |
5de3064b | 2594 | return 0 |
71353a5f | 2595 | } |
2596 | ||
2597 | aliroot() | |
2598 | { | |
e48b4924 | 2599 | args=("$@") |
6fc48f9e | 2600 | if [[ -n ${useProfilingCommand} ]]; then |
e48b4924 | 2601 | profilerLogFile="cpu.txt" |
1a0fcd62 | 2602 | [[ "${args[@]}" =~ rec ]] && profilerLogFile="cpu_rec.txt" |
2603 | [[ "${args[@]}" =~ Calib ]] && profilerLogFile="cpu_calib.txt" | |
2f2d99b8 | 2604 | echo running "${useProfilingCommand} aliroot ${args[@]} &> ${profilerLogFile}" |
e48b4924 | 2605 | ${useProfilingCommand} aliroot "${args[@]}" &> ${profilerLogFile} |
71353a5f | 2606 | else |
2607 | #to prevent an infinite recursion use "command aliroot" to disable | |
2608 | #aliases and functions | |
e48b4924 | 2609 | echo running command aliroot "${args[@]}" |
2610 | command aliroot "${args[@]}" | |
71353a5f | 2611 | fi |
5de3064b | 2612 | return 0 |
71353a5f | 2613 | } |
2614 | ||
13abb339 | 2615 | paranoidCp() |
2616 | ( | |
2617 | #recursively copy files and directories | |
2618 | #to avoid using find and the like as they kill | |
2619 | #the performance on some cluster file systems | |
2620 | #does not copy links to avoid problems | |
2621 | sourceFiles=("${@}") | |
2622 | destination="${sourceFiles[@]:(-1)}" #last element | |
2623 | unset sourceFiles[${#sourceFiles[@]}-1] #remove last element (dst) | |
2624 | for src in "${sourceFiles[@]}"; do | |
2625 | if [[ -f "${src}" && ! -h "${src}" ]]; then | |
2626 | paranoidCopyFile "${src}" "${destination}" | |
2627 | elif [[ -d "${src}" && ! -h "${src}" ]]; then | |
2628 | src="${src%/}" | |
2629 | dst="${destination}/${src##*/}" | |
2630 | mkdir -p "${dst}" | |
2631 | paranoidCp "${src}"/* "${dst}" | |
2632 | fi | |
2633 | done | |
2634 | ) | |
2635 | ||
2636 | paranoidCopyFile() | |
2637 | ( | |
2638 | #copy a single file to a target in an existing dir | |
2639 | #repeat a few times if copy fails | |
2640 | src="${1}" | |
2641 | dst="${2}" | |
2642 | [[ -d "${dst}" ]] && dst="${dst}/${src##*/}" | |
2643 | [[ -z "${maxCopyTries}" ]] && maxCopyTries=5 | |
054fdec5 | 2644 | #echo "maxCopyTries=${maxCopyTries}" |
13abb339 | 2645 | echo "cp ${src} ${dst}" |
2646 | cp "${src}" "${dst}" | |
2647 | i=0 | |
2648 | until cmp -s "${src}" "${dst}"; do | |
2649 | echo "try: ${i}" | |
2650 | [[ -f "${dst}" ]] && rm "${dst}" | |
2651 | cp "${src}" "${dst}" | |
2652 | [[ ${i} -gt ${maxCopyTries} ]] && ret=1 && return 1 | |
2653 | (( i++ )) | |
2654 | done | |
2655 | return 0 | |
2656 | ) | |
2657 | ||
71353a5f | 2658 | guessRunData() |
2659 | { | |
2660 | #guess the period from the path, pick the rightmost one | |
2661 | period="" | |
2662 | runNumber="" | |
2663 | year="" | |
2664 | pass="" | |
2665 | legoTrainRunNumber="" | |
2666 | dataType="" | |
2667 | ||
2668 | local shortRunNumber="" | |
2669 | local IFS="/" | |
2670 | declare -a path=( $1 ) | |
2671 | local dirDepth=$(( ${#path[*]}-1 )) | |
2672 | i=0 | |
2673 | #for ((x=${dirDepth};x>=0;x--)); do | |
2674 | for ((x=0;x<=${dirDepth};x++)); do | |
2675 | ||
2676 | [[ $((x-1)) -ge 0 ]] && local fieldPrev=${path[$((x-1))]} | |
2677 | local field=${path[${x}]} | |
2678 | local fieldNext=${path[$((x+1))]} | |
2679 | ||
2680 | [[ ${field} =~ ^[0-9]*$ && ${fieldNext} =~ (.*\.zip$|.*\.root$) ]] && legoTrainRunNumber=${field} | |
2681 | [[ -n ${legoTrainRunNumber} && -z ${pass} ]] && pass=${fieldPrev} | |
2682 | [[ ${field} =~ ^LHC[0-9][0-9][a-z].*$ ]] && period=${field%_*} | |
2683 | [[ ${field} =~ ^000[0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && runNumber=${field#000} | |
2684 | [[ ${field} =~ ^[0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && shortRunNumber=${field} | |
2685 | [[ ${field} =~ ^20[0-9][0-9]$ ]] && year=${field} | |
2686 | [[ ${field} =~ ^(^sim$|^data$) ]] && dataType=${field} | |
2687 | (( i++ )) | |
2688 | done | |
2689 | [[ -z ${legoTrainRunNumber} ]] && pass=${path[$((dirDepth-1))]} | |
2690 | [[ "${dataType}" =~ ^sim$ ]] && pass="passMC" && runNumber=${shortRunNumber} | |
2691 | ||
2692 | #if [[ -z ${dataType} || -z ${year} || -z ${period} || -z ${runNumber}} || -z ${pass} ]]; | |
2693 | if [[ -z ${runNumber}} ]]; | |
2694 | then | |
2695 | #error condition | |
2696 | return 1 | |
2697 | else | |
2698 | #ALL OK | |
2699 | return 0 | |
2700 | fi | |
5de3064b | 2701 | return 0 |
71353a5f | 2702 | } |
2703 | ||
839b07ca | 2704 | #these functions encode strings to and from a space-less form |
2705 | #use when spaces are not well handled (e.g. in arguments to | |
2706 | #commands in makeflow files, etc. | |
0eaf38c7 | 2707 | encSpaces()(echo "${1// /@@@@}") |
2708 | decSpaces()(echo "${1//@@@@/ }") | |
839b07ca | 2709 | |
fc7e1e45 | 2710 | main "$@" |