]>
Commit | Line | Data |
---|---|---|
1 | #!/bin/bash | |
2 | #include benchmark.config | |
3 | ||
4 | # this script runs the CPass0/CPass1 train | |
5 | # produced OCDB updates are local | |
6 | ||
7 | main() | |
8 | { | |
9 | #run in proper mode depending on the selection | |
10 | if [[ $# -lt 1 ]]; then | |
11 | if [[ ! "${0}" =~ "bash" ]]; then | |
12 | echo "uses makeflow:" | |
13 | echo " ${0} \"run\" productionID inputList configFile [extraOpts]" | |
14 | echo "uses a batch system (SGE):" | |
15 | echo " ${0} \"submit\" productionID inputList configFile [extraOpts]" | |
16 | echo "extraOpts if non-empty override the config file, e.g.:" | |
17 | echo " ${0} submit test1 benchmark.list benchmark.config runNumber=169123 nEvents=10" | |
18 | fi | |
19 | return | |
20 | fi | |
21 | ||
22 | #define some aliases - default is to call one of the functions directly | |
23 | runMode=${1} | |
24 | umask 0002 | |
25 | shift | |
26 | case ${runMode} in | |
27 | "CPass0") goCPass0 "$@";; | |
28 | "CPass1") goCPass1 "$@";; | |
29 | "MakeLocalOCDBaccessConfig") goMakeLocalOCDBaccessConfig "$@";; | |
30 | "MergeCPass0") goMergeCPass0 "$@";; | |
31 | "MergeCPass1") goMergeCPass1 "$@";; | |
32 | "MakeFilteredTrees") goMakeFilteredTrees "$@";; | |
33 | "MakeSummary") goMakeSummary "$@";; | |
34 | "run") goSubmitMakeflow "$@";; | |
35 | "submit") goSubmitBatch "$@";; | |
36 | "test") goTest "$@";; | |
37 | "GenerateMakeflow") goGenerateMakeflow "$@";; | |
38 | "PrintValues") goPrintValues "$@";; | |
39 | "CreateQAplots") goCreateQAplots "$@";; | |
40 | "WaitForOutput") goWaitForOutput "$@";; | |
41 | "Merge") goMerge "$@";; | |
42 | *) | |
43 | ${runMode} "$@" | |
44 | ;; | |
45 | esac | |
46 | return 0 | |
47 | } | |
48 | ||
49 | generateMC() | |
50 | { | |
51 | #generate one raw chunk in current directory | |
52 | SEED=${JOB_ID}${SGE_TASK_ID} | |
53 | export CONFIG_SEED=${SEED} | |
54 | runNumber=${1} | |
55 | OCDBpath=${2} | |
56 | nEventsim=${3} | |
57 | if [[ -n ${pretend} ]]; then | |
58 | sleep ${pretendDelay} | |
59 | touch galice.root | |
60 | else | |
61 | if [[ -f sim.C && -f Config.C ]] ; then | |
62 | time aliroot -b -q -x sim.C\(${runNumber},\"${OCDBpath}\",${nEventsim}\) >sim.log 2>&1 | |
63 | mv syswatch.log simwatch.log | |
64 | fi | |
65 | fi | |
66 | } | |
67 | ||
68 | goCPass0() | |
69 | ( | |
70 | umask 0002 | |
71 | ||
72 | targetDirectory=${1} | |
73 | inputList=${2} | |
74 | nEvents=${3} | |
75 | ocdbPath=${4} | |
76 | configFile=${5} | |
77 | runNumber=${6} | |
78 | jobindex=${7} | |
79 | shift 7 | |
80 | if ! parseConfig ${configFile} "$@"; then return 1; fi | |
81 | ||
82 | #record the working directory provided by the batch system | |
83 | batchWorkingDirectory=${PWD} | |
84 | ||
85 | #use the jobindex only if set and non-negative | |
86 | if [[ -z ${jobindex} || ${jobindex} -lt 0 ]]; then | |
87 | [[ -n "${LSB_JOBINDEX}" ]] && jobindex=${LSB_JOBINDEX} | |
88 | [[ -n "${SGE_TASK_ID}" ]] && jobindex=${SGE_TASK_ID} | |
89 | if [[ -z ${jobindex} ]]; then | |
90 | echo "no jobindex!" | |
91 | return 1 | |
92 | fi | |
93 | fi | |
94 | ||
95 | [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD} | |
96 | doneFile="${commonOutputPath}/meta/cpass0.job${jobindex}.run${runNumber}.done" | |
97 | [[ -n ${useProfilingCommand} ]] && doneFile="${commonOutputPath}/meta/profiling.cpass0.job${jobindex}.run${runNumber}.done" | |
98 | ||
99 | [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource} | |
100 | ||
101 | if [[ -n ${ALIROOT_FORCE_COREDUMP} ]]; then | |
102 | ulimit -c unlimited | |
103 | export ALIROOT_FORCE_COREDUMP | |
104 | fi | |
105 | ||
106 | #the contents of this is stored in the tree and used later (e.g. AliAnalysisTaskPIDResponse)! | |
107 | #at the QA stage the pass number is guessed from the path stored here. | |
108 | #The Format is: | |
109 | #Packages= ;OutputDir= ;LPMPass= ;TriggerAlias= ;LPMRunNumber= ;LPMProductionType= ;LPMInteractionType= ;LPMProductionTag= ;LPMAnchorRun= ;LPMAnchorProduction= ;LPMAnchorYear= | |
110 | export PRODUCTION_METADATA="OutputDir=cpass0" | |
111 | ||
112 | if [[ "${inputList}" =~ \.root$ ]]; then | |
113 | infile=${inputList} | |
114 | else | |
115 | infile=$(sed -ne "${jobindex}p" ${inputList} | egrep '\s*\w*/\w*') | |
116 | fi | |
117 | chunkName=${infile##*/} | |
118 | ||
119 | outputDir=${targetDirectory}/${jobindex}_${chunkName%.*} | |
120 | mkdir -p ${outputDir} | |
121 | if [[ ! -d ${outputDir} ]]; then | |
122 | touch ${doneFile} | |
123 | echo "cannot make ${outputDir}" >> ${doneFile} | |
124 | return 1 | |
125 | fi | |
126 | ||
127 | #runpath=${PWD}/rundir_cpass0_${runNumber}_${jobindex} | |
128 | runpath=${outputDir} | |
129 | #[[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR} | |
130 | [[ ${reconstructInTemporaryDir} -eq 1 ]] && runpath=$(mktemp -d -t cpass0.XXXXXX) | |
131 | mkdir -p ${runpath} | |
132 | if [[ ! -d ${runpath} ]]; then | |
133 | touch ${doneFile} | |
134 | echo "cannot make runpath ${runpath}" >> ${doneFile} | |
135 | return 1 | |
136 | fi | |
137 | if ! cd ${runpath}; then | |
138 | touch ${doneFile} | |
139 | echo "PWD=$PWD is not the runpath=${runpath}" >> ${doneFile} | |
140 | return 1 | |
141 | fi | |
142 | ||
143 | #runCPassX/C expects the raw chunk to be linked in the run dir | |
144 | #despite it being accessed by the full path | |
145 | ln -s ${infile} ${runpath}/${chunkName} | |
146 | ||
147 | #####MC | |
148 | if [[ -n ${generateMC} ]]; then | |
149 | olddir=${PWD} | |
150 | outputDirMC=${commonOutputPath}/000${runNumber}/sim/${jobindex} | |
151 | simrunpath=${outputDirMC} | |
152 | #[[ ${simulateInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && simrunpath=${TMPDIR} | |
153 | [[ ${simulateInTemporaryDir} -eq 1 ]] && simrunpath=$(mktemp -d -t cpass0MC.XXXXXX) | |
154 | mkdir -p ${outputDirMC} | |
155 | mkdir -p ${simrunpath} | |
156 | if cd ${simrunpath}; then | |
157 | ||
158 | filesMC=( | |
159 | "${batchWorkingDirectory}/sim.C" | |
160 | "${batchWorkingDirectory}/rec.C" | |
161 | "${batchWorkingDirectory}/Config.C" | |
162 | "${batchWorkingDirectory}/OCDB_*.root" | |
163 | ) | |
164 | for file in ${filesMC[*]}; do | |
165 | [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} . | |
166 | done | |
167 | ||
168 | generateMC ${runNumber} ${ocdbPath} ${nEvents} | |
169 | ||
170 | [[ ! "${simrunpath}" =~ "${outputDirMC}" ]] && mv * ${outputDirMC} #TODO check if it works | |
171 | cd ${olddir} | |
172 | ||
173 | ln -s ${outputDirMC}/* ${runpath}/ | |
174 | ||
175 | inputList=${outputDirMC}/galice.root #TODO not valid outside shell !!! | |
176 | infile="" | |
177 | fi | |
178 | fi | |
179 | ###### | |
180 | ||
181 | if [[ ! -f ${inputList} && -z ${pretend} ]]; then | |
182 | touch ${doneFile} | |
183 | echo "input file ${inputList} not found, exiting..." >> ${doneFile} | |
184 | return 1 | |
185 | fi | |
186 | ||
187 | logOutputDir=${runpath} | |
188 | [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir} | |
189 | [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/stdout | |
190 | #[[ -z ${dontRedirectStdOutToLog} ]] && exec 2> ${logOutputDir}/stderr | |
191 | echo "${0} $*" | |
192 | ||
193 | echo "#####################" | |
194 | echo CPass0: | |
195 | echo JOB setup | |
196 | echo nEvents ${nEvents} | |
197 | echo runNumber ${runNumber} | |
198 | echo ocdbPath ${ocdbPath} | |
199 | echo infile ${infile} | |
200 | echo chunkName ${chunkName} | |
201 | echo jobindex ${jobindex} | |
202 | echo recoTriggerOptions ${recoTriggerOptions} | |
203 | echo targetDirectory ${targetDirectory} | |
204 | echo commonOutputPath ${commonOutputPath} | |
205 | echo doneFile ${doneFile} | |
206 | echo batchWorkingDirectory=$batchWorkingDirectory | |
207 | echo runpath ${runpath} | |
208 | echo outputDir ${outputDir} | |
209 | echo PWD ${PWD} | |
210 | echo ALICE_ROOT ${ALICE_ROOT} | |
211 | echo "########## ###########" | |
212 | ||
213 | alirootInfo > ALICE_ROOT.log | |
214 | ||
215 | filesCPass0=( | |
216 | "${batchWorkingDirectory}/runCPass0.sh" | |
217 | "${batchWorkingDirectory}/recCPass0.C" | |
218 | "${batchWorkingDirectory}/runCalibTrain.C" | |
219 | "${batchWorkingDirectory}/localOCDBaccessConfig.C" | |
220 | "${batchWorkingDirectory}/OCDB.root" | |
221 | "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/runCPass0.sh" | |
222 | "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/recCPass0.C" | |
223 | "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/runCalibTrain.C" | |
224 | ) | |
225 | ||
226 | for file in ${filesCPass0[*]}; do | |
227 | [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} . | |
228 | [[ ${file##*/} =~ .*\.sh ]] && chmod +x ${file##*/} | |
229 | done | |
230 | ||
231 | echo "this directory (${PWD}) contents:" | |
232 | /bin/ls | |
233 | echo | |
234 | chmod u+x runCPass0.sh | |
235 | ||
236 | sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh | |
237 | ||
238 | if [[ -n ${postSetUpActionCPass0} ]]; then | |
239 | echo "running ${postSetUpActionCPass0}" | |
240 | eval ${postSetUpActionCPass0} | |
241 | fi | |
242 | ||
243 | #run CPass0 | |
244 | echo "${runpath}/runCPass0.sh ${infile} ${nEvents} ${runNumber} ${ocdbPath} ${recoTriggerOptions}" | |
245 | if [[ -n ${pretend} ]]; then | |
246 | sleep ${pretendDelay} | |
247 | touch AliESDs.root | |
248 | touch AliESDfriends.root | |
249 | touch AliESDfriends_v1.root | |
250 | touch rec.log | |
251 | touch calib.log | |
252 | else | |
253 | echo ./runCPass0.sh "${infile}" "${nEvents}" "${runNumber}" "${ocdbPath}" "${recoTriggerOptions}" | |
254 | ./runCPass0.sh "${infile}" "${nEvents}" "${runNumber}" "${ocdbPath}" "${recoTriggerOptions}" | |
255 | fi | |
256 | ||
257 | #move stuff to final destination | |
258 | echo "this directory (${PWD}) contents:" | |
259 | /bin/ls | |
260 | echo | |
261 | ||
262 | echo rm -f ./${chunkName} | |
263 | rm -f ./${chunkName} | |
264 | echo "cp -R ${runpath}/* ${outputDir}" | |
265 | cp -p -R ${runpath}/* ${outputDir} | |
266 | echo | |
267 | ||
268 | #validate CPass0 | |
269 | cd ${outputDir} | |
270 | touch ${doneFile} | |
271 | echo "dir ${outputDir}" >> ${doneFile} | |
272 | if summarizeLogs >> ${doneFile}; then | |
273 | [[ -f ${outputDirMC}/galice.root ]] && echo "sim ${outputDirMC}/galice.root" >> ${doneFile} | |
274 | [[ -f AliESDfriends_v1.root ]] && echo "calibfile ${outputDir}/AliESDfriends_v1.root" >> ${doneFile} | |
275 | [[ -f AliESDs.root ]] && echo "esd ${outputDir}/AliESDs.root" >> ${doneFile} | |
276 | fi | |
277 | ||
278 | [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath} && echo "removing ${runpath}" | |
279 | return 0 | |
280 | ) | |
281 | ||
282 | goCPass1() | |
283 | ( | |
284 | umask 0002 | |
285 | ||
286 | targetDirectory=${1} | |
287 | inputList=${2} | |
288 | nEvents=${3} | |
289 | ocdbPath=${4} | |
290 | configFile=${5} | |
291 | runNumber=${6} | |
292 | jobindex=${7} | |
293 | shift 7 | |
294 | extraOpts=("$@") | |
295 | if ! parseConfig ${configFile} "$@"; then return 1; fi | |
296 | ||
297 | #record the working directory provided by the batch system | |
298 | batchWorkingDirectory=${PWD} | |
299 | ||
300 | #use the jobindex only if set and non-negative | |
301 | if [[ -z ${jobindex} || ${jobindex} -lt 0 ]]; then | |
302 | [[ -n "${LSB_JOBINDEX}" ]] && jobindex=${LSB_JOBINDEX} | |
303 | [[ -n "${SGE_TASK_ID}" ]] && jobindex=${SGE_TASK_ID} | |
304 | if [[ -z ${jobindex} ]]; then | |
305 | echo "no jobindex!" | |
306 | return 1 | |
307 | fi | |
308 | fi | |
309 | ||
310 | [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD} | |
311 | doneFile="${commonOutputPath}/meta/cpass1.job${jobindex}.run${runNumber}.done" | |
312 | [[ -n ${useProfilingCommand} ]] && doneFile="${commonOutputPath}/meta/profiling.cpass1.job${jobindex}.run${runNumber}.done" | |
313 | ||
314 | [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource} | |
315 | ||
316 | if [[ -n ${ALIROOT_FORCE_COREDUMP} ]]; then | |
317 | ulimit -c unlimited | |
318 | export ALIROOT_FORCE_COREDUMP | |
319 | fi | |
320 | ||
321 | #the contents of this is stored in the tree and used later (e.g. AliAnalysisTaskPIDResponse)! | |
322 | #at the QA stage the pass number is guessed from the path stored here. | |
323 | #The Format is: | |
324 | #Packages= ;OutputDir= ;LPMPass= ;TriggerAlias= ;LPMRunNumber= ;LPMProductionType= ;LPMInteractionType= ;LPMProductionTag= ;LPMAnchorRun= ;LPMAnchorProduction= ;LPMAnchorYear= | |
325 | export PRODUCTION_METADATA="OutputDir=cpass1" | |
326 | ||
327 | if [[ ! -f ${inputList} && -z ${pretend} ]]; then | |
328 | touch ${doneFile} | |
329 | echo "input file ${inputList} not found, exiting..." >> ${doneFile} | |
330 | return 1 | |
331 | fi | |
332 | if [[ "${inputList}" =~ \.root$ ]]; then | |
333 | infile=${inputList} | |
334 | else | |
335 | infile=$(sed -ne "${jobindex}p" ${inputList} | egrep '\s*\w*/\w*') | |
336 | fi | |
337 | chunkName=${infile##*/} | |
338 | ||
339 | outputDir=${targetDirectory}/${jobindex}_${chunkName%.*} | |
340 | mkdir -p ${outputDir} | |
341 | if [[ ! -d ${outputDir} ]];then | |
342 | touch ${doneFile} | |
343 | echo "cannot make ${outputDir}" >> ${doneFile} | |
344 | return 1 | |
345 | fi | |
346 | ||
347 | #runpath=${PWD}/rundir_cpass1_${runNumber}_${jobindex} | |
348 | runpath=${outputDir} | |
349 | #[[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR} | |
350 | [[ ${reconstructInTemporaryDir} -eq 1 ]] && runpath=$(mktemp -d -t cpass1.XXXXXX) | |
351 | ||
352 | #MC | |
353 | if [[ "${infile}" =~ galice\.root ]]; then | |
354 | ln -s ${inputList%/*}/* ${runpath} | |
355 | infile="" | |
356 | fi | |
357 | ||
358 | #init the running path | |
359 | mkdir -p ${runpath} | |
360 | if [[ ! -d ${runpath} ]]; then | |
361 | touch ${doneFile} | |
362 | echo "cannot make runpath ${runpath}" >> ${doneFile} | |
363 | return 1 | |
364 | fi | |
365 | if ! cd ${runpath}; then | |
366 | touch ${doneFile} | |
367 | echo "PWD=$PWD is not the runpath=${runpath}" >> ${doneFile} | |
368 | return 1 | |
369 | fi | |
370 | ||
371 | #this is needed for runCPass1.sh | |
372 | ln -s ${infile} ${runpath}/${chunkName} | |
373 | ||
374 | logOutputDir=${runpath} | |
375 | [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir} | |
376 | [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/stdout | |
377 | #[[ -z ${dontRedirectStdOutToLog} ]] && exec 2> ${logOutputDir}/stderr | |
378 | echo "${0} $*" | |
379 | ||
380 | echo "#####################" | |
381 | echo CPass1: | |
382 | echo JOB setup | |
383 | echo nEvents ${nEvents} | |
384 | echo runNumber ${runNumber} | |
385 | echo ocdbPath ${ocdbPath} | |
386 | echo infile ${infile} | |
387 | echo chunkName ${chunkName} | |
388 | echo jobindex ${jobindex} | |
389 | echo recoTriggerOptions ${recoTriggerOptions} | |
390 | echo targetDirectory ${targetDirectory} | |
391 | echo commonOutputPath ${commonOutputPath} | |
392 | echo doneFile ${doneFile} | |
393 | echo runpath ${runpath} | |
394 | echo outputDir ${outputDir} | |
395 | echo batchWorkingDirectory ${batchWorkingDirectory} | |
396 | echo ALICE_ROOT ${ALICE_ROOT} | |
397 | echo PWD ${PWD} | |
398 | echo "########## ###########" | |
399 | ||
400 | alirootInfo > ALICE_ROOT.log | |
401 | ||
402 | filesCPass1=( | |
403 | "${batchWorkingDirectory}/runCPass1.sh" | |
404 | "${batchWorkingDirectory}/recCPass1.C" | |
405 | "${batchWorkingDirectory}/recCPass1_OuterDet.C" | |
406 | "${batchWorkingDirectory}/runCalibTrain.C" | |
407 | "${batchWorkingDirectory}/QAtrain_duo.C" | |
408 | "${batchWorkingDirectory}/localOCDBaccessConfig.C" | |
409 | "${batchWorkingDirectory}/${configFile}" | |
410 | "${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz" | |
411 | "${batchWorkingDirectory}/OCDB.root" | |
412 | "${trustedQAtrainMacro}" | |
413 | "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/runCPass1.sh" | |
414 | "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/recCPass1.C" | |
415 | "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/recCPass1_OuterDet.C" | |
416 | "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/runCalibTrain.C" | |
417 | "${ALICE_ROOT}/ANALYSIS/macros/QAtrain_duo.C" | |
418 | ) | |
419 | ||
420 | for file in "${filesCPass1[@]}"; do | |
421 | [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} . | |
422 | [[ ${file##*/} =~ .*\.sh ]] && echo "making ${file##*/} executable" && chmod +x ${file##*/} | |
423 | done | |
424 | ||
425 | echo "this directory (${PWD}) contents:" | |
426 | /bin/ls | |
427 | echo | |
428 | ||
429 | #remove spaces around commas from calls to root | |
430 | sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh | |
431 | ||
432 | if [[ -n ${postSetUpActionCPass1} ]]; then | |
433 | echo "running ${postSetUpActionCPass1}" | |
434 | eval ${postSetUpActionCPass1} | |
435 | echo | |
436 | fi | |
437 | ||
438 | #configure local OCDB storage from CPass0 (creates the localOCDBaccessConfig.C script) | |
439 | if [[ -f cpass0.localOCDB.${runNumber}.tgz ]]; then | |
440 | echo goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz" | |
441 | goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz" | |
442 | else | |
443 | echo "WARNING: file cpass0.localOCDB.${runNumber}.tgz not found!" | |
444 | fi | |
445 | ||
446 | if [[ ! $(/bin/ls -1 OCDB/*/*/*/*.root 2>/dev/null) ]]; then | |
447 | touch ${doneFile} | |
448 | echo "cpass0 produced no calibration! exiting..." >> ${doneFile} | |
449 | return 1 | |
450 | fi | |
451 | ||
452 | #create the Barrel and OuterDet directories for CPass1 and link the local OCDB directory | |
453 | #there to make the localOCDBaccessConfig.C file work, since it may point to the OCDB | |
454 | #entries using a relative path, e.g. local://./OCDB | |
455 | echo "linking the OCDB/ for Barrel and OuterDet directories" | |
456 | mkdir Barrel OuterDet | |
457 | ls -l | |
458 | ln -s ../OCDB Barrel/OCDB | |
459 | ln -s ../OCDB OuterDet/OCDB | |
460 | ||
461 | #setup the filtering | |
462 | #the following option enables the filtering task inside the QAtrain_duo.C | |
463 | [[ -n $runESDfiltering ]] && export QA_TaskFilteredTree=1 | |
464 | #set the downscaling factors during the filtering fro expert QA (overrides the previous values) | |
465 | if [[ -n ${filteringFactorHighPt} ]]; then | |
466 | export AliAnalysisTaskFilteredTree_fLowPtTrackDownscaligF=${filteringFactorHighPt} | |
467 | fi | |
468 | if [[ -n ${filteringFactorV0s} ]]; then | |
469 | export AliAnalysisTaskFilteredTree_fLowPtV0DownscaligF=${filteringFactorV0s} | |
470 | fi | |
471 | ||
472 | #run CPass1 | |
473 | chmod u+x runCPass1.sh | |
474 | echo "${runpath}/runCPass1.sh ${infile} ${nEvents} ${runNumber} ${ocdbPath} ${recoTriggerOptions}" | |
475 | if [[ -n ${pretend} ]]; then | |
476 | sleep ${pretendDelay} | |
477 | touch AliESDs_Barrel.root | |
478 | touch AliESDfriends_Barrel.root | |
479 | touch AliESDfriends_v1.root | |
480 | touch QAresults_barrel.root | |
481 | touch EventStat_temp_barrel.root | |
482 | touch AODtpITS.root | |
483 | touch AliESDs_Outer.root | |
484 | touch AliESDfriends_Outer.root | |
485 | touch QAresults_outer.root | |
486 | touch EventStat_temp_outer.root | |
487 | touch rec.log | |
488 | touch calib.log | |
489 | touch qa.log | |
490 | touch filtering.log FilterEvents_Trees.root | |
491 | else | |
492 | ./runCPass1.sh "${infile}" "${nEvents}" "${runNumber}" "${ocdbPath}" "${recoTriggerOptions}" | |
493 | ||
494 | [[ ! -f AliESDs_Barrel.root && -f Barrel/AliESDs.root ]] && mv Barrel/AliESDs.root AliESDs_Barrel.root | |
495 | [[ ! -f AliESDfriends_Barrel.root && -f Barrel/AliESDfriends.root ]] && mv Barrel/AliESDfriends.root AliESDfriends_Barrel.root | |
496 | [[ ! -f AliESDfriends_v1.root && -f Barrel/AliESDfriends_v1.root ]] && mv Barrel/AliESDfriends_v1.root . | |
497 | [[ ! -f QAresults_barrel.root && -f Barrel/QAresults_barrel.root ]] && mv Barrel/QAresults_barrel.root . | |
498 | [[ ! -f AliESDs_Outer.root && -f OuterDet/AliESDs.root ]] && mv OuterDet/AliESDs.root AliESDs_Outer.root | |
499 | [[ ! -f AliESDfriends_Outer.root && -f OuterDet/AliESDfriends.root ]] && mv OuterDet/AliESDfriends.root AliESDfriends_Outer.root | |
500 | [[ ! -f QAresults_outer.root && -f OuterDet/QAresults_outer.root ]] && mv OuterDet/QAresults_outer.root . | |
501 | [[ ! -f FilterEvents_Trees.root && -f Barrel/FilterEvents_Trees.root ]] && mv Barrel/FilterEvents_Trees.root . | |
502 | ||
503 | #make the filtered tree (if requested and not already produced by QA | |
504 | [[ -f AliESDs_Barrel.root ]] && echo "AliESDs_Barrel.root" > filtered.list | |
505 | if [[ -n ${runESDfiltering} && ! -f FilterEvents_Trees.root && -f filtered.list ]]; then | |
506 | goMakeFilteredTrees ${PWD} ${runNumber} "${PWD}/filtered.list" ${filteringFactorHighPt} ${filteringFactorV0s} ${ocdbPath} 1000000 0 10000000 0 ${configFile} AliESDs_Barrel.root "${extraOpts[@]}" >filtering.log | |
507 | else | |
508 | echo "" | |
509 | fi | |
510 | ||
511 | fi | |
512 | ||
513 | ##handle possible crashes in QA (happens often in trunk) | |
514 | ##rerun QA with a trusted aliroot version | |
515 | #if [[ $(validateLog qa_barrel.log) ]]; then | |
516 | # echo "qa_barrel.log not validated!" | |
517 | #fi | |
518 | #if [[ ! -f QAresults_barrel.root && -f ${setupTrustedAliROOTenvInCurrentShell} || $(validateLog qa_barrel.log) ]]; then | |
519 | # echo "WARNING: using trusted QA aliroot ${ALICE_ROOT}" | |
520 | # source ${setupTrustedAliROOTenvInCurrentShell} | |
521 | # cd Barrel | |
522 | # rm QAresults_barrel.root | |
523 | # rm EventStat_temp_barrel.root | |
524 | # rm AODtpITS.root | |
525 | # [[ ! -f AliESDs.root ]] && ln -s ../AliESDs_Barrel.root AliESDs.root | |
526 | # [[ ! -f AliESDfriends.root ]] && ln -s ../AliESDfriends_Barrel.root AliESDfriends.root | |
527 | # if [[ -n ${trustedQAtrainMacro} ]]; then | |
528 | # eval "cp ${trustedQAtrainMacro} QAtrain_duo_trusted.C" | |
529 | # fi | |
530 | # echo executing aliroot -b -q "QAtrain_duo_trusted.C(\"_barrel\",${runNumber},\"wn.xml\",0,\"${ocdbPath}\")" | |
531 | # time aliroot -b -q "QAtrain_duo.C(\"_barrel\",${runNumber},\"wn.xml\",0,\"${ocdbPath}\")" &> ../qa_barrel_trusted.log | |
532 | # cd ../ | |
533 | #fi | |
534 | ||
535 | #move stuff to final destination | |
536 | echo "this directory (${PWD}) contents:" | |
537 | /bin/ls | |
538 | echo rm -f ./${chunkName} | |
539 | rm -f ./${chunkName} | |
540 | echo "cp -R ${runpath}/* ${outputDir}" | |
541 | cp -pf -R ${runpath}/* ${outputDir} | |
542 | echo | |
543 | ||
544 | #validate CPass1 | |
545 | cd ${outputDir} | |
546 | touch ${doneFile} | |
547 | echo "dir ${outputDir}" >> ${doneFile} | |
548 | if summarizeLogs >> ${doneFile}; then | |
549 | [[ -f AliESDs_Barrel.root ]] && echo "esd ${outputDir}/AliESDs_Barrel.root" >> ${doneFile} | |
550 | [[ -f AliESDfriends_v1.root ]] && echo "calibfile ${outputDir}/AliESDfriends_v1.root" >> ${doneFile} | |
551 | [[ -f QAresults_Barrel.root ]] && echo "qafile ${outputDir}/QAresults_Barrel.root" >> ${doneFile} | |
552 | [[ -f QAresults_Outer.root ]] && echo "qafile ${outputDir}/QAresults_Outer.root" >> ${doneFile} | |
553 | [[ -f QAresults_barrel.root ]] && echo "qafile ${outputDir}/QAresults_barrel.root" >> ${doneFile} | |
554 | [[ -f QAresults_outer.root ]] && echo "qafile ${outputDir}/QAresults_outer.root" >> ${doneFile} | |
555 | [[ -f FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFile} | |
556 | else | |
557 | if grep "qa_outer.log.*OK" ${doneFile} > /dev/null; then | |
558 | [[ -f QAresults_Outer.root ]] && echo "qafile ${outputDir}/QAresults_Outer.root" >> ${doneFile} | |
559 | [[ -f QAresults_outer.root ]] && echo "qafile ${outputDir}/QAresults_outer.root" >> ${doneFile} | |
560 | fi | |
561 | if grep "qa_barrel.log.*OK" ${doneFile} > /dev/null; then | |
562 | [[ -f QAresults_Barrel.root ]] && echo "qafile ${outputDir}/QAresults_Barrel.root" >> ${doneFile} | |
563 | [[ -f QAresults_barrel.root ]] && echo "qafile ${outputDir}/QAresults_barrel.root" >> ${doneFile} | |
564 | fi | |
565 | if grep "filtering.log.*OK" ${doneFile} > /dev/null; then | |
566 | [[ -f FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFile} | |
567 | fi | |
568 | fi | |
569 | ||
570 | [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath} | |
571 | return 0 | |
572 | ) | |
573 | ||
574 | ||
575 | goMergeCPass0() | |
576 | ( | |
577 | # | |
578 | # find the output files and merge them | |
579 | # | |
580 | ||
581 | outputDir=${1} | |
582 | ocdbStorage=${2} | |
583 | configFile=${3} | |
584 | runNumber=${4} | |
585 | calibrationFilesToMerge=${5} #can be a non-existent file, will then be produced on the fly | |
586 | shift 5 | |
587 | if ! parseConfig ${configFile} "$@"; then return 1; fi | |
588 | ||
589 | #record the working directory provided by the batch system | |
590 | batchWorkingDirectory=${PWD} | |
591 | ||
592 | [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD} | |
593 | doneFile="${commonOutputPath}/meta/merge.cpass0.run${runNumber}.done" | |
594 | ||
595 | umask 0002 | |
596 | ulimit -c unlimited | |
597 | ||
598 | [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource} | |
599 | ||
600 | #runpath=${PWD}/rundir_cpass0_Merge_${runNumber} | |
601 | runpath=${outputDir} | |
602 | #[[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR} | |
603 | [[ ${reconstructInTemporaryDir} -eq 1 ]] && runpath=$(mktemp -d -t mergeCPass0.XXXXXX) | |
604 | ||
605 | mkdir -p ${runpath} | |
606 | if [[ ! -d ${runpath} ]]; then | |
607 | touch ${doneFile} | |
608 | echo "not able to make the runpath ${runpath}" >> ${doneFile} | |
609 | return 1 | |
610 | fi | |
611 | if ! cd ${runpath}; then | |
612 | touch ${doneFile} | |
613 | echo "PWD=$PWD is not the runpath=${runpath}" >> ${doneFile} | |
614 | return 1 | |
615 | fi | |
616 | ||
617 | logOutputDir=${runpath} | |
618 | [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir} | |
619 | [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/stdout | |
620 | echo "${0} $*" | |
621 | ||
622 | mergingScript="mergeMakeOCDB.byComponent.sh" | |
623 | ||
624 | echo goMergeCPass0 SETUP: | |
625 | echo runNumber=${runNumber} | |
626 | echo outputDir=${outputDir} | |
627 | echo ocdbStorage=${ocdbStorage} | |
628 | echo calibrationFilesToMerge=${calibrationFilesToMerge} | |
629 | echo mergingScript=${mergingScript} | |
630 | echo commonOutputPath=${commonOutputPath} | |
631 | echo runpath=${runpath} | |
632 | ||
633 | # copy files in case they are not already there | |
634 | filesMergeCPass0=( | |
635 | "${batchWorkingDirectory}/${calibrationFilesToMerge}" | |
636 | "${batchWorkingDirectory}/OCDB.root" | |
637 | "${batchWorkingDirectory}/localOCDBaccessConfig.C" | |
638 | "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/mergeMakeOCDB.byComponent.sh" | |
639 | "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/mergeByComponent.C" | |
640 | "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/makeOCDB.C" | |
641 | "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/merge.C" | |
642 | "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/mergeMakeOCDB.sh" | |
643 | ) | |
644 | for file in ${filesMergeCPass0[*]}; do | |
645 | [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} . | |
646 | [[ ${file##*/} =~ .*\.sh ]] && chmod +x ${file##*/} | |
647 | done | |
648 | ||
649 | sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh | |
650 | ||
651 | alirootInfo > ALICE_ROOT.log | |
652 | ||
653 | # | |
654 | echo "PWD" | |
655 | /bin/ls | |
656 | echo "PWD/.." | |
657 | /bin/ls ../ | |
658 | ||
659 | ||
660 | #merge calibration | |
661 | chmod u+x ${mergingScript} | |
662 | mkdir -p ./OCDB | |
663 | if [[ ! -f ${calibrationFilesToMerge} ]]; then | |
664 | echo "/bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root > ${calibrationFilesToMerge}" | |
665 | /bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root 2>/dev/null > ${calibrationFilesToMerge} | |
666 | fi | |
667 | ||
668 | echo "${mergingScript} ${calibrationFilesToMerge} ${runNumber} local://./OCDB ${ocdbStorage}" | |
669 | if [[ -n ${pretend} ]]; then | |
670 | sleep ${pretendDelay} | |
671 | touch CalibObjects.root | |
672 | touch ocdb.log | |
673 | touch merge.log | |
674 | touch dcsTime.root | |
675 | mkdir -p ./OCDB/TPC/Calib/TimeGain/ | |
676 | mkdir -p ./OCDB/TPC/Calib/TimeDrift/ | |
677 | echo "some calibration" >> ./OCDB/TPC/Calib/TimeGain/someCalibObject_0-999999_cpass0.root | |
678 | echo "some calibration" >> ./OCDB/TPC/Calib/TimeDrift/otherCalibObject_0-999999_cpass0.root | |
679 | else | |
680 | ./${mergingScript} ${calibrationFilesToMerge} ${runNumber} "local://./OCDB" ${ocdbStorage} >> "mergeMakeOCDB.log" | |
681 | ||
682 | #produce the calib trees for expert QA (dcsTime.root) | |
683 | goMakeLocalOCDBaccessConfig ./OCDB | |
684 | echo aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")" | |
685 | aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")" | |
686 | fi | |
687 | ||
688 | ### produce the output | |
689 | #tar the produced OCDB for reuse | |
690 | tar czf ${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz ./OCDB | |
691 | ||
692 | /bin/ls | |
693 | ||
694 | #copy all to output dir | |
695 | cp -pf -R ${runpath}/* ${outputDir} | |
696 | ||
697 | if [[ -n ${generateMC} ]]; then | |
698 | goPrintValues sim ${commonOutputPath}/meta/sim.run${runNumber}.list ${commonOutputPath}/meta/cpass0.job*.run${runNumber}.done | |
699 | fi | |
700 | ||
701 | #validate merging cpass0 | |
702 | cd ${outputDir} | |
703 | touch ${doneFile} | |
704 | echo "dir ${outputDir}" >> ${doneFile} | |
705 | if summarizeLogs >> ${doneFile}; then | |
706 | [[ -f CalibObjects.root ]] && echo "calibfile ${outputDir}/CalibObjects.root" >> ${doneFile} | |
707 | [[ -f dcsTime.root ]] && echo "dcsTree ${outputDir}/dcsTime.root" >> ${doneFile} | |
708 | fi | |
709 | ||
710 | [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath} | |
711 | return 0 | |
712 | ) | |
713 | ||
714 | goMergeCPass1() | |
715 | ( | |
716 | # | |
717 | # find the output files and merge them | |
718 | # | |
719 | ||
720 | outputDir=${1} | |
721 | ocdbStorage=${2} | |
722 | configFile=${3} | |
723 | runNumber=${4} | |
724 | calibrationFilesToMerge=${5} | |
725 | qaFilesToMerge=${6} | |
726 | filteredFilesToMerge=${7} | |
727 | shift 7 | |
728 | if ! parseConfig ${configFile} "$@"; then return 1; fi | |
729 | ||
730 | #record the working directory provided by the batch system | |
731 | batchWorkingDirectory=${PWD} | |
732 | ||
733 | [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD} | |
734 | doneFile="${commonOutputPath}/meta/merge.cpass1.run${runNumber}.done" | |
735 | ||
736 | umask 0002 | |
737 | ulimit -c unlimited | |
738 | ||
739 | #clean up first: | |
740 | rm -f ${outputDir}/*.log | |
741 | rm -f ${outputDir}/*.root | |
742 | rm -f ${outputDir}/*done | |
743 | ||
744 | [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource} | |
745 | ||
746 | #runpath=${PWD}/rundir_cpass1_Merge_${runNumber} | |
747 | runpath=${outputDir} | |
748 | #[[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR} | |
749 | [[ ${reconstructInTemporaryDir} -eq 1 ]] && runpath=$(mktemp -d -t mergeCPass1.XXXXXX) | |
750 | ||
751 | mkdir -p ${runpath} | |
752 | if [[ ! -d ${runpath} ]]; then | |
753 | touch ${doneFile} | |
754 | echo "not able to make the runpath ${runpath}" >> ${doneFile} | |
755 | return 1 | |
756 | fi | |
757 | if ! cd ${runpath}; then | |
758 | touch ${doneFile} | |
759 | echo "PWD=$PWD is not the runpath=${runpath}" >> ${doneFile} | |
760 | return 1 | |
761 | fi | |
762 | ||
763 | logOutputDir=${runpath} | |
764 | [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir} | |
765 | [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/mergeMakeOCDB.log | |
766 | echo "${0} $*" | |
767 | ||
768 | calibrationOutputFileName='AliESDfriends_v1.root' | |
769 | qaOutputFileName='QAresults*.root' | |
770 | mergingScript="mergeMakeOCDB.byComponent.sh" | |
771 | #important to have the string "Stage.txt" in the filename to trigger the merging | |
772 | #it has to be a list of directories containing the files | |
773 | qaMergedOutputFileName="QAresults_merged.root" | |
774 | ||
775 | echo goMergeCPass1 SETUP: | |
776 | echo runNumber=${runNumber} | |
777 | echo outputDir=${outputDir} | |
778 | echo ocdbStorage=${ocdbStorage} | |
779 | echo calibrationFilesToMerge=$calibrationFilesToMerge | |
780 | echo qaFilesToMerge=$qaFilesToMerge | |
781 | echo calibrationOutputFileName=${calibrationOutputFileName} | |
782 | echo mergingScript=${mergingScript} | |
783 | ||
784 | # copy files in case they are not already there | |
785 | filesMergeCPass1=( | |
786 | "${batchWorkingDirectory}/${calibrationFilesToMerge}" | |
787 | "${batchWorkingDirectory}/${qaFilesToMerge}" | |
788 | "${batchWorkingDirectory}/OCDB.root" | |
789 | "${batchWorkingDirectory}/localOCDBaccessConfig.C" | |
790 | "${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz" | |
791 | "${batchWorkingDirectory}/QAtrain_duo.C" | |
792 | "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/mergeMakeOCDB.byComponent.sh" | |
793 | "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/mergeByComponent.C" | |
794 | "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/makeOCDB.C" | |
795 | "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/merge.C" | |
796 | "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/mergeMakeOCDB.sh" | |
797 | "${trustedQAtrainMacro}" | |
798 | "${ALICE_ROOT}/ANALYSIS/macros/QAtrain_duo.C" | |
799 | ) | |
800 | for file in ${filesMergeCPass1[*]}; do | |
801 | [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} . | |
802 | [[ ${file##*/} =~ .*\.sh ]] && chmod +x ${file##*/} | |
803 | done | |
804 | ||
805 | sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh | |
806 | ||
807 | #configure local OCDB storage from CPass0 (creates the localOCDBaccessConfig.C script) | |
808 | if [[ -f cpass0.localOCDB.${runNumber}.tgz ]]; then | |
809 | echo goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz" | |
810 | goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz" | |
811 | else | |
812 | echo "WARNING: file cpass0.localOCDB.${runNumber}.tgz not found!" | |
813 | fi | |
814 | ||
815 | alirootInfo > ALICE_ROOT.log | |
816 | ||
817 | # | |
818 | /bin/ls | |
819 | ||
820 | #merge calibration | |
821 | chmod u+x ${mergingScript} | |
822 | mkdir -p OCDB | |
823 | ||
824 | #if not provided, create the lists of files to merge | |
825 | if [[ ! -f ${filteredFilesToMerge} ]]; then | |
826 | echo "/bin/ls -1 ${outputDir}/*/FilterEvents_Trees.root > ${filteredFilesToMerge}" | |
827 | /bin/ls -1 ${outputDir}/*/FilterEvents_Trees.root 2>/dev/null > ${filteredFilesToMerge} | |
828 | fi | |
829 | if [[ ! -f ${calibrationFilesToMerge} ]]; then | |
830 | echo "/bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root > ${calibrationFilesToMerge}" | |
831 | /bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root 2>/dev/null > ${calibrationFilesToMerge} | |
832 | fi | |
833 | if [[ ! -f ${qaFilesToMerge} ]]; then | |
834 | #find the files, but only store the directories (QAtrain_duo.C requires this) | |
835 | echo "/bin/ls -1 ${outputDir}/*/QAresults*.root | while read x; do echo ${x%/*}; done | sort | uniq > ${qaFilesToMerge}" | |
836 | /bin/ls -1 ${outputDir}/*/QAresults*.root | while read x; do echo ${x%/*}; done | sort | uniq > ${qaFilesToMerge} | |
837 | fi | |
838 | ||
839 | echo "${mergingScript} ${calibrationFilesToMerge} ${runNumber} local://./OCDB ${ocdbStorage}" | |
840 | if [[ -n ${pretend} ]]; then | |
841 | sleep ${pretendDelay} | |
842 | touch ocdb.log | |
843 | touch cpass1.localOCDB.${runNumber}.tgz | |
844 | touch ${qaMergedOutputFileName} | |
845 | touch merge.log | |
846 | touch trending.root | |
847 | touch FilterEvents_Trees.root | |
848 | touch CalibObjects.root | |
849 | touch dcsTime.root | |
850 | touch ${qaMergedOutputFileName} | |
851 | mkdir -p OCDB | |
852 | else | |
853 | ./${mergingScript} ${calibrationFilesToMerge} ${runNumber} "local://./OCDB" ${ocdbStorage} | |
854 | ||
855 | #merge QA (and filtered trees) | |
856 | [[ -n ${AliAnalysisTaskFilteredTree_fLowPtTrackDownscaligF} ]] && export AliAnalysisTaskFilteredTree_fLowPtTrackDownscaligF | |
857 | [[ -n ${AliAnalysisTaskFilteredTree_fLowPtV0DownscaligF} ]] && export AliAnalysisTaskFilteredTree_fLowPtV0DownscaligF | |
858 | ||
859 | #echo aliroot -l -b -q "merge.C(\"${qaFilesToMerge}\",\"\",kFALSE,\"${qaMergedOutputFileName}\")" | |
860 | echo aliroot -b -q "QAtrain_duo.C(\"_barrel\",${runNumber},\"${qaFilesToMerge}\",1,\"${ocdbStorage}\")" | |
861 | #aliroot -l -b -q "merge.C(\"${qaFilesToMerge}\",\"\",kFALSE,\"${qaMergedOutputFileName}\")" | |
862 | aliroot -b -q "QAtrain_duo.C(\"_barrel\",${runNumber},\"${qaFilesToMerge}\",1,\"${ocdbStorage}\")" > mergeQA.log | |
863 | mv QAresults_barrel.root ${qaMergedOutputFileName} | |
864 | mv trending_barrel.root trending.root | |
865 | ||
866 | #merge filtered trees | |
867 | echo aliroot -l -b -q "merge.C(\"${qaFilesToMerge}\",\"\",kFALSE,\"${qaMergedOutputFileName}\")" | |
868 | aliroot -l -b -q "merge.C(\"${filteredFilesToMerge}\",\"\",kFALSE,\"FilterEvents_Trees.root\")" > mergeFilteredTrees.log | |
869 | ||
870 | #produce the calib trees for expert QA | |
871 | echo aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")" | |
872 | aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")" > calibTree.log | |
873 | fi | |
874 | ||
875 | tar czf ${commonOutputPath}/meta/cpass1.localOCDB.${runNumber}.tgz ./OCDB | |
876 | ||
877 | /bin/ls | |
878 | ||
879 | #copy all to output dir | |
880 | cp -pf -R ${runpath}/* ${outputDir} | |
881 | ||
882 | #validate merge cpass1 | |
883 | cd ${outputDir} | |
884 | touch ${doneFile} | |
885 | echo "dir ${outputDir}" >> ${doneFile} | |
886 | if summarizeLogs >> ${doneFile}; then | |
887 | [[ -f CalibObjects.root ]] && echo "calibfile ${outputDir}/CalibObjects.root" >> ${doneFile} | |
888 | [[ -f ${qaMergedOutputFileName} ]] && echo "qafile ${outputDir}/${qaMergedOutputFileName}" >> ${doneFile} | |
889 | [[ -f trending.root ]] && echo "trendingfile ${outputDir}/trending.root" >> ${doneFile} | |
890 | [[ -f dcsTime.root ]] && echo "dcsTree ${outputDir}/dcsTime.root" >> ${doneFile} | |
891 | [[ -f FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFile} | |
892 | else | |
893 | if grep "mergeQA.log.*OK" ${doneFile} > /dev/null; then | |
894 | [[ -f ${qaMergedOutputFileName} ]] && echo "qafile ${outputDir}/${qaMergedOutputFileName}" >> ${doneFile} | |
895 | fi | |
896 | if grep "mergeFilteredTrees.log.*OK" ${doneFile} > /dev/null; then | |
897 | [[ -f FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFile} | |
898 | fi | |
899 | fi | |
900 | ||
901 | [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath} | |
902 | return 0 | |
903 | ) | |
904 | ||
905 | goMerge() | |
906 | ( | |
907 | #generic root merge using CPass1 merge.C script | |
908 | inputList=${1} | |
909 | outputFile=${2} | |
910 | configFile=${3-"benchmark.config"} | |
911 | shift 3 | |
912 | if ! parseConfig ${configFile} "$@"; then return 1; fi | |
913 | ||
914 | #record the working directory provided by the batch system | |
915 | batchWorkingDirectory=${PWD} | |
916 | ||
917 | [[ ! -f ${inputList} ]] && echo "inputList ${inputList} does not exist!" && return 1 | |
918 | [[ ! -f ${configFile} ]] && echo "configFile ${configFile} does not exist!" && return 1 | |
919 | umask 0002 | |
920 | [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource} | |
921 | rm -f ${outputFile} | |
922 | aliroot -b -q "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/merge.C(\"${inputList}\",\"\",kFALSE,\"${outputFile}\")" > merge_${inputList}.log | |
923 | return 0 | |
924 | ) | |
925 | ||
926 | goSubmitMakeflow() | |
927 | { | |
928 | #run | |
929 | productionID=${1} | |
930 | inputFileList=${2} | |
931 | configFile=${3} | |
932 | shift 3 | |
933 | extraOpts=("$@") | |
934 | if ! parseConfig ${configFile} "${extraOpts[@]}"; then return 1; fi | |
935 | ||
936 | #record the working directory provided by the batch system | |
937 | batchWorkingDirectory=${PWD} | |
938 | ||
939 | [[ -z ${configFile} ]] && configFile="benchmark.config" | |
940 | [[ ! -f ${configFile} ]] && echo "no config file found (${configFile})" && return 1 | |
941 | ||
942 | if [[ ! $(which makeflow &>/dev/null) && -n ${makeflowPath} ]]; then | |
943 | echo "setting the makflow path from the config: " | |
944 | echo " export PATH=${makeflowPath}:${PATH}" | |
945 | export PATH=${makeflowPath}:${PATH} | |
946 | fi | |
947 | ||
948 | #create the common output dir and the meta dir | |
949 | commonOutputPath=${baseOutputDirectory}/${productionID} | |
950 | if [[ -d ${commonOutputPath} ]]; then | |
951 | echo "output dir ${commonOutputPath} exists!" | |
952 | #return 1 | |
953 | else | |
954 | mkdir -p ${commonOutputPath} | |
955 | fi | |
956 | mkdir -p ${commonOutputPath}/meta | |
957 | ||
958 | self=${0} | |
959 | #if which greadlink; then self=$(greadlink -f "${0}"); fi | |
960 | ||
961 | #for reference copy the setup to the output dir | |
962 | cp ${self} ${commonOutputPath} | |
963 | cp ${configFile} ${commonOutputPath} | |
964 | cp ${inputFileList} ${commonOutputPath} | |
965 | ||
966 | #submit - use makeflow if available, fall back to old stuff when makeflow not there | |
967 | if which makeflow; then | |
968 | goGenerateMakeflow ${productionID} ${inputFileList} ${configFile} "${extraOpts[@]}" commonOutputPath=${commonOutputPath} > benchmark.makeflow | |
969 | cp benchmark.makeflow ${commonOutputPath} | |
970 | makeflow ${makeflowOptions} benchmark.makeflow | |
971 | else | |
972 | echo "no makeflow!" | |
973 | fi | |
974 | return 0 | |
975 | } | |
976 | ||
977 | goGenerateMakeflow() | |
978 | ( | |
979 | #generate the makeflow file | |
980 | [[ $# -lt 3 ]] && echo "args: id inputFileList configFile" && return 1 | |
981 | productionID=${1} | |
982 | inputFileList=${2} | |
983 | configFile=${3} | |
984 | shift 3 | |
985 | extraOpts=("$@") | |
986 | ||
987 | #batch systems/makeflow sometimes handle spaces in arguments poorly, so encode them | |
988 | for (( i=0;i<${#extraOpts[@]};i++ )); do | |
989 | extraOpts[i]=$(encSpaces "${extraOpts[i]}") | |
990 | done | |
991 | extraOpts+=("encodedSpaces=1") | |
992 | ||
993 | if ! parseConfig ${configFile} "${extraOpts[@]}" &>/dev/null; then return 1; fi | |
994 | ||
995 | #extra safety | |
996 | if [[ -z ${commonOutputPath} ]]; then | |
997 | commonOutputPath=${baseOutputDirectory}/${productionID} | |
998 | extraOpts=( "${extraOpts[@]}" "commonOutputPath=${commonOutputPath}" ) | |
999 | fi | |
1000 | ||
1001 | #record the working directory provided by the batch system | |
1002 | batchWorkingDirectory=${PWD} | |
1003 | ||
1004 | [[ -z ${configFile} ]] && configFile="benchmark.config" | |
1005 | [[ ! -f ${configFile} ]] && echo "no config file found (${configFile})" && return 1 | |
1006 | ||
1007 | #these files will be made a dependency - will be copied to the working dir of the jobs | |
1008 | declare -a copyFiles | |
1009 | inputFiles=( | |
1010 | "OCDB.root" | |
1011 | "localOCDBaccessConfig.C" | |
1012 | "QAtrain_duo.C" | |
1013 | "runCPass1.sh" | |
1014 | "recCPass1.C" | |
1015 | "recCPass1_OuterDet.C" | |
1016 | "runCalibTrain.C" | |
1017 | "runCPass0.sh" | |
1018 | "recCPass0.C" | |
1019 | ) | |
1020 | for file in ${inputFiles[*]}; do | |
1021 | [[ -f ${file} ]] && copyFiles+=("${file}") | |
1022 | done | |
1023 | ||
1024 | #create the makeflow file | |
1025 | [[ -n ${batchFlags} ]] && echo "BATCH_OPTIONS = ${batchFlags}" | |
1026 | declare -A arr_cpass0_merged arr_cpass1_merged | |
1027 | declare -A arr_cpass0_calib_list arr_cpass1_calib_list | |
1028 | declare -A arr_cpass1_QA_list arr_cpass1_ESD_list arr_cpass1_filtered_list | |
1029 | declare -A arr_cpass0_profiled_outputs | |
1030 | declare -A listOfRuns | |
1031 | [[ -n ${runNumber} ]] && listOfRuns[${runNumber}]=1 | |
1032 | while read x; do tmpRun=$(guessRunNumber ${x}); [[ -n ${tmpRun} ]] && listOfRuns[${tmpRun}]=1; done < ${inputFileList} | |
1033 | for runNumber in "${!listOfRuns[@]}"; do | |
1034 | [[ -z ${runNumber} ]] && continue | |
1035 | [[ ! ${runNumber} =~ ^[0-9]*[0-9]$ ]] && continue | |
1036 | ||
1037 | unset arr_cpass0_outputs | |
1038 | unset arr_cpass1_outputs | |
1039 | declare -a arr_cpass0_outputs | |
1040 | declare -a arr_cpass1_outputs | |
1041 | ||
1042 | jobindex=0 | |
1043 | inputFile="" | |
1044 | while read inputFile; do | |
1045 | currentDefaultOCDB=${defaultOCDB} | |
1046 | [[ -z ${autoOCDB} ]] && autoOCDB=1 | |
1047 | if [[ ${autoOCDB} -ne 0 ]]; then | |
1048 | currentDefaultOCDB=$(setYear ${inputFile} ${defaultOCDB}) | |
1049 | fi | |
1050 | ||
1051 | #CPass0 | |
1052 | arr_cpass0_outputs[${jobindex}]="${commonOutputPath}/meta/cpass0.job${jobindex}.run${runNumber}.done" | |
1053 | echo "${arr_cpass0_outputs[${jobindex}]} : benchmark.sh ${configFile} ${copyFiles[@]}" | |
1054 | echo " ${alirootEnv} ./benchmark.sh CPass0 ${commonOutputPath}/000${runNumber}/cpass0 ${inputFile} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} ${jobindex} ${extraOpts[@]}"" " | |
1055 | ||
1056 | #CPass1 | |
1057 | arr_cpass1_outputs[${jobindex}]="${commonOutputPath}/meta/cpass1.job${jobindex}.run${runNumber}.done" | |
1058 | echo "${arr_cpass1_outputs[${jobindex}]} : benchmark.sh ${configFile} ${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz ${copyFiles[@]}" | |
1059 | echo " ${alirootEnv} ./benchmark.sh CPass1 ${commonOutputPath}/000${runNumber}/cpass1 ${inputFile} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} ${jobindex} ${extraOpts[@]}"" " | |
1060 | ((jobindex++)) | |
1061 | ||
1062 | done< <(grep "/000${runNumber}/" ${inputFileList}) | |
1063 | ||
1064 | #CPass0 list of Calib files to merge | |
1065 | arr_cpass0_calib_list[${runNumber}]="${commonOutputPath}/meta/cpass0.calib.run${runNumber}.list" | |
1066 | echo "${arr_cpass0_calib_list[${runNumber}]} : benchmark.sh ${arr_cpass0_outputs[*]}" | |
1067 | echo " ./benchmark.sh PrintValues calibfile ${arr_cpass0_calib_list[${runNumber}]} ${arr_cpass0_outputs[*]}" | |
1068 | echo | |
1069 | ||
1070 | #CPass0 merging | |
1071 | arr_cpass0_merged[${runNumber}]="${commonOutputPath}/meta/merge.cpass0.run${runNumber}.done" | |
1072 | echo "${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz ${arr_cpass0_merged[${runNumber}]} : benchmark.sh ${configFile} ${arr_cpass0_calib_list[${runNumber}]} ${copyFiles[@]}" | |
1073 | echo " ${alirootEnv} ./benchmark.sh MergeCPass0 ${commonOutputPath}/000${runNumber}/cpass0 ${currentDefaultOCDB} ${configFile} ${runNumber} ${arr_cpass0_calib_list[${runNumber}]} ${extraOpts[@]}"" " | |
1074 | ||
1075 | #CPass1 list of Calib/QA/ESD/filtered files | |
1076 | # the trick with QA is to have the string "Stage.txt" in the file name of the list of directories with QA output to trigger | |
1077 | # the production of the QA trending tree (only then the task->Finish() will be called in QAtrain_duo.C, on the grid | |
1078 | # this corresponds to the last merging stage) | |
1079 | arr_cpass1_QA_list[${runNumber}]="${commonOutputPath}/meta/cpass1.QA.run${runNumber}.lastMergingStage.txt.list" | |
1080 | echo "${arr_cpass1_QA_list[${runNumber}]}: benchmark.sh ${arr_cpass1_outputs[*]}" | |
1081 | echo " ./benchmark.sh PrintValues dir ${arr_cpass1_QA_list[${runNumber}]} ${arr_cpass1_outputs[*]}" | |
1082 | echo | |
1083 | arr_cpass1_calib_list[${runNumber}]="${commonOutputPath}/meta/cpass1.calib.run${runNumber}.list" | |
1084 | echo "${arr_cpass1_calib_list[${runNumber}]} : benchmark.sh ${arr_cpass1_outputs[*]}" | |
1085 | echo " ./benchmark.sh PrintValues calibfile ${arr_cpass1_calib_list[${runNumber}]} ${arr_cpass1_outputs[*]};" | |
1086 | echo | |
1087 | arr_cpass1_ESD_list[${runNumber}]="${commonOutputPath}/meta/cpass1.ESD.run${runNumber}.list" | |
1088 | echo "${arr_cpass1_ESD_list[${runNumber}]} : benchmark.sh ${arr_cpass1_outputs[*]}" | |
1089 | echo " ./benchmark.sh PrintValues esd ${arr_cpass1_ESD_list[${runNumber}]} ${arr_cpass1_outputs[*]}" | |
1090 | echo | |
1091 | arr_cpass1_filtered_list[${runNumber}]="${commonOutputPath}/meta/cpass1.filtered.run${runNumber}.list" | |
1092 | echo "${arr_cpass1_filtered_list[${runNumber}]} : benchmark.sh ${arr_cpass1_outputs[*]}" | |
1093 | echo " ./benchmark.sh PrintValues filteredTree ${arr_cpass1_filtered_list[${runNumber}]} ${arr_cpass1_outputs[*]}" | |
1094 | echo | |
1095 | ||
1096 | #CPass1 merging | |
1097 | arr_cpass1_merged[${runNumber}]="${commonOutputPath}/meta/merge.cpass1.run${runNumber}.done" | |
1098 | echo "${commonOutputPath}/meta/cpass1.localOCDB.${runNumber}.tgz ${arr_cpass1_merged[${runNumber}]} : benchmark.sh ${configFile} ${arr_cpass1_calib_list[${runNumber}]} ${arr_cpass1_QA_list[${runNumber}]} ${copyFiles[@]}" | |
1099 | echo " ${alirootEnv} ./benchmark.sh MergeCPass1 ${commonOutputPath}/000${runNumber}/cpass1 ${currentDefaultOCDB} ${configFile} ${runNumber} ${arr_cpass1_calib_list[${runNumber}]} ${arr_cpass1_QA_list[${runNumber}]} ${arr_cpass1_filtered_list[${runNumber}]} ${extraOpts[@]}"" " | |
1100 | ||
1101 | #CPass0 wrapped in a profiling tool (valgrind,....) | |
1102 | if [[ -n ${profilingCommand} ]]; then | |
1103 | inputFile=$(grep -m1 "${runNumber}/" ${inputFileList}) | |
1104 | [[ -z ${nEventsProfiling} ]] && nEventsProfiling=2 | |
1105 | currentDefaultOCDB=$(setYear ${inputFile} ${defaultOCDB}) | |
1106 | jobindex="profiling" | |
1107 | ||
1108 | arr_cpass0_profiled_outputs[${runNumber}]="${commonOutputPath}/meta/profiling.cpass0.job${jobindex}.run${runNumber}.done" | |
1109 | echo "${arr_cpass0_profiled_outputs[${runNumber}]} : benchmark.sh ${configFile} ${copyFiles[@]}" | |
1110 | profilingCommand=$(encSpaces "${profilingCommand}") | |
1111 | echo " ${alirootEnv} ./benchmark.sh CPass0 ${commonOutputPath}/000${runNumber}/${jobindex} ${inputFile} ${nEventsProfiling} ${currentDefaultOCDB} ${configFile} ${runNumber} ${jobindex} ${extraOpts[@]} useProfilingCommand=${profilingCommand}" | |
1112 | fi | |
1113 | ||
1114 | done #runs | |
1115 | ||
1116 | #Summary | |
1117 | echo "${commonOutputPath}/summary.log : benchmark.sh ${configFile} ${arr_cpass1_merged[*]}" | |
1118 | echo " ${alirootEnv} ./benchmark.sh MakeSummary ${configFile} ${extraOpts[@]}" | |
1119 | echo | |
1120 | ||
1121 | return 0 | |
1122 | ) | |
1123 | ||
1124 | goPrintValues() | |
1125 | ( | |
1126 | #print the values given the key from any number of files (store in output file on second argument) | |
1127 | if [[ $# -lt 3 ]]; then | |
1128 | echo "goPrintValues key outputFile inputFiles" | |
1129 | echo "if outputFile is \"-\" dont redirect to a file" | |
1130 | return | |
1131 | fi | |
1132 | key=${1} | |
1133 | outputFile=${2} | |
1134 | [[ ${outputFile} =~ "-" ]] && outputFile="" | |
1135 | shift 2 #remove 2 first arguments from arg list to only pass the input files to awk | |
1136 | awk -v key=${key} '$0 ~ key" " {print $2}' "$@" | tee ${outputFile} | |
1137 | return 0 | |
1138 | ) | |
1139 | ||
1140 | goCreateQAplots() | |
1141 | ( | |
1142 | umask 0002 | |
1143 | mergedQAfileList=${1} | |
1144 | productionID=${2} | |
1145 | outputDir=${3} | |
1146 | configFile=${4} | |
1147 | shift 4 | |
1148 | if ! parseConfig ${configFile} "$@"; then return 1; fi | |
1149 | ||
1150 | #record the working directory provided by the batch system | |
1151 | batchWorkingDirectory=${PWD} | |
1152 | ||
1153 | [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource} | |
1154 | ||
1155 | [[ -z ${logOutputDir} ]] && logOutputDir=${PWD} | |
1156 | [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/makeQAplots.log | |
1157 | echo "${0} $*" | |
1158 | ||
1159 | olddir=${PWD} | |
1160 | mkdir -p ${outputDir} | |
1161 | cd ${outputDir} | |
1162 | [[ ! "${PWD}" =~ "${outputDir}" ]] && echo "PWD is not equal to outputDir=${outputDir}" && cd ${olddir} && return 1 | |
1163 | ||
1164 | if [[ -z ${pretend} ]]; then | |
1165 | ${ALICE_ROOT}/PWGPP/QA/scripts/runQA.sh inputList="${mergedQAfileList}" inputListHighPtTrees="${filteringList}" ocdbStorage=${defaultOCDB} | |
1166 | else | |
1167 | touch pretendCreateQAplots.done | |
1168 | fi | |
1169 | cd ${olddir} | |
1170 | return 0 | |
1171 | ) | |
1172 | ||
1173 | goTest() | |
1174 | ( | |
1175 | echo AA | |
1176 | ) | |
1177 | ||
1178 | alirootInfo() | |
1179 | ( | |
1180 | umask 0002 | |
1181 | # save aliroot repository info | |
1182 | [[ -z "${ALICE_ROOT}" ]] && return 1 | |
1183 | ||
1184 | echo "\${ALICE_ROOT}=${ALICE_ROOT}" | |
1185 | echo "\${ROOTSYS}=${ROOTSYS}" | |
1186 | echo "\${PATH}=${PATH}" | |
1187 | echo "\${LD_LIBRARY_PATH}=${LD_LIBRARY_PATH}" | |
1188 | echo | |
1189 | ||
1190 | pushd ${PWD} | |
1191 | cd ${ALICE_ROOT} | |
1192 | ||
1193 | currentBranch=$(git rev-parse --abbrev-ref HEAD) | |
1194 | git status | |
1195 | echo "" | |
1196 | echo "" | |
1197 | git diff ${currentBranch} | |
1198 | popd | |
1199 | return 0 | |
1200 | ) | |
1201 | ||
1202 | setYear() | |
1203 | ( | |
1204 | #set the year | |
1205 | # ${1} - year to be set | |
1206 | # ${2} - where to set the year | |
1207 | year1=$(guessYear ${1}) | |
1208 | year2=$(guessYear ${2}) | |
1209 | local path=${2} | |
1210 | [[ ${year1} -ne ${year2} && -n ${year2} && -n ${year1} ]] && path=${2/\/${year2}\//\/${year1}\/} | |
1211 | echo ${path} | |
1212 | return 0 | |
1213 | ) | |
1214 | ||
1215 | guessPeriod() | |
1216 | ( | |
1217 | #guess the period from the path, pick the rightmost one | |
1218 | local IFS="/" | |
1219 | declare -a path=( ${1} ) | |
1220 | local dirDepth=${#path[*]} | |
1221 | for ((x=${dirDepth}-1;x>=0;x--)); do | |
1222 | local field=${path[${x}]} | |
1223 | [[ ${field} =~ ^LHC[0-9][0-9][a-z]$ ]] && period=${field} && break | |
1224 | done | |
1225 | echo ${period} | |
1226 | return 0 | |
1227 | ) | |
1228 | ||
1229 | guessYear() | |
1230 | ( | |
1231 | #guess the year from the path, pick the rightmost one | |
1232 | local IFS="/" | |
1233 | declare -a path=( ${1} ) | |
1234 | local dirDepth=${#path[*]} | |
1235 | for ((x=${dirDepth}-1;x>=0;x--)); do | |
1236 | local field=${path[${x}]} | |
1237 | [[ ${field} =~ ^20[0-9][0-9]$ ]] && year=${field} && break | |
1238 | done | |
1239 | echo ${year} | |
1240 | return 0 | |
1241 | ) | |
1242 | ||
1243 | guessRunNumber() | |
1244 | ( | |
1245 | #guess the run number from the path, pick the rightmost one | |
1246 | #works for /path/foo/000123456/bar/... | |
1247 | #and /path/foo.run123456.bar | |
1248 | local IFS="/." | |
1249 | declare -a path=( ${1} ) | |
1250 | local dirDepth=${#path[*]} | |
1251 | for ((x=${dirDepth}-1;x>=0;x--)); do | |
1252 | local field=${path[${x}]} | |
1253 | field=${field/run/000} | |
1254 | [[ ${field} =~ [0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && runNumber=${field#000} && break | |
1255 | done | |
1256 | echo ${runNumber} | |
1257 | return 0 | |
1258 | ) | |
1259 | ||
1260 | validateLog() | |
1261 | ( | |
1262 | log=${1} | |
1263 | errorConditions=( | |
1264 | 'There was a crash' | |
1265 | 'floating' | |
1266 | 'error while loading shared libraries' | |
1267 | 'std::bad_alloc' | |
1268 | 's_err_syswatch_' | |
1269 | 'Thread [0-9]* (Thread' | |
1270 | 'AliFatal' | |
1271 | 'core dumped' | |
1272 | '\.C.*error:.*\.h: No such file' | |
1273 | 'line.*Aborted' | |
1274 | ) | |
1275 | ||
1276 | warningConditions=( | |
1277 | 'This is serious !' | |
1278 | 'rocVoltage out of range:' | |
1279 | ) | |
1280 | ||
1281 | local logstatus=0 | |
1282 | local errorSummary="" | |
1283 | local warningSummary="" | |
1284 | ||
1285 | for ((i=0; i<${#errorConditions[@]};i++)); do | |
1286 | local tmp=$(grep -m1 -e "${errorConditions[${i}]}" ${log}) | |
1287 | [[ -n ${tmp} ]] && tmp+=" : " | |
1288 | errorSummary+=${tmp} | |
1289 | done | |
1290 | ||
1291 | for ((i=0; i<${#warningConditions[@]};i++)); do | |
1292 | local tmp=$(grep -m1 -e "${warningConditions[${i}]}" ${log}) | |
1293 | [[ -n ${tmp} ]] && tmp+=" : " | |
1294 | warningSummary+=${tmp} | |
1295 | done | |
1296 | ||
1297 | if [[ -n ${errorSummary} ]]; then | |
1298 | echo "${errorSummary}" | |
1299 | return 1 | |
1300 | fi | |
1301 | ||
1302 | if [[ -n ${warningSummary} ]]; then | |
1303 | echo "${warningSummary}" | |
1304 | return 2 | |
1305 | fi | |
1306 | ||
1307 | return 0 | |
1308 | ) | |
1309 | ||
1310 | summarizeLogs() | |
1311 | ( | |
1312 | #print a summary of logs | |
1313 | logFiles=( | |
1314 | "*.log" | |
1315 | "stdout" | |
1316 | "stderr" | |
1317 | ) | |
1318 | ||
1319 | #check logs | |
1320 | local logstatus=0 | |
1321 | for log in ${logFiles[*]}; do | |
1322 | finallog=${PWD%/}/${log} | |
1323 | [[ ! -f ${log} ]] && continue | |
1324 | errorSummary=$(validateLog ${log}) | |
1325 | validationStatus=$? | |
1326 | if [[ ${validationStatus} -eq 0 ]]; then | |
1327 | #in pretend mode randomly report an error in rec.log some cases | |
1328 | if [[ -n ${pretend} && "${log}" == "rec.log" ]]; then | |
1329 | #[[ $(( ${RANDOM}%2 )) -ge 1 ]] && echo "${finallog} BAD random error" || echo "${finallog} OK" | |
1330 | echo "${finallog} OK" | |
1331 | else | |
1332 | echo "${finallog} OK" | |
1333 | fi | |
1334 | elif [[ ${validationStatus} -eq 1 ]]; then | |
1335 | echo "${finallog} BAD ${errorSummary}" | |
1336 | logstatus=1 | |
1337 | elif [[ ${validationStatus} -eq 2 ]]; then | |
1338 | echo "${finallog} OK MWAH ${errorSummary}" | |
1339 | fi | |
1340 | done | |
1341 | ||
1342 | #report core files | |
1343 | while read x; do | |
1344 | echo ${x} | |
1345 | chmod 644 ${x} | |
1346 | gdb --batch --quiet -ex "bt" -ex "quit" aliroot ${x} > stacktrace_${x//\//_}.log | |
1347 | done < <(/bin/ls ${PWD}/*/core 2>/dev/null; /bin/ls ${PWD}/core 2>/dev/null) | |
1348 | ||
1349 | return ${logstatus} | |
1350 | ) | |
1351 | ||
1352 | spitOutLocalOCDBaccessConfig() | |
1353 | { | |
1354 | umask 0002 | |
1355 | #find ${1} -name "*root" | \ | |
1356 | /bin/ls -1 ${1}/*/*/*/*.root 2>/dev/null | \ | |
1357 | while read line | |
1358 | do | |
1359 | local tmp=${line#${1}} | |
1360 | echo ${tmp%/*} | \ | |
1361 | awk -v ocdb=${1} '{print " man->SetSpecificStorage(\""$1"\",\"local://"ocdb"\");"}' | |
1362 | done | |
1363 | return 0 | |
1364 | } | |
1365 | ||
1366 | goMakeLocalOCDBaccessConfig() | |
1367 | { | |
1368 | umask 0002 | |
1369 | # make a script that sets the specific storages form all the root files produced by CPass0 | |
1370 | local localOCDBpathCPass0=${1} | |
1371 | local OCDBpathPrefix=${2} | |
1372 | [[ -z ${OCDBpathPrefix} ]] && OCDBpathPrefix="." | |
1373 | ||
1374 | if [[ -f ${localOCDBpathCPass0} && ${localOCDBpathCPass0} =~ \.tgz$ ]]; then | |
1375 | tar xzf ${localOCDBpathCPass0} | |
1376 | local localOCDBpathCPass0="${OCDBpathPrefix}/OCDB" | |
1377 | fi | |
1378 | ||
1379 | echo | |
1380 | echo creating the specific storage script | |
1381 | echo localOCDBaccessConfig.C | |
1382 | echo based on OCDB: ${localOCDBaccessConfig} | |
1383 | echo | |
1384 | ||
1385 | local tempLocalOCDB="" | |
1386 | if [[ -f localOCDBaccessConfig.C ]]; then | |
1387 | tempLocalOCDB=$(mktemp -t tempLocalOCDB.XXXXXX) | |
1388 | echo "egrep "SetSpecificStorage" localOCDBaccessConfig.C > ${tempLocalOCDB}" | |
1389 | egrep "SetSpecificStorage" localOCDBaccessConfig.C > ${tempLocalOCDB} | |
1390 | fi | |
1391 | ||
1392 | echo "localOCDBaccessConfig()" > localOCDBaccessConfig.C | |
1393 | echo "{" >> localOCDBaccessConfig.C | |
1394 | echo " AliCDBManager* man = AliCDBManager::Instance();" >> localOCDBaccessConfig.C | |
1395 | spitOutLocalOCDBaccessConfig ${localOCDBpathCPass0}|sort|uniq >> localOCDBaccessConfig.C | |
1396 | [[ -f "${tempLocalOCDB}" ]] && cat ${tempLocalOCDB} >> localOCDBaccessConfig.C | |
1397 | echo "}" >> localOCDBaccessConfig.C | |
1398 | ||
1399 | [[ -f "${tempLocalOCDB}" ]] && rm -f ${tempLocalOCDB} | |
1400 | ||
1401 | if ! grep SetSpecificStorage localOCDBaccessConfig.C; then | |
1402 | echo | |
1403 | echo "!!!!!!! CPass0 produced no OCDB entries" | |
1404 | return 1 | |
1405 | fi | |
1406 | return 0 | |
1407 | } | |
1408 | ||
1409 | goMakeFilteredTrees() | |
1410 | ( | |
1411 | outputDir=${1} | |
1412 | runNumber=${2} | |
1413 | #get path to input list | |
1414 | inputListfiles=${3} | |
1415 | #get scale number for tracks | |
1416 | filterT=${4} | |
1417 | #get scale number for V0s | |
1418 | filterV=${5} | |
1419 | #get OCDB path (optional) | |
1420 | OCDBpath=${6} | |
1421 | #get max number of files | |
1422 | maxFiles=${7-"1000000"} | |
1423 | #get offset of first file | |
1424 | offsetFile=${8-"0"} | |
1425 | #get max number of events | |
1426 | maxEvents=${9-"30000000"} | |
1427 | #get offset of first event | |
1428 | offsetEvent=${10-"0"} | |
1429 | configFile=${11-"benchmark.config"} | |
1430 | esdFileName=${12-"AliESDs_Barrel.root"} | |
1431 | shift 12 | |
1432 | if ! parseConfig ${configFile} "$@"; then return 1; fi | |
1433 | ||
1434 | #record the working directory provided by the batch system | |
1435 | batchWorkingDirectory=${PWD} | |
1436 | ||
1437 | [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD} | |
1438 | doneFile=${commonOutputPath}/meta/filtering.cpass1.run${runNumber}.done | |
1439 | ||
1440 | cat > filtering.log << EOF | |
1441 | goMakeFilteredTrees config: | |
1442 | runpath=${runpath} | |
1443 | outputDir=${outputDir} | |
1444 | commonOutputPath=${commonOutputPath} | |
1445 | ALICE_ROOT=${ALICE_ROOT} | |
1446 | PATH=${PATH} | |
1447 | offsetEvent=$offsetEvent | |
1448 | configFile=$configFile | |
1449 | esdFileName=$esdFileName | |
1450 | inputListfiles=$inputListfiles | |
1451 | doneFile=$doneFile | |
1452 | EOF | |
1453 | ||
1454 | #runpath=${outputDir} | |
1455 | #[[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR} | |
1456 | #[[ ${reconstructInTemporaryDir} -eq 1 ]] && runpath=$(mktemp -d -t goMakeFilteredTrees.XXXXXX) | |
1457 | #mkdir -p ${outputDir} | |
1458 | #mkdir -p ${runpath} | |
1459 | #if ! cd ${runpath}; then | |
1460 | # echo "PWD=$PWD is not the runpath=${runpath}" | |
1461 | # touch ${doneFile} | |
1462 | # return 1 | |
1463 | #fi | |
1464 | ||
1465 | if [[ -z ${pretend} ]];then | |
1466 | aliroot -l -b -q "${ALICE_ROOT}/PWGPP/macros/runFilteringTask.C(\"${inputListfiles}\",${filterT},${filterV},\"${OCDBpath}\",${maxFiles},${offsetFile},${maxEvents},${offsetEvent},\"${esdFileName}\")" &>> filtering.log | |
1467 | else | |
1468 | sleep ${pretendDelay} | |
1469 | touch filtering.log FilterEvents_Trees.root | |
1470 | fi | |
1471 | pwd | |
1472 | /bin/ls | |
1473 | touch ${doneFile} | |
1474 | summarizeLogs >> ${doneFile} | |
1475 | ||
1476 | #echo mv -f * ${outputDir} | |
1477 | #mv -f * ${outputDir} | |
1478 | #[[ -f ${outputDir}/FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFile} | |
1479 | #cd ${commonOutputPath} | |
1480 | #[[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath} | |
1481 | ||
1482 | return 0 | |
1483 | ) | |
1484 | ||
1485 | submit() | |
1486 | { | |
1487 | umask 0002 | |
1488 | [[ $# -lt 5 ]] && echo "at least 5 args needed, you supplied $#" && return 1 | |
1489 | JobID=${1} | |
1490 | startID=${2} | |
1491 | endID=${3} | |
1492 | waitForJOBID=${4} | |
1493 | command=${5} | |
1494 | shift 5 | |
1495 | local commandArgs=("$@") | |
1496 | ||
1497 | #add quote strings around the extra arguments | |
1498 | for ((i=0; i<${#commandArgs[@]}; i++)); do | |
1499 | commandArgs[i]=\"${commandArgs[i]}\" | |
1500 | done | |
1501 | ||
1502 | [[ -z ${waitForJOBID} ]] && waitForJOBID=0 | |
1503 | ||
1504 | newFarm=$(which qsub|grep "^/usr/bin/qsub") | |
1505 | ||
1506 | batchSystem="SGE" | |
1507 | ||
1508 | if [[ -z "${newFarm}" ]] | |
1509 | then | |
1510 | #old LSF | |
1511 | # submit it (as job array) | |
1512 | nFiles=$(( ${endID}-${startID}+1 )) | |
1513 | while [ ${startID} -le ${nFiles} ] ; do | |
1514 | if [ $(expr ${nFiles} - ${startID}) -gt 999 ] ; then | |
1515 | endID=$(expr ${startID} + 999) | |
1516 | else | |
1517 | endID=${nFiles} | |
1518 | fi | |
1519 | if [[ ${waitForJOBID} -eq 0 ]]; then | |
1520 | echo ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}" | |
1521 | ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}" | |
1522 | else | |
1523 | echo ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -w "ended(${waitForJOBID})" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}" | |
1524 | ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -w "ended(${waitForJOBID})" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}" | |
1525 | fi | |
1526 | startID=$(expr ${endID} + 1) | |
1527 | done | |
1528 | else | |
1529 | #new SGE farm | |
1530 | if [[ ${waitForJOBID} =~ "000" ]]; then | |
1531 | echo ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}" | |
1532 | ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}" | |
1533 | else | |
1534 | echo ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -hold_jid "${waitForJOBID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}" | |
1535 | ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -hold_jid "${waitForJOBID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}" | |
1536 | fi | |
1537 | fi | |
1538 | return 0 | |
1539 | } | |
1540 | ||
1541 | goSubmitBatch() | |
1542 | { | |
1543 | if [[ $# -lt 3 ]]; then | |
1544 | echo "minimal use:" | |
1545 | echo " ${0} submit fileList productionID configFile" | |
1546 | return 0 | |
1547 | fi | |
1548 | ||
1549 | productionID=${1} | |
1550 | inputList=${2} | |
1551 | configFile=${3:-"benchmark.config"} | |
1552 | #if which greadlink; then configFile=$(greadlink -f ${configFile}); fi | |
1553 | shift 3 | |
1554 | extraOpts=("$@") | |
1555 | if ! parseConfig ${configFile} "${extraOpts[@]}"; then return 1; fi | |
1556 | ||
1557 | #batch systems/makeflow sometimes handle spaces in arguments poorly, so encode them | |
1558 | for (( i=0;i<${#extraOpts[@]};i++ )); do | |
1559 | extraOpts[i]=$(encSpaces "${extraOpts[i]}") | |
1560 | done | |
1561 | extraOpts+=("encodedSpaces=1") | |
1562 | ||
1563 | #record the working directory provided by the batch system | |
1564 | batchWorkingDirectory=${PWD} | |
1565 | ||
1566 | #redirect all output to submit.log | |
1567 | echo "redirecting all output to ${PWD}/submit_${productionID//"/"/_}.log" | |
1568 | exec 7>&1 | |
1569 | exec 1>submit_${productionID//"/"/_}.log 2>&1 | |
1570 | ||
1571 | umask 0002 | |
1572 | echo ${0}" submit $*" | |
1573 | if [[ -z "${inputList}" || -z "${productionID}" ]] | |
1574 | then | |
1575 | echo | |
1576 | echo " Usage: ${0} submit inputList productionID [configFile=benchmark.config]" | |
1577 | echo | |
1578 | return | |
1579 | fi | |
1580 | ||
1581 | # check if config file is there | |
1582 | if [ ! -f ${configFile} ]; then | |
1583 | echo "ERROR! Config File '${configFile}' not found" >&2 | |
1584 | return | |
1585 | else | |
1586 | echo "Using Config File: '${configFile}'" | |
1587 | fi | |
1588 | ||
1589 | [[ ! -f ${alirootEnv} ]] && echo "alirootEnv script ${alirootEnv} not found!..." && return 1 | |
1590 | ||
1591 | #move the script, config and some other stuff to ${commonOutputPath} first, then use them from there | |
1592 | self=${0} | |
1593 | #if which greadlink; then self=$(greadlink -f "${0}"); fi | |
1594 | configPath=$(dirname ${configFile}) | |
1595 | export commonOutputPath=${baseOutputDirectory}/${productionID} | |
1596 | ||
1597 | mkdir -p ${commonOutputPath} | |
1598 | mkdir -p ${commonOutputPath}/logs | |
1599 | mkdir -p ${commonOutputPath}/meta | |
1600 | ||
1601 | cp ${self} ${commonOutputPath} | |
1602 | cp ${configFile} ${commonOutputPath} | |
1603 | cp ${inputList} ${commonOutputPath} | |
1604 | self=${commonOutputPath}/${self##*/} | |
1605 | chmod u+x ${self} | |
1606 | configFile=${commonOutputPath}/${configFile##*/} | |
1607 | inputList=${commonOutputPath}/${inputList##*/} | |
1608 | ||
1609 | #convert to absolut pathnames | |
1610 | #if which greadlink; then inputList=$(greadlink -f "${inputList}"); fi | |
1611 | #make list of runs | |
1612 | if [[ -z ${runNumber} ]]; then | |
1613 | listOfRuns=($(while read x; do guessRunNumber ${x}; done < ${inputList} | sort | uniq)) | |
1614 | else | |
1615 | listOfRuns=${runNumber} | |
1616 | fi | |
1617 | ||
1618 | #if which greadlink; then alirootSource=$(greadlink -f "${alirootSource}"); fi | |
1619 | ||
1620 | echo "" | |
1621 | echo "### BEGIN CONFIGURATION ###" | |
1622 | echo "" | |
1623 | echo "GENERAL:" | |
1624 | echo "" | |
1625 | echo " productionID: ${productionID}" | |
1626 | echo " batchCommand: ${batchCommand}" | |
1627 | echo " batchFlags: ${batchFlags}" | |
1628 | echo " alirootEnv: ${alirootEnv}" | |
1629 | ${alirootEnv} echo ' ALICE_ROOT: ${ALICE_ROOT}' | |
1630 | ${alirootEnv} echo ' ALIROOT_RELEASE: ${ALICE_RELEASE}' | |
1631 | echo " inputList: ${inputList}" | |
1632 | echo " configPath: ${configPath}" | |
1633 | echo " commonOutputPath: ${commonOutputPath}" | |
1634 | echo " defaultOCDB: ${defaultOCDB}" | |
1635 | echo " autoOCDB: ${autoOCDB}" | |
1636 | echo " recoTriggerOptions: ${recoTriggerOptions}" | |
1637 | echo " runs:" | |
1638 | echo " ${listOfRuns[*]}" | |
1639 | echo "" | |
1640 | echo "THE TRAIN WILL RUN:" | |
1641 | ||
1642 | if [ ${runCPass0reco} -eq 1 ]; then | |
1643 | echo " Pass0 - Recontruction" | |
1644 | fi | |
1645 | ||
1646 | if [ ${runCPass0MergeMakeOCDB} -eq 1 ]; then | |
1647 | echo " Pass0 - merging and OCDB export" | |
1648 | fi | |
1649 | ||
1650 | if [ ${runCPass1reco} -eq 1 ]; then | |
1651 | echo " Pass1 - Recontruction" | |
1652 | fi | |
1653 | if [ ${runCPass1MergeMakeOCDB} -eq 1 ]; then | |
1654 | echo " Pass1 - merging and OCDB export" | |
1655 | fi | |
1656 | ||
1657 | echo "" | |
1658 | echo "LIMITS:" | |
1659 | echo " max. Events/Chunk: ${nEvents}" | |
1660 | echo " max. Number of Chunks per Run: ${nMaxChunks}" | |
1661 | echo "" | |
1662 | echo "### END CONFIGURATION ###" | |
1663 | echo "" | |
1664 | ||
1665 | ||
1666 | # check if input file is there | |
1667 | if [ ! -f ${inputList} ]; then | |
1668 | echo "ERROR! Input List '${inputList}' not found" >&2 | |
1669 | return | |
1670 | fi | |
1671 | ||
1672 | # define jobid (for dependent jobs) | |
1673 | date=$(date +%Y_%m_%d_%H%M%S) | |
1674 | #for each run we submit one jobarray: | |
1675 | for runNumber in ${listOfRuns[*]}; do | |
1676 | ||
1677 | [[ -z ${runNumber} ]] && continue | |
1678 | [[ ! ${runNumber} =~ ^[0-9]*[0-9]$ ]] && continue | |
1679 | ||
1680 | JOBpostfix="${productionID//"/"/_}_${runNumber}_${date}" | |
1681 | JOBID1="p0_${JOBpostfix}" | |
1682 | JOBID1wait="w0_${JOBpostfix}" | |
1683 | JOBID2="m0_${JOBpostfix}" | |
1684 | JOBID2wait="wm0_${JOBpostfix}" | |
1685 | JOBID3="op0_${JOBpostfix}" | |
1686 | JOBID3wait="wop0_${JOBpostfix}" | |
1687 | JOBID4="p1_${JOBpostfix}" | |
1688 | JOBID4wait="w1_${JOBpostfix}" | |
1689 | JOBID5="m1_${JOBpostfix}" | |
1690 | JOBID5wait="wm1_${JOBpostfix}" | |
1691 | JOBID6="s1_${JOBpostfix}" | |
1692 | JOBID6wait="ws1_${JOBpostfix}" | |
1693 | JOBID7="QA_${JOBpostfix}" | |
1694 | JOBmakeESDlistCPass1="lp1_${JOBpostfix}" | |
1695 | JOBfilterESDcpass1="fp1_${JOBpostfix}" | |
1696 | LASTJOB="000" | |
1697 | ||
1698 | oneInputFile=$(egrep -m1 "${runNumber}/" ${inputList}) | |
1699 | ||
1700 | currentDefaultOCDB=${defaultOCDB} | |
1701 | [[ -z ${autoOCDB} ]] && autoOCDB=1 | |
1702 | if [[ ${autoOCDB} -ne 0 ]]; then | |
1703 | currentDefaultOCDB=$(setYear ${oneInputFile} ${defaultOCDB}) | |
1704 | fi | |
1705 | ||
1706 | echo "submitting run ${runNumber} with OCDB ${currentDefaultOCDB}" | |
1707 | ||
1708 | ############################################################################### | |
1709 | #run one chunk with valgrind: | |
1710 | if [[ -n ${profilingCommand} ]]; then | |
1711 | [[ -z ${nEventsProfiling} ]] && nEventsProfiling=2 | |
1712 | [[ -z ${profilingCommand} ]] && profilingCommand="/usr/bin/valgrind --tool=callgrind --num-callers=40 -v --trace-children=yes" | |
1713 | submit "profile-${JOBpostfix}" 1 1 000 "${alirootEnv} ${self}" CPass0 ${commonOutputPath}/000${runNumber}/${jobindex} ${oneInputFile} ${nEventsProfiling} ${currentDefaultOCDB} ${configFile} ${runNumber} ${jobindex} useProfilingCommand=$(encSpaces "${profilingCommand}") "${extraOpts[@]}" | |
1714 | fi | |
1715 | ||
1716 | ################################################################################ | |
1717 | ################################################################################ | |
1718 | # run the CPass0 if requested | |
1719 | ||
1720 | if [ ${runCPass0reco} -eq 1 ]; then | |
1721 | ||
1722 | echo | |
1723 | echo "starting CPass0... for run ${runNumber}" | |
1724 | echo | |
1725 | ||
1726 | # create directory and copy all files that are needed | |
1727 | targetDirectory="${commonOutputPath}/000${runNumber}/cpass0" | |
1728 | mkdir -p ${targetDirectory} | |
1729 | ||
1730 | filesCPass0=( | |
1731 | "${configPath}/runCPass0.sh" | |
1732 | "${configPath}/recCPass0.C" | |
1733 | "${configPath}/runCalibTrain.C" | |
1734 | "${configPath}/localOCDBaccessConfig.C" | |
1735 | "${configPath}/OCDB*.root" | |
1736 | "${configPath}/sim.C" | |
1737 | "${configPath}/rec.C" | |
1738 | "${configPath}/Config.C" | |
1739 | ) | |
1740 | for file in ${filesCPass0[*]}; do | |
1741 | [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath} | |
1742 | done | |
1743 | ||
1744 | localInputList=${targetDirectory}/${inputList##*/} | |
1745 | [[ ! -f ${localInputList} ]] && egrep "\/000${runNumber}\/" ${inputList} > ${localInputList} | |
1746 | # limit nFiles to nMaxChunks | |
1747 | nFiles=$(wc -l < ${localInputList}) | |
1748 | [[ ${nFiles} -eq 0 ]] && echo "list contains ZERO files! exiting..." && return 1 | |
1749 | echo "raw files in list: ${nFiles}" | |
1750 | if [[ ${nMaxChunks} -gt 0 && ${nMaxChunks} -le ${nFiles} ]]; then | |
1751 | nFiles=${nMaxChunks} | |
1752 | fi | |
1753 | echo "raw files to process: ${nFiles}" | |
1754 | [[ -z "${percentProcessedFilesToContinue}" ]] && percentProcessedFilesToContinue=100 | |
1755 | if [[ ${percentProcessedFilesToContinue} -eq 100 ]]; then | |
1756 | nFilesToWaitFor=${nFiles} | |
1757 | else | |
1758 | nFilesToWaitFor=$(( ${nFiles}-${nFiles}/(100/(100-${percentProcessedFilesToContinue})) )) | |
1759 | fi | |
1760 | echo "requested success rate is ${percentProcessedFilesToContinue}%" | |
1761 | echo "merging will start after ${nFilesToWaitFor} jobs are done" | |
1762 | ||
1763 | submit ${JOBID1} 1 ${nFiles} 000 "${alirootEnv} ${self}" CPass0 ${targetDirectory} ${localInputList} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} -1 "${extraOpts[@]}" | |
1764 | ||
1765 | ## submit a monitoring job that will run until a certain number of jobs are done with reconstruction | |
1766 | submit "${JOBID1wait}" 1 1 000 "${alirootEnv} ${self}" WaitForOutput ${commonOutputPath} "meta/cpass0.job*.run${runNumber}.done" ${nFilesToWaitFor} ${maxSecondsToWait} | |
1767 | LASTJOB=${JOBID1wait} | |
1768 | ||
1769 | fi #end running CPass0 | |
1770 | ################################################################################ | |
1771 | ||
1772 | ||
1773 | ################################################################################ | |
1774 | # submit merging of CPass0, depends on the reconstruction | |
1775 | ||
1776 | if [ ${runCPass0MergeMakeOCDB} -eq 1 ]; then | |
1777 | ||
1778 | echo | |
1779 | echo "submit CPass0 merging for run ${runNumber}" | |
1780 | echo | |
1781 | ||
1782 | targetDirectory="${commonOutputPath}/000${runNumber}/cpass0" | |
1783 | mkdir -p ${targetDirectory} | |
1784 | ||
1785 | #copy the scripts | |
1786 | filesMergeCPass0=( | |
1787 | "${configPath}/OCDB.root" | |
1788 | "${configPath}/mergeMakeOCDB.byComponent.sh" | |
1789 | "${configPath}/mergeMakeOCDB.sh" | |
1790 | "${configPath}/localOCDBaccessConfig.C" | |
1791 | "${configPath}/mergeByComponent.C" | |
1792 | "${configPath}/makeOCDB.C" | |
1793 | "${configPath}/merge.C" | |
1794 | ) | |
1795 | for file in ${filesMergeCPass0[*]}; do | |
1796 | [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath} | |
1797 | done | |
1798 | ||
1799 | submit ${JOBID2} 1 1 "${LASTJOB}" "${alirootEnv} ${self}" MergeCPass0 ${targetDirectory} ${currentDefaultOCDB} ${configFile} ${runNumber} cpass0.calib.run${runNumber}.list "${extraOpts[@]}" | |
1800 | LASTJOB=${JOBID2} | |
1801 | ||
1802 | if [[ -n ${generateMC} ]]; then | |
1803 | submit "mrl${JOBpostfix}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" PrintValues sim ${commonOutputPath}/meta/sim.run${runNumber}.list ${commonOutputPath}/meta/cpass0.job*.run${runNumber}.done | |
1804 | LASTJOB="mrl${JOBpostfix}" | |
1805 | fi | |
1806 | ||
1807 | echo | |
1808 | fi | |
1809 | # end of merging CPass0 | |
1810 | ################################################################################ | |
1811 | ||
1812 | ################################################################################ | |
1813 | ################################################################################ | |
1814 | # run the CPass1 if requested | |
1815 | ||
1816 | if [ ${runCPass1reco} -eq 1 ]; then | |
1817 | ||
1818 | targetDirectory="${commonOutputPath}/000${runNumber}/cpass1" | |
1819 | rm -f ${commonOutputPath}/meta/cpass1.job*.run${runNumber}.done | |
1820 | ||
1821 | # safety feature: if we are re-running for any reason we want to delete the previous output first. | |
1822 | [[ -d ${targetDirectory} ]] && rm -rf ${targetDirectory}/* && echo "removed old output at ${targetDirectory}/*" | |
1823 | ||
1824 | echo | |
1825 | echo "starting CPass1... for run ${runNumber}" | |
1826 | echo | |
1827 | ||
1828 | # create directory and copy all files that are needed | |
1829 | mkdir -p ${targetDirectory} | |
1830 | ||
1831 | filesCPass1=( | |
1832 | "${configPath}/runCPass1.sh" | |
1833 | "${configPath}/recCPass1.C" | |
1834 | "${configPath}/recCPass1_OuterDet.C" | |
1835 | "${configPath}/runCalibTrain.C" | |
1836 | "${configPath}/QAtrain_duo.C" | |
1837 | "${configPath}/localOCDBaccessConfig.C" | |
1838 | "${configPath}/OCDB.root" | |
1839 | ) | |
1840 | for file in ${filesCPass1[*]}; do | |
1841 | [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath} | |
1842 | done | |
1843 | ||
1844 | if [[ -n ${generateMC} ]]; then | |
1845 | localInputList=${commonOutputPath}/meta/sim.run${runNumber}.list | |
1846 | else | |
1847 | localInputList=${targetDirectory}/${inputList##*/} | |
1848 | [[ ! -f ${localInputList} ]] && egrep "\/000${runNumber}\/" ${inputList} > ${localInputList} | |
1849 | fi | |
1850 | # limit nFiles to nMaxChunks | |
1851 | nFiles=$(wc -l < ${localInputList}) | |
1852 | [[ ${nFiles} -eq 0 ]] && echo "list contains ZERO files! continuing..." && continue | |
1853 | echo "raw files in list: ${nFiles}" | |
1854 | if [[ ${nMaxChunks} -gt 0 && ${nMaxChunks} -le ${nFiles} ]]; then | |
1855 | nFiles=${nMaxChunks} | |
1856 | fi | |
1857 | echo "raw files to process: ${nFiles}" | |
1858 | [[ -z "${percentProcessedFilesToContinue}" ]] && percentProcessedFilesToContinue=100 | |
1859 | if [[ ${percentProcessedFilesToContinue} -eq 100 ]]; then | |
1860 | nFilesToWaitFor=${nFiles} | |
1861 | else | |
1862 | nFilesToWaitFor=$(( ${nFiles}-${nFiles}/(100/(100-${percentProcessedFilesToContinue})) )) | |
1863 | fi | |
1864 | echo "requested success rate is ${percentProcessedFilesToContinue}%" | |
1865 | echo "merging will start after ${nFilesToWaitFor} jobs are done" | |
1866 | ||
1867 | submit ${JOBID4} 1 ${nFiles} "${LASTJOB}" "${alirootEnv} ${self}" CPass1 ${targetDirectory} ${localInputList} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} -1 "${extraOpts[@]}" | |
1868 | ||
1869 | ################################################################################ | |
1870 | ## submit a monitoring job that will run until a certain number of jobs are done with reconstruction | |
1871 | submit "${JOBID4wait}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" WaitForOutput ${commonOutputPath} "meta/cpass1.job*.run${runNumber}.done" ${nFilesToWaitFor} ${maxSecondsToWait} | |
1872 | LASTJOB=${JOBID4wait} | |
1873 | ################################################################################ | |
1874 | ||
1875 | echo | |
1876 | fi #end running CPass1 | |
1877 | ||
1878 | ################################################################################ | |
1879 | # submit merging of CPass1, depends on the reconstruction | |
1880 | if [ ${runCPass1MergeMakeOCDB} -eq 1 ]; then | |
1881 | ||
1882 | echo | |
1883 | echo "submit CPass1 merging for run ${runNumber}" | |
1884 | echo | |
1885 | ||
1886 | targetDirectory="${commonOutputPath}/000${runNumber}/cpass1" | |
1887 | rm -f ${commonOutputPath}/meta/merge.cpass1.run${runNumber}.done | |
1888 | mkdir -p ${targetDirectory} | |
1889 | ||
1890 | # copy files | |
1891 | filesMergeCPass1=( | |
1892 | "${configPath}/OCDB.root" | |
1893 | "${configPath}/localOCDBaccessConfig.C" | |
1894 | "${configPath}/mergeMakeOCDB.byComponent.sh" | |
1895 | "${configPath}/mergeByComponent.C" | |
1896 | "${configPath}/makeOCDB.C" | |
1897 | "${configPath}/merge.C" | |
1898 | "${configPath}/mergeMakeOCDB.sh" | |
1899 | "${configPath}/QAtrain_duo.C" | |
1900 | ) | |
1901 | for file in ${filesMergeCPass1[*]}; do | |
1902 | [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath} | |
1903 | done | |
1904 | ||
1905 | submit "${JOBID5}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" MergeCPass1 ${targetDirectory} ${currentDefaultOCDB} ${configFile} ${runNumber} cpass1.calib.run${runNumber}.list cpass1.QA.run${runNumber}.lastMergingStage.txt.list cpass1.filtered.run${runNumber}.list "${extraOpts[@]}" | |
1906 | LASTJOB=${JOBID5} | |
1907 | echo | |
1908 | fi | |
1909 | ||
1910 | ############################### | |
1911 | #if [ ${runESDfiltering} -eq 1 ]; then | |
1912 | # rm -f ${commonOutputPath}/cpass1.ESD.run${runNumber}.list | |
1913 | # rm -f ${commonOutputPath}/meta/filtering.cpass1.run*.done | |
1914 | # echo | |
1915 | # echo submitting filtering for run ${runNumber} | |
1916 | # echo | |
1917 | # submit "${JOBmakeESDlistCPass1}" 1 1 "${LASTJOB}" "${self}" PrintValues esd ${commonOutputPath}/meta/cpass1.ESD.run${runNumber}.list ${commonOutputPath}/meta/cpass1.job*.run${runNumber}.done | |
1918 | # submit "${JOBfilterESDcpass1}" 1 1 "${JOBmakeESDlistCPass1}" "${alirootEnv} ${self}" MakeFilteredTrees ${commonOutputPath}/000${runNumber}/cpass1 ${runNumber} ${commonOutputPath}/meta/cpass1.ESD.run${runNumber}.list ${filteringFactorHighPt} ${filteringFactorV0s} ${currentDefaultOCDB} 1000000 0 10000000 0 ${configFile} AliESDs_Barrel.root "${extraOpts[@]}" | |
1919 | # LASTJOB=${JOBfilterESDcpass1} | |
1920 | #fi | |
1921 | ||
1922 | done | |
1923 | ||
1924 | ################################################################################# | |
1925 | ################################################################################# | |
1926 | #if [ ${runESDfiltering} -eq 1 ]; then | |
1927 | # submit "${JOBID5wait}" 1 1 "${LASTJOB}" "${self}" WaitForOutput ${commonOutputPath} "meta/filtering.cpass1.run*.done" "${#listOfRuns[@]}" ${maxSecondsToWait} | |
1928 | #else | |
1929 | submit "${JOBID5wait}" 1 1 "${LASTJOB}" "${self}" WaitForOutput ${commonOutputPath} "meta/merge.cpass1.run*.done" ${#listOfRuns[@]} ${maxSecondsToWait} | |
1930 | #fi | |
1931 | LASTJOB=${JOBID5wait} | |
1932 | ||
1933 | ################################################################################# | |
1934 | echo | |
1935 | echo "submit make a summary" | |
1936 | echo | |
1937 | ||
1938 | submit "${JOBID6}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" MakeSummary ${configFile} | |
1939 | LASTJOB=${JOBID6} | |
1940 | ################################################################################# | |
1941 | ||
1942 | #restore stdout | |
1943 | exec 1>&7 7>&- | |
1944 | echo "jobs submitted." | |
1945 | return 0 | |
1946 | } | |
1947 | ||
1948 | goWaitForOutput() | |
1949 | ( | |
1950 | umask 0002 | |
1951 | [[ $# -lt 3 ]] && echo "goWaitForOutput() wrong number of arguments, exiting.." && return 1 | |
1952 | echo searchPath=${1} | |
1953 | echo fileName=${2} | |
1954 | echo numberOfFiles=${3} | |
1955 | echo maxSecondsToWait=${4} | |
1956 | searchPath=${1} | |
1957 | fileName=${2} | |
1958 | numberOfFiles=${3} | |
1959 | maxSecondsToWait=${4} | |
1960 | echo "command to be executed: /bin/ls -1 ${searchPath}/${fileName}" | |
1961 | [[ -z "${maxSecondsToWait}" ]] && maxSecondsToWait=$(( 3600*12 )) | |
1962 | while true; do | |
1963 | n=$(/bin/ls -1 ${searchPath}/${fileName} 2>/dev/null | wc -l) | |
1964 | [[ ${n} -gt 0 ]] && echo "found ${n} X ${fileName}" | |
1965 | [[ ${n} -ge ${numberOfFiles} ]] && break | |
1966 | [[ ${SECONDS} -gt ${maxSecondsToWait} ]] && echo "timeout of ${maxSecondsToWait}s!" && break | |
1967 | sleep 60 | |
1968 | done | |
1969 | echo "DONE! exiting..." | |
1970 | return 0 | |
1971 | ) | |
1972 | ||
1973 | mergeSysLogs() | |
1974 | ( | |
1975 | outputFile=${1} | |
1976 | shift | |
1977 | inputFiles="$@" | |
1978 | i=0 | |
1979 | if ! ls -1 ${inputFiles} &>/dev/null; then echo "the files dont exist!: ${inputFiles}"; return 1; fi | |
1980 | while read x; do | |
1981 | runNumber=$(guessRunNumber ${x}) | |
1982 | [[ -z ${runNumber} ]] && echo "run number cannot be guessed for ${x}" && continue | |
1983 | awk -v run=${runNumber} -v i=${i} 'NR > 1 {print run" "$0} NR==1 && i==0 {print "run/I:"$0}' ${x} | |
1984 | (( i++ )) | |
1985 | done < <(ls -1 ${inputFiles}) > ${outputFile} | |
1986 | return 0 | |
1987 | ) | |
1988 | ||
1989 | goMakeMergedSummaryTree() | |
1990 | ( | |
1991 | # create list of calibration entries | |
1992 | # takes no arguments, just run it in the base output | |
1993 | # directory with the following files in the working directory | |
1994 | # | |
1995 | # Calibration file lists: | |
1996 | # cpass0.dcsTree.list, cpass1.dcsTree.list | |
1997 | # QA trending root file: | |
1998 | # trending.root | |
1999 | # | |
2000 | # Production infoo ascii files: | |
2001 | # summary_pass0.tree | |
2002 | # summary_pass1.tree | |
2003 | # | |
2004 | ||
2005 | [[ ! -f cpass0.dcsTree.list ]] && echo "no cpass0.dcsTree.list" && return 1 | |
2006 | [[ ! -f cpass1.dcsTree.list ]] && echo "no cpass1.dcsTree.list" && return 1 | |
2007 | [[ ! -f trending.root ]] && echo "no trending.root" && return 1 | |
2008 | [[ ! -f summary_pass0.tree ]] && echo "no summary_pass0.tree" && return 1 | |
2009 | [[ ! -f summary_pass1.tree ]] && echo "no summary_pass1.tree" && return 1 | |
2010 | ||
2011 | #first, dump the C macro to file | |
2012 | cat << EOF > mergeTree.C | |
2013 | // | |
2014 | // Merge summary information | |
2015 | // Following files are expected to be in the working directory | |
2016 | // | |
2017 | // Calibration file lists: | |
2018 | // cpass0.dcsTree.list, cpass1.dcsTree.list | |
2019 | // QA trending root file: | |
2020 | // trending.root | |
2021 | // | |
2022 | // Production infoo ascii files: | |
2023 | // summary_pass0.tree | |
2024 | // summary_pass1.tree | |
2025 | // | |
2026 | void mergeTree(){ | |
2027 | // | |
2028 | // | |
2029 | // | |
2030 | // Calibration values dump | |
2031 | // | |
2032 | //Printf("MakeTreeFromList cpass0.dcsTree.list"); | |
2033 | AliXRDPROOFtoolkit::MakeTreeFromList("Calib.TPC.CPass0.root", "dcs","dcs","cpass0.dcsTree.list",1); | |
2034 | //Printf("MakeTreeFromList cpass1.dcsTree.list"); | |
2035 | AliXRDPROOFtoolkit::MakeTreeFromList("Calib.TPC.CPass1.root", "dcs","dcs","cpass1.dcsTree.list",1); | |
2036 | // | |
2037 | // Calibration status dump | |
2038 | // | |
2039 | TFile *fprod = TFile::Open("fproduction.root","recreate"); | |
2040 | TTree tree0, tree1; | |
2041 | //Printf("reading summary_pass0.tree"); | |
2042 | tree0.ReadFile("summary_pass0.tree"); | |
2043 | //Printf("reading summary_pass1.tree"); | |
2044 | tree1.ReadFile("summary_pass1.tree"); | |
2045 | tree0.Write("CPass0"); | |
2046 | tree1.Write("CPass1"); | |
2047 | fprod->Close(); | |
2048 | // | |
2049 | // | |
2050 | // | |
2051 | TString stringSetup=""; | |
2052 | stringSetup+="1#QA.TPC#run#SummaryTPCQA/tpcQA#trending.root+"; // | |
2053 | stringSetup+="1#Calib.TPC.CPass0#run#dcs#Calib.TPC.CPass0.root+"; // | |
2054 | stringSetup+="1#Calib.TPC.CPass1#run#dcs#Calib.TPC.CPass1.root+"; // | |
2055 | // | |
2056 | stringSetup+="1#CPass0#runnumber#CPass0#fproduction.root+"; // | |
2057 | stringSetup+="1#CPass1#runnumber#CPass1#fproduction.root+"; // | |
2058 | // | |
2059 | //Printf("stringSetup: %s", stringSetup.Data()); | |
2060 | AliXRDPROOFtoolkit::JoinTreesIndex("outAll.root","joinAll","run",stringSetup.Data(), 1); | |
2061 | } | |
2062 | EOF | |
2063 | ||
2064 | aliroot -b -q "mergeTree.C" > mergeTrees.log | |
2065 | return $? | |
2066 | ) | |
2067 | ||
2068 | stackTraceTree() | |
2069 | ( | |
2070 | awk ' | |
2071 | BEGIN { | |
2072 | print "frame/I:method/C:line/C:cpass/I:aliroot/I"; | |
2073 | RS="#[0-9]*"; | |
2074 | aliroot=0; | |
2075 | } | |
2076 | { | |
2077 | if ($3 ~ /Ali*/) aliroot=1; else aliroot=0; | |
2078 | gsub("#","",RT); | |
2079 | if ($NF!="" && RT!="" && $3!="") print RT" "$3" "$NF" "0" "aliroot | |
2080 | } | |
2081 | ' "$@" 2>/dev/null | |
2082 | ) | |
2083 | ||
2084 | goMakeSummary() | |
2085 | ( | |
2086 | #all the final stuff goes in here for ease of use: | |
2087 | # summary logs | |
2088 | # qa plot making | |
2089 | # final file lists | |
2090 | #some defaults: | |
2091 | log="summary.log" | |
2092 | productionID="qa" | |
2093 | ||
2094 | configFile=${1} | |
2095 | shift 1 | |
2096 | extraOpts=("$@") | |
2097 | if ! parseConfig ${configFile} "${extraOpts[@]}"; then return 1; fi | |
2098 | ||
2099 | #if which greadlink; then configFile=$(greadlink -f ${configFile}); fi | |
2100 | ||
2101 | #record the working directory provided by the batch system | |
2102 | batchWorkingDirectory=${PWD} | |
2103 | ||
2104 | [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource} | |
2105 | ||
2106 | [[ ! -f ${configFile} ]] && echo "no config file ${configFile}!" && return | |
2107 | ||
2108 | [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD} | |
2109 | ||
2110 | #copy some useful stuff | |
2111 | #and go to the commonOutputPath | |
2112 | cp ${configFile} ${commonOutputPath} | |
2113 | cd ${commonOutputPath} | |
2114 | ||
2115 | exec &> >(tee ${log}) | |
2116 | ||
2117 | #summarize the global stuff | |
2118 | echo "env script: ${alirootSource} ${alirootEnv}" | |
2119 | echo "\$ALICE_ROOT=${ALICE_ROOT}" | |
2120 | echo "commonOutputPath=${commonOutputPath}" | |
2121 | ||
2122 | #summarize the stacktraces | |
2123 | stackTraceTree 000*/cpass0/*/stacktrace* > stacktrace_cpass0.tree | |
2124 | stackTraceTree 000*/cpass1/*/stacktrace* > stacktrace_cpass1.tree | |
2125 | ||
2126 | echo total numbers for the production: | |
2127 | echo | |
2128 | awk 'BEGIN {nFiles=0;nCore=0;} | |
2129 | /^calibfile/ {nFiles++;} | |
2130 | /core dumped/ {nCore++i;} | |
2131 | END {print "cpass0 produced "nFiles" calib files, "nCore" core files";}' meta/cpass0.job*done 2>/dev/null | |
2132 | awk 'BEGIN {nOK=0; nBAD=0; } | |
2133 | /\/rec.log OK/ {nOK++;} | |
2134 | /\/rec.log BAD/ {nBAD++;} | |
2135 | /stderr BAD/ {if ($0 ~ /rec.log/){nBAD++;}} | |
2136 | END {print "cpass0 reco: OK: "nOK"\tBAD: "nBAD;}' meta/cpass0.job*done 2>/dev/null | |
2137 | awk 'BEGIN {nOK=0; nBAD=0; } | |
2138 | /\/calib.log OK/ {nOK++;} | |
2139 | /\/calib.log BAD/ {nBAD++;} | |
2140 | END {print "cpass0 calib: OK: "nOK"\tBAD: "nBAD;}' meta/cpass0.job*done 2>/dev/null | |
2141 | ||
2142 | awk 'BEGIN {nOK=0; nBAD=0; } | |
2143 | /merge.log OK/ {nOK++;} | |
2144 | /merge.log BAD/ {nBAD++;} | |
2145 | END {print "cpass0 merge: OK: "nOK"\tBAD: "nBAD;}' meta/merge.cpass0*done 2>/dev/null | |
2146 | awk 'BEGIN {nOK=0; nBAD=0; } | |
2147 | /ocdb.log OK/ {nOK++;} | |
2148 | /ocdb.log BAD/ {nBAD++;} | |
2149 | END {print "cpass0 OCDB: OK: "nOK"\tBAD: "nBAD;}' meta/merge.cpass0*done 2>/dev/null | |
2150 | ||
2151 | echo | |
2152 | awk 'BEGIN {nFiles=0;nCore=0;} | |
2153 | /^calibfile/ {nFiles++;} | |
2154 | /core dumped/ {nCore++;} | |
2155 | END {print "cpass1 produced "nFiles" calib files, "nCore" core files";}' meta/cpass1.job*done 2>/dev/null | |
2156 | awk 'BEGIN {nOK=0; nBAD=0; } | |
2157 | /\/rec.log OK/ {nOK++;} | |
2158 | /\/rec.log BAD/ {nBAD++;} | |
2159 | /stderr BAD/ {if ($0 ~ /rec.log/){nBAD++;}} | |
2160 | END {print "cpass1 reco: OK: "nOK"\tBAD: "nBAD;}' meta/cpass1.job*done 2>/dev/null | |
2161 | awk 'BEGIN {nOK=0; nBAD=0; } | |
2162 | /\/calib.log OK/ {nOK++;} | |
2163 | /\/calib.log BAD/ {nBAD++;} | |
2164 | END {print "cpass1 calib: OK: "nOK"\tBAD: "nBAD;}' meta/cpass1.job*done 2>/dev/null | |
2165 | ||
2166 | awk 'BEGIN {nOK=0; nBAD=0; } | |
2167 | /merge.log OK/ {nOK++;} | |
2168 | /merge.log BAD/ {nBAD++;} | |
2169 | END {print "cpass1 merge: OK: "nOK"\tBAD: "nBAD;}' meta/merge.cpass1*done 2>/dev/null | |
2170 | awk 'BEGIN {nOK=0; nBAD=0; } | |
2171 | /ocdb.log OK/ {nOK++;} | |
2172 | /ocdb.log BAD/ {nBAD++;} | |
2173 | END {print "cpass1 OCDB: OK: "nOK"\tBAD: "nBAD;}' meta/merge.cpass1*done 2>/dev/null | |
2174 | ||
2175 | echo | |
2176 | echo per run stats: | |
2177 | /bin/ls -1 meta/merge.cpass0.run*.done | while read x | |
2178 | do | |
2179 | dir=$(goPrintValues calibfile - ${x}) | |
2180 | runNumber=$(guessRunNumber ${dir}) | |
2181 | [[ -z ${runNumber} ]] && continue | |
2182 | ||
2183 | if $(/bin/ls meta/cpass0.job*.run${runNumber}.done &> /dev/null); then | |
2184 | statusCPass0=( $( | |
2185 | awk 'BEGIN {nOKrec=0;nBADrec=0;nOKcalib=0;nBADcalib=0;nOKstderr=0;nBADstderr=0;} | |
2186 | /\/rec.log OK/ {nOKrec++;} | |
2187 | /\/rec.log BAD/ {nBADrec++;} | |
2188 | /stderr BAD/ {if ($0 ~ /rec.log/) {nBADrec++;} nBADstderr++;} | |
2189 | /stderr OK/ {nOKstderr++;} | |
2190 | /\/calib.log OK/ {nOKcalib++;} | |
2191 | /\/calib.log BAD/ {nBADcalib++} | |
2192 | END {print ""nOKrec" "nBADrec" "nOKstderr" "nBADstderr" "nOKcalib" "nBADcalib;}' meta/cpass0.job*.run${runNumber}.done 2>/dev/null | |
2193 | ) ) | |
2194 | fi | |
2195 | ||
2196 | if $(/bin/ls meta/cpass1.job*.run${runNumber}.done &>/dev/null); then | |
2197 | statusCPass1=( $( | |
2198 | awk 'BEGIN {nOKrec=0;nBADrec=0;nOKcalib=0;nBADcalib=0;nOKstderr=0;nBADstderr=0;nQAbarrelOK=0;nQAbarrelBAD=0;nQAouterOK=0;nQAouterBAD=0;} | |
2199 | /\/rec.log OK/ {nOKrec++;} | |
2200 | /\/rec.log BAD/ {nBADrec++;} | |
2201 | /stderr BAD/ {if ($0 ~ /rec.log/) nBADrec++;nBADstderr++;} | |
2202 | /stderr OK/ {nOKstderr++;} | |
2203 | /\/calib.log OK/ {nOKcalib++;} | |
2204 | /\/calib.log BAD/ {nBADcalib++} | |
2205 | /\/qa_barrel.log OK/ {nQAbarrelOK++;} | |
2206 | /\/qa_barrel.log BAD/ {nQAbarrelBAD++;} | |
2207 | /\/qa_outer.log OK/ {nQAouterOK++;} | |
2208 | /\/qa_outer.log BAD/ {nQAouterBAD++;} | |
2209 | END {print ""nOKrec" "nBADrec" "nOKstderr" "nBADstderr" "nOKcalib" "nBADcalib" "nQAbarrelOK" "nQAbarrelBAD" "nQAouterOK" "nQAouterBAD;}' meta/cpass1.job*.run${runNumber}.done 2>/dev/null | |
2210 | ) ) | |
2211 | fi | |
2212 | ||
2213 | statusOCDBcpass0=$(awk '/ocdb.log/ {print $2} ' ${x} 2>/dev/null) | |
2214 | statusOCDBcpass1=$(awk '/ocdb.log/ {print $2}' ${x/cpass0/cpass1} 2>/dev/null) | |
2215 | statusQA=$(awk '/mergeMakeOCDB.log/ {print $2}' ${x/cpass0/cpass1} 2>/dev/null) | |
2216 | ||
2217 | printf "%s\t ocdb.log cpass0: %s\t ocdb.log cpass1: %s\tqa.log:%s\t| cpass0: rec:%s/%s stderr:%s/%s calib:%s/%s cpass1: rec:%s/%s stderr:%s/%s calib:%s/%s QAbarrel:%s/%s QAouter:%s/%s\n" ${runNumber} ${statusOCDBcpass0} ${statusOCDBcpass1} ${statusQA} ${statusCPass0[0]} ${statusCPass0[1]} ${statusCPass0[2]} ${statusCPass0[3]} ${statusCPass0[4]} ${statusCPass0[5]} ${statusCPass1[0]} ${statusCPass1[1]} ${statusCPass1[2]} ${statusCPass1[3]} ${statusCPass1[4]} ${statusCPass1[5]} ${statusCPass1[6]} ${statusCPass1[7]} ${statusCPass1[8]} ${statusCPass1[9]} | |
2218 | done | |
2219 | ||
2220 | #make lists with output files - QA, trending, filtering and calibration | |
2221 | ### wait for the merging of all runs to be over ### | |
2222 | rm -f qa.list | |
2223 | goPrintValues qafile qa.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null | |
2224 | rm -f calib.list | |
2225 | goPrintValues calibfile calib.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null | |
2226 | rm -f trending.list | |
2227 | goPrintValues trendingfile trending.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null | |
2228 | rm -f filtering.list | |
2229 | goPrintValues filteredTree filtering.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null | |
2230 | rm -f cpass0.dcsTree.list | |
2231 | goPrintValues dcsTree cpass0.dcsTree.list ${commonOutputPath}/meta/merge.cpass0.run*.done &>/dev/null | |
2232 | rm -f cpass1.dcsTree.list | |
2233 | goPrintValues dcsTree cpass1.dcsTree.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null | |
2234 | ||
2235 | #merge trending | |
2236 | rm -f ${commonOutputPath}/trending_merged.root | |
2237 | goMerge trending.list ${commonOutputPath}/trending.root ${configFile} "${extraOpts[@]}" &> mergeTrending.log | |
2238 | ||
2239 | goMakeSummaryTree ${commonOutputPath} 0 | |
2240 | goMakeSummaryTree ${commonOutputPath} 1 | |
2241 | ||
2242 | goCreateQAplots "${PWD}/qa.list" "${productionID}" "QAplots" "${configFile}" "${extraOpts[@]}" filteringList="${PWD}/filtering.list" &>createQAplots.log | |
2243 | ||
2244 | #make a merged summary tree out of the QA trending, dcs trees and log summary trees | |
2245 | goMakeMergedSummaryTree | |
2246 | ||
2247 | #if set, email the summary | |
2248 | [[ -n ${MAILTO} ]] && cat ${log} | mail -s "benchmark ${productionID} done" ${MAILTO} | |
2249 | ||
2250 | return 0 | |
2251 | ) | |
2252 | ||
2253 | goMakeSummaryTree() | |
2254 | ( | |
2255 | if [[ $# -lt 1 ]] ; then | |
2256 | return | |
2257 | fi | |
2258 | #1. define vars/arrays | |
2259 | DIR=${1} #use input or exec in current dir | |
2260 | pass=${2-"0"} #pass from input | |
2261 | outfile="summary_pass${pass}.tree" | |
2262 | Ncolumns=0 | |
2263 | test -f ${outfile} && : >${outfile} | |
2264 | errfile=${outfile/tree/err} | |
2265 | test -f ${errfile} && : >${errfile} | |
2266 | ||
2267 | declare -a counterString=(TOFevents TOFtracks TPCevents TPCtracks TRDevents TRDtracks T0events SDDevents SDDtracks MeanVertexevents) | |
2268 | Ncounter=${#counterString[@]} | |
2269 | ||
2270 | declare -a statusString=(TRDStatus TOFStatus TPCStatus T0Status MeanVertexStatus) | |
2271 | Nstatus=${#statusString[@]} | |
2272 | ||
2273 | ||
2274 | declare -a ratesString=(rec stderr calib qa_barrel qa_outer) | |
2275 | Nrates=${#ratesString[@]} | |
2276 | ||
2277 | runs=( $(ls -1 ${DIR}/meta/merge.cpass0* | while read x; do guessRunNumber $x; done) ) | |
2278 | Nruns=${#runs[@]} | |
2279 | ||
2280 | echo -n runnumber/I >>${outfile} | |
2281 | echo -n :cpass${pass}status/I >>${outfile} | |
2282 | echo -n :cpass${pass}QAstatus/I >>${outfile} | |
2283 | for i in ${ratesString[@]}; do | |
2284 | echo -n :${i}OK/I >>${outfile} | |
2285 | echo -n :${i}BAD/I >>${outfile} | |
2286 | ||
2287 | done | |
2288 | for i in ${counterString[@]} ${statusString[@]} ; do | |
2289 | echo -n :${i}/I >>${outfile} | |
2290 | done | |
2291 | Ncolumns=$((2 + 2*Nrates + Ncounter + Nstatus)) | |
2292 | echo >> ${outfile} | |
2293 | ||
2294 | #2. loop runs | |
2295 | ||
2296 | for runnumber in ${runs[@]} ; do | |
2297 | ||
2298 | ||
2299 | ||
2300 | filejob="${DIR}/meta/cpass${pass}.job*.run${runnumber}.done" | |
2301 | filemerge="${DIR}/meta/merge.cpass${pass}.run${runnumber}.done" | |
2302 | fileOCDB=$(grep /ocdb.log ${filemerge} | awk '{print $1}') | |
2303 | if ! $(/bin/ls ${filemerge} &>/dev/null) ; then | |
2304 | echo "${filemerge} does not exist!" >>${errfile} | |
2305 | continue | |
2306 | elif ! $(/bin/ls ${filejob} &>/dev/null) ; then | |
2307 | echo "${filejob} does not exist!" >>${errfile} | |
2308 | echo -n ${runnumber} >> ${outfile} | |
2309 | for i in $(seq ${Ncolumns}) ; do | |
2310 | echo -n "-1" >> ${outfile} | |
2311 | done | |
2312 | echo >> ${outfile} | |
2313 | continue | |
2314 | fi | |
2315 | echo -n ${runnumber} >> ${outfile} | |
2316 | #pass0status= grep '/ocdb.log' ${filemerge} | cut -d' ' -f2 | tr OK x1 | tr BAD xx0 | tr -d 'x' | |
2317 | passStatus=$(grep '/ocdb.log' ${filemerge} | grep OK | wc -l) | |
2318 | echo -n " ${passStatus}" >> ${outfile} | |
2319 | qaStatus=$(grep '/mergeMakeOCDB.log' ${filemerge} | grep OK | wc -l) | |
2320 | echo -n " ${qaStatus}" >> ${outfile} | |
2321 | ||
2322 | ||
2323 | #fill OK/BAD rates | |
2324 | for i in $(seq 0 $((${Nrates}-1))) ; do | |
2325 | var1=$(grep "/${ratesString[${i}]}.log" ${filejob} | grep OK | wc -l) | |
2326 | var2=$(grep "/${ratesString[${i}]}.log" ${filejob} | grep BAD | wc -l) | |
2327 | ||
2328 | if [[ ${ratesString[${i}]} == "stderr" ]] ; then | |
2329 | var1=$(grep "stderr" ${filejob} | grep OK | wc -l) | |
2330 | var2=$(grep "stderr" ${filejob} | grep "rec.log" | grep BAD | wc -l) | |
2331 | fi | |
2332 | echo -n " ${var1}" >> ${outfile} | |
2333 | echo -n " ${var2}" >> ${outfile} | |
2334 | done | |
2335 | ||
2336 | if [[ -f ${fileOCDB} ]] ; then | |
2337 | #fill counter | |
2338 | for i in $(seq 0 $((${Ncounter}-1))) ; do | |
2339 | var1=$(grep Monalisa ${fileOCDB} | grep ${counterString[${i}]} | cut -f2) | |
2340 | echo -n " ${var1:-"-1"}" >> ${outfile} | |
2341 | done | |
2342 | ||
2343 | #fill status | |
2344 | for i in $(seq 0 $((${Nstatus}-1))) ; do | |
2345 | var1=$(grep "calibration status=" ${fileOCDB} | grep ${statusString[${i}]/Status/} | cut -d'=' -f2) | |
2346 | echo -n " ${var1:-"-1"}" >> ${outfile} | |
2347 | done | |
2348 | fi | |
2349 | echo >> ${outfile} | |
2350 | done | |
2351 | ||
2352 | return 0 | |
2353 | ) | |
2354 | ||
2355 | parseConfig() | |
2356 | { | |
2357 | configFile=${1} | |
2358 | shift | |
2359 | args=("$@") | |
2360 | ||
2361 | ||
2362 | #some defaults | |
2363 | #autoOCDB=0 | |
2364 | defaultOCDB="raw://" | |
2365 | #runNumber=167123 | |
2366 | #makeflowPath="/hera/alice/aux/cctools/bin" | |
2367 | #makeflowOptions="-T wq -N alice -d all -C ali-copilot.cern.ch:9097" | |
2368 | #makeflowOptions="-T wq -N alice -C ali-copilot.cern.ch:9097" | |
2369 | makeflowOptions="" | |
2370 | #batchCommand="/usr/bin/qsub" | |
2371 | batchFlags="-b y -cwd -l h_rt=24:0:0,h_rss=4G " | |
2372 | baseOutputDirectory="$PWD/output" | |
2373 | #alirootEnv="/cvmfs/alice.cern.ch/bin/alienv setenv AliRoot/v5-04-34-AN -c" | |
2374 | #alirootEnv="/home/mkrzewic/alisoft/balice_master.sh" | |
2375 | #trustedQAtrainMacro='/hera/alice/mkrzewic/gsisvn/Calibration/QAtrain_duo.C' | |
2376 | reconstructInTemporaryDir=0 | |
2377 | recoTriggerOptions="\"\"" | |
2378 | percentProcessedFilesToContinue=100 | |
2379 | maxSecondsToWait=$(( 3600*24 )) | |
2380 | nEvents=-1 | |
2381 | nMaxChunks=0 | |
2382 | postSetUpActionCPass0="" | |
2383 | postSetUpActionCPass1="" | |
2384 | runCPass0reco=1 | |
2385 | runCPass0MergeMakeOCDB=1 | |
2386 | runCPass1reco=1 | |
2387 | runCPass1MergeMakeOCDB=1 | |
2388 | runESDfiltering=1 | |
2389 | filteringFactorHighPt=1e2 | |
2390 | filteringFactorV0s=1e1 | |
2391 | MAILTO="" | |
2392 | #pretend=1 | |
2393 | #dontRedirectStdOutToLog=1 | |
2394 | logToFinalDestination=1 | |
2395 | ALIROOT_FORCE_COREDUMP=1 | |
2396 | pretendDelay=0 | |
2397 | ||
2398 | #first, source the config file | |
2399 | if [ -f ${configFile} ]; then | |
2400 | source ${configFile} | |
2401 | else | |
2402 | echo "config file ${configFile} not found!" | |
2403 | return 1 | |
2404 | fi | |
2405 | ||
2406 | unset encodedSpaces | |
2407 | for opt in "${args[@]}"; do | |
2408 | [[ "${opt}" =~ encodedSpaces=.* ]] && encodedSpaces=1 && echo "encodedSpaces!" && break | |
2409 | done | |
2410 | ||
2411 | #then, parse the options as they override the options from file | |
2412 | for opt in "${args[@]}"; do | |
2413 | [[ -z ${opt} ]] && continue | |
2414 | [[ -n ${encodedSpaces} ]] && opt="$(decSpaces ${opt})" | |
2415 | [[ "${opt}" =~ ^[[:space:]]*$ ]] && continue | |
2416 | if [[ ! "${opt}" =~ .*=.* ]]; then | |
2417 | echo "badly formatted option \"${opt}\" should be: option=value, stopping..." | |
2418 | return 1 | |
2419 | fi | |
2420 | local var="${opt%%=*}" | |
2421 | local value="${opt#*=}" | |
2422 | echo "${var}=${value}" | |
2423 | export ${var}="${value}" | |
2424 | done | |
2425 | ||
2426 | #do some checking | |
2427 | [[ -z ${alirootEnv} ]] && echo "alirootEnv not defined!" && return 1 | |
2428 | ||
2429 | #export the aliroot function if defined to override normal behaviour | |
2430 | [[ $(type -t aliroot) =~ "function" ]] && export -f aliroot | |
2431 | ||
2432 | return 0 | |
2433 | } | |
2434 | ||
2435 | aliroot() | |
2436 | { | |
2437 | args=("$@") | |
2438 | if [[ -n ${useProfilingCommand} ]]; then | |
2439 | profilerLogFile="cpu.txt" | |
2440 | [[ "${args}" =~ rec ]] && profilerLogFile="cpu_rec.txt" | |
2441 | [[ "${args}}" =~ Calib ]] && profilerLogFile="cpu_calib.txt" | |
2442 | echo running "${useProfilingCommand} aliroot ${args} &> ${profilerLogFile}" | |
2443 | ${useProfilingCommand} aliroot "${args[@]}" &> ${profilerLogFile} | |
2444 | else | |
2445 | #to prevent an infinite recursion use "command aliroot" to disable | |
2446 | #aliases and functions | |
2447 | echo running command aliroot "${args[@]}" | |
2448 | command aliroot "${args[@]}" | |
2449 | fi | |
2450 | return 0 | |
2451 | } | |
2452 | ||
2453 | guessRunData() | |
2454 | { | |
2455 | #guess the period from the path, pick the rightmost one | |
2456 | period="" | |
2457 | runNumber="" | |
2458 | year="" | |
2459 | pass="" | |
2460 | legoTrainRunNumber="" | |
2461 | dataType="" | |
2462 | ||
2463 | local shortRunNumber="" | |
2464 | local IFS="/" | |
2465 | declare -a path=( $1 ) | |
2466 | local dirDepth=$(( ${#path[*]}-1 )) | |
2467 | i=0 | |
2468 | #for ((x=${dirDepth};x>=0;x--)); do | |
2469 | for ((x=0;x<=${dirDepth};x++)); do | |
2470 | ||
2471 | [[ $((x-1)) -ge 0 ]] && local fieldPrev=${path[$((x-1))]} | |
2472 | local field=${path[${x}]} | |
2473 | local fieldNext=${path[$((x+1))]} | |
2474 | ||
2475 | [[ ${field} =~ ^[0-9]*$ && ${fieldNext} =~ (.*\.zip$|.*\.root$) ]] && legoTrainRunNumber=${field} | |
2476 | [[ -n ${legoTrainRunNumber} && -z ${pass} ]] && pass=${fieldPrev} | |
2477 | [[ ${field} =~ ^LHC[0-9][0-9][a-z].*$ ]] && period=${field%_*} | |
2478 | [[ ${field} =~ ^000[0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && runNumber=${field#000} | |
2479 | [[ ${field} =~ ^[0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && shortRunNumber=${field} | |
2480 | [[ ${field} =~ ^20[0-9][0-9]$ ]] && year=${field} | |
2481 | [[ ${field} =~ ^(^sim$|^data$) ]] && dataType=${field} | |
2482 | (( i++ )) | |
2483 | done | |
2484 | [[ -z ${legoTrainRunNumber} ]] && pass=${path[$((dirDepth-1))]} | |
2485 | [[ "${dataType}" =~ ^sim$ ]] && pass="passMC" && runNumber=${shortRunNumber} | |
2486 | ||
2487 | #if [[ -z ${dataType} || -z ${year} || -z ${period} || -z ${runNumber}} || -z ${pass} ]]; | |
2488 | if [[ -z ${runNumber}} ]]; | |
2489 | then | |
2490 | #error condition | |
2491 | return 1 | |
2492 | else | |
2493 | #ALL OK | |
2494 | return 0 | |
2495 | fi | |
2496 | return 0 | |
2497 | } | |
2498 | ||
2499 | #these functions encode strings to and from a space-less form | |
2500 | #use when spaces are not well handled (e.g. in arguments to | |
2501 | #commands in makeflow files, etc. | |
2502 | encSpaces()(echo "${1// /@@@@}") | |
2503 | decSpaces()(echo "${1//@@@@/ }") | |
2504 | ||
2505 | main "$@" |