]>
Commit | Line | Data |
---|---|---|
1 | #!/bin/bash | |
2 | #include benchmark.config | |
3 | ||
4 | # this script runs the CPass0/CPass1 train | |
5 | # produced OCDB updates are local | |
6 | ||
7 | main() | |
8 | { | |
9 | #run in proper mode depending on the selection | |
10 | if [[ $# -lt 1 ]]; then | |
11 | if [[ ! "${0}" =~ "bash" ]]; then | |
12 | echo "uses makeflow:" | |
13 | echo " ${0} \"run\" productionID inputList configFile [extraOpts]" | |
14 | echo "uses a batch system (SGE):" | |
15 | echo " ${0} \"submit\" productionID inputList configFile [extraOpts]" | |
16 | echo "extraOpts if non-empty override the config file, e.g.:" | |
17 | echo " ${0} submit test1 benchmark.list benchmark.config runNumber=169123 nEvents=10" | |
18 | fi | |
19 | return | |
20 | fi | |
21 | ||
22 | #define some aliases - default is to call one of the functions directly | |
23 | runMode=${1} | |
24 | umask 0002 | |
25 | shift | |
26 | case ${runMode} in | |
27 | "CPass0") goCPass0 "$@";; | |
28 | "CPass1") goCPass1 "$@";; | |
29 | "MakeLocalOCDBaccessConfig") goMakeLocalOCDBaccessConfig "$@";; | |
30 | "MergeCPass0") goMergeCPass0 "$@";; | |
31 | "MergeCPass1") goMergeCPass1 "$@";; | |
32 | "MakeFilteredTrees") goMakeFilteredTrees "$@";; | |
33 | "MakeSummary") goMakeSummary "$@";; | |
34 | "run") goSubmitMakeflow "$@";; | |
35 | "submit") goSubmitBatch "$@";; | |
36 | "test") goTest "$@";; | |
37 | "GenerateMakeflow") goGenerateMakeflow "$@";; | |
38 | "PrintValues") goPrintValues "$@";; | |
39 | "CreateQAplots") goCreateQAplots "$@";; | |
40 | "WaitForOutput") goWaitForOutput "$@";; | |
41 | "Merge") goMerge "$@";; | |
42 | *) | |
43 | ${runMode} "$@" | |
44 | ;; | |
45 | esac | |
46 | } | |
47 | ||
48 | generateMC() | |
49 | { | |
50 | #generate one raw chunk in current directory | |
51 | SEED=${JOB_ID}${SGE_TASK_ID} | |
52 | export CONFIG_SEED=${SEED} | |
53 | runNumber=${1} | |
54 | OCDBpath=${2} | |
55 | nEventsim=${3} | |
56 | if [[ -n ${pretend} ]]; then | |
57 | touch galice.root | |
58 | else | |
59 | if [[ -f sim.C && -f Config.C ]] ; then | |
60 | time aliroot -b -q -x sim.C\(${runNumber},\"${OCDBpath}\",${nEventsim}\) >sim.log 2>&1 | |
61 | mv syswatch.log simwatch.log | |
62 | fi | |
63 | fi | |
64 | } | |
65 | ||
66 | goCPass0() | |
67 | { | |
68 | umask 0002 | |
69 | ||
70 | targetDirectory=${1} | |
71 | inputList=${2} | |
72 | nEvents=${3} | |
73 | ocdbPath=${4} | |
74 | configFile=${5} | |
75 | runNumber=${6} | |
76 | jobindex=${7} | |
77 | shift 7 | |
78 | if ! parseConfig ${configFile} "$@"; then return 1; fi | |
79 | ||
80 | #use the jobindex only if set and non-negative | |
81 | if [[ -z ${jobindex} || ${jobindex} -lt 0 ]]; then | |
82 | [[ -n "${LSB_JOBINDEX}" ]] && jobindex=${LSB_JOBINDEX} | |
83 | [[ -n "${SGE_TASK_ID}" ]] && jobindex=${SGE_TASK_ID} | |
84 | fi | |
85 | ||
86 | [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD} | |
87 | doneFile="${commonOutputPath}/meta/cpass0.job${jobindex}.run${runNumber}.done" | |
88 | ||
89 | [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource} | |
90 | ||
91 | if [[ -n ${ALIROOT_FORCE_COREDUMP} ]]; then | |
92 | ulimit -c unlimited | |
93 | export ALIROOT_FORCE_COREDUMP | |
94 | fi | |
95 | ||
96 | #the contents of this is stored in the tree and used later (e.g. AliAnalysisTaskPIDResponse)! | |
97 | #at the QA stage the pass number is guessed from the path stored here. | |
98 | #The Format is: | |
99 | #Packages= ;OutputDir= ;LPMPass= ;TriggerAlias= ;LPMRunNumber= ;LPMProductionType= ;LPMInteractionType= ;LPMProductionTag= ;LPMAnchorRun= ;LPMAnchorProduction= ;LPMAnchorYear= | |
100 | export PRODUCTION_METADATA="OutputDir=cpass0" | |
101 | ||
102 | if [[ "${inputList}" =~ \.root$ ]]; then | |
103 | infile=${inputList} | |
104 | else | |
105 | infile=$(sed -ne "${jobindex}p" ${inputList} | egrep '\s*\w*/\w*') | |
106 | fi | |
107 | chunkName=${infile##*/} | |
108 | ||
109 | outputDir=${targetDirectory}/${jobindex}_${chunkName%.*} | |
110 | mkdir -p ${outputDir} | |
111 | [[ ! -d ${outputDir} ]] && echo "cannot make ${outputDir}" && touch ${doneFile} && return 1 | |
112 | ||
113 | #runpath=${PWD}/rundir_cpass0_${runNumber}_${jobindex} | |
114 | runpath=${outputDir} | |
115 | [[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR} | |
116 | [[ ${reconstructInTemporaryDir} -eq 1 && -z ${TMPDIR} ]] && runpath=$(mktemp -d) | |
117 | mkdir -p ${runpath} | |
118 | [[ ! -d ${runpath} ]] && echo "cannot make runpath ${runpath}" && touch ${doneFile} && return 1 | |
119 | cd ${runpath} | |
120 | ||
121 | #runCPassX/C expects the raw chunk to be linked in the run dir | |
122 | #despite it being accessed by the full path | |
123 | ln -s ${infile} ${runpath}/${chunkName} | |
124 | ||
125 | #####MC | |
126 | if [[ -n ${generateMC} ]]; then | |
127 | olddir=${PWD} | |
128 | outputDirMC=${commonOutputPath}/000${runNumber}/sim/${jobindex} | |
129 | simrunpath=${outputDirMC} | |
130 | [[ ${simulateInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && simrunpath=${TMPDIR} | |
131 | [[ ${simulateInTemporaryDir} -eq 1 && -z ${TMPDIR} ]] && simrunpath=$(mktemp -d) | |
132 | mkdir -p ${outputDirMC} | |
133 | mkdir -p ${simrunpath} | |
134 | cd ${simrunpath} | |
135 | ||
136 | filesMC=( | |
137 | "${commonOutputPath}/sim.C" | |
138 | "${commonOutputPath}/rec.C" | |
139 | "${commonOutputPath}/Config.C" | |
140 | "${commonOutputPath}/OCDB_*.root" | |
141 | ) | |
142 | for file in ${filesMC[*]}; do | |
143 | [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} . | |
144 | done | |
145 | ||
146 | generateMC ${runNumber} ${ocdbPath} ${nEvents} | |
147 | ||
148 | [[ ! "${simrunpath}" =~ "${outputDirMC}" ]] && mv * ${outputDirMC} #TODO check if it works | |
149 | cd ${olddir} | |
150 | ||
151 | ln -s ${outputDirMC}/* ${runpath}/ | |
152 | ||
153 | inputList=${outputDirMC}/galice.root #TODO not valid outside shell !!! | |
154 | infile="" | |
155 | fi | |
156 | ###### | |
157 | ||
158 | [[ ! -f ${inputList} && -z ${pretend} ]] && echo "input file ${inputList} not found, exiting..." && touch ${doneFile} && return 1 | |
159 | ||
160 | logOutputDir=${runpath} | |
161 | [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir} | |
162 | [[ -z ${dontRedirectStdOutToLog} ]] && exec 1> ${logOutputDir}/stdout | |
163 | [[ -z ${dontRedirectStdOutToLog} ]] && exec 2> ${logOutputDir}/stderr | |
164 | echo "${0} $*" | |
165 | ||
166 | echo "#####################" | |
167 | echo CPass0: | |
168 | echo JOB setup | |
169 | echo nEvents ${nEvents} | |
170 | echo runNumber ${runNumber} | |
171 | echo ocdbPath ${ocdbPath} | |
172 | echo infile ${infile} | |
173 | echo chunkName ${chunkName} | |
174 | echo jobindex ${jobindex} | |
175 | echo recoTriggerOptions ${recoTriggerOptions} | |
176 | echo targetDirectory ${targetDirectory} | |
177 | echo commonOutputPath ${commonOutputPath} | |
178 | echo doneFile ${doneFile} | |
179 | echo runpath ${runpath} | |
180 | echo outputDir ${outputDir} | |
181 | echo ALICE_ROOT ${ALICE_ROOT} | |
182 | echo PWD ${PWD} | |
183 | echo "########## ###########" | |
184 | ||
185 | alirootInfo > ALICE_ROOT.log | |
186 | ||
187 | filesCPass0=( | |
188 | "${commonOutputPath}/runCPass0.sh" | |
189 | "${commonOutputPath}/recCPass0.C" | |
190 | "${commonOutputPath}/runCalibTrain.C" | |
191 | "${commonOutputPath}/localOCDBaccessConfig.C" | |
192 | "${commonOutputPath}/OCDB.root" | |
193 | "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/runCPass0.sh" | |
194 | "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/recCPass0.C" | |
195 | "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/runCalibTrain.C" | |
196 | ) | |
197 | ||
198 | for file in ${filesCPass0[*]}; do | |
199 | [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} . | |
200 | done | |
201 | ||
202 | echo "this directory (${PWD}) contents:" | |
203 | /bin/ls | |
204 | echo | |
205 | chmod u+x runCPass0.sh | |
206 | ||
207 | sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh | |
208 | ||
209 | if [[ -n ${postSetUpActionCPass0} ]]; then | |
210 | echo "running ${postSetUpActionCPass0}" | |
211 | eval ${postSetUpActionCPass0} | |
212 | fi | |
213 | ||
214 | #run CPass0 | |
215 | echo "${runpath}/runCPass0.sh ${infile} ${nEvents} ${runNumber} ${ocdbPath} ${recoTriggerOptions}" | |
216 | if [[ -n ${pretend} ]]; then | |
217 | touch AliESDs.root | |
218 | touch AliESDfriends.root | |
219 | touch AliESDfriends_v1.root | |
220 | touch rec.log | |
221 | touch calib.log | |
222 | else | |
223 | echo ./runCPass0.sh "${infile}" "${nEvents}" "${runNumber}" "${ocdbPath}" "${recoTriggerOptions}" | |
224 | ./runCPass0.sh "${infile}" "${nEvents}" "${runNumber}" "${ocdbPath}" "${recoTriggerOptions}" | |
225 | fi | |
226 | ||
227 | #move stuff to final destination | |
228 | echo "this directory (${PWD}) contents:" | |
229 | /bin/ls | |
230 | echo | |
231 | ||
232 | echo rm -f ./${chunkName} | |
233 | rm -f ./${chunkName} | |
234 | echo "cp --recursive ${runpath}/* ${outputDir}" | |
235 | cp -p --recursive ${runpath}/* ${outputDir} | |
236 | echo | |
237 | ||
238 | #validate CPass0 | |
239 | cd ${outputDir} | |
240 | touch ${doneFile} | |
241 | echo "dir ${outputDir}" >> ${doneFile} | |
242 | if summarizeLogs >> ${doneFile}; then | |
243 | [[ -f ${outputDirMC}/galice.root ]] && echo "sim ${outputDirMC}/galice.root" >> ${doneFile} | |
244 | [[ -f AliESDfriends_v1.root ]] && echo "calibfile ${outputDir}/AliESDfriends_v1.root" >> ${doneFile} | |
245 | [[ -f AliESDs.root ]] && echo "esd ${outputDir}/AliESDs.root" >> ${doneFile} | |
246 | fi | |
247 | ||
248 | [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath} | |
249 | return 0 | |
250 | } | |
251 | ||
252 | goCPass1() | |
253 | { | |
254 | umask 0002 | |
255 | ||
256 | targetDirectory=${1} | |
257 | inputList=${2} | |
258 | nEvents=${3} | |
259 | ocdbPath=${4} | |
260 | configFile=${5} | |
261 | runNumber=${6} | |
262 | jobindex=${7} | |
263 | shift 7 | |
264 | extraOpts=("$@") | |
265 | if ! parseConfig ${configFile} "$@"; then return 1; fi | |
266 | ||
267 | #use the jobindex only if set and non-negative | |
268 | if [[ -z ${jobindex} || ${jobindex} -lt 0 ]]; then | |
269 | [[ -n "${LSB_JOBINDEX}" ]] && jobindex=${LSB_JOBINDEX} | |
270 | [[ -n "${SGE_TASK_ID}" ]] && jobindex=${SGE_TASK_ID} | |
271 | fi | |
272 | ||
273 | [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD} | |
274 | doneFile="${commonOutputPath}/meta/cpass1.job${jobindex}.run${runNumber}.done" | |
275 | ||
276 | [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource} | |
277 | ||
278 | if [[ -n ${ALIROOT_FORCE_COREDUMP} ]]; then | |
279 | ulimit -c unlimited | |
280 | export ALIROOT_FORCE_COREDUMP | |
281 | fi | |
282 | ||
283 | #the contents of this is stored in the tree and used later (e.g. AliAnalysisTaskPIDResponse)! | |
284 | #at the QA stage the pass number is guessed from the path stored here. | |
285 | #The Format is: | |
286 | #Packages= ;OutputDir= ;LPMPass= ;TriggerAlias= ;LPMRunNumber= ;LPMProductionType= ;LPMInteractionType= ;LPMProductionTag= ;LPMAnchorRun= ;LPMAnchorProduction= ;LPMAnchorYear= | |
287 | export PRODUCTION_METADATA="OutputDir=cpass1" | |
288 | ||
289 | [[ ! -f ${inputList} && -z ${pretend} ]] && echo "input file ${inputList} not found, exiting..." && touch ${doneFile} && return 1 | |
290 | if [[ "${inputList}" =~ \.root$ ]]; then | |
291 | infile=${inputList} | |
292 | else | |
293 | infile=$(sed -ne "${jobindex}p" ${inputList} | egrep '\s*\w*/\w*') | |
294 | fi | |
295 | chunkName=${infile##*/} | |
296 | ||
297 | outputDir=${targetDirectory}/${jobindex}_${chunkName%.*} | |
298 | mkdir -p ${outputDir} | |
299 | [[ ! -d ${outputDir} ]] && echo "cannot make ${outputDir}" && touch ${doneFile} && return 1 | |
300 | ||
301 | #runpath=${PWD}/rundir_cpass1_${runNumber}_${jobindex} | |
302 | runpath=${outputDir} | |
303 | [[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR} | |
304 | [[ ${reconstructInTemporaryDir} -eq 1 && -z ${TMPDIR} ]] && runpath=$(mktemp -d) | |
305 | ||
306 | #MC | |
307 | if [[ "${infile}" =~ galice\.root ]]; then | |
308 | ln -s ${inputList%/*}/* ${runpath} | |
309 | infile="" | |
310 | fi | |
311 | ||
312 | #init the running path | |
313 | mkdir -p ${runpath} | |
314 | [[ ! -d ${runpath} ]] && echo "cannot make runpath ${runpath}" && touch ${doneFile} && return 1 | |
315 | cd ${runpath} | |
316 | ||
317 | #this is needed for runCPass1.sh | |
318 | ln -s ${infile} ${runpath}/${chunkName} | |
319 | ||
320 | logOutputDir=${runpath} | |
321 | [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir} | |
322 | [[ -z ${dontRedirectStdOutToLog} ]] && exec 1> ${logOutputDir}/stdout | |
323 | [[ -z ${dontRedirectStdOutToLog} ]] && exec 2> ${logOutputDir}/stderr | |
324 | echo "${0} $*" | |
325 | ||
326 | echo "#####################" | |
327 | echo CPass1: | |
328 | echo JOB setup | |
329 | echo nEvents ${nEvents} | |
330 | echo runNumber ${runNumber} | |
331 | echo ocdbPath ${ocdbPath} | |
332 | echo infile ${infile} | |
333 | echo chunkName ${chunkName} | |
334 | echo jobindex ${jobindex} | |
335 | echo recoTriggerOptions ${recoTriggerOptions} | |
336 | echo targetDirectory ${targetDirectory} | |
337 | echo commonOutputPath ${commonOutputPath} | |
338 | echo doneFile ${doneFile} | |
339 | echo runpath ${runpath} | |
340 | echo outputDir ${outputDir} | |
341 | echo ALICE_ROOT ${ALICE_ROOT} | |
342 | echo PWD ${PWD} | |
343 | echo "########## ###########" | |
344 | ||
345 | alirootInfo > ALICE_ROOT.log | |
346 | ||
347 | filesCPass1=( | |
348 | "${commonOutputPath}/runCPass1.sh" | |
349 | "${commonOutputPath}/recCPass1.C" | |
350 | "${commonOutputPath}/recCPass1_OuterDet.C" | |
351 | "${commonOutputPath}/runCalibTrain.C" | |
352 | "${commonOutputPath}/QAtrain_duo.C" | |
353 | "${commonOutputPath}/localOCDBaccessConfig.C" | |
354 | "${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz" | |
355 | "${commonOutputPath}/OCDB.root" | |
356 | "${trustedQAtrainMacro}" | |
357 | "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/runCPass1.sh" | |
358 | "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/recCPass1.C" | |
359 | "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/recCPass1_OuterDet.C" | |
360 | "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/runCalibTrain.C" | |
361 | "${ALICE_ROOT}/ANALYSIS/macros/QAtrain_duo.C" | |
362 | ) | |
363 | ||
364 | for file in ${filesCPass1[*]}; do | |
365 | [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} . | |
366 | done | |
367 | ||
368 | echo "this directory (${PWD}) contents:" | |
369 | /bin/ls | |
370 | echo | |
371 | ||
372 | sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh | |
373 | ||
374 | if [[ -n ${postSetUpActionCPass1} ]]; then | |
375 | echo "running ${postSetUpActionCPass1}" | |
376 | eval ${postSetUpActionCPass1} | |
377 | echo | |
378 | fi | |
379 | ||
380 | #configure local OCDB storage from CPass0 (creates the localOCDBaccessConfig.C script) | |
381 | if [[ -f cpass0.localOCDB.${runNumber}.tgz ]]; then | |
382 | echo goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz" | |
383 | goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz" | |
384 | else | |
385 | echo "WARNING: file cpass0.localOCDB.${runNumber}.tgz not found!" | |
386 | fi | |
387 | ||
388 | if [[ ! $(/bin/ls -1 OCDB/*/*/*/*.root 2>/dev/null) ]]; then | |
389 | echo "cpass0 produced no calibration! exiting..." | |
390 | touch ${doneFile} | |
391 | return 1 | |
392 | fi | |
393 | ||
394 | #create the Barrel and OuterDet directories for CPass1 and link the local OCDB directory | |
395 | #there to make the localOCDBaccessConfig.C file work, since it may point to the OCDB | |
396 | #entries using a relative path, e.g. local://./OCDB | |
397 | mkdir Barrel OuterDet | |
398 | ln -s ../OCDB Barrel/OCDB | |
399 | ln -s ../OCDB OuterDet/OCDB | |
400 | ||
401 | #setup the filtering | |
402 | #the following option enables the filtering task inside the QAtrain_duo.C | |
403 | [[ -n $runESDfiltering ]] && export QA_TaskFilteredTree=1 | |
404 | #set the downscaling factors during the filtering fro expert QA (overrides the previous values) | |
405 | if [[ -n ${filteringFactorHighPt} ]]; then | |
406 | export AliAnalysisTaskFilteredTree_fLowPtTrackDownscaligF=${filteringFactorHighPt} | |
407 | fi | |
408 | if [[ -n ${filteringFactorV0s} ]]; then | |
409 | export AliAnalysisTaskFilteredTree_fLowPtV0DownscaligF=${filteringFactorV0s} | |
410 | fi | |
411 | ||
412 | #run CPass1 | |
413 | chmod u+x runCPass1.sh | |
414 | echo "${runpath}/runCPass1.sh ${infile} ${nEvents} ${runNumber} ${ocdbPath} ${recoTriggerOptions}" | |
415 | if [[ -n ${pretend} ]]; then | |
416 | touch AliESDs_Barrel.root | |
417 | touch AliESDfriends_Barrel.root | |
418 | touch AliESDfriends_v1.root | |
419 | touch QAresults_barrel.root | |
420 | touch EventStat_temp_barrel.root | |
421 | touch AODtpITS.root | |
422 | touch AliESDs_Outer.root | |
423 | touch AliESDfriends_Outer.root | |
424 | touch QAresults_outer.root | |
425 | touch EventStat_temp_outer.root | |
426 | touch rec.log | |
427 | touch calib.log | |
428 | touch qa.log | |
429 | touch filtering.log FilterEvents_Trees.root | |
430 | else | |
431 | ./runCPass1.sh "${infile}" "${nEvents}" "${runNumber}" "${ocdbPath}" "${recoTriggerOptions}" | |
432 | ||
433 | [[ ! -f AliESDs_Barrel.root && -f Barrel/AliESDs.root ]] && mv Barrel/AliESDs.root AliESDs_Barrel.root | |
434 | [[ ! -f AliESDfriends_Barrel.root && -f Barrel/AliESDfriends.root ]] && mv Barrel/AliESDfriends.root AliESDfriends_Barrel.root | |
435 | [[ ! -f AliESDfriends_v1.root && -f Barrel/AliESDfriends_v1.root ]] && mv Barrel/AliESDfriends_v1.root . | |
436 | [[ ! -f QAresults_barrel.root && -f Barrel/QAresults_barrel.root ]] && mv Barrel/QAresults_barrel.root . | |
437 | [[ ! -f AliESDs_Outer.root && -f OuterDet/AliESDs.root ]] && mv OuterDet/AliESDs.root AliESDs_Outer.root | |
438 | [[ ! -f AliESDfriends_Outer.root && -f OuterDet/AliESDfriends.root ]] && mv OuterDet/AliESDfriends.root AliESDfriends_Outer.root | |
439 | [[ ! -f QAresults_outer.root && -f OuterDet/QAresults_outer.root ]] && mv OuterDet/QAresults_outer.root . | |
440 | [[ ! -f FilterEvents_Trees.root && -f Barrel/FilterEvents_Trees.root ]] && mv Barrel/FilterEvents_Trees.root . | |
441 | ||
442 | #make the filtered tree (if requested and not already produced by QA | |
443 | [[ -f AliESDs_Barrel.root ]] && echo "AliESDs_Barrel.root" > filtered.list | |
444 | if [[ -n ${runESDfiltering} && ! -f FilterEvents_Trees.root ]]; then | |
445 | goMakeFilteredTrees ${PWD} ${runNumber} "filtered.list" ${filteringFactorHighPt} ${filteringFactorV0s} ${ocdbPath} 1000000 0 10000000 0 ${configFile} AliESDs_Barrel.root "${extraOpts[@]}" | |
446 | fi | |
447 | ||
448 | fi | |
449 | ||
450 | ##handle possible crashes in QA (happens often in trunk) | |
451 | ##rerun QA with a trusted aliroot version | |
452 | #if [[ $(validateLog qa_barrel.log) ]]; then | |
453 | # echo "qa_barrel.log not validated!" | |
454 | #fi | |
455 | #if [[ ! -f QAresults_barrel.root && -f ${setupTrustedAliROOTenvInCurrentShell} || $(validateLog qa_barrel.log) ]]; then | |
456 | # echo "WARNING: using trusted QA aliroot ${ALICE_ROOT}" | |
457 | # source ${setupTrustedAliROOTenvInCurrentShell} | |
458 | # cd Barrel | |
459 | # rm QAresults_barrel.root | |
460 | # rm EventStat_temp_barrel.root | |
461 | # rm AODtpITS.root | |
462 | # [[ ! -f AliESDs.root ]] && ln -s ../AliESDs_Barrel.root AliESDs.root | |
463 | # [[ ! -f AliESDfriends.root ]] && ln -s ../AliESDfriends_Barrel.root AliESDfriends.root | |
464 | # if [[ -n ${trustedQAtrainMacro} ]]; then | |
465 | # eval "cp ${trustedQAtrainMacro} QAtrain_duo_trusted.C" | |
466 | # fi | |
467 | # echo executing aliroot -b -q "QAtrain_duo_trusted.C(\"_barrel\",${runNumber},\"wn.xml\",0,\"${ocdbPath}\")" | |
468 | # time aliroot -b -q "QAtrain_duo.C(\"_barrel\",${runNumber},\"wn.xml\",0,\"${ocdbPath}\")" &> ../qa_barrel_trusted.log | |
469 | # cd ../ | |
470 | #fi | |
471 | ||
472 | #move stuff to final destination | |
473 | echo "this directory (${PWD}) contents:" | |
474 | /bin/ls | |
475 | echo rm -f ./${chunkName} | |
476 | rm -f ./${chunkName} | |
477 | echo "cp --recursive ${runpath}/* ${outputDir}" | |
478 | cp -pf --recursive ${runpath}/* ${outputDir} | |
479 | echo | |
480 | ||
481 | #validate CPass1 | |
482 | cd ${outputDir} | |
483 | touch ${doneFile} | |
484 | echo "dir ${outputDir}" >> ${doneFile} | |
485 | if summarizeLogs >> ${doneFile}; then | |
486 | [[ -f AliESDs_Barrel.root ]] && echo "esd ${outputDir}/AliESDs_Barrel.root" >> ${doneFile} | |
487 | [[ -f AliESDfriends_v1.root ]] && echo "calibfile ${outputDir}/AliESDfriends_v1.root" >> ${doneFile} | |
488 | [[ -f QAresults_Barrel.root ]] && echo "qafile ${outputDir}/QAresults_Barrel.root" >> ${doneFile} | |
489 | [[ -f QAresults_Outer.root ]] && echo "qafile ${outputDir}/QAresults_Outer.root" >> ${doneFile} | |
490 | [[ -f FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFile} | |
491 | fi | |
492 | ||
493 | [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath} | |
494 | return 0 | |
495 | } | |
496 | ||
497 | ||
498 | goMergeCPass0() | |
499 | { | |
500 | # | |
501 | # find the output files and merge them | |
502 | # | |
503 | ||
504 | outputDir=${1} | |
505 | ocdbStorage=${2} | |
506 | configFile=${3} | |
507 | runNumber=${4} | |
508 | calibrationFilesToMerge=${5} #can be a non-existent file, will then be produced on the fly | |
509 | shift 5 | |
510 | if ! parseConfig ${configFile} "$@"; then return 1; fi | |
511 | ||
512 | [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD} | |
513 | doneFile="${commonOutputPath}/meta/merge.cpass0.run${runNumber}.done" | |
514 | ||
515 | umask 0002 | |
516 | ulimit -c unlimited | |
517 | ||
518 | [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource} | |
519 | ||
520 | #runpath=${PWD}/rundir_cpass0_Merge_${runNumber} | |
521 | runpath=${outputDir} | |
522 | [[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR} | |
523 | [[ ${reconstructInTemporaryDir} -eq 1 && -z ${TMPDIR} ]] && runpath=$(mktemp -d) | |
524 | ||
525 | mkdir -p ${runpath} | |
526 | [[ ! -d ${runpath} ]] && echo "not able to make the runpath ${runpath}" && touch ${doneFile} && return 1 | |
527 | cd ${runpath} | |
528 | ||
529 | logOutputDir=${runpath} | |
530 | [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir} | |
531 | [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/mergeMakeOCDB.log | |
532 | echo "${0} $*" | |
533 | ||
534 | mergingScript="mergeMakeOCDB.byComponent.sh" | |
535 | ||
536 | echo goMergeCPass0 SETUP: | |
537 | echo runNumber=${runNumber} | |
538 | echo outputDir=${outputDir} | |
539 | echo ocdbStorage=${ocdbStorage} | |
540 | echo calibrationFilesToMerge=${calibrationFilesToMerge} | |
541 | echo mergingScript=${mergingScript} | |
542 | echo commonOutputPath=${commonOutputPath} | |
543 | echo runpath=${runpath} | |
544 | ||
545 | # copy files in case they are not already there | |
546 | filesMergeCPass0=( | |
547 | "${commonOutputPath}/${calibrationFilesToMerge}" | |
548 | "${commonOutputPath}/OCDB.root" | |
549 | "${commonOutputPath}/localOCDBaccessConfig.C" | |
550 | "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/mergeMakeOCDB.byComponent.sh" | |
551 | "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/mergeByComponent.C" | |
552 | "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/makeOCDB.C" | |
553 | "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/merge.C" | |
554 | "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/mergeMakeOCDB.sh" | |
555 | ) | |
556 | for file in ${filesMergeCPass0[*]}; do | |
557 | [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} . | |
558 | done | |
559 | ||
560 | sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh | |
561 | ||
562 | alirootInfo > ALICE_ROOT.log | |
563 | ||
564 | # | |
565 | echo "PWD" | |
566 | /bin/ls | |
567 | echo "PWD/.." | |
568 | /bin/ls ../ | |
569 | ||
570 | ||
571 | #merge calibration | |
572 | chmod u+x ${mergingScript} | |
573 | mkdir -p ./OCDB | |
574 | if [[ ! -f ${calibrationFilesToMerge} ]]; then | |
575 | echo "/bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root > ${calibrationFilesToMerge}" | |
576 | /bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root 2>/dev/null > ${calibrationFilesToMerge} | |
577 | fi | |
578 | ||
579 | echo "${mergingScript} ${calibrationFilesToMerge} ${runNumber} local://./OCDB ${ocdbStorage}" | |
580 | if [[ -n ${pretend} ]]; then | |
581 | touch CalibObjects.root | |
582 | touch ocdb.log | |
583 | touch merge.log | |
584 | mkdir -p ./OCDB/TPC/Calib/TimeGain/ | |
585 | mkdir -p ./OCDB/TPC/Calib/TimeDrift/ | |
586 | touch ./OCDB/TPC/Calib/TimeGain/someCalibObject_0-999999_cpass0.root | |
587 | touch ./OCDB/TPC/Calib/TimeDrift/otherCalibObject_0-999999_cpass0.root | |
588 | else | |
589 | ./${mergingScript} ${calibrationFilesToMerge} ${runNumber} "local://./OCDB" ${ocdbStorage} | |
590 | fi | |
591 | ||
592 | ### produce the output | |
593 | #tar the produced OCDB for reuse | |
594 | tar czf ${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz ./OCDB | |
595 | ||
596 | #produce the calib trees for expert QA (dcsTime.root) | |
597 | goMakeLocalOCDBaccessConfig ./OCDB | |
598 | echo aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")" | |
599 | aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")" | |
600 | ||
601 | /bin/ls | |
602 | ||
603 | #copy all to output dir | |
604 | cp -pf --recursive ${runpath}/* ${outputDir} | |
605 | ||
606 | if [[ -n ${generateMC} ]]; then | |
607 | goPrintValues sim ${commonOutputPath}/meta/sim.run${runNumber}.list ${commonOutputPath}/meta/cpass0.job*.run${runNumber}.done | |
608 | fi | |
609 | ||
610 | #validate merging cpass0 | |
611 | cd ${outputDir} | |
612 | touch ${doneFile} | |
613 | echo "dir ${outputDir}" >> ${doneFile} | |
614 | if summarizeLogs >> ${doneFile}; then | |
615 | [[ -f CalibObjects.root ]] && echo "calibfile ${outputDir}/CalibObjects.root" >> ${doneFile} | |
616 | [[ -f dcsTime.root ]] && echo "dcsTree ${outputDir}/dcsTime.root" >> ${doneFile} | |
617 | fi | |
618 | ||
619 | [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath} | |
620 | return 0 | |
621 | } | |
622 | ||
623 | goMergeCPass1() | |
624 | { | |
625 | # | |
626 | # find the output files and merge them | |
627 | # | |
628 | ||
629 | outputDir=${1} | |
630 | ocdbStorage=${2} | |
631 | configFile=${3} | |
632 | runNumber=${4} | |
633 | calibrationFilesToMerge=${5} | |
634 | qaFilesToMerge=${6} | |
635 | filteredFilesToMerge=${7} | |
636 | shift 7 | |
637 | if ! parseConfig ${configFile} "$@"; then return 1; fi | |
638 | ||
639 | [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD} | |
640 | doneFile="${commonOutputPath}/meta/merge.cpass1.run${runNumber}.done" | |
641 | ||
642 | umask 0002 | |
643 | ulimit -c unlimited | |
644 | ||
645 | #clean up first: | |
646 | rm -f ${outputDir}/*.log | |
647 | rm -f ${outputDir}/*.root | |
648 | rm -f ${outputDir}/*done | |
649 | ||
650 | [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource} | |
651 | ||
652 | #runpath=${PWD}/rundir_cpass1_Merge_${runNumber} | |
653 | runpath=${outputDir} | |
654 | [[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR} | |
655 | [[ ${reconstructInTemporaryDir} -eq 1 && -z ${TMPDIR} ]] && runpath=$(mktemp -d) | |
656 | ||
657 | mkdir -p ${runpath} | |
658 | [[ ! -d ${runpath} ]] && echo "not able to make the runpath ${runpath}" && touch ${doneFile} && return 1 | |
659 | cd ${runpath} | |
660 | ||
661 | logOutputDir=${runpath} | |
662 | [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir} | |
663 | [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/mergeMakeOCDB.log | |
664 | echo "${0} $*" | |
665 | ||
666 | calibrationOutputFileName='AliESDfriends_v1.root' | |
667 | qaOutputFileName='QAresults*.root' | |
668 | mergingScript="mergeMakeOCDB.byComponent.sh" | |
669 | #important to have the string "Stage.txt" in the filename to trigger the merging | |
670 | #it has to be a list of directories containing the files | |
671 | qaMergedOutputFileName="QAresults_merged.root" | |
672 | ||
673 | echo goMergeCPass1 SETUP: | |
674 | echo runNumber=${runNumber} | |
675 | echo outputDir=${outputDir} | |
676 | echo ocdbStorage=${ocdbStorage} | |
677 | echo calibrationFilesToMerge=$calibrationFilesToMerge | |
678 | echo qaFilesToMerge=$qaFilesToMerge | |
679 | echo calibrationOutputFileName=${calibrationOutputFileName} | |
680 | echo mergingScript=${mergingScript} | |
681 | ||
682 | # copy files in case they are not already there | |
683 | filesMergeCPass1=( | |
684 | "${commonOutputPath}/${calibrationFilesToMerge}" | |
685 | "${commonOutputPath}/${qaFilesToMerge}" | |
686 | "${commonOutputPath}/OCDB.root" | |
687 | "${commonOutputPath}/localOCDBaccessConfig.C" | |
688 | "${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz" | |
689 | "${commonOutputPath}/QAtrain_duo.C" | |
690 | "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/mergeMakeOCDB.byComponent.sh" | |
691 | "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/mergeByComponent.C" | |
692 | "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/makeOCDB.C" | |
693 | "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/merge.C" | |
694 | "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/mergeMakeOCDB.sh" | |
695 | "${trustedQAtrainMacro}" | |
696 | "${ALICE_ROOT}/ANALYSIS/macros/QAtrain_duo.C" | |
697 | ) | |
698 | for file in ${filesMergeCPass1[*]}; do | |
699 | [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} . | |
700 | done | |
701 | ||
702 | sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh | |
703 | ||
704 | #configure local OCDB storage from CPass0 (creates the localOCDBaccessConfig.C script) | |
705 | if [[ -f cpass0.localOCDB.${runNumber}.tgz ]]; then | |
706 | echo goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz" | |
707 | goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz" | |
708 | else | |
709 | echo "WARNING: file cpass0.localOCDB.${runNumber}.tgz not found!" | |
710 | fi | |
711 | ||
712 | alirootInfo > ALICE_ROOT.log | |
713 | ||
714 | # | |
715 | /bin/ls | |
716 | ||
717 | #merge calibration | |
718 | chmod u+x ${mergingScript} | |
719 | mkdir -p OCDB | |
720 | ||
721 | #if not provided, create the lists of files to merge | |
722 | if [[ ! -f ${filteredFilesToMerge} ]]; then | |
723 | echo "/bin/ls -1 ${outputDir}/*/FilterEvents_Trees.root > ${filteredFilesToMerge}" | |
724 | /bin/ls -1 ${outputDir}/*/FilterEvents_Trees.root 2>/dev/null > ${filteredFilesToMerge} | |
725 | fi | |
726 | if [[ ! -f ${calibrationFilesToMerge} ]]; then | |
727 | echo "/bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root > ${calibrationFilesToMerge}" | |
728 | /bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root 2>/dev/null > ${calibrationFilesToMerge} | |
729 | fi | |
730 | if [[ ! -f ${qaFilesToMerge} ]]; then | |
731 | #find the files, but only store the directories (QAtrain_duo.C requires this) | |
732 | echo "/bin/ls -1 ${outputDir}/*/QAresults*.root | while read x; do echo ${x%/*}; done | sort | uniq > ${qaFilesToMerge}" | |
733 | /bin/ls -1 ${outputDir}/*/QAresults*.root | while read x; do echo ${x%/*}; done | sort | uniq > ${qaFilesToMerge} | |
734 | fi | |
735 | ||
736 | echo "${mergingScript} ${calibrationFilesToMerge} ${runNumber} local://./OCDB ${ocdbStorage}" | |
737 | if [[ -n ${pretend} ]]; then | |
738 | touch CalibObjects.root | |
739 | touch ocdb.log | |
740 | touch merge.log | |
741 | touch cpass1.localOCDB.${runNumber}.tgz | |
742 | else | |
743 | ./${mergingScript} ${calibrationFilesToMerge} ${runNumber} "local://./OCDB" ${ocdbStorage} | |
744 | fi | |
745 | ||
746 | tar czf ${commonOutputPath}/meta/cpass1.localOCDB.${runNumber}.tgz ./OCDB | |
747 | ||
748 | #merge QA (and filtered trees) | |
749 | [[ -n ${AliAnalysisTaskFilteredTree_fLowPtTrackDownscaligF} ]] && export AliAnalysisTaskFilteredTree_fLowPtTrackDownscaligF | |
750 | [[ -n ${AliAnalysisTaskFilteredTree_fLowPtV0DownscaligF} ]] && export AliAnalysisTaskFilteredTree_fLowPtV0DownscaligF | |
751 | ||
752 | #echo aliroot -l -b -q "merge.C(\"${qaFilesToMerge}\",\"\",kFALSE,\"${qaMergedOutputFileName}\")" | |
753 | echo aliroot -b -q "QAtrain_duo.C(\"_barrel\",${runNumber},\"${qaFilesToMerge}\",1,\"${ocdbStorage}\")" | |
754 | if [[ -n ${pretend} ]]; then | |
755 | touch ${qaMergedOutputFileName} | |
756 | touch merge.log | |
757 | touch trending.root | |
758 | touch FilterEvents_Trees.root | |
759 | touch CalibObjects.root | |
760 | touch dcsTime.root | |
761 | touch ${qaMergedOutputFileName} | |
762 | else | |
763 | #aliroot -l -b -q "merge.C(\"${qaFilesToMerge}\",\"\",kFALSE,\"${qaMergedOutputFileName}\")" | |
764 | aliroot -b -q "QAtrain_duo.C(\"_barrel\",${runNumber},\"${qaFilesToMerge}\",1,\"${ocdbStorage}\")" > mergeQA.log | |
765 | mv QAresults_barrel.root ${qaMergedOutputFileName} | |
766 | mv trending_barrel.root trending.root | |
767 | ||
768 | #merge filtered trees | |
769 | echo aliroot -l -b -q "merge.C(\"${qaFilesToMerge}\",\"\",kFALSE,\"${qaMergedOutputFileName}\")" | |
770 | aliroot -l -b -q "merge.C(\"${filteredFilesToMerge}\",\"\",kFALSE,\"FilterEvents_Trees.root\")" | |
771 | fi | |
772 | ||
773 | #produce the calib trees for expert QA | |
774 | echo aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")" | |
775 | aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")" | |
776 | ||
777 | /bin/ls | |
778 | ||
779 | #copy all to output dir | |
780 | cp -pf --recursive ${runpath}/* ${outputDir} | |
781 | ||
782 | #validate merge cpass1 | |
783 | cd ${outputDir} | |
784 | touch ${doneFile} | |
785 | echo "dir ${outputDir}" >> ${doneFile} | |
786 | if summarizeLogs >> ${doneFile}; then | |
787 | [[ -f CalibObjects.root ]] && echo "calibfile ${outputDir}/CalibObjects.root" >> ${doneFile} | |
788 | [[ -f ${qaMergedOutputFileName} ]] && echo "qafile ${outputDir}/${qaMergedOutputFileName}" >> ${doneFile} | |
789 | [[ -f trending.root ]] && echo "trendingfile ${outputDir}/trending.root" >> ${doneFile} | |
790 | [[ -f dcsTime.root ]] && echo "dcsTree ${outputDir}/dcsTime.root" >> ${doneFile} | |
791 | [[ -f FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFile} | |
792 | fi | |
793 | ||
794 | [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath} | |
795 | return 0 | |
796 | } | |
797 | ||
798 | goMerge() | |
799 | { | |
800 | #generic root merge using CPass1 merge.C script | |
801 | inputList=${1} | |
802 | outputFile=${2} | |
803 | configFile=${3-"becnhmark.config"} | |
804 | shift 3 | |
805 | if ! parseConfig ${configFile} "$@"; then return 1; fi | |
806 | ||
807 | [[ ! -f ${inputList} ]] && echo "inputList ${inputList} does not exist!" && return 1 | |
808 | [[ ! -f ${configFile} ]] && echo "configFile ${configFile} does not exist!" && return 1 | |
809 | umask 0002 | |
810 | [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource} | |
811 | rm -f ${outputFile} | |
812 | aliroot -b -q "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/merge.C(\"${inputList}\",\"\",kFALSE,\"${outputFile}\")" > merge_${inputList}.log | |
813 | return 0 | |
814 | } | |
815 | ||
816 | goSubmitMakeflow() | |
817 | { | |
818 | #run | |
819 | productionID=${1} | |
820 | inputFileList=${2} | |
821 | configFile=${3} | |
822 | shift 3 | |
823 | extraOpts=("$@") | |
824 | if ! parseConfig ${configFile} "${extraOpts[@]}"; then return 1; fi | |
825 | ||
826 | [[ -z ${configFile} ]] && configFile="benchmark.config" | |
827 | [[ ! -f ${configFile} ]] && echo "no config file found (${configFile})" && return 1 | |
828 | ||
829 | #create the directopry for the metadata | |
830 | mkdir meta | |
831 | ||
832 | if [[ ! $(which makeflow &>/dev/null) && -n ${makeflowPath} ]]; then | |
833 | echo "setting the makflow path from the config: " | |
834 | echo " export PATH=${makeflowPath}:${PATH}" | |
835 | export PATH=${makeflowPath}:${PATH} | |
836 | fi | |
837 | ||
838 | #submit - use makeflow if available, fall back to old stuff when makeflow not there | |
839 | if which makeflow; then | |
840 | ||
841 | goGenerateMakeflow ${productionID} ${inputFileList} ${configFile} "${extraOpts[@]}" > benchmark.makeflow | |
842 | ||
843 | makeflow ${makeflowOptions} benchmark.makeflow | |
844 | cd ../ | |
845 | else | |
846 | echo "no makeflow!" | |
847 | fi | |
848 | return 0 | |
849 | } | |
850 | ||
851 | goGenerateMakeflow() | |
852 | { | |
853 | #generate the makeflow file | |
854 | [[ $# -lt 3 ]] && echo "args: id inputFileList configFile" && return 1 | |
855 | productionID=${1} | |
856 | inputFileList=${2} | |
857 | configFile=${3} | |
858 | shift 3 | |
859 | extraOpts=("$@") | |
860 | if ! parseConfig ${configFile} "${extraOpts[@]}"; then return 1; fi | |
861 | ||
862 | [[ -z ${configFile} ]] && configFile="benchmark.config" | |
863 | [[ ! -f ${configFile} ]] && echo "no config file found (${configFile})" && return 1 | |
864 | ||
865 | commonOutputPath=${baseOutputDirectory}/${productionID} | |
866 | ||
867 | self=$(readlink -f "${0}") | |
868 | echo "self: $self" | |
869 | #these files will be made a dependency - will be copied to the working dir of the jobs | |
870 | declare -a copyFiles | |
871 | inputFiles=( | |
872 | "OCDB.root" | |
873 | "localOCDBaccessConfig.C" | |
874 | "QAtrain_duo.C" | |
875 | "runCPass1.sh" | |
876 | "recCPass1.C" | |
877 | "recCPass1_OuterDet.C" | |
878 | "runCalibTrain.C" | |
879 | "runCPass0.sh" | |
880 | "recCPass0.C" | |
881 | ) | |
882 | for file in ${inputFiles[*]}; do | |
883 | [[ -f ${file} ]] && copyFiles+=("${file}") | |
884 | done | |
885 | ||
886 | #create the makeflow file | |
887 | [[ -n ${batchFlags} ]] && echo "BATCH_OPTIONS = ${batchFlags}" | |
888 | declare -A arr_cpass0_merged arr_cpass1_merged | |
889 | declare -A arr_cpass0_calib_list arr_cpass1_calib_list | |
890 | declare -A arr_cpass1_QA_list arr_cpass1_ESD_list arr_cpass1_filtered_list | |
891 | declare -A arr_cpass0_profiled_outputs | |
892 | declare -A listOfRuns | |
893 | [[ -n ${runNumber} ]] && listOfRuns[${runNumber}]=1 | |
894 | while read x; do listOfRuns[$(guessRunNumber ${x})]=1; done < ${inputFileList} | |
895 | for runNumber in "${!listOfRuns[@]}"; do | |
896 | [[ -z ${runNumber} ]] && continue | |
897 | [[ ! ${runNumber} =~ ^[0-9]*[0-9]$ ]] && continue | |
898 | ||
899 | unset arr_cpass0_outputs | |
900 | unset arr_cpass1_outputs | |
901 | declare -a arr_cpass0_outputs | |
902 | declare -a arr_cpass1_outputs | |
903 | ||
904 | jobindex=0 | |
905 | inputFile="" | |
906 | while read inputFile; do | |
907 | currentDefaultOCDB=${defaultOCDB} | |
908 | [[ -z ${autoOCDB} ]] && autoOCDB=1 | |
909 | if [[ ${autoOCDB} -ne 0 ]]; then | |
910 | currentDefaultOCDB=$(setYear ${inputFile} ${defaultOCDB}) | |
911 | fi | |
912 | ||
913 | #CPass0 | |
914 | arr_cpass0_outputs[${jobindex}]="meta/cpass0.job${jobindex}.run${runNumber}.done" | |
915 | echo "${arr_cpass0_outputs[${jobindex}]} : benchmark.sh ${configFile} ${copyFiles[@]}" | |
916 | echo -n " ${alirootEnv} ./benchmark.sh CPass0 ${commonOutputPath}/000${runNumber}/cpass0 ${inputFile} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} ${jobindex}"" " | |
917 | for extraOption in "${extraOpts[@]}"; do echo -n \"${extraOption}\"" "; done; echo | |
918 | echo | |
919 | ||
920 | #CPass1 | |
921 | arr_cpass1_outputs[${jobindex}]="meta/cpass1.job${jobindex}.run${runNumber}.done" | |
922 | echo "${arr_cpass1_outputs[${jobindex}]} : benchmark.sh ${configFile} meta/cpass0.localOCDB.${runNumber}.tgz ${copyFiles[@]}" | |
923 | echo -n " ${alirootEnv} ./benchmark.sh CPass1 ${commonOutputPath}/000${runNumber}/cpass1 ${inputFile} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} ${jobindex}"" " | |
924 | for extraOption in "${extraOpts[@]}"; do echo -n \"${extraOption}\"" "; done; echo | |
925 | echo | |
926 | ((jobindex++)) | |
927 | ||
928 | done< <(grep "/000${runNumber}/" ${inputFileList}) | |
929 | ||
930 | #CPass0 list of Calib files to merge | |
931 | arr_cpass0_calib_list[${runNumber}]="meta/cpass0.calib.run${runNumber}.list" | |
932 | echo "${arr_cpass0_calib_list[${runNumber}]} : benchmark.sh ${arr_cpass0_outputs[*]}" | |
933 | echo " ./benchmark.sh PrintValues calibfile ${arr_cpass0_calib_list[${runNumber}]} ${arr_cpass0_outputs[*]}" | |
934 | echo | |
935 | ||
936 | #CPass0 merging | |
937 | arr_cpass0_merged[${runNumber}]="meta/merge.cpass0.run${runNumber}.done" | |
938 | echo "meta/cpass0.localOCDB.${runNumber}.tgz ${arr_cpass0_merged[${runNumber}]} : benchmark.sh ${configFile} ${arr_cpass0_calib_list[${runNumber}]} ${copyFiles[@]}" | |
939 | echo -n " ${alirootEnv} ./benchmark.sh MergeCPass0 ${commonOutputPath}/000${runNumber}/cpass0 ${currentDefaultOCDB} ${configFile} ${runNumber} ${arr_cpass0_calib_list[${runNumber}]}"" " | |
940 | for extraOption in "${extraOpts[@]}"; do echo -n \"${extraOption}\"" "; done; echo | |
941 | echo | |
942 | ||
943 | #CPass1 list of Calib/QA/ESD/filtered files | |
944 | # the trick with QA is to have the string "Stage.txt" in the file name of the list of directories with QA output to trigger | |
945 | # the production of the QA trending tree (only then the task->Finish() will be called in QAtrain_duo.C, on the grid | |
946 | # this corresponds to the last merging stage) | |
947 | arr_cpass1_QA_list[${runNumber}]="meta/cpass1.QA.run${runNumber}.lastMergingStage.txt.list" | |
948 | echo "${arr_cpass1_QA_list[${runNumber}]}: benchmark.sh ${arr_cpass1_outputs[*]}" | |
949 | echo " ./benchmark.sh PrintValues dir ${arr_cpass1_QA_list[${runNumber}]} ${arr_cpass1_outputs[*]}" | |
950 | echo | |
951 | arr_cpass1_calib_list[${runNumber}]="meta/cpass1.calib.run${runNumber}.list" | |
952 | echo "${arr_cpass1_calib_list[${runNumber}]} : benchmark.sh ${arr_cpass1_outputs[*]}" | |
953 | echo " ./benchmark.sh PrintValues calibfile ${arr_cpass1_calib_list[${runNumber}]} ${arr_cpass1_outputs[*]};" | |
954 | echo | |
955 | arr_cpass1_ESD_list[${runNumber}]="meta/cpass1.ESD.run${runNumber}.list" | |
956 | echo "${arr_cpass1_ESD_list[${runNumber}]} : benchmark.sh ${arr_cpass1_outputs[*]}" | |
957 | echo " ./benchmark.sh PrintValues esd ${arr_cpass1_ESD_list[${runNumber}]} ${arr_cpass1_outputs[*]}" | |
958 | echo | |
959 | arr_cpass1_filtered_list[${runNumber}]="meta/cpass1.filtered.run${runNumber}.list" | |
960 | echo "${arr_cpass1_filtered_list[${runNumber}]} : benchmark.sh ${arr_cpass1_outputs[*]}" | |
961 | echo " ./benchmark.sh PrintValues filteredTree ${arr_cpass1_filtered_list[${runNumber}]} ${arr_cpass1_outputs[*]}" | |
962 | echo | |
963 | ||
964 | #CPass1 merging | |
965 | arr_cpass1_merged[${runNumber}]="meta/merge.cpass1.run${runNumber}.done" | |
966 | echo "meta/cpass1.localOCDB.${runNumber}.tgz ${arr_cpass1_merged[${runNumber}]} : benchmark.sh ${configFile} ${arr_cpass1_calib_list[${runNumber}]} ${arr_cpass1_QA_list[${runNumber}]} ${copyFiles[@]}" | |
967 | echo -n " ${alirootEnv} ./benchmark.sh MergeCPass1 ${commonOutputPath}/000${runNumber}/cpass1 ${currentDefaultOCDB} ${configFile} ${runNumber} ${arr_cpass1_calib_list[${runNumber}]} ${arr_cpass1_QA_list[${runNumber}]} ${arr_cpass1_filtered_list[${runNumber}]}"" " | |
968 | for extraOption in "${extraOpts[@]}"; do echo -n \"${extraOption}\"" "; done; echo | |
969 | echo | |
970 | ||
971 | #CPass0 wrapped in a profiling tool (valgrind,....) | |
972 | if [[ -n profilingCommand ]]; then | |
973 | arr_cpass0_profiled_outputs[${runNumber}]="meta/cpass0.jobProfiled.run${runNumber}.done" | |
974 | echo "${arr_cpass0_profiled_outputs[${runNumber}]} : benchmark.sh ${configFile} ${copyFiles[@]}" | |
975 | echo -n " ${alirootEnv} ./benchmark.sh CPass0 ${commonOutputPath}/000${runNumber}/profiled ${inputFile} ${nEventsProfiling} ${currentDefaultOCDB} ${configFile} ${runNumber} profiling"" " | |
976 | for extraOption in "${extraOpts[@]}"; do echo -n \"${extraOption}\"" "; done; | |
977 | echo "\"useProfilingCommand=${profilingCommand}\"" | |
978 | echo | |
979 | fi | |
980 | ||
981 | done #runs | |
982 | ||
983 | #Summary | |
984 | echo "summary.log : benchmark.sh ${configFile} ${arr_cpass1_merged[*]}" | |
985 | echo " LOCAL ./benchmark.sh MakeSummary ${configFile}" | |
986 | echo | |
987 | ||
988 | return 0 | |
989 | } | |
990 | ||
991 | goPrintValues() | |
992 | { | |
993 | #print the values given the key from any number of files (store in output file on second argument) | |
994 | if [[ $# -lt 3 ]]; then | |
995 | echo "goPrintValues key outputFile inputFiles" | |
996 | echo "if outputFile is \"-\" dont redirect to a file" | |
997 | return | |
998 | fi | |
999 | key=${1} | |
1000 | outputFile=${2} | |
1001 | [[ ${outputFile} =~ "-" ]] && outputFile="" | |
1002 | shift 2 #remove 2 first arguments from arg list to only pass the input files to awk | |
1003 | awk -v key=${key} '$0 ~ key" " {print $2}' "$@" | tee ${outputFile} | |
1004 | return 0 | |
1005 | } | |
1006 | ||
1007 | goCreateQAplots() | |
1008 | { | |
1009 | umask 0002 | |
1010 | mergedQAfileList=${1} | |
1011 | productionID=${2} | |
1012 | outputDir=${3} | |
1013 | configFile=${4} | |
1014 | shift 4 | |
1015 | if ! parseConfig ${configFile} ${@}; then return 1; fi | |
1016 | ||
1017 | [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource} | |
1018 | ||
1019 | [[ -z ${logOutputDir} ]] && logOutputDir=${PWD} | |
1020 | [[ -z ${dontRedirectStdOutToLog} ]] && exec 2>&1 > ${logOutputDir}/makeQAplots.log | |
1021 | echo "${0} $*" | |
1022 | ||
1023 | olddir=${PWD} | |
1024 | mkdir -p ${outputDir} | |
1025 | cd ${outputDir} | |
1026 | [[ ! "${PWD}" = "${outputDir}" ]] && echo "cannot make ${outputDir}... exiting" && return 1 | |
1027 | ||
1028 | echo ${ALICE_ROOT}/PWGPP/QA/scripts/runQA.sh inputList=${mergedQAfileList} | |
1029 | ${ALICE_ROOT}/PWGPP/QA/scripts/runQA.sh inputList="${mergedQAfileList}" inputListHighPtTrees="${filteringList}" | |
1030 | ||
1031 | cd ${olddir} | |
1032 | return 0 | |
1033 | } | |
1034 | ||
1035 | goTest() | |
1036 | { | |
1037 | umask 0002 | |
1038 | exec 2>&1 | |
1039 | exec > >(tee test.log) | |
1040 | echo "$@" | |
1041 | echo something | |
1042 | return 0 | |
1043 | } | |
1044 | ||
1045 | alirootInfo() | |
1046 | { | |
1047 | umask 0002 | |
1048 | # save aliroot repository info | |
1049 | [[ -z "${ALICE_ROOT}" ]] && return 1 | |
1050 | ||
1051 | echo "\${ALICE_ROOT}=${ALICE_ROOT}" | |
1052 | echo "\${ROOTSYS}=${ROOTSYS}" | |
1053 | echo "\${PATH}=${PATH}" | |
1054 | echo "\${LD_LIBRARY_PATH}=${LD_LIBRARY_PATH}" | |
1055 | ||
1056 | pushd ${PWD} | |
1057 | cd ${ALICE_ROOT} | |
1058 | echo | |
1059 | ||
1060 | currentBranch=$(git rev-parse --abbrev-ref HEAD) | |
1061 | git status | |
1062 | echo "" | |
1063 | echo "" | |
1064 | git diff ${currentBranch} | |
1065 | popd | |
1066 | return 0 | |
1067 | } | |
1068 | ||
1069 | setYear() | |
1070 | { | |
1071 | #set the year | |
1072 | # ${1} - year to be set | |
1073 | # ${2} - where to set the year | |
1074 | year1=$(guessYear ${1}) | |
1075 | year2=$(guessYear ${2}) | |
1076 | local path=${2} | |
1077 | [[ ${year1} -ne ${year2} && -n ${year2} && -n ${year1} ]] && path=${2/\/${year2}\//\/${year1}\/} | |
1078 | echo ${path} | |
1079 | return 0 | |
1080 | } | |
1081 | ||
1082 | guessPeriod() | |
1083 | { | |
1084 | #guess the period from the path, pick the rightmost one | |
1085 | local IFS="/" | |
1086 | declare -a path=( ${1} ) | |
1087 | local dirDepth=${#path[*]} | |
1088 | for ((x=${dirDepth}-1;x>=0;x--)); do | |
1089 | local field=${path[${x}]} | |
1090 | [[ ${field} =~ ^LHC[0-9][0-9][a-z]$ ]] && period=${field} && break | |
1091 | done | |
1092 | echo ${period} | |
1093 | return 0 | |
1094 | } | |
1095 | ||
1096 | guessYear() | |
1097 | { | |
1098 | #guess the year from the path, pick the rightmost one | |
1099 | local IFS="/" | |
1100 | declare -a path=( ${1} ) | |
1101 | local dirDepth=${#path[*]} | |
1102 | for ((x=${dirDepth}-1;x>=0;x--)); do | |
1103 | local field=${path[${x}]} | |
1104 | [[ ${field} =~ ^20[0-9][0-9]$ ]] && year=${field} && break | |
1105 | done | |
1106 | echo ${year} | |
1107 | return 0 | |
1108 | } | |
1109 | ||
1110 | guessRunNumber() | |
1111 | { | |
1112 | #guess the run number from the path, pick the rightmost one | |
1113 | #works for /path/foo/000123456/bar/... | |
1114 | #and /path/foo.run123456.bar | |
1115 | local IFS="/." | |
1116 | declare -a path=( ${1} ) | |
1117 | local dirDepth=${#path[*]} | |
1118 | for ((x=${dirDepth}-1;x>=0;x--)); do | |
1119 | local field=${path[${x}]} | |
1120 | field=${field/run/000} | |
1121 | [[ ${field} =~ ^000[0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && runNumber=${field#000} && break | |
1122 | done | |
1123 | echo ${runNumber} | |
1124 | return 0 | |
1125 | } | |
1126 | ||
1127 | validateLog() | |
1128 | { | |
1129 | log=${1} | |
1130 | errorConditions=( | |
1131 | 'There was a crash' | |
1132 | 'floating' | |
1133 | 'error while loading shared libraries' | |
1134 | 'std::bad_alloc' | |
1135 | 's_err_syswatch_' | |
1136 | 'Thread [0-9]* (Thread' | |
1137 | 'AliFatal' | |
1138 | 'core dumped' | |
1139 | '\.C.*error:.*\.h: No such file' | |
1140 | ) | |
1141 | ||
1142 | warningConditions=( | |
1143 | 'This is serious !' | |
1144 | 'rocVoltage out of range:' | |
1145 | ) | |
1146 | ||
1147 | local logstatus=0 | |
1148 | local errorSummary="" | |
1149 | local warningSummary="" | |
1150 | ||
1151 | for ((i=0; i<${#errorConditions[@]};i++)); do | |
1152 | local tmp=$(grep -m1 -e "${errorConditions[${i}]}" ${log}) | |
1153 | [[ -n ${tmp} ]] && tmp+=" : " | |
1154 | errorSummary+=${tmp} | |
1155 | done | |
1156 | ||
1157 | for ((i=0; i<${#warningConditions[@]};i++)); do | |
1158 | local tmp=$(grep -m1 -e "${warningConditions[${i}]}" ${log}) | |
1159 | [[ -n ${tmp} ]] && tmp+=" : " | |
1160 | warningSummary+=${tmp} | |
1161 | done | |
1162 | ||
1163 | if [[ -n ${errorSummary} ]]; then | |
1164 | echo "${errorSummary}" | |
1165 | return 1 | |
1166 | fi | |
1167 | ||
1168 | if [[ -n ${warningSummary} ]]; then | |
1169 | echo "${warningSummary}" | |
1170 | return 2 | |
1171 | fi | |
1172 | ||
1173 | return 0 | |
1174 | } | |
1175 | ||
1176 | summarizeLogs() | |
1177 | { | |
1178 | #print a summary of logs | |
1179 | logFiles=( | |
1180 | "*.log" | |
1181 | "stdout" | |
1182 | "stderr" | |
1183 | ) | |
1184 | ||
1185 | #check logs | |
1186 | local logstatus=0 | |
1187 | for log in ${logFiles[*]}; do | |
1188 | finallog=${PWD%/}/${log} | |
1189 | [[ ! -f ${log} ]] && continue | |
1190 | errorSummary=$(validateLog ${log}) | |
1191 | validationStatus=$? | |
1192 | if [[ ${validationStatus} -eq 0 ]]; then | |
1193 | #in pretend mode randomly report an error in rec.log some cases | |
1194 | if [[ -n ${pretend} && "${log}" == "rec.log" ]]; then | |
1195 | [[ $(( ${RANDOM}%2 )) -ge 1 ]] && echo "${finallog} BAD random error" || echo "${finallog} OK" | |
1196 | else | |
1197 | echo "${finallog} OK" | |
1198 | fi | |
1199 | elif [[ ${validationStatus} -eq 1 ]]; then | |
1200 | echo "${finallog} BAD ${errorSummary}" | |
1201 | logstatus=1 | |
1202 | elif [[ ${validationStatus} -eq 2 ]]; then | |
1203 | echo "${finallog} OK MWAH ${errorSummary}" | |
1204 | fi | |
1205 | done | |
1206 | ||
1207 | #report core files | |
1208 | while read x; do | |
1209 | echo ${x} | |
1210 | chmod 644 ${x} | |
1211 | gdb --batch --quiet -ex "bt" -ex "quit" aliroot ${x} > stacktrace_${x//\//_}.log | |
1212 | done < <(/bin/ls ${PWD}/*/core 2>/dev/null; /bin/ls ${PWD}/core 2>/dev/null) | |
1213 | ||
1214 | return ${logstatus} | |
1215 | } | |
1216 | ||
1217 | spitOutLocalOCDBaccessConfig() | |
1218 | { | |
1219 | umask 0002 | |
1220 | #find ${1} -name "*root" | \ | |
1221 | /bin/ls -1 ${1}/*/*/*/*.root 2>/dev/null | \ | |
1222 | while read line | |
1223 | do | |
1224 | local tmp=${line#${1}} | |
1225 | echo ${tmp%/*} | \ | |
1226 | awk -v ocdb=${1} '{print " man->SetSpecificStorage(\""$1"\",\"local://"ocdb"\");"}' | |
1227 | done | |
1228 | return 0 | |
1229 | } | |
1230 | ||
1231 | goMakeLocalOCDBaccessConfig() | |
1232 | { | |
1233 | umask 0002 | |
1234 | # make a script that sets the specific storages form all the root files produced by CPass0 | |
1235 | local localOCDBpathCPass0=${1} | |
1236 | local OCDBpathPrefix=${2} | |
1237 | [[ -z ${OCDBpathPrefix} ]] && OCDBpathPrefix="." | |
1238 | ||
1239 | if [[ -f ${localOCDBpathCPass0} && ${localOCDBpathCPass0} =~ \.tgz$ ]]; then | |
1240 | tar xzf ${localOCDBpathCPass0} | |
1241 | local localOCDBpathCPass0="${OCDBpathPrefix}/OCDB" | |
1242 | fi | |
1243 | ||
1244 | echo | |
1245 | echo creating the specific storage script | |
1246 | echo localOCDBaccessConfig.C | |
1247 | echo based on OCDB: ${localOCDBaccessConfig} | |
1248 | echo | |
1249 | ||
1250 | local tempLocalOCDB="" | |
1251 | if [[ -f localOCDBaccessConfig.C ]]; then | |
1252 | tempLocalOCDB=$(mktemp) | |
1253 | echo "egrep "SetSpecificStorage" localOCDBaccessConfig.C > ${tempLocalOCDB}" | |
1254 | egrep "SetSpecificStorage" localOCDBaccessConfig.C > ${tempLocalOCDB} | |
1255 | fi | |
1256 | ||
1257 | echo "localOCDBaccessConfig()" > localOCDBaccessConfig.C | |
1258 | echo "{" >> localOCDBaccessConfig.C | |
1259 | echo " AliCDBManager* man = AliCDBManager::Instance();" >> localOCDBaccessConfig.C | |
1260 | spitOutLocalOCDBaccessConfig ${localOCDBpathCPass0}|sort|uniq >> localOCDBaccessConfig.C | |
1261 | [[ -f "${tempLocalOCDB}" ]] && cat ${tempLocalOCDB} >> localOCDBaccessConfig.C | |
1262 | echo "}" >> localOCDBaccessConfig.C | |
1263 | ||
1264 | [[ -f "${tempLocalOCDB}" ]] && rm -f ${tempLocalOCDB} | |
1265 | ||
1266 | if ! grep SetSpecificStorage localOCDBaccessConfig.C; then | |
1267 | echo | |
1268 | echo "!!!!!!! CPass0 produced no OCDB entries" | |
1269 | return 1 | |
1270 | fi | |
1271 | return 0 | |
1272 | } | |
1273 | ||
1274 | goMakeFilteredTrees() | |
1275 | { | |
1276 | outputDir=${1} | |
1277 | runNumber=${2} | |
1278 | #get path to input list | |
1279 | inputListfiles=${3} | |
1280 | #get scale number for tracks | |
1281 | filterT=${4} | |
1282 | #get scale number for V0s | |
1283 | filterV=${5} | |
1284 | #get OCDB path (optional) | |
1285 | OCDBpath=${6} | |
1286 | #get max number of files | |
1287 | maxFiles=${7-"1000000"} | |
1288 | #get offset of first file | |
1289 | offsetFile=${8-"0"} | |
1290 | #get max number of events | |
1291 | maxEvents=${9-"30000000"} | |
1292 | #get offset of first event | |
1293 | offsetEvent=${10-"0"} | |
1294 | configFile=${11-"benchmark.config"} | |
1295 | esdFileName=${12-"AliESDs_Barrel.root"} | |
1296 | shift 12 | |
1297 | if ! parseConfig ${configFile} "$@"; then return 1; fi | |
1298 | ||
1299 | commonOutputPath=${PWD} | |
1300 | doneFile=${commonOutputPath}/meta/filtering.cpass1.run${runNumber}.done | |
1301 | ||
1302 | runpath=${outputDir} | |
1303 | [[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR} | |
1304 | [[ ${reconstructInTemporaryDir} -eq 1 && -z ${TMPDIR} ]] && runpath=$(mktemp -d) | |
1305 | ||
1306 | ||
1307 | [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource} | |
1308 | ||
1309 | mkdir -p ${outputDir} | |
1310 | mkdir -p ${runpath} | |
1311 | ||
1312 | cd ${runpath} | |
1313 | ||
1314 | cat > filtering.log << EOF | |
1315 | goMakeFilteredTrees config: | |
1316 | runpath=${runpath} | |
1317 | outputDir=${outputDir} | |
1318 | commonOutputPath=${commonOutputPath} | |
1319 | ALICE_ROOT=${ALICE_ROOT} | |
1320 | PATH=${PATH} | |
1321 | offsetEvent=$offsetEvent | |
1322 | configFile=$configFile | |
1323 | esdFileName=$esdFileName | |
1324 | EOF | |
1325 | ||
1326 | if [[ -z ${pretend} ]];then | |
1327 | aliroot -l -b -q "${ALICE_ROOT}/PWGPP/macros/runFilteringTask.C(\"${inputListfiles}\",${filterT},${filterV},\"${OCDBpath}\",${maxFiles},${offsetFile},${maxEvents},${offsetEvent},\"${esdFileName}\")" &>> filtering.log | |
1328 | else | |
1329 | touch filtering.log FilterEvents_Trees.root | |
1330 | fi | |
1331 | pwd | |
1332 | /bin/ls | |
1333 | touch ${doneFile} | |
1334 | summarizeLogs >> ${doneFile} | |
1335 | echo mv -f * ${outputDir} | |
1336 | mv -f * ${outputDir} | |
1337 | [[ -f ${outputDir}/FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFile} | |
1338 | cd ${commonOutputPath} | |
1339 | [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath} | |
1340 | return 0 | |
1341 | } | |
1342 | ||
1343 | submit() | |
1344 | { | |
1345 | umask 0002 | |
1346 | [[ $# -lt 5 ]] && echo "at least 5 args needed, you supplied $#" && return 1 | |
1347 | JobID=${1} | |
1348 | startID=${2} | |
1349 | endID=${3} | |
1350 | waitForJOBID=${4} | |
1351 | command=${5} | |
1352 | shift 5 | |
1353 | commandArgs=("$@") | |
1354 | ||
1355 | #add quote strings around the extra arguments | |
1356 | for ((i=0; i<${#commandArgs[@]}; i++)); do | |
1357 | commandArgs[i]=\"${commandArgs[i]}\" | |
1358 | done | |
1359 | ||
1360 | [[ -z ${waitForJOBID} ]] && waitForJOBID=0 | |
1361 | ||
1362 | newFarm=$(which qsub|grep "^/usr/bin/qsub") | |
1363 | ||
1364 | batchSystem="SGE" | |
1365 | ||
1366 | if [[ -z "${newFarm}" ]] | |
1367 | then | |
1368 | #old LSF | |
1369 | # submit it (as job array) | |
1370 | nFiles=$(( ${endID}-${startID}+1 )) | |
1371 | while [ ${startID} -le ${nFiles} ] ; do | |
1372 | if [ $(expr ${nFiles} - ${startID}) -gt 999 ] ; then | |
1373 | endID=$(expr ${startID} + 999) | |
1374 | else | |
1375 | endID=${nFiles} | |
1376 | fi | |
1377 | if [[ ${waitForJOBID} -eq 0 ]]; then | |
1378 | echo ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}" | |
1379 | ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}" | |
1380 | else | |
1381 | echo ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -w "ended(${waitForJOBID})" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}" | |
1382 | ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -w "ended(${waitForJOBID})" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}" | |
1383 | fi | |
1384 | startID=$(expr ${endID} + 1) | |
1385 | done | |
1386 | else | |
1387 | #new SGE farm | |
1388 | if [[ ${waitForJOBID} =~ "000" ]]; then | |
1389 | echo ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}" | |
1390 | ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}" | |
1391 | else | |
1392 | echo ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -hold_jid "${waitForJOBID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}" | |
1393 | ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -hold_jid "${waitForJOBID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}" | |
1394 | fi | |
1395 | fi | |
1396 | return 0 | |
1397 | } | |
1398 | ||
1399 | goSubmitBatch() | |
1400 | { | |
1401 | if [[ $# -lt 3 ]]; then | |
1402 | echo "minimal use:" | |
1403 | echo " ${0} submit fileList productionID configFile" | |
1404 | return 0 | |
1405 | fi | |
1406 | ||
1407 | productionID=${1} | |
1408 | inputList=${2} | |
1409 | configFile=${3:-"benchmark.config"} | |
1410 | configFile=$(readlink -f ${configFile}) | |
1411 | shift 3 | |
1412 | extraOpts=("$@") | |
1413 | if ! parseConfig ${configFile} "${extraOpts[@]}"; then return 1; fi | |
1414 | ||
1415 | #redirect all output to submit.log | |
1416 | echo "redirecting all output to ${PWD}/submit_${productionID//"/"/_}.log" | |
1417 | exec 7>&1 | |
1418 | exec 1>submit_${productionID//"/"/_}.log 2>&1 | |
1419 | ||
1420 | umask 0002 | |
1421 | echo ${0}" submit $*" | |
1422 | if [[ -z "${inputList}" || -z "${productionID}" ]] | |
1423 | then | |
1424 | echo | |
1425 | echo " Usage: ${0} submit inputList productionID [configFile=benchmark.config]" | |
1426 | echo | |
1427 | return | |
1428 | fi | |
1429 | ||
1430 | # check if config file is there | |
1431 | if [ ! -f ${configFile} ]; then | |
1432 | echo "ERROR! Config File '${configFile}' not found" >&2 | |
1433 | return | |
1434 | else | |
1435 | echo "Using Config File: '${configFile}'" | |
1436 | fi | |
1437 | ||
1438 | [[ ! -f ${alirootEnv} ]] && echo "alirootEnv script ${alirootEnv} not found!..." && return 1 | |
1439 | ||
1440 | #move the script, config and some other stuff to ${commonOutputPath} first, then use them from there | |
1441 | self=$(readlink -f "${0}") | |
1442 | configPath=$(dirname ${configFile}) | |
1443 | export commonOutputPath=${baseOutputDirectory}/${productionID} | |
1444 | ||
1445 | mkdir -p ${commonOutputPath} | |
1446 | mkdir -p ${commonOutputPath}/logs | |
1447 | mkdir -p ${commonOutputPath}/meta | |
1448 | ||
1449 | cp ${self} ${commonOutputPath} | |
1450 | cp ${configFile} ${commonOutputPath} | |
1451 | cp ${inputList} ${commonOutputPath} | |
1452 | self=${commonOutputPath}/${self##*/} | |
1453 | chmod u+x ${self} | |
1454 | configFile=${commonOutputPath}/${configFile##*/} | |
1455 | inputList=${commonOutputPath}/${inputList##*/} | |
1456 | ||
1457 | #convert to absolut pathnames | |
1458 | inputList=$(readlink -f "${inputList}") | |
1459 | #make list of runs | |
1460 | if [[ -z ${runNumber} ]]; then | |
1461 | listOfRuns=($(while read x; do guessRunNumber ${x}; done < ${inputList} | sort | uniq)) | |
1462 | else | |
1463 | listOfRuns=${runNumber} | |
1464 | fi | |
1465 | ||
1466 | alirootSource=$(readlink -f "${alirootSource}") | |
1467 | ||
1468 | echo "" | |
1469 | echo "### BEGIN CONFIGURATION ###" | |
1470 | echo "" | |
1471 | echo "GENERAL:" | |
1472 | echo "" | |
1473 | echo " productionID: ${productionID}" | |
1474 | echo " batchCommand: ${batchCommand}" | |
1475 | echo " batchFlags: ${batchFlags}" | |
1476 | echo " alirootEnv: ${alirootEnv}" | |
1477 | ${alirootEnv} echo ' ALICE_ROOT: ${ALICE_ROOT}' | |
1478 | ${alirootEnv} echo ' ALIROOT_RELEASE: ${ALICE_RELEASE}' | |
1479 | echo " inputList: ${inputList}" | |
1480 | echo " configPath: ${configPath}" | |
1481 | echo " commonOutputPath: ${commonOutputPath}" | |
1482 | echo " defaultOCDB: ${defaultOCDB}" | |
1483 | echo " autoOCDB: ${autoOCDB}" | |
1484 | echo " recoTriggerOptions: ${recoTriggerOptions}" | |
1485 | echo " runs:" | |
1486 | echo " ${listOfRuns[*]}" | |
1487 | echo "" | |
1488 | echo "THE TRAIN WILL RUN:" | |
1489 | ||
1490 | if [ ${runCPass0reco} -eq 1 ]; then | |
1491 | echo " Pass0 - Recontruction" | |
1492 | fi | |
1493 | ||
1494 | if [ ${runCPass0MergeMakeOCDB} -eq 1 ]; then | |
1495 | echo " Pass0 - merging and OCDB export" | |
1496 | fi | |
1497 | ||
1498 | if [ ${runCPass1reco} -eq 1 ]; then | |
1499 | echo " Pass1 - Recontruction" | |
1500 | fi | |
1501 | if [ ${runCPass1MergeMakeOCDB} -eq 1 ]; then | |
1502 | echo " Pass1 - merging and OCDB export" | |
1503 | fi | |
1504 | ||
1505 | echo "" | |
1506 | echo "LIMITS:" | |
1507 | echo " max. Events/Chunk: ${nEvents}" | |
1508 | echo " max. Number of Chunks per Run: ${nMaxChunks}" | |
1509 | echo "" | |
1510 | echo "### END CONFIGURATION ###" | |
1511 | echo "" | |
1512 | ||
1513 | ||
1514 | # check if input file is there | |
1515 | if [ ! -f ${inputList} ]; then | |
1516 | echo "ERROR! Input List '${inputList}' not found" >&2 | |
1517 | return | |
1518 | fi | |
1519 | ||
1520 | # define jobid (for dependent jobs) | |
1521 | date=$(date +%Y_%m_%d_%H%M%S) | |
1522 | #for each run we submit one jobarray: | |
1523 | for runNumber in ${listOfRuns[*]}; do | |
1524 | ||
1525 | [[ -z ${runNumber} ]] && continue | |
1526 | [[ ! ${runNumber} =~ ^[0-9]*[0-9]$ ]] && continue | |
1527 | ||
1528 | JOBpostfix="${productionID//"/"/_}_${runNumber}_${date}" | |
1529 | JOBID1="p0_${JOBpostfix}" | |
1530 | JOBID1wait="w0_${JOBpostfix}" | |
1531 | JOBID2="m0_${JOBpostfix}" | |
1532 | JOBID2wait="wm0_${JOBpostfix}" | |
1533 | JOBID3="op0_${JOBpostfix}" | |
1534 | JOBID3wait="wop0_${JOBpostfix}" | |
1535 | JOBID4="p1_${JOBpostfix}" | |
1536 | JOBID4wait="w1_${JOBpostfix}" | |
1537 | JOBID5="m1_${JOBpostfix}" | |
1538 | JOBID5wait="wm1_${JOBpostfix}" | |
1539 | JOBID6="s1_${JOBpostfix}" | |
1540 | JOBID6wait="ws1_${JOBpostfix}" | |
1541 | JOBID7="QA_${JOBpostfix}" | |
1542 | JOBmakeESDlistCPass1="lp1_${JOBpostfix}" | |
1543 | JOBfilterESDcpass1="fp1_${JOBpostfix}" | |
1544 | LASTJOB="000" | |
1545 | ||
1546 | oneInputFile=$(egrep -m1 "${runNumber}\/" ${inputList}) | |
1547 | ||
1548 | currentDefaultOCDB=${defaultOCDB} | |
1549 | [[ -z ${autoOCDB} ]] && autoOCDB=1 | |
1550 | if [[ ${autoOCDB} -ne 0 ]]; then | |
1551 | currentDefaultOCDB=$(setYear ${oneInputFile} ${defaultOCDB}) | |
1552 | fi | |
1553 | ||
1554 | echo "submitting run ${runNumber} with OCDB ${currentDefaultOCDB}" | |
1555 | ||
1556 | ############################################################################### | |
1557 | #run one chunk with valgrind: | |
1558 | if [[ -n ${profilingCommand} ]]; then | |
1559 | [[ -z ${nEventsProfiling} ]] && nEventsProfiling=2 | |
1560 | [[ -z ${profilingCommand} ]] && profilingCommand="/usr/bin/valgrind --tool=callgrind --num-callers=40 -v --trace-children=yes" | |
1561 | submit "valgrind" 1 1 000 "${alirootEnv} ${self}" CPass0 ${commonOutputPath}/000${runNumber}/valgrind ${oneInputFile} ${nEventsProfiling} ${currentDefaultOCDB} ${configFile} ${runNumber} valgrind useProfilingCommand=${profilingCommand} "${extraOpts[@]}" | |
1562 | fi | |
1563 | ||
1564 | ################################################################################ | |
1565 | ################################################################################ | |
1566 | # run the CPass0 if requested | |
1567 | ||
1568 | if [ ${runCPass0reco} -eq 1 ]; then | |
1569 | ||
1570 | echo | |
1571 | echo "starting CPass0... for run ${runNumber}" | |
1572 | echo | |
1573 | ||
1574 | # create directory and copy all files that are needed | |
1575 | targetDirectory="${commonOutputPath}/000${runNumber}/cpass0" | |
1576 | mkdir -p ${targetDirectory} | |
1577 | ||
1578 | filesCPass0=( | |
1579 | "${configPath}/runCPass0.sh" | |
1580 | "${configPath}/recCPass0.C" | |
1581 | "${configPath}/runCalibTrain.C" | |
1582 | "${configPath}/localOCDBaccessConfig.C" | |
1583 | "${configPath}/OCDB*.root" | |
1584 | "${configPath}/sim.C" | |
1585 | "${configPath}/rec.C" | |
1586 | "${configPath}/Config.C" | |
1587 | ) | |
1588 | for file in ${filesCPass0[*]}; do | |
1589 | [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath} | |
1590 | done | |
1591 | ||
1592 | localInputList=${targetDirectory}/${inputList##*/} | |
1593 | [[ ! -f ${localInputList} ]] && egrep "\/000${runNumber}\/" ${inputList} > ${localInputList} | |
1594 | # limit nFiles to nMaxChunks | |
1595 | nFiles=$(wc -l < ${localInputList}) | |
1596 | [[ ${nFiles} -eq 0 ]] && echo "list contains ZERO files! exiting..." && return 1 | |
1597 | echo "raw files in list: ${nFiles}" | |
1598 | if [[ ${nMaxChunks} -gt 0 && ${nMaxChunks} -le ${nFiles} ]]; then | |
1599 | nFiles=${nMaxChunks} | |
1600 | fi | |
1601 | echo "raw files to process: ${nFiles}" | |
1602 | [[ -z "${percentProcessedFilesToContinue}" ]] && percentProcessedFilesToContinue=100 | |
1603 | if [[ ${percentProcessedFilesToContinue} -eq 100 ]]; then | |
1604 | nFilesToWaitFor=${nFiles} | |
1605 | else | |
1606 | nFilesToWaitFor=$(( ${nFiles}-${nFiles}/(100/(100-${percentProcessedFilesToContinue})) )) | |
1607 | fi | |
1608 | echo "requested success rate is ${percentProcessedFilesToContinue}%" | |
1609 | echo "merging will start after ${nFilesToWaitFor} jobs are done" | |
1610 | ||
1611 | submit ${JOBID1} 1 ${nFiles} 000 "${alirootEnv} ${self}" CPass0 ${targetDirectory} ${localInputList} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} -1 "${extraOpts[@]}" | |
1612 | ||
1613 | ## submit a monitoring job that will run until a certain number of jobs are done with reconstruction | |
1614 | submit "${JOBID1wait}" 1 1 000 "${alirootEnv} ${self}" WaitForOutput ${commonOutputPath} "meta/cpass0.job*.run${runNumber}.done" ${nFilesToWaitFor} ${maxSecondsToWait} '-maxdepth 1' | |
1615 | LASTJOB=${JOBID1wait} | |
1616 | ||
1617 | fi #end running CPass0 | |
1618 | ################################################################################ | |
1619 | ||
1620 | ||
1621 | ################################################################################ | |
1622 | # submit merging of CPass0, depends on the reconstruction | |
1623 | ||
1624 | if [ ${runCPass0MergeMakeOCDB} -eq 1 ]; then | |
1625 | ||
1626 | echo | |
1627 | echo "submit CPass0 merging for run ${runNumber}" | |
1628 | echo | |
1629 | ||
1630 | targetDirectory="${commonOutputPath}/000${runNumber}/cpass0" | |
1631 | mkdir -p ${targetDirectory} | |
1632 | ||
1633 | #copy the scripts | |
1634 | filesMergeCPass0=( | |
1635 | "${configPath}/OCDB.root" | |
1636 | "${configPath}/mergeMakeOCDB.byComponent.sh" | |
1637 | "${configPath}/mergeMakeOCDB.sh" | |
1638 | "${configPath}/localOCDBaccessConfig.C" | |
1639 | "${configPath}/mergeByComponent.C" | |
1640 | "${configPath}/makeOCDB.C" | |
1641 | "${configPath}/merge.C" | |
1642 | ) | |
1643 | for file in ${filesMergeCPass0[*]}; do | |
1644 | [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath} | |
1645 | done | |
1646 | ||
1647 | submit ${JOBID2} 1 1 "${LASTJOB}" "${alirootEnv} ${self}" MergeCPass0 ${targetDirectory} ${currentDefaultOCDB} ${configFile} ${runNumber} cpass0.calib.run${runNumber}.list "${extraOpts[@]}" | |
1648 | LASTJOB=${JOBID2} | |
1649 | ||
1650 | if [[ -n ${generateMC} ]]; then | |
1651 | submit "mrl${JOBpostfix}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" PrintValues sim ${commonOutputPath}/meta/sim.run${runNumber}.list ${commonOutputPath}/meta/cpass0.job*.run${runNumber}.done | |
1652 | LASTJOB="mrl${JOBpostfix}" | |
1653 | fi | |
1654 | ||
1655 | echo | |
1656 | fi | |
1657 | # end of merging CPass0 | |
1658 | ################################################################################ | |
1659 | ||
1660 | ################################################################################ | |
1661 | ################################################################################ | |
1662 | # run the CPass1 if requested | |
1663 | ||
1664 | if [ ${runCPass1reco} -eq 1 ]; then | |
1665 | ||
1666 | targetDirectory="${commonOutputPath}/000${runNumber}/cpass1" | |
1667 | rm -f ${commonOutputPath}/meta/cpass1.job*.run${runNumber}.done | |
1668 | ||
1669 | # safety feature: if we are re-running for any reason we want to delete the previous output first. | |
1670 | [[ -d ${targetDirectory} ]] && rm -rf ${targetDirectory}/* && echo "removed old output at ${targetDirectory}/*" | |
1671 | ||
1672 | echo | |
1673 | echo "starting CPass1... for run ${runNumber}" | |
1674 | echo | |
1675 | ||
1676 | # create directory and copy all files that are needed | |
1677 | mkdir -p ${targetDirectory} | |
1678 | ||
1679 | filesCPass1=( | |
1680 | "${configPath}/runCPass1.sh" | |
1681 | "${configPath}/recCPass1.C" | |
1682 | "${configPath}/recCPass1_OuterDet.C" | |
1683 | "${configPath}/runCalibTrain.C" | |
1684 | "${configPath}/QAtrain_duo.C" | |
1685 | "${configPath}/localOCDBaccessConfig.C" | |
1686 | "${configPath}/OCDB.root" | |
1687 | ) | |
1688 | for file in ${filesCPass1[*]}; do | |
1689 | [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath} | |
1690 | done | |
1691 | ||
1692 | if [[ -n ${generateMC} ]]; then | |
1693 | localInputList=${commonOutputPath}/meta/sim.run${runNumber}.list | |
1694 | else | |
1695 | localInputList=${targetDirectory}/${inputList##*/} | |
1696 | [[ ! -f ${localInputList} ]] && egrep "\/000${runNumber}\/" ${inputList} > ${localInputList} | |
1697 | fi | |
1698 | # limit nFiles to nMaxChunks | |
1699 | nFiles=$(wc -l < ${localInputList}) | |
1700 | [[ ${nFiles} -eq 0 ]] && echo "list contains ZERO files! continuing..." && continue | |
1701 | echo "raw files in list: ${nFiles}" | |
1702 | if [[ ${nMaxChunks} -gt 0 && ${nMaxChunks} -le ${nFiles} ]]; then | |
1703 | nFiles=${nMaxChunks} | |
1704 | fi | |
1705 | echo "raw files to process: ${nFiles}" | |
1706 | [[ -z "${percentProcessedFilesToContinue}" ]] && percentProcessedFilesToContinue=100 | |
1707 | if [[ ${percentProcessedFilesToContinue} -eq 100 ]]; then | |
1708 | nFilesToWaitFor=${nFiles} | |
1709 | else | |
1710 | nFilesToWaitFor=$(( ${nFiles}-${nFiles}/(100/(100-${percentProcessedFilesToContinue})) )) | |
1711 | fi | |
1712 | echo "requested success rate is ${percentProcessedFilesToContinue}%" | |
1713 | echo "merging will start after ${nFilesToWaitFor} jobs are done" | |
1714 | ||
1715 | submit ${JOBID4} 1 ${nFiles} "${LASTJOB}" "${alirootEnv} ${self}" CPass1 ${targetDirectory} ${localInputList} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} -1 "${extraOpts[@]}" | |
1716 | ||
1717 | ################################################################################ | |
1718 | ## submit a monitoring job that will run until a certain number of jobs are done with reconstruction | |
1719 | submit "${JOBID4wait}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" WaitForOutput ${commonOutputPath} "meta/cpass1.job*.run${runNumber}.done" ${nFilesToWaitFor} ${maxSecondsToWait} '-maxdepth 1' | |
1720 | LASTJOB=${JOBID4wait} | |
1721 | ################################################################################ | |
1722 | ||
1723 | echo | |
1724 | fi #end running CPass1 | |
1725 | ||
1726 | ################################################################################ | |
1727 | # submit merging of CPass1, depends on the reconstruction | |
1728 | if [ ${runCPass1MergeMakeOCDB} -eq 1 ]; then | |
1729 | ||
1730 | echo | |
1731 | echo "submit CPass1 merging for run ${runNumber}" | |
1732 | echo | |
1733 | ||
1734 | targetDirectory="${commonOutputPath}/000${runNumber}/cpass1" | |
1735 | rm -f ${commonOutputPath}/meta/merge.cpass1.run${runNumber}.done | |
1736 | mkdir -p ${targetDirectory} | |
1737 | ||
1738 | # copy files | |
1739 | filesMergeCPass1=( | |
1740 | "${configPath}/OCDB.root" | |
1741 | "${configPath}/localOCDBaccessConfig.C" | |
1742 | "${configPath}/mergeMakeOCDB.byComponent.sh" | |
1743 | "${configPath}/mergeByComponent.C" | |
1744 | "${configPath}/makeOCDB.C" | |
1745 | "${configPath}/merge.C" | |
1746 | "${configPath}/mergeMakeOCDB.sh" | |
1747 | "${configPath}/QAtrain_duo.C" | |
1748 | ) | |
1749 | for file in ${filesMergeCPass1[*]}; do | |
1750 | [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath} | |
1751 | done | |
1752 | ||
1753 | submit "${JOBID5}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" MergeCPass1 ${targetDirectory} ${currentDefaultOCDB} ${configFile} ${runNumber} cpass1.calib.run${runNumber}.list cpass1.QA.run${runNumber}.lastMergingStage.txt.list cpass1.filtered.run${runNumber}.list "${extraOpts[@]}" | |
1754 | LASTJOB=${JOBID5} | |
1755 | echo | |
1756 | fi | |
1757 | ||
1758 | ############################### | |
1759 | #if [ ${runESDfiltering} -eq 1 ]; then | |
1760 | # rm -f ${commonOutputPath}/cpass1.ESD.run${runNumber}.list | |
1761 | # rm -f ${commonOutputPath}/meta/filtering.cpass1.run*.done | |
1762 | # echo | |
1763 | # echo submitting filtering for run ${runNumber} | |
1764 | # echo | |
1765 | # submit "${JOBmakeESDlistCPass1}" 1 1 "${LASTJOB}" "${self}" PrintValues esd ${commonOutputPath}/meta/cpass1.ESD.run${runNumber}.list ${commonOutputPath}/meta/cpass1.job*.run${runNumber}.done | |
1766 | # submit "${JOBfilterESDcpass1}" 1 1 "${JOBmakeESDlistCPass1}" "${alirootEnv} ${self}" MakeFilteredTrees ${commonOutputPath}/000${runNumber}/cpass1 ${runNumber} ${commonOutputPath}/meta/cpass1.ESD.run${runNumber}.list ${filteringFactorHighPt} ${filteringFactorV0s} ${currentDefaultOCDB} 1000000 0 10000000 0 ${configFile} AliESDs_Barrel.root "${extraOpts[@]}" | |
1767 | # LASTJOB=${JOBfilterESDcpass1} | |
1768 | #fi | |
1769 | ||
1770 | done | |
1771 | ||
1772 | ################################################################################# | |
1773 | ################################################################################# | |
1774 | #if [ ${runESDfiltering} -eq 1 ]; then | |
1775 | # submit "${JOBID5wait}" 1 1 "${LASTJOB}" "${self}" WaitForOutput ${commonOutputPath} "meta/filtering.cpass1.run*.done" "${#listOfRuns[@]}" ${maxSecondsToWait} | |
1776 | #else | |
1777 | submit "${JOBID5wait}" 1 1 "${LASTJOB}" "${self}" WaitForOutput ${commonOutputPath} "meta/merge.cpass1.run*.done" ${#listOfRuns[@]} ${maxSecondsToWait} | |
1778 | #fi | |
1779 | LASTJOB=${JOBID5wait} | |
1780 | ||
1781 | ################################################################################# | |
1782 | echo | |
1783 | echo "submit make a summary" | |
1784 | echo | |
1785 | ||
1786 | submit "${JOBID6}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" MakeSummary ${configFile} | |
1787 | LASTJOB=${JOBID6} | |
1788 | ################################################################################# | |
1789 | ||
1790 | #restore stdout | |
1791 | exec 1>&7 7>&- | |
1792 | echo "jobs submitted." | |
1793 | return 0 | |
1794 | } | |
1795 | ||
1796 | goWaitForOutput() | |
1797 | { | |
1798 | umask 0002 | |
1799 | [[ $# -lt 3 ]] && echo "goWaitForOutput() wrong number of arguments, exiting.." && return 1 | |
1800 | echo searchPath=${1} | |
1801 | echo fileName=${2} | |
1802 | echo numberOfFiles=${3} | |
1803 | echo maxSecondsToWait=${4} | |
1804 | searchPath=${1} | |
1805 | fileName=${2} | |
1806 | numberOfFiles=${3} | |
1807 | maxSecondsToWait=${4} | |
1808 | extraFindOptions=${5} | |
1809 | echo "command to be executed: /bin/ls -1 ${searchPath}/${fileName} ${extraFindOptions}" | |
1810 | [[ -z "${maxSecondsToWait}" ]] && maxSecondsToWait=$(( 3600*12 )) | |
1811 | while true; do | |
1812 | #n=$(find ${searchPath} ${extraFindOptions} -name "${fileName}" | wc -l) | |
1813 | n=$(/bin/ls -1 ${searchPath}/${fileName} 2>/dev/null | wc -l) | |
1814 | [[ ${n} -gt 0 ]] && echo "found ${n} X ${fileName}" | |
1815 | [[ ${n} -ge ${numberOfFiles} ]] && break | |
1816 | [[ ${SECONDS} -gt ${maxSecondsToWait} ]] && echo "timeout of ${maxSecondsToWait}s!" && break | |
1817 | sleep 60 | |
1818 | done | |
1819 | echo "DONE! exiting..." | |
1820 | return 0 | |
1821 | } | |
1822 | ||
1823 | mergeSysLogs() | |
1824 | { | |
1825 | outputFile=${1} | |
1826 | shift | |
1827 | inputFiles="$@" | |
1828 | i=0 | |
1829 | if ! ls -1 ${inputFiles} &>/dev/null; then echo "the files dont exist!: ${inputFiles}"; return 1; fi | |
1830 | while read x; do | |
1831 | runNumber=$(guessRunNumber ${x}) | |
1832 | [[ -z ${runNumber} ]] && echo "run number cannot be guessed for ${x}" && continue | |
1833 | awk -v run=${runNumber} -v i=${i} 'NR > 1 {print run" "$0} NR==1 && i==0 {print "run/I:"$0}' ${x} | |
1834 | (( i++ )) | |
1835 | done < <(ls -1 ${inputFiles}) > ${outputFile} | |
1836 | return 0 | |
1837 | } | |
1838 | ||
1839 | goMakeMergedSummaryTree() | |
1840 | { | |
1841 | # create list of calibration entries | |
1842 | # takes no arguments, just run it in the base output | |
1843 | # directory with the following files in the working directory | |
1844 | # | |
1845 | # Calibration file lists: | |
1846 | # cpass0.dcsTree.list, cpass1.dcsTree.list | |
1847 | # QA trending root file: | |
1848 | # trending.root | |
1849 | # | |
1850 | # Production infoo ascii files: | |
1851 | # summary_pass0.tree | |
1852 | # summary_pass1.tree | |
1853 | # | |
1854 | ||
1855 | [[ ! -f cpass0.dcsTree.list ]] && echo "no cpass0.dcsTree.list" && return 1 | |
1856 | [[ ! -f cpass1.dcsTree.list ]] && echo "no cpass1.dcsTree.list" && return 1 | |
1857 | [[ ! -f trending.root ]] && echo "no trending.root" && return 1 | |
1858 | [[ ! -f summary_pass0.tree ]] && echo "no summary_pass0.tree" && return 1 | |
1859 | [[ ! -f summary_pass1.tree ]] && echo "no summary_pass1.tree" && return 1 | |
1860 | ||
1861 | #first, dump the C macro to file | |
1862 | cat << EOF > mergeTree.C | |
1863 | // | |
1864 | // Merge summary information | |
1865 | // Following files are expected to be in the working directory | |
1866 | // | |
1867 | // Calibration file lists: | |
1868 | // cpass0.dcsTree.list, cpass1.dcsTree.list | |
1869 | // QA trending root file: | |
1870 | // trending.root | |
1871 | // | |
1872 | // Production infoo ascii files: | |
1873 | // summary_pass0.tree | |
1874 | // summary_pass1.tree | |
1875 | // | |
1876 | void mergeTree(){ | |
1877 | // | |
1878 | // | |
1879 | // | |
1880 | // Calibration values dump | |
1881 | // | |
1882 | AliXRDPROOFtoolkit::MakeTreeFromList("Calib.TPC.CPass0.root", "dcs","dcs","cpass0.dcsTree.list",1); | |
1883 | AliXRDPROOFtoolkit::MakeTreeFromList("Calib.TPC.CPass1.root", "dcs","dcs","cpass1.dcsTree.list",1); | |
1884 | // | |
1885 | // Calibration status dump | |
1886 | // | |
1887 | TFile *fprod = TFile::Open("fproduction.root","recreate"); | |
1888 | TTree tree0, tree1; | |
1889 | tree0.ReadFile("summary_pass0.tree"); | |
1890 | tree1.ReadFile("summary_pass1.tree"); | |
1891 | tree0.Write("CPass0"); | |
1892 | tree1.Write("CPass1"); | |
1893 | fprod->Close(); | |
1894 | // | |
1895 | // | |
1896 | // | |
1897 | TString stringSetup=""; | |
1898 | stringSetup+="1#QA.TPC#run#SummaryTPCQA/tpcQA#trending.root+"; // | |
1899 | stringSetup+="1#Calib.TPC.CPass0#run#dcs#Calib.TPC.CPass0.root+"; // | |
1900 | stringSetup+="1#Calib.TPC.CPass1#run#dcs#Calib.TPC.CPass1.root+"; // | |
1901 | // | |
1902 | stringSetup+="1#CPass0#runnumber#CPass0#fproduction.root+"; // | |
1903 | stringSetup+="1#CPass1#runnumber#CPass1#fproduction.root+"; // | |
1904 | // | |
1905 | AliXRDPROOFtoolkit::JoinTreesIndex("outAll.root","joinAll","run",stringSetup.Data(), 1); | |
1906 | } | |
1907 | EOF | |
1908 | ||
1909 | aliroot -b -q "mergeTree.C" > mergeTrees.log | |
1910 | return $? | |
1911 | } | |
1912 | ||
1913 | goMakeSummary() | |
1914 | { | |
1915 | #all the final stuff goes in here for ease of use: | |
1916 | # summary logs | |
1917 | # qa plot making | |
1918 | # final file lists | |
1919 | #some defaults: | |
1920 | log="summary.log" | |
1921 | productionID="qa" | |
1922 | ||
1923 | configFile=${1} | |
1924 | shift 1 | |
1925 | extraOpts=("$@") | |
1926 | if ! parseConfig ${configFile} "${extraOpts[@]}"; then return 1; fi | |
1927 | ||
1928 | [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource} | |
1929 | ||
1930 | exec &> >(tee ${log}) | |
1931 | ||
1932 | [[ ! -f ${configFile} ]] && echo "no config file ${configFile}!" && return | |
1933 | ||
1934 | [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD} | |
1935 | ||
1936 | #summarize the global stuff | |
1937 | echo "env script: ${alirootSource} ${alirootEnv}" | |
1938 | echo "\$ALICE_ROOT=${ALICE_ROOT}" | |
1939 | echo "commonOutputPath=${commonOutputPath}" | |
1940 | ||
1941 | #summarize the stacktraces | |
1942 | awk ' | |
1943 | BEGIN { | |
1944 | print "frame/I:method/C:line/C:cpass/I:aliroot/I"; | |
1945 | RS="#[0-9]*"; | |
1946 | aliroot=0; | |
1947 | } | |
1948 | { | |
1949 | if ($3 ~ /Ali*/) aliroot=1; else aliroot=0; | |
1950 | gsub("#","",RT); | |
1951 | if ($NF!="" && RT!="" && $3!="") print RT" "$3" "$NF" "0" "aliroot | |
1952 | } | |
1953 | ' 000*/cpass0/*/stacktrace* 2>/dev/null > stacktrace.tree | |
1954 | awk ' | |
1955 | BEGIN { | |
1956 | RS="#[0-9]*"; | |
1957 | aliroot=0; | |
1958 | } | |
1959 | { | |
1960 | if ($3 ~ /Ali*/) aliroot=1; else aliroot=0; | |
1961 | gsub("#","",RT); | |
1962 | if ($NF!="" && RT!="" && $3!="") print RT" "$3" "$NF" "1" "aliroot | |
1963 | } | |
1964 | ' 000*/cpass1/*/stacktrace* 2>/dev/null >> stacktrace.tree | |
1965 | ||
1966 | echo total numbers for the production: | |
1967 | echo | |
1968 | awk 'BEGIN {nFiles=0;nCore=0;} | |
1969 | /^calibfile/ {nFiles++;} | |
1970 | /core dumped/ {nCore++i;} | |
1971 | END {print "cpass0 produced "nFiles" calib files, "nCore" core files";}' meta/cpass0.job*done 2>/dev/null | |
1972 | awk 'BEGIN {nOK=0; nBAD=0; } | |
1973 | /\/rec.log OK/ {nOK++;} | |
1974 | /\/rec.log BAD/ {nBAD++;} | |
1975 | /stderr BAD/ {if ($0 ~ /rec.log/){nBAD++;}} | |
1976 | END {print "cpass0 reco: OK: "nOK"\tBAD: "nBAD;}' meta/cpass0.job*done 2>/dev/null | |
1977 | awk 'BEGIN {nOK=0; nBAD=0; } | |
1978 | /\/calib.log OK/ {nOK++;} | |
1979 | /\/calib.log BAD/ {nBAD++;} | |
1980 | END {print "cpass0 calib: OK: "nOK"\tBAD: "nBAD;}' meta/cpass0.job*done 2>/dev/null | |
1981 | ||
1982 | awk 'BEGIN {nOK=0; nBAD=0; } | |
1983 | /merge.log OK/ {nOK++;} | |
1984 | /merge.log BAD/ {nBAD++;} | |
1985 | END {print "cpass0 merge: OK: "nOK"\tBAD: "nBAD;}' meta/merge.cpass0*done 2>/dev/null | |
1986 | awk 'BEGIN {nOK=0; nBAD=0; } | |
1987 | /ocdb.log OK/ {nOK++;} | |
1988 | /ocdb.log BAD/ {nBAD++;} | |
1989 | END {print "cpass0 OCDB: OK: "nOK"\tBAD: "nBAD;}' meta/merge.cpass0*done 2>/dev/null | |
1990 | ||
1991 | echo | |
1992 | awk 'BEGIN {nFiles=0;nCore=0;} | |
1993 | /^calibfile/ {nFiles++;} | |
1994 | /core dumped/ {nCore++;} | |
1995 | END {print "cpass1 produced "nFiles" calib files, "nCore" core files";}' meta/cpass1.job*done 2>/dev/null | |
1996 | awk 'BEGIN {nOK=0; nBAD=0; } | |
1997 | /\/rec.log OK/ {nOK++;} | |
1998 | /\/rec.log BAD/ {nBAD++;} | |
1999 | /stderr BAD/ {if ($0 ~ /rec.log/){nBAD++;}} | |
2000 | END {print "cpass1 reco: OK: "nOK"\tBAD: "nBAD;}' meta/cpass1.job*done 2>/dev/null | |
2001 | awk 'BEGIN {nOK=0; nBAD=0; } | |
2002 | /\/calib.log OK/ {nOK++;} | |
2003 | /\/calib.log BAD/ {nBAD++;} | |
2004 | END {print "cpass1 calib: OK: "nOK"\tBAD: "nBAD;}' meta/cpass1.job*done 2>/dev/null | |
2005 | ||
2006 | awk 'BEGIN {nOK=0; nBAD=0; } | |
2007 | /merge.log OK/ {nOK++;} | |
2008 | /merge.log BAD/ {nBAD++;} | |
2009 | END {print "cpass1 merge: OK: "nOK"\tBAD: "nBAD;}' meta/merge.cpass1*done 2>/dev/null | |
2010 | awk 'BEGIN {nOK=0; nBAD=0; } | |
2011 | /ocdb.log OK/ {nOK++;} | |
2012 | /ocdb.log BAD/ {nBAD++;} | |
2013 | END {print "cpass1 OCDB: OK: "nOK"\tBAD: "nBAD;}' meta/merge.cpass1*done 2>/dev/null | |
2014 | ||
2015 | echo | |
2016 | echo per run stats: | |
2017 | /bin/ls -1 meta/merge.cpass0.run*.done | while read x | |
2018 | do | |
2019 | dir=$(goPrintValues calibfile - ${x}) | |
2020 | runNumber=$(guessRunNumber ${dir}) | |
2021 | [[ -z ${runNumber} ]] && continue | |
2022 | ||
2023 | if $(/bin/ls meta/cpass0.job*.run${runNumber}.done &> /dev/null); then | |
2024 | statusCPass0=( $( | |
2025 | awk 'BEGIN {nOKrec=0;nBADrec=0;nOKcalib=0;nBADcalib=0;nOKstderr=0;nBADstderr=0;} | |
2026 | /\/rec.log OK/ {nOKrec++;} | |
2027 | /\/rec.log BAD/ {nBADrec++;} | |
2028 | /stderr BAD/ {if ($0 ~ /rec.log/) {nBADrec++;} nBADstderr++;} | |
2029 | /stderr OK/ {nOKstderr++;} | |
2030 | /\/calib.log OK/ {nOKcalib++;} | |
2031 | /\/calib.log BAD/ {nBADcalib++} | |
2032 | END {print ""nOKrec" "nBADrec" "nOKstderr" "nBADstderr" "nOKcalib" "nBADcalib;}' meta/cpass0.job*.run${runNumber}.done 2>/dev/null | |
2033 | ) ) | |
2034 | fi | |
2035 | ||
2036 | if $(/bin/ls meta/cpass1.job*.run${runNumber}.done &>/dev/null); then | |
2037 | statusCPass1=( $( | |
2038 | awk 'BEGIN {nOKrec=0;nBADrec=0;nOKcalib=0;nBADcalib=0;nOKstderr=0;nBADstderr=0;nQAbarrelOK=0;nQAbarrelBAD=0;nQAouterOK=0;nQAouterBAD=0;} | |
2039 | /\/rec.log OK/ {nOKrec++;} | |
2040 | /\/rec.log BAD/ {nBADrec++;} | |
2041 | /stderr BAD/ {if ($0 ~ /rec.log/) nBADrec++;nBADstderr++;} | |
2042 | /stderr OK/ {nOKstderr++;} | |
2043 | /\/calib.log OK/ {nOKcalib++;} | |
2044 | /\/calib.log BAD/ {nBADcalib++} | |
2045 | /\/qa_barrel.log OK/ {nQAbarrelOK++;} | |
2046 | /\/qa_barrel.log BAD/ {nQAbarrelBAD++;} | |
2047 | /\/qa_outer.log OK/ {nQAouterOK++;} | |
2048 | /\/qa_outer.log BAD/ {nQAouterBAD++;} | |
2049 | END {print ""nOKrec" "nBADrec" "nOKstderr" "nBADstderr" "nOKcalib" "nBADcalib" "nQAbarrelOK" "nQAbarrelBAD" "nQAouterOK" "nQAouterBAD;}' meta/cpass1.job*.run${runNumber}.done 2>/dev/null | |
2050 | ) ) | |
2051 | fi | |
2052 | ||
2053 | statusOCDBcpass0=$(awk '/ocdb.log/ {print $2} ' ${x} 2>/dev/null) | |
2054 | statusOCDBcpass1=$(awk '/ocdb.log/ {print $2}' ${x/cpass0/cpass1} 2>/dev/null) | |
2055 | statusQA=$(awk '/mergeMakeOCDB.log/ {print $2}' ${x/cpass0/cpass1} 2>/dev/null) | |
2056 | ||
2057 | printf "%s\t ocdb.log cpass0: %s\t ocdb.log cpass1: %s\tqa.log:%s\t| cpass0: rec:%s/%s stderr:%s/%s calib:%s/%s cpass1: rec:%s/%s stderr:%s/%s calib:%s/%s QAbarrel:%s/%s QAouter:%s/%s\n" ${runNumber} ${statusOCDBcpass0} ${statusOCDBcpass1} ${statusQA} ${statusCPass0[0]} ${statusCPass0[1]} ${statusCPass0[2]} ${statusCPass0[3]} ${statusCPass0[4]} ${statusCPass0[5]} ${statusCPass1[0]} ${statusCPass1[1]} ${statusCPass1[2]} ${statusCPass1[3]} ${statusCPass1[4]} ${statusCPass1[5]} ${statusCPass1[6]} ${statusCPass1[7]} ${statusCPass1[8]} ${statusCPass1[9]} | |
2058 | done | |
2059 | ||
2060 | #make lists with output files - QA, trending, filtering and calibration | |
2061 | ### wait for the merging of all runs to be over ### | |
2062 | rm -f qa.list | |
2063 | goPrintValues qafile qa.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null | |
2064 | rm -f calib.list | |
2065 | goPrintValues calibfile calib.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null | |
2066 | rm -f trending.list | |
2067 | goPrintValues trendingfile trending.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null | |
2068 | rm -f filtering.list | |
2069 | goPrintValues filteredTree filtering.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null | |
2070 | #/bin/ls ${commonOutputPath}/*/cpass0/dcs* > cpass0.dcsTree.list | |
2071 | rm -f cpass0.dcsTree.list | |
2072 | goPrintValues dcsTree cpass0.dcsTree.list ${commonOutputPath}/meta/merge.cpass0.run*.done &>/dev/null | |
2073 | #/bin/ls ${commonOutputPath}/*/cpass1/dcs* > cpass1.dcsTree.list | |
2074 | rm -f cpass1.dcsTree.list | |
2075 | goPrintValues dcsTree cpass1.dcsTree.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null | |
2076 | ||
2077 | #merge trending | |
2078 | rm -f ${commonOutputPath}/trending_merged.root | |
2079 | goMerge trending.list ${commonOutputPath}/trending.root ${configFile} "${extraOpts[@]}" &> mergeTrending.log | |
2080 | ||
2081 | goMakeSummaryTree ${commonOutputPath} 0 | |
2082 | goMakeSummaryTree ${commonOutputPath} 1 | |
2083 | ||
2084 | goCreateQAplots "qa.list" "${productionID}" "QAplots" "${configFile}" "${extraOpts[@]}" filteringList="filtering.list" &>createQAplots.log | |
2085 | ||
2086 | #make a merged summary tree out of the QA trending, dcs trees and log summary trees | |
2087 | goMakeMergedSummaryTree | |
2088 | ||
2089 | #if set, email the summary | |
2090 | [[ -n ${MAILTO} ]] && cat ${log} | mail -s "benchmark ${productionID} done" ${MAILTO} | |
2091 | ||
2092 | return 0 | |
2093 | } | |
2094 | ||
2095 | goMakeSummaryTree() | |
2096 | { | |
2097 | if [[ $# -lt 1 ]] ; then | |
2098 | return | |
2099 | fi | |
2100 | #1. define vars/arrays | |
2101 | DIR=${1} #use input or exec in current dir | |
2102 | pass=${2-"0"} #pass from input | |
2103 | outfile="summary_pass${pass}.tree" | |
2104 | Ncolumns=0 | |
2105 | test -f ${outfile} && : >${outfile} | |
2106 | errfile=${outfile/tree/err} | |
2107 | test -f ${errfile} && : >${errfile} | |
2108 | ||
2109 | declare -a counterString=(TOFevents TOFtracks TPCevents TPCtracks TRDevents TRDtracks T0events SDDevents SDDtracks MeanVertexevents) | |
2110 | Ncounter=${#counterString[@]} | |
2111 | ||
2112 | declare -a statusString=(TRDStatus TOFStatus TPCStatus T0Status MeanVertexStatus) | |
2113 | Nstatus=${#statusString[@]} | |
2114 | ||
2115 | ||
2116 | declare -a ratesString=(rec stderr calib qa_barrel qa_outer) | |
2117 | Nrates=${#ratesString[@]} | |
2118 | ||
2119 | runs=( $(ls -1 ${DIR}/meta/merge.cpass0* | while read x; do guessRunNumber $x; done) ) | |
2120 | Nruns=${#runs[@]} | |
2121 | ||
2122 | echo -n runnumber/I >>${outfile} | |
2123 | echo -n :cpass${pass}status/I >>${outfile} | |
2124 | echo -n :cpass${pass}QAstatus/I >>${outfile} | |
2125 | for i in ${ratesString[@]}; do | |
2126 | echo -n :${i}OK/I >>${outfile} | |
2127 | echo -n :${i}BAD/I >>${outfile} | |
2128 | ||
2129 | done | |
2130 | for i in ${counterString[@]} ${statusString[@]} ; do | |
2131 | echo -n :${i}/I >>${outfile} | |
2132 | done | |
2133 | Ncolumns=$((2 + 2*Nrates + Ncounter + Nstatus)) | |
2134 | echo >> ${outfile} | |
2135 | ||
2136 | #2. loop runs | |
2137 | ||
2138 | for runnumber in ${runs[@]} ; do | |
2139 | ||
2140 | ||
2141 | ||
2142 | filejob="${DIR}/meta/cpass${pass}.job*.run${runnumber}.done" | |
2143 | filemerge="${DIR}/meta/merge.cpass${pass}.run${runnumber}.done" | |
2144 | fileOCDB=$(grep /ocdb.log ${filemerge} | awk '{print $1}') | |
2145 | if ! $(/bin/ls ${filemerge} &>/dev/null) ; then | |
2146 | echo "${filemerge} does not exist!" >>${errfile} | |
2147 | continue | |
2148 | elif ! $(/bin/ls ${filejob} &>/dev/null) ; then | |
2149 | echo "${filejob} does not exist!" >>${errfile} | |
2150 | echo -n ${runnumber} >> ${outfile} | |
2151 | for i in $(seq ${Ncolumns}) ; do | |
2152 | echo -n "-1" >> ${outfile} | |
2153 | done | |
2154 | echo >> ${outfile} | |
2155 | continue | |
2156 | fi | |
2157 | echo -n ${runnumber} >> ${outfile} | |
2158 | #pass0status= grep '/ocdb.log' ${filemerge} | cut -d' ' -f2 | tr OK x1 | tr BAD xx0 | tr -d 'x' | |
2159 | passStatus=$(grep '/ocdb.log' ${filemerge} | grep OK | wc -l) | |
2160 | echo -n " ${passStatus}" >> ${outfile} | |
2161 | qaStatus=$(grep '/mergeMakeOCDB.log' ${filemerge} | grep OK | wc -l) | |
2162 | echo -n " ${qaStatus}" >> ${outfile} | |
2163 | ||
2164 | ||
2165 | #fill OK/BAD rates | |
2166 | for i in $(seq 0 $((${Nrates}-1))) ; do | |
2167 | var1=$(grep "/${ratesString[${i}]}.log" ${filejob} | grep OK | wc -l) | |
2168 | var2=$(grep "/${ratesString[${i}]}.log" ${filejob} | grep BAD | wc -l) | |
2169 | ||
2170 | if [[ ${ratesString[${i}]} == "stderr" ]] ; then | |
2171 | var1=$(grep "stderr" ${filejob} | grep OK | wc -l) | |
2172 | var2=$(grep "stderr" ${filejob} | grep "rec.log" | grep BAD | wc -l) | |
2173 | fi | |
2174 | echo -n " ${var1}" >> ${outfile} | |
2175 | echo -n " ${var2}" >> ${outfile} | |
2176 | done | |
2177 | ||
2178 | if [[ -f ${fileOCDB} ]] ; then | |
2179 | #fill counter | |
2180 | for i in $(seq 0 $((${Ncounter}-1))) ; do | |
2181 | var1=$(grep Monalisa ${fileOCDB} | grep ${counterString[${i}]} | cut -f2) | |
2182 | echo -n " ${var1:-"-1"}" >> ${outfile} | |
2183 | done | |
2184 | ||
2185 | #fill status | |
2186 | for i in $(seq 0 $((${Nstatus}-1))) ; do | |
2187 | var1=$(grep "calibration status=" ${fileOCDB} | grep ${statusString[${i}]/Status/} | cut -d'=' -f2) | |
2188 | echo -n " ${var1:-"-1"}" >> ${outfile} | |
2189 | done | |
2190 | fi | |
2191 | echo >> ${outfile} | |
2192 | done | |
2193 | ||
2194 | return 0 | |
2195 | } | |
2196 | ||
2197 | parseConfig() | |
2198 | { | |
2199 | configFile=${1} | |
2200 | shift | |
2201 | args=("$@") | |
2202 | ||
2203 | #some defaults | |
2204 | #autoOCDB=0 | |
2205 | defaultOCDB="raw://" | |
2206 | #runNumber=167123 | |
2207 | #makeflowPath="/hera/alice/aux/cctools/bin" | |
2208 | #makeflowOptions="-T wq -N alice -d all -C ali-copilot.cern.ch:9097" | |
2209 | #makeflowOptions="-T wq -N alice -C ali-copilot.cern.ch:9097" | |
2210 | makeflowOptions="" | |
2211 | batchCommand="/usr/bin/qsub" | |
2212 | batchFlags="-b y -cwd -l h_rt=24:0:0,h_rss=4G " | |
2213 | baseOutputDirectory="$PWD/output" | |
2214 | #alirootEnv="/cvmfs/alice.cern.ch/bin/alienv setenv AliRoot/v5-04-34-AN -c" | |
2215 | #alirootEnv="/home/mkrzewic/alisoft/balice_master.sh" | |
2216 | #trustedQAtrainMacro='/hera/alice/mkrzewic/gsisvn/Calibration/QAtrain_duo.C' | |
2217 | reconstructInTemporaryDir=0 | |
2218 | recoTriggerOptions="\"\"" | |
2219 | percentProcessedFilesToContinue=100 | |
2220 | maxSecondsToWait=$(( 3600*24 )) | |
2221 | nEvents=-1 | |
2222 | nMaxChunks=0 | |
2223 | postSetUpActionCPass0="" | |
2224 | postSetUpActionCPass1="" | |
2225 | runCPass0reco=1 | |
2226 | runCPass0MergeMakeOCDB=1 | |
2227 | runCPass1reco=1 | |
2228 | runCPass1MergeMakeOCDB=1 | |
2229 | runESDfiltering=1 | |
2230 | filteringFactorHighPt=1e2 | |
2231 | filteringFactorV0s=1e1 | |
2232 | MAILTO="" | |
2233 | #pretend=1 | |
2234 | #dontRedirectStdOutToLog=1 | |
2235 | logToFinalDestination=1 | |
2236 | ALIROOT_FORCE_COREDUMP=1 | |
2237 | ||
2238 | #first, source the config file | |
2239 | if [ -f ${configFile} ]; then | |
2240 | source ${configFile} | |
2241 | else | |
2242 | echo "config file ${configFile} not found!, skipping..." | |
2243 | fi | |
2244 | ||
2245 | #then, parse the options as they override the options from file | |
2246 | for opt in "${args[@]}"; do | |
2247 | if [[ ! "${opt}" =~ .*=.* ]]; then | |
2248 | echo "badly formatted option ${var}, should be: option=value, stopping..." | |
2249 | return 1 | |
2250 | fi | |
2251 | local var="${opt%%=*}" | |
2252 | local value="${opt#*=}" | |
2253 | echo "${var} = ${value}" | |
2254 | export ${var}="${value}" | |
2255 | done | |
2256 | ||
2257 | #do some checking | |
2258 | [[ -z ${alirootEnv} ]] && echo "alirootEnv not defined!" && return 1 | |
2259 | ||
2260 | #export the aliroot function if defined to override normal behaviour | |
2261 | [[ $(type -t aliroot) =~ "function" ]] && export -f aliroot | |
2262 | ||
2263 | return 0 | |
2264 | } | |
2265 | ||
2266 | aliroot() | |
2267 | { | |
2268 | args="$@" | |
2269 | if [[ -n ${useProfilingCommand} ]]; then | |
2270 | valgrindLogFile="cpu.txt" | |
2271 | [[ "${args}" =~ rec ]] && valgrindLogFile="cpu_rec.txt" | |
2272 | [[ "${args}}" =~ Calib ]] && valgrindLogFile="cpu_calib.txt" | |
2273 | [[ -n ${useProfilingCommand} ]] && useProfilingCommand="${useProfilingCommand} --log-file=${valgrindLogFile}" | |
2274 | echo running ${useProfilingCommand} aliroot ${args} | |
2275 | ${useProfilingCommand} aliroot ${args} | |
2276 | else | |
2277 | #to prevent an infinite recursion use "command aliroot" to disable | |
2278 | #aliases and functions | |
2279 | echo running command aliroot ${args} | |
2280 | command aliroot "$@" | |
2281 | fi | |
2282 | return 0 | |
2283 | } | |
2284 | ||
2285 | guessRunData() | |
2286 | { | |
2287 | #guess the period from the path, pick the rightmost one | |
2288 | period="" | |
2289 | runNumber="" | |
2290 | year="" | |
2291 | pass="" | |
2292 | legoTrainRunNumber="" | |
2293 | dataType="" | |
2294 | ||
2295 | local shortRunNumber="" | |
2296 | local IFS="/" | |
2297 | declare -a path=( $1 ) | |
2298 | local dirDepth=$(( ${#path[*]}-1 )) | |
2299 | i=0 | |
2300 | #for ((x=${dirDepth};x>=0;x--)); do | |
2301 | for ((x=0;x<=${dirDepth};x++)); do | |
2302 | ||
2303 | [[ $((x-1)) -ge 0 ]] && local fieldPrev=${path[$((x-1))]} | |
2304 | local field=${path[${x}]} | |
2305 | local fieldNext=${path[$((x+1))]} | |
2306 | ||
2307 | [[ ${field} =~ ^[0-9]*$ && ${fieldNext} =~ (.*\.zip$|.*\.root$) ]] && legoTrainRunNumber=${field} | |
2308 | [[ -n ${legoTrainRunNumber} && -z ${pass} ]] && pass=${fieldPrev} | |
2309 | [[ ${field} =~ ^LHC[0-9][0-9][a-z].*$ ]] && period=${field%_*} | |
2310 | [[ ${field} =~ ^000[0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && runNumber=${field#000} | |
2311 | [[ ${field} =~ ^[0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && shortRunNumber=${field} | |
2312 | [[ ${field} =~ ^20[0-9][0-9]$ ]] && year=${field} | |
2313 | [[ ${field} =~ ^(^sim$|^data$) ]] && dataType=${field} | |
2314 | (( i++ )) | |
2315 | done | |
2316 | [[ -z ${legoTrainRunNumber} ]] && pass=${path[$((dirDepth-1))]} | |
2317 | [[ "${dataType}" =~ ^sim$ ]] && pass="passMC" && runNumber=${shortRunNumber} | |
2318 | ||
2319 | #if [[ -z ${dataType} || -z ${year} || -z ${period} || -z ${runNumber}} || -z ${pass} ]]; | |
2320 | if [[ -z ${runNumber}} ]]; | |
2321 | then | |
2322 | #error condition | |
2323 | return 1 | |
2324 | else | |
2325 | #ALL OK | |
2326 | return 0 | |
2327 | fi | |
2328 | return 0 | |
2329 | } | |
2330 | ||
2331 | main "$@" |