2 #include benchmark.config
4 # this script runs the CPass0/CPass1 train
5 # produced OCDB updates are local
9 #run in proper mode depending on the selection
10 if [[ $# -lt 1 ]]; then
11 if [[ ! "${0}" =~ "bash" ]]; then
13 echo " ${0} \"run\" productionID inputList configFile [extraOpts]"
14 echo "uses a batch system (SGE):"
15 echo " ${0} \"submit\" productionID inputList configFile [extraOpts]"
16 echo "extraOpts if non-empty override the config file, e.g.:"
17 echo " ${0} submit test1 benchmark.list benchmark.config runNumber=169123 nEvents=10"
22 #define some aliases - default is to call one of the functions directly
27 "CPass0") goCPass0 "$@";;
28 "CPass1") goCPass1 "$@";;
29 "MakeLocalOCDBaccessConfig") goMakeLocalOCDBaccessConfig "$@";;
30 "MergeCPass0") goMergeCPass0 "$@";;
31 "MergeCPass1") goMergeCPass1 "$@";;
32 "MakeFilteredTrees") goMakeFilteredTrees "$@";;
33 "MakeSummary") goMakeSummary "$@";;
34 "run") goSubmitMakeflow "$@";;
35 "submit") goSubmitBatch "$@";;
37 "GenerateMakeflow") goGenerateMakeflow "$@";;
38 "PrintValues") goPrintValues "$@";;
39 "CreateQAplots") goCreateQAplots "$@";;
40 "WaitForOutput") goWaitForOutput "$@";;
41 "Merge") goMerge "$@";;
51 #generate one raw chunk in current directory
52 SEED=${JOB_ID}${SGE_TASK_ID}
53 export CONFIG_SEED=${SEED}
57 if [[ -n ${pretend} ]]; then
61 if [[ -f sim.C && -f Config.C ]] ; then
62 time aliroot -b -q -x sim.C\(${runNumber},\"${OCDBpath}\",${nEventsim}\) >sim.log 2>&1
63 mv syswatch.log simwatch.log
80 if ! parseConfig ${configFile} "$@"; then return 1; fi
82 #record the working directory provided by the batch system
83 batchWorkingDirectory=${PWD}
85 #use the jobindex only if set and non-negative
86 if [[ -z ${jobindex} || ${jobindex} -lt 0 ]]; then
87 [[ -n "${LSB_JOBINDEX}" ]] && jobindex=${LSB_JOBINDEX}
88 [[ -n "${SGE_TASK_ID}" ]] && jobindex=${SGE_TASK_ID}
89 if [[ -z ${jobindex} ]]; then
95 [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
96 doneFile="${commonOutputPath}/meta/cpass0.job${jobindex}.run${runNumber}.done"
97 [[ -n ${useProfilingCommand} ]] && doneFile="${commonOutputPath}/meta/profiling.cpass0.job${jobindex}.run${runNumber}.done"
99 [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
101 if [[ -n ${ALIROOT_FORCE_COREDUMP} ]]; then
103 export ALIROOT_FORCE_COREDUMP
106 #the contents of this is stored in the tree and used later (e.g. AliAnalysisTaskPIDResponse)!
107 #at the QA stage the pass number is guessed from the path stored here.
109 #Packages= ;OutputDir= ;LPMPass= ;TriggerAlias= ;LPMRunNumber= ;LPMProductionType= ;LPMInteractionType= ;LPMProductionTag= ;LPMAnchorRun= ;LPMAnchorProduction= ;LPMAnchorYear=
110 export PRODUCTION_METADATA="OutputDir=cpass0"
112 if [[ "${inputList}" =~ \.root$ ]]; then
115 infile=$(sed -ne "${jobindex}p" ${inputList} | egrep '\s*\w*/\w*')
117 chunkName=${infile##*/}
119 outputDir=${targetDirectory}/${jobindex}_${chunkName%.*}
120 mkdir -p ${outputDir}
121 [[ ! -d ${outputDir} ]] && echo "cannot make ${outputDir}" && touch ${doneFile} && return 1
123 #runpath=${PWD}/rundir_cpass0_${runNumber}_${jobindex}
125 #[[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
126 [[ ${reconstructInTemporaryDir} -eq 1 ]] && runpath=$(mktemp -d -t cpass0.XXXXXX)
128 [[ ! -d ${runpath} ]] && echo "cannot make runpath ${runpath}" && touch ${doneFile} && return 1
129 if ! cd ${runpath}; then
130 echo "PWD=$PWD is not the runpath=${runpath}"
135 #runCPassX/C expects the raw chunk to be linked in the run dir
136 #despite it being accessed by the full path
137 ln -s ${infile} ${runpath}/${chunkName}
140 if [[ -n ${generateMC} ]]; then
142 outputDirMC=${commonOutputPath}/000${runNumber}/sim/${jobindex}
143 simrunpath=${outputDirMC}
144 #[[ ${simulateInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && simrunpath=${TMPDIR}
145 [[ ${simulateInTemporaryDir} -eq 1 ]] && simrunpath=$(mktemp -d -t cpass0MC.XXXXXX)
146 mkdir -p ${outputDirMC}
147 mkdir -p ${simrunpath}
148 if cd ${simrunpath}; then
151 "${batchWorkingDirectory}/sim.C"
152 "${batchWorkingDirectory}/rec.C"
153 "${batchWorkingDirectory}/Config.C"
154 "${batchWorkingDirectory}/OCDB_*.root"
156 for file in ${filesMC[*]}; do
157 [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
160 generateMC ${runNumber} ${ocdbPath} ${nEvents}
162 [[ ! "${simrunpath}" =~ "${outputDirMC}" ]] && mv * ${outputDirMC} #TODO check if it works
165 ln -s ${outputDirMC}/* ${runpath}/
167 inputList=${outputDirMC}/galice.root #TODO not valid outside shell !!!
173 [[ ! -f ${inputList} && -z ${pretend} ]] && echo "input file ${inputList} not found, exiting..." && touch ${doneFile} && return 1
175 logOutputDir=${runpath}
176 [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir}
177 [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/stdout
178 #[[ -z ${dontRedirectStdOutToLog} ]] && exec 2> ${logOutputDir}/stderr
181 echo "#####################"
184 echo nEvents ${nEvents}
185 echo runNumber ${runNumber}
186 echo ocdbPath ${ocdbPath}
187 echo infile ${infile}
188 echo chunkName ${chunkName}
189 echo jobindex ${jobindex}
190 echo recoTriggerOptions ${recoTriggerOptions}
191 echo targetDirectory ${targetDirectory}
192 echo commonOutputPath ${commonOutputPath}
193 echo doneFile ${doneFile}
194 echo runpath ${runpath}
195 echo outputDir ${outputDir}
196 echo ALICE_ROOT ${ALICE_ROOT}
198 echo "########## ###########"
200 alirootInfo > ALICE_ROOT.log
203 "${batchWorkingDirectory}/runCPass0.sh"
204 "${batchWorkingDirectory}/recCPass0.C"
205 "${batchWorkingDirectory}/runCalibTrain.C"
206 "${batchWorkingDirectory}/localOCDBaccessConfig.C"
207 "${batchWorkingDirectory}/OCDB.root"
208 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/runCPass0.sh"
209 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/recCPass0.C"
210 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/runCalibTrain.C"
213 for file in ${filesCPass0[*]}; do
214 [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
215 [[ ${file##*/} =~ .*\.sh ]] && chmod +x ${file##*/}
218 echo "this directory (${PWD}) contents:"
221 chmod u+x runCPass0.sh
223 sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh
225 if [[ -n ${postSetUpActionCPass0} ]]; then
226 echo "running ${postSetUpActionCPass0}"
227 eval ${postSetUpActionCPass0}
231 echo "${runpath}/runCPass0.sh ${infile} ${nEvents} ${runNumber} ${ocdbPath} ${recoTriggerOptions}"
232 if [[ -n ${pretend} ]]; then
233 sleep ${pretendDelay}
235 touch AliESDfriends.root
236 touch AliESDfriends_v1.root
240 echo ./runCPass0.sh "${infile}" "${nEvents}" "${runNumber}" "${ocdbPath}" "${recoTriggerOptions}"
241 ./runCPass0.sh "${infile}" "${nEvents}" "${runNumber}" "${ocdbPath}" "${recoTriggerOptions}"
244 #move stuff to final destination
245 echo "this directory (${PWD}) contents:"
249 echo rm -f ./${chunkName}
251 echo "cp -R ${runpath}/* ${outputDir}"
252 cp -p -R ${runpath}/* ${outputDir}
258 echo "dir ${outputDir}" >> ${doneFile}
259 if summarizeLogs >> ${doneFile}; then
260 [[ -f ${outputDirMC}/galice.root ]] && echo "sim ${outputDirMC}/galice.root" >> ${doneFile}
261 [[ -f AliESDfriends_v1.root ]] && echo "calibfile ${outputDir}/AliESDfriends_v1.root" >> ${doneFile}
262 [[ -f AliESDs.root ]] && echo "esd ${outputDir}/AliESDs.root" >> ${doneFile}
265 [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath} && echo "removing ${runpath}"
282 if ! parseConfig ${configFile} "$@"; then return 1; fi
284 #record the working directory provided by the batch system
285 batchWorkingDirectory=${PWD}
287 #use the jobindex only if set and non-negative
288 if [[ -z ${jobindex} || ${jobindex} -lt 0 ]]; then
289 [[ -n "${LSB_JOBINDEX}" ]] && jobindex=${LSB_JOBINDEX}
290 [[ -n "${SGE_TASK_ID}" ]] && jobindex=${SGE_TASK_ID}
291 if [[ -z ${jobindex} ]]; then
297 [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
298 doneFile="${commonOutputPath}/meta/cpass1.job${jobindex}.run${runNumber}.done"
299 [[ -n ${useProfilingCommand} ]] && doneFile="${commonOutputPath}/meta/profiling.cpass1.job${jobindex}.run${runNumber}.done"
301 [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
303 if [[ -n ${ALIROOT_FORCE_COREDUMP} ]]; then
305 export ALIROOT_FORCE_COREDUMP
308 #the contents of this is stored in the tree and used later (e.g. AliAnalysisTaskPIDResponse)!
309 #at the QA stage the pass number is guessed from the path stored here.
311 #Packages= ;OutputDir= ;LPMPass= ;TriggerAlias= ;LPMRunNumber= ;LPMProductionType= ;LPMInteractionType= ;LPMProductionTag= ;LPMAnchorRun= ;LPMAnchorProduction= ;LPMAnchorYear=
312 export PRODUCTION_METADATA="OutputDir=cpass1"
314 [[ ! -f ${inputList} && -z ${pretend} ]] && echo "input file ${inputList} not found, exiting..." && touch ${doneFile} && return 1
315 if [[ "${inputList}" =~ \.root$ ]]; then
318 infile=$(sed -ne "${jobindex}p" ${inputList} | egrep '\s*\w*/\w*')
320 chunkName=${infile##*/}
322 outputDir=${targetDirectory}/${jobindex}_${chunkName%.*}
323 mkdir -p ${outputDir}
324 [[ ! -d ${outputDir} ]] && echo "cannot make ${outputDir}" && touch ${doneFile} && return 1
326 #runpath=${PWD}/rundir_cpass1_${runNumber}_${jobindex}
328 #[[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
329 [[ ${reconstructInTemporaryDir} -eq 1 ]] && runpath=$(mktemp -d -t cpass1.XXXXXX)
332 if [[ "${infile}" =~ galice\.root ]]; then
333 ln -s ${inputList%/*}/* ${runpath}
337 #init the running path
339 [[ ! -d ${runpath} ]] && echo "cannot make runpath ${runpath}" && touch ${doneFile} && return 1
340 if ! cd ${runpath}; then
341 echo "PWD=$PWD is not the runpath=${runpath}"
346 #this is needed for runCPass1.sh
347 ln -s ${infile} ${runpath}/${chunkName}
349 logOutputDir=${runpath}
350 [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir}
351 [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/stdout
352 #[[ -z ${dontRedirectStdOutToLog} ]] && exec 2> ${logOutputDir}/stderr
355 echo "#####################"
358 echo nEvents ${nEvents}
359 echo runNumber ${runNumber}
360 echo ocdbPath ${ocdbPath}
361 echo infile ${infile}
362 echo chunkName ${chunkName}
363 echo jobindex ${jobindex}
364 echo recoTriggerOptions ${recoTriggerOptions}
365 echo targetDirectory ${targetDirectory}
366 echo commonOutputPath ${commonOutputPath}
367 echo doneFile ${doneFile}
368 echo runpath ${runpath}
369 echo outputDir ${outputDir}
370 echo batchWorkingDirectory ${batchWorkingDirectory}
371 echo ALICE_ROOT ${ALICE_ROOT}
373 echo "########## ###########"
375 alirootInfo > ALICE_ROOT.log
378 "${batchWorkingDirectory}/runCPass1.sh"
379 "${batchWorkingDirectory}/recCPass1.C"
380 "${batchWorkingDirectory}/recCPass1_OuterDet.C"
381 "${batchWorkingDirectory}/runCalibTrain.C"
382 "${batchWorkingDirectory}/QAtrain_duo.C"
383 "${batchWorkingDirectory}/localOCDBaccessConfig.C"
384 "${batchWorkingDirectory}/${configFile}"
385 "${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz"
386 "${batchWorkingDirectory}/OCDB.root"
387 "${trustedQAtrainMacro}"
388 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/runCPass1.sh"
389 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/recCPass1.C"
390 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/recCPass1_OuterDet.C"
391 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/runCalibTrain.C"
392 "${ALICE_ROOT}/ANALYSIS/macros/QAtrain_duo.C"
395 for file in "${filesCPass1[@]}"; do
396 [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
397 [[ ${file##*/} =~ .*\.sh ]] && echo "making ${file##*/} executable" && chmod +x ${file##*/}
400 echo "this directory (${PWD}) contents:"
404 #remove spaces around commas from calls to root
405 sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh
407 if [[ -n ${postSetUpActionCPass1} ]]; then
408 echo "running ${postSetUpActionCPass1}"
409 eval ${postSetUpActionCPass1}
413 #configure local OCDB storage from CPass0 (creates the localOCDBaccessConfig.C script)
414 if [[ -f cpass0.localOCDB.${runNumber}.tgz ]]; then
415 echo goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz"
416 goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz"
418 echo "WARNING: file cpass0.localOCDB.${runNumber}.tgz not found!"
421 if [[ ! $(/bin/ls -1 OCDB/*/*/*/*.root 2>/dev/null) ]]; then
422 echo "cpass0 produced no calibration! exiting..."
427 #create the Barrel and OuterDet directories for CPass1 and link the local OCDB directory
428 #there to make the localOCDBaccessConfig.C file work, since it may point to the OCDB
429 #entries using a relative path, e.g. local://./OCDB
430 echo "linking the OCDB/ for Barrel and OuterDet directories"
431 mkdir Barrel OuterDet
433 ln -s ../OCDB Barrel/OCDB
434 ln -s ../OCDB OuterDet/OCDB
437 #the following option enables the filtering task inside the QAtrain_duo.C
438 [[ -n $runESDfiltering ]] && export QA_TaskFilteredTree=1
439 #set the downscaling factors during the filtering fro expert QA (overrides the previous values)
440 if [[ -n ${filteringFactorHighPt} ]]; then
441 export AliAnalysisTaskFilteredTree_fLowPtTrackDownscaligF=${filteringFactorHighPt}
443 if [[ -n ${filteringFactorV0s} ]]; then
444 export AliAnalysisTaskFilteredTree_fLowPtV0DownscaligF=${filteringFactorV0s}
448 chmod u+x runCPass1.sh
449 echo "${runpath}/runCPass1.sh ${infile} ${nEvents} ${runNumber} ${ocdbPath} ${recoTriggerOptions}"
450 if [[ -n ${pretend} ]]; then
451 sleep ${pretendDelay}
452 touch AliESDs_Barrel.root
453 touch AliESDfriends_Barrel.root
454 touch AliESDfriends_v1.root
455 touch QAresults_barrel.root
456 touch EventStat_temp_barrel.root
458 touch AliESDs_Outer.root
459 touch AliESDfriends_Outer.root
460 touch QAresults_outer.root
461 touch EventStat_temp_outer.root
465 touch filtering.log FilterEvents_Trees.root
467 ./runCPass1.sh "${infile}" "${nEvents}" "${runNumber}" "${ocdbPath}" "${recoTriggerOptions}"
469 [[ ! -f AliESDs_Barrel.root && -f Barrel/AliESDs.root ]] && mv Barrel/AliESDs.root AliESDs_Barrel.root
470 [[ ! -f AliESDfriends_Barrel.root && -f Barrel/AliESDfriends.root ]] && mv Barrel/AliESDfriends.root AliESDfriends_Barrel.root
471 [[ ! -f AliESDfriends_v1.root && -f Barrel/AliESDfriends_v1.root ]] && mv Barrel/AliESDfriends_v1.root .
472 [[ ! -f QAresults_barrel.root && -f Barrel/QAresults_barrel.root ]] && mv Barrel/QAresults_barrel.root .
473 [[ ! -f AliESDs_Outer.root && -f OuterDet/AliESDs.root ]] && mv OuterDet/AliESDs.root AliESDs_Outer.root
474 [[ ! -f AliESDfriends_Outer.root && -f OuterDet/AliESDfriends.root ]] && mv OuterDet/AliESDfriends.root AliESDfriends_Outer.root
475 [[ ! -f QAresults_outer.root && -f OuterDet/QAresults_outer.root ]] && mv OuterDet/QAresults_outer.root .
476 [[ ! -f FilterEvents_Trees.root && -f Barrel/FilterEvents_Trees.root ]] && mv Barrel/FilterEvents_Trees.root .
478 #make the filtered tree (if requested and not already produced by QA
479 [[ -f AliESDs_Barrel.root ]] && echo "AliESDs_Barrel.root" > filtered.list
480 if [[ -n ${runESDfiltering} && ! -f FilterEvents_Trees.root && -f filtered.list ]]; then
481 goMakeFilteredTrees ${PWD} ${runNumber} "${PWD}/filtered.list" ${filteringFactorHighPt} ${filteringFactorV0s} ${ocdbPath} 1000000 0 10000000 0 ${configFile} AliESDs_Barrel.root "${extraOpts[@]}" >filtering.log
488 ##handle possible crashes in QA (happens often in trunk)
489 ##rerun QA with a trusted aliroot version
490 #if [[ $(validateLog qa_barrel.log) ]]; then
491 # echo "qa_barrel.log not validated!"
493 #if [[ ! -f QAresults_barrel.root && -f ${setupTrustedAliROOTenvInCurrentShell} || $(validateLog qa_barrel.log) ]]; then
494 # echo "WARNING: using trusted QA aliroot ${ALICE_ROOT}"
495 # source ${setupTrustedAliROOTenvInCurrentShell}
497 # rm QAresults_barrel.root
498 # rm EventStat_temp_barrel.root
500 # [[ ! -f AliESDs.root ]] && ln -s ../AliESDs_Barrel.root AliESDs.root
501 # [[ ! -f AliESDfriends.root ]] && ln -s ../AliESDfriends_Barrel.root AliESDfriends.root
502 # if [[ -n ${trustedQAtrainMacro} ]]; then
503 # eval "cp ${trustedQAtrainMacro} QAtrain_duo_trusted.C"
505 # echo executing aliroot -b -q "QAtrain_duo_trusted.C(\"_barrel\",${runNumber},\"wn.xml\",0,\"${ocdbPath}\")"
506 # time aliroot -b -q "QAtrain_duo.C(\"_barrel\",${runNumber},\"wn.xml\",0,\"${ocdbPath}\")" &> ../qa_barrel_trusted.log
510 #move stuff to final destination
511 echo "this directory (${PWD}) contents:"
513 echo rm -f ./${chunkName}
515 echo "cp -R ${runpath}/* ${outputDir}"
516 cp -pf -R ${runpath}/* ${outputDir}
522 echo "dir ${outputDir}" >> ${doneFile}
523 if summarizeLogs >> ${doneFile}; then
524 [[ -f AliESDs_Barrel.root ]] && echo "esd ${outputDir}/AliESDs_Barrel.root" >> ${doneFile}
525 [[ -f AliESDfriends_v1.root ]] && echo "calibfile ${outputDir}/AliESDfriends_v1.root" >> ${doneFile}
526 [[ -f QAresults_Barrel.root ]] && echo "qafile ${outputDir}/QAresults_Barrel.root" >> ${doneFile}
527 [[ -f QAresults_Outer.root ]] && echo "qafile ${outputDir}/QAresults_Outer.root" >> ${doneFile}
528 [[ -f QAresults_barrel.root ]] && echo "qafile ${outputDir}/QAresults_barrel.root" >> ${doneFile}
529 [[ -f QAresults_outer.root ]] && echo "qafile ${outputDir}/QAresults_outer.root" >> ${doneFile}
530 [[ -f FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFile}
532 if grep "qa_outer.log.*OK" ${doneFile} > /dev/null; then
533 [[ -f QAresults_Outer.root ]] && echo "qafile ${outputDir}/QAresults_Outer.root" >> ${doneFile}
534 [[ -f QAresults_outer.root ]] && echo "qafile ${outputDir}/QAresults_outer.root" >> ${doneFile}
536 if grep "qa_barrel.log.*OK" ${doneFile} > /dev/null; then
537 [[ -f QAresults_Barrel.root ]] && echo "qafile ${outputDir}/QAresults_Barrel.root" >> ${doneFile}
538 [[ -f QAresults_barrel.root ]] && echo "qafile ${outputDir}/QAresults_barrel.root" >> ${doneFile}
540 if grep "filtering.log.*OK" ${doneFile} > /dev/null; then
541 [[ -f FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFile}
545 [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
553 # find the output files and merge them
560 calibrationFilesToMerge=${5} #can be a non-existent file, will then be produced on the fly
562 if ! parseConfig ${configFile} "$@"; then return 1; fi
564 #record the working directory provided by the batch system
565 batchWorkingDirectory=${PWD}
567 [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
568 doneFile="${commonOutputPath}/meta/merge.cpass0.run${runNumber}.done"
573 [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
575 #runpath=${PWD}/rundir_cpass0_Merge_${runNumber}
577 #[[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
578 [[ ${reconstructInTemporaryDir} -eq 1 ]] && runpath=$(mktemp -d -t mergeCPass0.XXXXXX)
581 [[ ! -d ${runpath} ]] && echo "not able to make the runpath ${runpath}" && touch ${doneFile} && return 1
582 if ! cd ${runpath}; then
583 echo "PWD=$PWD is not the runpath=${runpath}"
588 logOutputDir=${runpath}
589 [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir}
590 [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/mergeMakeOCDB.log
593 mergingScript="mergeMakeOCDB.byComponent.sh"
595 echo goMergeCPass0 SETUP:
596 echo runNumber=${runNumber}
597 echo outputDir=${outputDir}
598 echo ocdbStorage=${ocdbStorage}
599 echo calibrationFilesToMerge=${calibrationFilesToMerge}
600 echo mergingScript=${mergingScript}
601 echo commonOutputPath=${commonOutputPath}
602 echo runpath=${runpath}
604 # copy files in case they are not already there
606 "${batchWorkingDirectory}/${calibrationFilesToMerge}"
607 "${batchWorkingDirectory}/OCDB.root"
608 "${batchWorkingDirectory}/localOCDBaccessConfig.C"
609 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/mergeMakeOCDB.byComponent.sh"
610 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/mergeByComponent.C"
611 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/makeOCDB.C"
612 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/merge.C"
613 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/mergeMakeOCDB.sh"
615 for file in ${filesMergeCPass0[*]}; do
616 [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
617 [[ ${file##*/} =~ .*\.sh ]] && chmod +x ${file##*/}
620 sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh
622 alirootInfo > ALICE_ROOT.log
632 chmod u+x ${mergingScript}
634 if [[ ! -f ${calibrationFilesToMerge} ]]; then
635 echo "/bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root > ${calibrationFilesToMerge}"
636 /bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root 2>/dev/null > ${calibrationFilesToMerge}
639 echo "${mergingScript} ${calibrationFilesToMerge} ${runNumber} local://./OCDB ${ocdbStorage}"
640 if [[ -n ${pretend} ]]; then
641 sleep ${pretendDelay}
642 touch CalibObjects.root
646 mkdir -p ./OCDB/TPC/Calib/TimeGain/
647 mkdir -p ./OCDB/TPC/Calib/TimeDrift/
648 touch ./OCDB/TPC/Calib/TimeGain/someCalibObject_0-999999_cpass0.root
649 touch ./OCDB/TPC/Calib/TimeDrift/otherCalibObject_0-999999_cpass0.root
651 ./${mergingScript} ${calibrationFilesToMerge} ${runNumber} "local://./OCDB" ${ocdbStorage}
653 #produce the calib trees for expert QA (dcsTime.root)
654 goMakeLocalOCDBaccessConfig ./OCDB
655 echo aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")"
656 aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")"
659 ### produce the output
660 #tar the produced OCDB for reuse
661 tar czf ${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz ./OCDB
665 #copy all to output dir
666 cp -pf -R ${runpath}/* ${outputDir}
668 if [[ -n ${generateMC} ]]; then
669 goPrintValues sim ${commonOutputPath}/meta/sim.run${runNumber}.list ${commonOutputPath}/meta/cpass0.job*.run${runNumber}.done
672 #validate merging cpass0
675 echo "dir ${outputDir}" >> ${doneFile}
676 if summarizeLogs >> ${doneFile}; then
677 [[ -f CalibObjects.root ]] && echo "calibfile ${outputDir}/CalibObjects.root" >> ${doneFile}
678 [[ -f dcsTime.root ]] && echo "dcsTree ${outputDir}/dcsTime.root" >> ${doneFile}
681 [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
688 # find the output files and merge them
695 calibrationFilesToMerge=${5}
697 filteredFilesToMerge=${7}
699 if ! parseConfig ${configFile} "$@"; then return 1; fi
701 #record the working directory provided by the batch system
702 batchWorkingDirectory=${PWD}
704 [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
705 doneFile="${commonOutputPath}/meta/merge.cpass1.run${runNumber}.done"
711 rm -f ${outputDir}/*.log
712 rm -f ${outputDir}/*.root
713 rm -f ${outputDir}/*done
715 [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
717 #runpath=${PWD}/rundir_cpass1_Merge_${runNumber}
719 #[[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
720 [[ ${reconstructInTemporaryDir} -eq 1 ]] && runpath=$(mktemp -d -t mergeCPass1.XXXXXX)
723 [[ ! -d ${runpath} ]] && echo "not able to make the runpath ${runpath}" && touch ${doneFile} && return 1
724 if ! cd ${runpath}; then
725 echo "PWD=$PWD is not the runpath=${runpath}"
730 logOutputDir=${runpath}
731 [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir}
732 [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/mergeMakeOCDB.log
735 calibrationOutputFileName='AliESDfriends_v1.root'
736 qaOutputFileName='QAresults*.root'
737 mergingScript="mergeMakeOCDB.byComponent.sh"
738 #important to have the string "Stage.txt" in the filename to trigger the merging
739 #it has to be a list of directories containing the files
740 qaMergedOutputFileName="QAresults_merged.root"
742 echo goMergeCPass1 SETUP:
743 echo runNumber=${runNumber}
744 echo outputDir=${outputDir}
745 echo ocdbStorage=${ocdbStorage}
746 echo calibrationFilesToMerge=$calibrationFilesToMerge
747 echo qaFilesToMerge=$qaFilesToMerge
748 echo calibrationOutputFileName=${calibrationOutputFileName}
749 echo mergingScript=${mergingScript}
751 # copy files in case they are not already there
753 "${batchWorkingDirectory}/${calibrationFilesToMerge}"
754 "${batchWorkingDirectory}/${qaFilesToMerge}"
755 "${batchWorkingDirectory}/OCDB.root"
756 "${batchWorkingDirectory}/localOCDBaccessConfig.C"
757 "${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz"
758 "${batchWorkingDirectory}/QAtrain_duo.C"
759 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/mergeMakeOCDB.byComponent.sh"
760 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/mergeByComponent.C"
761 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/makeOCDB.C"
762 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/merge.C"
763 "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/mergeMakeOCDB.sh"
764 "${trustedQAtrainMacro}"
765 "${ALICE_ROOT}/ANALYSIS/macros/QAtrain_duo.C"
767 for file in ${filesMergeCPass1[*]}; do
768 [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
769 [[ ${file##*/} =~ .*\.sh ]] && chmod +x ${file##*/}
772 sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh
774 #configure local OCDB storage from CPass0 (creates the localOCDBaccessConfig.C script)
775 if [[ -f cpass0.localOCDB.${runNumber}.tgz ]]; then
776 echo goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz"
777 goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz"
779 echo "WARNING: file cpass0.localOCDB.${runNumber}.tgz not found!"
782 alirootInfo > ALICE_ROOT.log
788 chmod u+x ${mergingScript}
791 #if not provided, create the lists of files to merge
792 if [[ ! -f ${filteredFilesToMerge} ]]; then
793 echo "/bin/ls -1 ${outputDir}/*/FilterEvents_Trees.root > ${filteredFilesToMerge}"
794 /bin/ls -1 ${outputDir}/*/FilterEvents_Trees.root 2>/dev/null > ${filteredFilesToMerge}
796 if [[ ! -f ${calibrationFilesToMerge} ]]; then
797 echo "/bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root > ${calibrationFilesToMerge}"
798 /bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root 2>/dev/null > ${calibrationFilesToMerge}
800 if [[ ! -f ${qaFilesToMerge} ]]; then
801 #find the files, but only store the directories (QAtrain_duo.C requires this)
802 echo "/bin/ls -1 ${outputDir}/*/QAresults*.root | while read x; do echo ${x%/*}; done | sort | uniq > ${qaFilesToMerge}"
803 /bin/ls -1 ${outputDir}/*/QAresults*.root | while read x; do echo ${x%/*}; done | sort | uniq > ${qaFilesToMerge}
806 echo "${mergingScript} ${calibrationFilesToMerge} ${runNumber} local://./OCDB ${ocdbStorage}"
807 if [[ -n ${pretend} ]]; then
808 sleep ${pretendDelay}
810 touch cpass1.localOCDB.${runNumber}.tgz
811 touch ${qaMergedOutputFileName}
814 touch FilterEvents_Trees.root
815 touch CalibObjects.root
817 touch ${qaMergedOutputFileName}
820 ./${mergingScript} ${calibrationFilesToMerge} ${runNumber} "local://./OCDB" ${ocdbStorage}
822 #merge QA (and filtered trees)
823 [[ -n ${AliAnalysisTaskFilteredTree_fLowPtTrackDownscaligF} ]] && export AliAnalysisTaskFilteredTree_fLowPtTrackDownscaligF
824 [[ -n ${AliAnalysisTaskFilteredTree_fLowPtV0DownscaligF} ]] && export AliAnalysisTaskFilteredTree_fLowPtV0DownscaligF
826 #echo aliroot -l -b -q "merge.C(\"${qaFilesToMerge}\",\"\",kFALSE,\"${qaMergedOutputFileName}\")"
827 echo aliroot -b -q "QAtrain_duo.C(\"_barrel\",${runNumber},\"${qaFilesToMerge}\",1,\"${ocdbStorage}\")"
828 #aliroot -l -b -q "merge.C(\"${qaFilesToMerge}\",\"\",kFALSE,\"${qaMergedOutputFileName}\")"
829 aliroot -b -q "QAtrain_duo.C(\"_barrel\",${runNumber},\"${qaFilesToMerge}\",1,\"${ocdbStorage}\")" > mergeQA.log
830 mv QAresults_barrel.root ${qaMergedOutputFileName}
831 mv trending_barrel.root trending.root
833 #merge filtered trees
834 echo aliroot -l -b -q "merge.C(\"${qaFilesToMerge}\",\"\",kFALSE,\"${qaMergedOutputFileName}\")"
835 aliroot -l -b -q "merge.C(\"${filteredFilesToMerge}\",\"\",kFALSE,\"FilterEvents_Trees.root\")" > mergeFilteredTrees.log
837 #produce the calib trees for expert QA
838 echo aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")"
839 aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")" > calibTree.log
842 tar czf ${commonOutputPath}/meta/cpass1.localOCDB.${runNumber}.tgz ./OCDB
846 #copy all to output dir
847 cp -pf -R ${runpath}/* ${outputDir}
849 #validate merge cpass1
852 echo "dir ${outputDir}" >> ${doneFile}
853 if summarizeLogs >> ${doneFile}; then
854 [[ -f CalibObjects.root ]] && echo "calibfile ${outputDir}/CalibObjects.root" >> ${doneFile}
855 [[ -f ${qaMergedOutputFileName} ]] && echo "qafile ${outputDir}/${qaMergedOutputFileName}" >> ${doneFile}
856 [[ -f trending.root ]] && echo "trendingfile ${outputDir}/trending.root" >> ${doneFile}
857 [[ -f dcsTime.root ]] && echo "dcsTree ${outputDir}/dcsTime.root" >> ${doneFile}
858 [[ -f FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFile}
860 if grep "mergeQA.log.*OK" ${doneFile} > /dev/null; then
861 [[ -f ${qaMergedOutputFileName} ]] && echo "qafile ${outputDir}/${qaMergedOutputFileName}" >> ${doneFile}
863 if grep "mergeFilteredTrees.log.*OK" ${doneFile} > /dev/null; then
864 [[ -f FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFile}
868 [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
874 #generic root merge using CPass1 merge.C script
877 configFile=${3-"benchmark.config"}
879 if ! parseConfig ${configFile} "$@"; then return 1; fi
881 #record the working directory provided by the batch system
882 batchWorkingDirectory=${PWD}
884 [[ ! -f ${inputList} ]] && echo "inputList ${inputList} does not exist!" && return 1
885 [[ ! -f ${configFile} ]] && echo "configFile ${configFile} does not exist!" && return 1
887 [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
889 aliroot -b -q "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/merge.C(\"${inputList}\",\"\",kFALSE,\"${outputFile}\")" > merge_${inputList}.log
901 if ! parseConfig ${configFile} "${extraOpts[@]}"; then return 1; fi
903 #record the working directory provided by the batch system
904 batchWorkingDirectory=${PWD}
906 [[ -z ${configFile} ]] && configFile="benchmark.config"
907 [[ ! -f ${configFile} ]] && echo "no config file found (${configFile})" && return 1
909 if [[ ! $(which makeflow &>/dev/null) && -n ${makeflowPath} ]]; then
910 echo "setting the makflow path from the config: "
911 echo " export PATH=${makeflowPath}:${PATH}"
912 export PATH=${makeflowPath}:${PATH}
915 #create the common output dir and the meta dir
916 commonOutputPath=${baseOutputDirectory}/${productionID}
917 if [[ -d ${commonOutputPath} ]]; then
918 echo "output dir ${commonOutputPath} exists!"
921 mkdir -p ${commonOutputPath}
923 mkdir -p ${commonOutputPath}/meta
926 #if which greadlink; then self=$(greadlink -f "${0}"); fi
928 #for reference copy the setup to the output dir
929 cp ${self} ${commonOutputPath}
930 cp ${configFile} ${commonOutputPath}
931 cp ${inputList} ${commonOutputPath}
933 #submit - use makeflow if available, fall back to old stuff when makeflow not there
934 if which makeflow; then
936 goGenerateMakeflow ${productionID} ${inputFileList} ${configFile} "${extraOpts[@]}" commonOutputPath=${commonOutputPath} > benchmark.makeflow
938 makeflow ${makeflowOptions} benchmark.makeflow
948 #generate the makeflow file
949 [[ $# -lt 3 ]] && echo "args: id inputFileList configFile" && return 1
956 #batch systems/makeflow sometimes handle spaces in arguments poorly, so encode them
957 for (( i=0;i<${#extraOpts[@]};i++ )); do
958 extraOpts[i]=$(encSpaces "${extraOpts[i]}")
960 extraOpts+=("encodedSpaces=1")
962 if ! parseConfig ${configFile} "${extraOpts[@]}" &>/dev/null; then return 1; fi
965 if [[ -z ${commonOutputPath} ]]; then
966 commonOutputPath=${baseOutputDirectory}/${productionID}
967 extraOpts=( "${extraOpts[@]}" "commonOutputPath=${commonOutputPath}" )
970 #record the working directory provided by the batch system
971 batchWorkingDirectory=${PWD}
973 [[ -z ${configFile} ]] && configFile="benchmark.config"
974 [[ ! -f ${configFile} ]] && echo "no config file found (${configFile})" && return 1
976 #these files will be made a dependency - will be copied to the working dir of the jobs
980 "localOCDBaccessConfig.C"
984 "recCPass1_OuterDet.C"
989 for file in ${inputFiles[*]}; do
990 [[ -f ${file} ]] && copyFiles+=("${file}")
993 #create the makeflow file
994 [[ -n ${batchFlags} ]] && echo "BATCH_OPTIONS = ${batchFlags}"
995 declare -A arr_cpass0_merged arr_cpass1_merged
996 declare -A arr_cpass0_calib_list arr_cpass1_calib_list
997 declare -A arr_cpass1_QA_list arr_cpass1_ESD_list arr_cpass1_filtered_list
998 declare -A arr_cpass0_profiled_outputs
999 declare -A listOfRuns
1000 [[ -n ${runNumber} ]] && listOfRuns[${runNumber}]=1
1001 while read x; do tmpRun=$(guessRunNumber ${x}); [[ -n ${tmpRun} ]] && listOfRuns[${tmpRun}]=1; done < ${inputFileList}
1002 for runNumber in "${!listOfRuns[@]}"; do
1003 [[ -z ${runNumber} ]] && continue
1004 [[ ! ${runNumber} =~ ^[0-9]*[0-9]$ ]] && continue
1006 unset arr_cpass0_outputs
1007 unset arr_cpass1_outputs
1008 declare -a arr_cpass0_outputs
1009 declare -a arr_cpass1_outputs
1013 while read inputFile; do
1014 currentDefaultOCDB=${defaultOCDB}
1015 [[ -z ${autoOCDB} ]] && autoOCDB=1
1016 if [[ ${autoOCDB} -ne 0 ]]; then
1017 currentDefaultOCDB=$(setYear ${inputFile} ${defaultOCDB})
1021 arr_cpass0_outputs[${jobindex}]="${commonOutputPath}/meta/cpass0.job${jobindex}.run${runNumber}.done"
1022 echo "${arr_cpass0_outputs[${jobindex}]} : benchmark.sh ${configFile} ${copyFiles[@]}"
1023 echo " ${alirootEnv} ./benchmark.sh CPass0 ${commonOutputPath}/000${runNumber}/cpass0 ${inputFile} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} ${jobindex} ${extraOpts[@]}"" "
1026 arr_cpass1_outputs[${jobindex}]="${commonOutputPath}/meta/cpass1.job${jobindex}.run${runNumber}.done"
1027 echo "${arr_cpass1_outputs[${jobindex}]} : benchmark.sh ${configFile} ${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz ${copyFiles[@]}"
1028 echo " ${alirootEnv} ./benchmark.sh CPass1 ${commonOutputPath}/000${runNumber}/cpass1 ${inputFile} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} ${jobindex} ${extraOpts[@]}"" "
1031 done< <(grep "/000${runNumber}/" ${inputFileList})
1033 #CPass0 list of Calib files to merge
1034 arr_cpass0_calib_list[${runNumber}]="${commonOutputPath}/meta/cpass0.calib.run${runNumber}.list"
1035 echo "${arr_cpass0_calib_list[${runNumber}]} : benchmark.sh ${arr_cpass0_outputs[*]}"
1036 echo " ./benchmark.sh PrintValues calibfile ${arr_cpass0_calib_list[${runNumber}]} ${arr_cpass0_outputs[*]}"
1040 arr_cpass0_merged[${runNumber}]="${commonOutputPath}/meta/merge.cpass0.run${runNumber}.done"
1041 echo "${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz ${arr_cpass0_merged[${runNumber}]} : benchmark.sh ${configFile} ${arr_cpass0_calib_list[${runNumber}]} ${copyFiles[@]}"
1042 echo " ${alirootEnv} ./benchmark.sh MergeCPass0 ${commonOutputPath}/000${runNumber}/cpass0 ${currentDefaultOCDB} ${configFile} ${runNumber} ${arr_cpass0_calib_list[${runNumber}]} ${extraOpts[@]}"" "
1044 #CPass1 list of Calib/QA/ESD/filtered files
1045 # the trick with QA is to have the string "Stage.txt" in the file name of the list of directories with QA output to trigger
1046 # the production of the QA trending tree (only then the task->Finish() will be called in QAtrain_duo.C, on the grid
1047 # this corresponds to the last merging stage)
1048 arr_cpass1_QA_list[${runNumber}]="${commonOutputPath}/meta/cpass1.QA.run${runNumber}.lastMergingStage.txt.list"
1049 echo "${arr_cpass1_QA_list[${runNumber}]}: benchmark.sh ${arr_cpass1_outputs[*]}"
1050 echo " ./benchmark.sh PrintValues dir ${arr_cpass1_QA_list[${runNumber}]} ${arr_cpass1_outputs[*]}"
1052 arr_cpass1_calib_list[${runNumber}]="${commonOutputPath}/meta/cpass1.calib.run${runNumber}.list"
1053 echo "${arr_cpass1_calib_list[${runNumber}]} : benchmark.sh ${arr_cpass1_outputs[*]}"
1054 echo " ./benchmark.sh PrintValues calibfile ${arr_cpass1_calib_list[${runNumber}]} ${arr_cpass1_outputs[*]};"
1056 arr_cpass1_ESD_list[${runNumber}]="${commonOutputPath}/meta/cpass1.ESD.run${runNumber}.list"
1057 echo "${arr_cpass1_ESD_list[${runNumber}]} : benchmark.sh ${arr_cpass1_outputs[*]}"
1058 echo " ./benchmark.sh PrintValues esd ${arr_cpass1_ESD_list[${runNumber}]} ${arr_cpass1_outputs[*]}"
1060 arr_cpass1_filtered_list[${runNumber}]="${commonOutputPath}/meta/cpass1.filtered.run${runNumber}.list"
1061 echo "${arr_cpass1_filtered_list[${runNumber}]} : benchmark.sh ${arr_cpass1_outputs[*]}"
1062 echo " ./benchmark.sh PrintValues filteredTree ${arr_cpass1_filtered_list[${runNumber}]} ${arr_cpass1_outputs[*]}"
1066 arr_cpass1_merged[${runNumber}]="${commonOutputPath}/meta/merge.cpass1.run${runNumber}.done"
1067 echo "${commonOutputPath}/meta/cpass1.localOCDB.${runNumber}.tgz ${arr_cpass1_merged[${runNumber}]} : benchmark.sh ${configFile} ${arr_cpass1_calib_list[${runNumber}]} ${arr_cpass1_QA_list[${runNumber}]} ${copyFiles[@]}"
1068 echo " ${alirootEnv} ./benchmark.sh MergeCPass1 ${commonOutputPath}/000${runNumber}/cpass1 ${currentDefaultOCDB} ${configFile} ${runNumber} ${arr_cpass1_calib_list[${runNumber}]} ${arr_cpass1_QA_list[${runNumber}]} ${arr_cpass1_filtered_list[${runNumber}]} ${extraOpts[@]}"" "
1070 #CPass0 wrapped in a profiling tool (valgrind,....)
1071 if [[ -n ${profilingCommand} ]]; then
1072 inputFile=$(grep -m1 "${runNumber}/" ${inputFileList})
1073 [[ -z ${nEventsProfiling} ]] && nEventsProfiling=2
1074 currentDefaultOCDB=$(setYear ${inputFile} ${defaultOCDB})
1075 jobindex="profiling"
1077 arr_cpass0_profiled_outputs[${runNumber}]="${commonOutputPath}/meta/cpass0.job${jobindex}.run${runNumber}.done"
1078 echo "${arr_cpass0_profiled_outputs[${runNumber}]} : benchmark.sh ${configFile} ${copyFiles[@]}"
1079 profilingCommand=$(encSpaces "${profilingCommand}")
1080 echo " ${alirootEnv} ./benchmark.sh CPass0 ${commonOutputPath}/000${runNumber}/${jobindex} ${inputFile} ${nEventsProfiling} ${currentDefaultOCDB} ${configFile} ${runNumber} ${jobindex} ${extraOpts[@]} useProfilingCommand=${profilingCommand}"
1086 echo "${commonOutputPath}/summary.log : benchmark.sh ${configFile} ${arr_cpass1_merged[*]}"
1087 echo " ${alirootEnv} ./benchmark.sh MakeSummary ${configFile} ${extraOpts[@]}"
1095 #print the values given the key from any number of files (store in output file on second argument)
1096 if [[ $# -lt 3 ]]; then
1097 echo "goPrintValues key outputFile inputFiles"
1098 echo "if outputFile is \"-\" dont redirect to a file"
1103 [[ ${outputFile} =~ "-" ]] && outputFile=""
1104 shift 2 #remove 2 first arguments from arg list to only pass the input files to awk
1105 awk -v key=${key} '$0 ~ key" " {print $2}' "$@" | tee ${outputFile}
1112 mergedQAfileList=${1}
1117 if ! parseConfig ${configFile} ${@}; then return 1; fi
1119 #record the working directory provided by the batch system
1120 batchWorkingDirectory=${PWD}
1122 [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
1124 [[ -z ${logOutputDir} ]] && logOutputDir=${PWD}
1125 [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/makeQAplots.log
1129 mkdir -p ${outputDir}
1131 [[ ! "${PWD}" =~ "${outputDir}" ]] && echo "PWD is not equal to outputDir=${outputDir}" && cd ${olddir} && return 1
1133 ${ALICE_ROOT}/PWGPP/QA/scripts/runQA.sh inputList="${mergedQAfileList}" inputListHighPtTrees="${filteringList}"
1147 # save aliroot repository info
1148 [[ -z "${ALICE_ROOT}" ]] && return 1
1150 echo "\${ALICE_ROOT}=${ALICE_ROOT}"
1151 echo "\${ROOTSYS}=${ROOTSYS}"
1152 echo "\${PATH}=${PATH}"
1153 echo "\${LD_LIBRARY_PATH}=${LD_LIBRARY_PATH}"
1159 currentBranch=$(git rev-parse --abbrev-ref HEAD)
1163 git diff ${currentBranch}
1171 # ${1} - year to be set
1172 # ${2} - where to set the year
1173 year1=$(guessYear ${1})
1174 year2=$(guessYear ${2})
1176 [[ ${year1} -ne ${year2} && -n ${year2} && -n ${year1} ]] && path=${2/\/${year2}\//\/${year1}\/}
1183 #guess the period from the path, pick the rightmost one
1185 declare -a path=( ${1} )
1186 local dirDepth=${#path[*]}
1187 for ((x=${dirDepth}-1;x>=0;x--)); do
1188 local field=${path[${x}]}
1189 [[ ${field} =~ ^LHC[0-9][0-9][a-z]$ ]] && period=${field} && break
1197 #guess the year from the path, pick the rightmost one
1199 declare -a path=( ${1} )
1200 local dirDepth=${#path[*]}
1201 for ((x=${dirDepth}-1;x>=0;x--)); do
1202 local field=${path[${x}]}
1203 [[ ${field} =~ ^20[0-9][0-9]$ ]] && year=${field} && break
1211 #guess the run number from the path, pick the rightmost one
1212 #works for /path/foo/000123456/bar/...
1213 #and /path/foo.run123456.bar
1215 declare -a path=( ${1} )
1216 local dirDepth=${#path[*]}
1217 for ((x=${dirDepth}-1;x>=0;x--)); do
1218 local field=${path[${x}]}
1219 field=${field/run/000}
1220 [[ ${field} =~ [0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && runNumber=${field#000} && break
1232 'error while loading shared libraries'
1235 'Thread [0-9]* (Thread'
1238 '\.C.*error:.*\.h: No such file'
1244 'rocVoltage out of range:'
1248 local errorSummary=""
1249 local warningSummary=""
1251 for ((i=0; i<${#errorConditions[@]};i++)); do
1252 local tmp=$(grep -m1 -e "${errorConditions[${i}]}" ${log})
1253 [[ -n ${tmp} ]] && tmp+=" : "
1254 errorSummary+=${tmp}
1257 for ((i=0; i<${#warningConditions[@]};i++)); do
1258 local tmp=$(grep -m1 -e "${warningConditions[${i}]}" ${log})
1259 [[ -n ${tmp} ]] && tmp+=" : "
1260 warningSummary+=${tmp}
1263 if [[ -n ${errorSummary} ]]; then
1264 echo "${errorSummary}"
1268 if [[ -n ${warningSummary} ]]; then
1269 echo "${warningSummary}"
1278 #print a summary of logs
1287 for log in ${logFiles[*]}; do
1288 finallog=${PWD%/}/${log}
1289 [[ ! -f ${log} ]] && continue
1290 errorSummary=$(validateLog ${log})
1292 if [[ ${validationStatus} -eq 0 ]]; then
1293 #in pretend mode randomly report an error in rec.log some cases
1294 if [[ -n ${pretend} && "${log}" == "rec.log" ]]; then
1295 #[[ $(( ${RANDOM}%2 )) -ge 1 ]] && echo "${finallog} BAD random error" || echo "${finallog} OK"
1296 echo "${finallog} OK"
1298 echo "${finallog} OK"
1300 elif [[ ${validationStatus} -eq 1 ]]; then
1301 echo "${finallog} BAD ${errorSummary}"
1303 elif [[ ${validationStatus} -eq 2 ]]; then
1304 echo "${finallog} OK MWAH ${errorSummary}"
1312 gdb --batch --quiet -ex "bt" -ex "quit" aliroot ${x} > stacktrace_${x//\//_}.log
1313 done < <(/bin/ls ${PWD}/*/core 2>/dev/null; /bin/ls ${PWD}/core 2>/dev/null)
1318 spitOutLocalOCDBaccessConfig()
1321 #find ${1} -name "*root" | \
1322 /bin/ls -1 ${1}/*/*/*/*.root 2>/dev/null | \
1325 local tmp=${line#${1}}
1327 awk -v ocdb=${1} '{print " man->SetSpecificStorage(\""$1"\",\"local://"ocdb"\");"}'
1332 goMakeLocalOCDBaccessConfig()
1335 # make a script that sets the specific storages form all the root files produced by CPass0
1336 local localOCDBpathCPass0=${1}
1337 local OCDBpathPrefix=${2}
1338 [[ -z ${OCDBpathPrefix} ]] && OCDBpathPrefix="."
1340 if [[ -f ${localOCDBpathCPass0} && ${localOCDBpathCPass0} =~ \.tgz$ ]]; then
1341 tar xzf ${localOCDBpathCPass0}
1342 local localOCDBpathCPass0="${OCDBpathPrefix}/OCDB"
1346 echo creating the specific storage script
1347 echo localOCDBaccessConfig.C
1348 echo based on OCDB: ${localOCDBaccessConfig}
1351 local tempLocalOCDB=""
1352 if [[ -f localOCDBaccessConfig.C ]]; then
1353 tempLocalOCDB=$(mktemp -t tempLocalOCDB.XXXXXX)
1354 echo "egrep "SetSpecificStorage" localOCDBaccessConfig.C > ${tempLocalOCDB}"
1355 egrep "SetSpecificStorage" localOCDBaccessConfig.C > ${tempLocalOCDB}
1358 echo "localOCDBaccessConfig()" > localOCDBaccessConfig.C
1359 echo "{" >> localOCDBaccessConfig.C
1360 echo " AliCDBManager* man = AliCDBManager::Instance();" >> localOCDBaccessConfig.C
1361 spitOutLocalOCDBaccessConfig ${localOCDBpathCPass0}|sort|uniq >> localOCDBaccessConfig.C
1362 [[ -f "${tempLocalOCDB}" ]] && cat ${tempLocalOCDB} >> localOCDBaccessConfig.C
1363 echo "}" >> localOCDBaccessConfig.C
1365 [[ -f "${tempLocalOCDB}" ]] && rm -f ${tempLocalOCDB}
1367 if ! grep SetSpecificStorage localOCDBaccessConfig.C; then
1369 echo "!!!!!!! CPass0 produced no OCDB entries"
1375 goMakeFilteredTrees()
1379 #get path to input list
1381 #get scale number for tracks
1383 #get scale number for V0s
1385 #get OCDB path (optional)
1387 #get max number of files
1388 maxFiles=${7-"1000000"}
1389 #get offset of first file
1391 #get max number of events
1392 maxEvents=${9-"30000000"}
1393 #get offset of first event
1394 offsetEvent=${10-"0"}
1395 configFile=${11-"benchmark.config"}
1396 esdFileName=${12-"AliESDs_Barrel.root"}
1398 if ! parseConfig ${configFile} "$@"; then return 1; fi
1400 #record the working directory provided by the batch system
1401 batchWorkingDirectory=${PWD}
1403 [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
1404 doneFile=${commonOutputPath}/meta/filtering.cpass1.run${runNumber}.done
1406 cat > filtering.log << EOF
1407 goMakeFilteredTrees config:
1409 outputDir=${outputDir}
1410 commonOutputPath=${commonOutputPath}
1411 ALICE_ROOT=${ALICE_ROOT}
1413 offsetEvent=$offsetEvent
1414 configFile=$configFile
1415 esdFileName=$esdFileName
1416 inputListfiles=$inputListfiles
1420 #runpath=${outputDir}
1421 #[[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
1422 #[[ ${reconstructInTemporaryDir} -eq 1 ]] && runpath=$(mktemp -d -t goMakeFilteredTrees.XXXXXX)
1423 #mkdir -p ${outputDir}
1424 #mkdir -p ${runpath}
1425 #if ! cd ${runpath}; then
1426 # echo "PWD=$PWD is not the runpath=${runpath}"
1431 if [[ -z ${pretend} ]];then
1432 aliroot -l -b -q "${ALICE_ROOT}/PWGPP/macros/runFilteringTask.C(\"${inputListfiles}\",${filterT},${filterV},\"${OCDBpath}\",${maxFiles},${offsetFile},${maxEvents},${offsetEvent},\"${esdFileName}\")" &>> filtering.log
1434 sleep ${pretendDelay}
1435 touch filtering.log FilterEvents_Trees.root
1440 summarizeLogs >> ${doneFile}
1442 #echo mv -f * ${outputDir}
1443 #mv -f * ${outputDir}
1444 #[[ -f ${outputDir}/FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFile}
1445 #cd ${commonOutputPath}
1446 #[[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
1454 [[ $# -lt 5 ]] && echo "at least 5 args needed, you supplied $#" && return 1
1461 local commandArgs=("$@")
1463 #add quote strings around the extra arguments
1464 for ((i=0; i<${#commandArgs[@]}; i++)); do
1465 commandArgs[i]=\"${commandArgs[i]}\"
1468 [[ -z ${waitForJOBID} ]] && waitForJOBID=0
1470 newFarm=$(which qsub|grep "^/usr/bin/qsub")
1474 if [[ -z "${newFarm}" ]]
1477 # submit it (as job array)
1478 nFiles=$(( ${endID}-${startID}+1 ))
1479 while [ ${startID} -le ${nFiles} ] ; do
1480 if [ $(expr ${nFiles} - ${startID}) -gt 999 ] ; then
1481 endID=$(expr ${startID} + 999)
1485 if [[ ${waitForJOBID} -eq 0 ]]; then
1486 echo ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}"
1487 ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}"
1489 echo ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -w "ended(${waitForJOBID})" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}"
1490 ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -w "ended(${waitForJOBID})" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}"
1492 startID=$(expr ${endID} + 1)
1496 if [[ ${waitForJOBID} =~ "000" ]]; then
1497 echo ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}"
1498 ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}"
1500 echo ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -hold_jid "${waitForJOBID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}"
1501 ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -hold_jid "${waitForJOBID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}"
1509 if [[ $# -lt 3 ]]; then
1511 echo " ${0} submit fileList productionID configFile"
1517 configFile=${3:-"benchmark.config"}
1518 #if which greadlink; then configFile=$(greadlink -f ${configFile}); fi
1521 if ! parseConfig ${configFile} "${extraOpts[@]}"; then return 1; fi
1523 #batch systems/makeflow sometimes handle spaces in arguments poorly, so encode them
1524 for (( i=0;i<${#extraOpts[@]};i++ )); do
1525 extraOpts[i]=$(encSpaces "${extraOpts[i]}")
1527 extraOpts+=("encodedSpaces=1")
1529 #record the working directory provided by the batch system
1530 batchWorkingDirectory=${PWD}
1532 #redirect all output to submit.log
1533 echo "redirecting all output to ${PWD}/submit_${productionID//"/"/_}.log"
1535 exec 1>submit_${productionID//"/"/_}.log 2>&1
1538 echo ${0}" submit $*"
1539 if [[ -z "${inputList}" || -z "${productionID}" ]]
1542 echo " Usage: ${0} submit inputList productionID [configFile=benchmark.config]"
1547 # check if config file is there
1548 if [ ! -f ${configFile} ]; then
1549 echo "ERROR! Config File '${configFile}' not found" >&2
1552 echo "Using Config File: '${configFile}'"
1555 [[ ! -f ${alirootEnv} ]] && echo "alirootEnv script ${alirootEnv} not found!..." && return 1
1557 #move the script, config and some other stuff to ${commonOutputPath} first, then use them from there
1559 #if which greadlink; then self=$(greadlink -f "${0}"); fi
1560 configPath=$(dirname ${configFile})
1561 export commonOutputPath=${baseOutputDirectory}/${productionID}
1563 mkdir -p ${commonOutputPath}
1564 mkdir -p ${commonOutputPath}/logs
1565 mkdir -p ${commonOutputPath}/meta
1567 cp ${self} ${commonOutputPath}
1568 cp ${configFile} ${commonOutputPath}
1569 cp ${inputList} ${commonOutputPath}
1570 self=${commonOutputPath}/${self##*/}
1572 configFile=${commonOutputPath}/${configFile##*/}
1573 inputList=${commonOutputPath}/${inputList##*/}
1575 #convert to absolut pathnames
1576 #if which greadlink; then inputList=$(greadlink -f "${inputList}"); fi
1578 if [[ -z ${runNumber} ]]; then
1579 listOfRuns=($(while read x; do guessRunNumber ${x}; done < ${inputList} | sort | uniq))
1581 listOfRuns=${runNumber}
1584 #if which greadlink; then alirootSource=$(greadlink -f "${alirootSource}"); fi
1587 echo "### BEGIN CONFIGURATION ###"
1591 echo " productionID: ${productionID}"
1592 echo " batchCommand: ${batchCommand}"
1593 echo " batchFlags: ${batchFlags}"
1594 echo " alirootEnv: ${alirootEnv}"
1595 ${alirootEnv} echo ' ALICE_ROOT: ${ALICE_ROOT}'
1596 ${alirootEnv} echo ' ALIROOT_RELEASE: ${ALICE_RELEASE}'
1597 echo " inputList: ${inputList}"
1598 echo " configPath: ${configPath}"
1599 echo " commonOutputPath: ${commonOutputPath}"
1600 echo " defaultOCDB: ${defaultOCDB}"
1601 echo " autoOCDB: ${autoOCDB}"
1602 echo " recoTriggerOptions: ${recoTriggerOptions}"
1604 echo " ${listOfRuns[*]}"
1606 echo "THE TRAIN WILL RUN:"
1608 if [ ${runCPass0reco} -eq 1 ]; then
1609 echo " Pass0 - Recontruction"
1612 if [ ${runCPass0MergeMakeOCDB} -eq 1 ]; then
1613 echo " Pass0 - merging and OCDB export"
1616 if [ ${runCPass1reco} -eq 1 ]; then
1617 echo " Pass1 - Recontruction"
1619 if [ ${runCPass1MergeMakeOCDB} -eq 1 ]; then
1620 echo " Pass1 - merging and OCDB export"
1625 echo " max. Events/Chunk: ${nEvents}"
1626 echo " max. Number of Chunks per Run: ${nMaxChunks}"
1628 echo "### END CONFIGURATION ###"
1632 # check if input file is there
1633 if [ ! -f ${inputList} ]; then
1634 echo "ERROR! Input List '${inputList}' not found" >&2
1638 # define jobid (for dependent jobs)
1639 date=$(date +%Y_%m_%d_%H%M%S)
1640 #for each run we submit one jobarray:
1641 for runNumber in ${listOfRuns[*]}; do
1643 [[ -z ${runNumber} ]] && continue
1644 [[ ! ${runNumber} =~ ^[0-9]*[0-9]$ ]] && continue
1646 JOBpostfix="${productionID//"/"/_}_${runNumber}_${date}"
1647 JOBID1="p0_${JOBpostfix}"
1648 JOBID1wait="w0_${JOBpostfix}"
1649 JOBID2="m0_${JOBpostfix}"
1650 JOBID2wait="wm0_${JOBpostfix}"
1651 JOBID3="op0_${JOBpostfix}"
1652 JOBID3wait="wop0_${JOBpostfix}"
1653 JOBID4="p1_${JOBpostfix}"
1654 JOBID4wait="w1_${JOBpostfix}"
1655 JOBID5="m1_${JOBpostfix}"
1656 JOBID5wait="wm1_${JOBpostfix}"
1657 JOBID6="s1_${JOBpostfix}"
1658 JOBID6wait="ws1_${JOBpostfix}"
1659 JOBID7="QA_${JOBpostfix}"
1660 JOBmakeESDlistCPass1="lp1_${JOBpostfix}"
1661 JOBfilterESDcpass1="fp1_${JOBpostfix}"
1664 oneInputFile=$(egrep -m1 "${runNumber}/" ${inputList})
1666 currentDefaultOCDB=${defaultOCDB}
1667 [[ -z ${autoOCDB} ]] && autoOCDB=1
1668 if [[ ${autoOCDB} -ne 0 ]]; then
1669 currentDefaultOCDB=$(setYear ${oneInputFile} ${defaultOCDB})
1672 echo "submitting run ${runNumber} with OCDB ${currentDefaultOCDB}"
1674 ###############################################################################
1675 #run one chunk with valgrind:
1676 if [[ -n ${profilingCommand} ]]; then
1677 [[ -z ${nEventsProfiling} ]] && nEventsProfiling=2
1678 [[ -z ${profilingCommand} ]] && profilingCommand="/usr/bin/valgrind --tool=callgrind --num-callers=40 -v --trace-children=yes"
1679 submit "profile-${JOBpostfix}" 1 1 000 "${alirootEnv} ${self}" CPass0 ${commonOutputPath}/000${runNumber}/valgrind ${oneInputFile} ${nEventsProfiling} ${currentDefaultOCDB} ${configFile} ${runNumber} valgrind useProfilingCommand=$(encSpaces "${profilingCommand}") "${extraOpts[@]}"
1682 ################################################################################
1683 ################################################################################
1684 # run the CPass0 if requested
1686 if [ ${runCPass0reco} -eq 1 ]; then
1689 echo "starting CPass0... for run ${runNumber}"
1692 # create directory and copy all files that are needed
1693 targetDirectory="${commonOutputPath}/000${runNumber}/cpass0"
1694 mkdir -p ${targetDirectory}
1697 "${configPath}/runCPass0.sh"
1698 "${configPath}/recCPass0.C"
1699 "${configPath}/runCalibTrain.C"
1700 "${configPath}/localOCDBaccessConfig.C"
1701 "${configPath}/OCDB*.root"
1702 "${configPath}/sim.C"
1703 "${configPath}/rec.C"
1704 "${configPath}/Config.C"
1706 for file in ${filesCPass0[*]}; do
1707 [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
1710 localInputList=${targetDirectory}/${inputList##*/}
1711 [[ ! -f ${localInputList} ]] && egrep "\/000${runNumber}\/" ${inputList} > ${localInputList}
1712 # limit nFiles to nMaxChunks
1713 nFiles=$(wc -l < ${localInputList})
1714 [[ ${nFiles} -eq 0 ]] && echo "list contains ZERO files! exiting..." && return 1
1715 echo "raw files in list: ${nFiles}"
1716 if [[ ${nMaxChunks} -gt 0 && ${nMaxChunks} -le ${nFiles} ]]; then
1717 nFiles=${nMaxChunks}
1719 echo "raw files to process: ${nFiles}"
1720 [[ -z "${percentProcessedFilesToContinue}" ]] && percentProcessedFilesToContinue=100
1721 if [[ ${percentProcessedFilesToContinue} -eq 100 ]]; then
1722 nFilesToWaitFor=${nFiles}
1724 nFilesToWaitFor=$(( ${nFiles}-${nFiles}/(100/(100-${percentProcessedFilesToContinue})) ))
1726 echo "requested success rate is ${percentProcessedFilesToContinue}%"
1727 echo "merging will start after ${nFilesToWaitFor} jobs are done"
1729 submit ${JOBID1} 1 ${nFiles} 000 "${alirootEnv} ${self}" CPass0 ${targetDirectory} ${localInputList} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} -1 "${extraOpts[@]}"
1731 ## submit a monitoring job that will run until a certain number of jobs are done with reconstruction
1732 submit "${JOBID1wait}" 1 1 000 "${alirootEnv} ${self}" WaitForOutput ${commonOutputPath} "meta/cpass0.job*.run${runNumber}.done" ${nFilesToWaitFor} ${maxSecondsToWait}
1733 LASTJOB=${JOBID1wait}
1735 fi #end running CPass0
1736 ################################################################################
1739 ################################################################################
1740 # submit merging of CPass0, depends on the reconstruction
1742 if [ ${runCPass0MergeMakeOCDB} -eq 1 ]; then
1745 echo "submit CPass0 merging for run ${runNumber}"
1748 targetDirectory="${commonOutputPath}/000${runNumber}/cpass0"
1749 mkdir -p ${targetDirectory}
1753 "${configPath}/OCDB.root"
1754 "${configPath}/mergeMakeOCDB.byComponent.sh"
1755 "${configPath}/mergeMakeOCDB.sh"
1756 "${configPath}/localOCDBaccessConfig.C"
1757 "${configPath}/mergeByComponent.C"
1758 "${configPath}/makeOCDB.C"
1759 "${configPath}/merge.C"
1761 for file in ${filesMergeCPass0[*]}; do
1762 [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
1765 submit ${JOBID2} 1 1 "${LASTJOB}" "${alirootEnv} ${self}" MergeCPass0 ${targetDirectory} ${currentDefaultOCDB} ${configFile} ${runNumber} cpass0.calib.run${runNumber}.list "${extraOpts[@]}"
1768 if [[ -n ${generateMC} ]]; then
1769 submit "mrl${JOBpostfix}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" PrintValues sim ${commonOutputPath}/meta/sim.run${runNumber}.list ${commonOutputPath}/meta/cpass0.job*.run${runNumber}.done
1770 LASTJOB="mrl${JOBpostfix}"
1775 # end of merging CPass0
1776 ################################################################################
1778 ################################################################################
1779 ################################################################################
1780 # run the CPass1 if requested
1782 if [ ${runCPass1reco} -eq 1 ]; then
1784 targetDirectory="${commonOutputPath}/000${runNumber}/cpass1"
1785 rm -f ${commonOutputPath}/meta/cpass1.job*.run${runNumber}.done
1787 # safety feature: if we are re-running for any reason we want to delete the previous output first.
1788 [[ -d ${targetDirectory} ]] && rm -rf ${targetDirectory}/* && echo "removed old output at ${targetDirectory}/*"
1791 echo "starting CPass1... for run ${runNumber}"
1794 # create directory and copy all files that are needed
1795 mkdir -p ${targetDirectory}
1798 "${configPath}/runCPass1.sh"
1799 "${configPath}/recCPass1.C"
1800 "${configPath}/recCPass1_OuterDet.C"
1801 "${configPath}/runCalibTrain.C"
1802 "${configPath}/QAtrain_duo.C"
1803 "${configPath}/localOCDBaccessConfig.C"
1804 "${configPath}/OCDB.root"
1806 for file in ${filesCPass1[*]}; do
1807 [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
1810 if [[ -n ${generateMC} ]]; then
1811 localInputList=${commonOutputPath}/meta/sim.run${runNumber}.list
1813 localInputList=${targetDirectory}/${inputList##*/}
1814 [[ ! -f ${localInputList} ]] && egrep "\/000${runNumber}\/" ${inputList} > ${localInputList}
1816 # limit nFiles to nMaxChunks
1817 nFiles=$(wc -l < ${localInputList})
1818 [[ ${nFiles} -eq 0 ]] && echo "list contains ZERO files! continuing..." && continue
1819 echo "raw files in list: ${nFiles}"
1820 if [[ ${nMaxChunks} -gt 0 && ${nMaxChunks} -le ${nFiles} ]]; then
1821 nFiles=${nMaxChunks}
1823 echo "raw files to process: ${nFiles}"
1824 [[ -z "${percentProcessedFilesToContinue}" ]] && percentProcessedFilesToContinue=100
1825 if [[ ${percentProcessedFilesToContinue} -eq 100 ]]; then
1826 nFilesToWaitFor=${nFiles}
1828 nFilesToWaitFor=$(( ${nFiles}-${nFiles}/(100/(100-${percentProcessedFilesToContinue})) ))
1830 echo "requested success rate is ${percentProcessedFilesToContinue}%"
1831 echo "merging will start after ${nFilesToWaitFor} jobs are done"
1833 submit ${JOBID4} 1 ${nFiles} "${LASTJOB}" "${alirootEnv} ${self}" CPass1 ${targetDirectory} ${localInputList} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} -1 "${extraOpts[@]}"
1835 ################################################################################
1836 ## submit a monitoring job that will run until a certain number of jobs are done with reconstruction
1837 submit "${JOBID4wait}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" WaitForOutput ${commonOutputPath} "meta/cpass1.job*.run${runNumber}.done" ${nFilesToWaitFor} ${maxSecondsToWait}
1838 LASTJOB=${JOBID4wait}
1839 ################################################################################
1842 fi #end running CPass1
1844 ################################################################################
1845 # submit merging of CPass1, depends on the reconstruction
1846 if [ ${runCPass1MergeMakeOCDB} -eq 1 ]; then
1849 echo "submit CPass1 merging for run ${runNumber}"
1852 targetDirectory="${commonOutputPath}/000${runNumber}/cpass1"
1853 rm -f ${commonOutputPath}/meta/merge.cpass1.run${runNumber}.done
1854 mkdir -p ${targetDirectory}
1858 "${configPath}/OCDB.root"
1859 "${configPath}/localOCDBaccessConfig.C"
1860 "${configPath}/mergeMakeOCDB.byComponent.sh"
1861 "${configPath}/mergeByComponent.C"
1862 "${configPath}/makeOCDB.C"
1863 "${configPath}/merge.C"
1864 "${configPath}/mergeMakeOCDB.sh"
1865 "${configPath}/QAtrain_duo.C"
1867 for file in ${filesMergeCPass1[*]}; do
1868 [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
1871 submit "${JOBID5}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" MergeCPass1 ${targetDirectory} ${currentDefaultOCDB} ${configFile} ${runNumber} cpass1.calib.run${runNumber}.list cpass1.QA.run${runNumber}.lastMergingStage.txt.list cpass1.filtered.run${runNumber}.list "${extraOpts[@]}"
1876 ###############################
1877 #if [ ${runESDfiltering} -eq 1 ]; then
1878 # rm -f ${commonOutputPath}/cpass1.ESD.run${runNumber}.list
1879 # rm -f ${commonOutputPath}/meta/filtering.cpass1.run*.done
1881 # echo submitting filtering for run ${runNumber}
1883 # submit "${JOBmakeESDlistCPass1}" 1 1 "${LASTJOB}" "${self}" PrintValues esd ${commonOutputPath}/meta/cpass1.ESD.run${runNumber}.list ${commonOutputPath}/meta/cpass1.job*.run${runNumber}.done
1884 # submit "${JOBfilterESDcpass1}" 1 1 "${JOBmakeESDlistCPass1}" "${alirootEnv} ${self}" MakeFilteredTrees ${commonOutputPath}/000${runNumber}/cpass1 ${runNumber} ${commonOutputPath}/meta/cpass1.ESD.run${runNumber}.list ${filteringFactorHighPt} ${filteringFactorV0s} ${currentDefaultOCDB} 1000000 0 10000000 0 ${configFile} AliESDs_Barrel.root "${extraOpts[@]}"
1885 # LASTJOB=${JOBfilterESDcpass1}
1890 #################################################################################
1891 #################################################################################
1892 #if [ ${runESDfiltering} -eq 1 ]; then
1893 # submit "${JOBID5wait}" 1 1 "${LASTJOB}" "${self}" WaitForOutput ${commonOutputPath} "meta/filtering.cpass1.run*.done" "${#listOfRuns[@]}" ${maxSecondsToWait}
1895 submit "${JOBID5wait}" 1 1 "${LASTJOB}" "${self}" WaitForOutput ${commonOutputPath} "meta/merge.cpass1.run*.done" ${#listOfRuns[@]} ${maxSecondsToWait}
1897 LASTJOB=${JOBID5wait}
1899 #################################################################################
1901 echo "submit make a summary"
1904 submit "${JOBID6}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" MakeSummary ${configFile}
1906 #################################################################################
1910 echo "jobs submitted."
1917 [[ $# -lt 3 ]] && echo "goWaitForOutput() wrong number of arguments, exiting.." && return 1
1918 echo searchPath=${1}
1920 echo numberOfFiles=${3}
1921 echo maxSecondsToWait=${4}
1925 maxSecondsToWait=${4}
1926 echo "command to be executed: /bin/ls -1 ${searchPath}/${fileName}"
1927 [[ -z "${maxSecondsToWait}" ]] && maxSecondsToWait=$(( 3600*12 ))
1929 n=$(/bin/ls -1 ${searchPath}/${fileName} 2>/dev/null | wc -l)
1930 [[ ${n} -gt 0 ]] && echo "found ${n} X ${fileName}"
1931 [[ ${n} -ge ${numberOfFiles} ]] && break
1932 [[ ${SECONDS} -gt ${maxSecondsToWait} ]] && echo "timeout of ${maxSecondsToWait}s!" && break
1935 echo "DONE! exiting..."
1945 if ! ls -1 ${inputFiles} &>/dev/null; then echo "the files dont exist!: ${inputFiles}"; return 1; fi
1947 runNumber=$(guessRunNumber ${x})
1948 [[ -z ${runNumber} ]] && echo "run number cannot be guessed for ${x}" && continue
1949 awk -v run=${runNumber} -v i=${i} 'NR > 1 {print run" "$0} NR==1 && i==0 {print "run/I:"$0}' ${x}
1951 done < <(ls -1 ${inputFiles}) > ${outputFile}
1955 goMakeMergedSummaryTree()
1957 # create list of calibration entries
1958 # takes no arguments, just run it in the base output
1959 # directory with the following files in the working directory
1961 # Calibration file lists:
1962 # cpass0.dcsTree.list, cpass1.dcsTree.list
1963 # QA trending root file:
1966 # Production infoo ascii files:
1967 # summary_pass0.tree
1968 # summary_pass1.tree
1971 [[ ! -f cpass0.dcsTree.list ]] && echo "no cpass0.dcsTree.list" && return 1
1972 [[ ! -f cpass1.dcsTree.list ]] && echo "no cpass1.dcsTree.list" && return 1
1973 [[ ! -f trending.root ]] && echo "no trending.root" && return 1
1974 [[ ! -f summary_pass0.tree ]] && echo "no summary_pass0.tree" && return 1
1975 [[ ! -f summary_pass1.tree ]] && echo "no summary_pass1.tree" && return 1
1977 #first, dump the C macro to file
1978 cat << EOF > mergeTree.C
1980 // Merge summary information
1981 // Following files are expected to be in the working directory
1983 // Calibration file lists:
1984 // cpass0.dcsTree.list, cpass1.dcsTree.list
1985 // QA trending root file:
1988 // Production infoo ascii files:
1989 // summary_pass0.tree
1990 // summary_pass1.tree
1996 // Calibration values dump
1998 //Printf("MakeTreeFromList cpass0.dcsTree.list");
1999 AliXRDPROOFtoolkit::MakeTreeFromList("Calib.TPC.CPass0.root", "dcs","dcs","cpass0.dcsTree.list",1);
2000 //Printf("MakeTreeFromList cpass1.dcsTree.list");
2001 AliXRDPROOFtoolkit::MakeTreeFromList("Calib.TPC.CPass1.root", "dcs","dcs","cpass1.dcsTree.list",1);
2003 // Calibration status dump
2005 TFile *fprod = TFile::Open("fproduction.root","recreate");
2007 //Printf("reading summary_pass0.tree");
2008 tree0.ReadFile("summary_pass0.tree");
2009 //Printf("reading summary_pass1.tree");
2010 tree1.ReadFile("summary_pass1.tree");
2011 tree0.Write("CPass0");
2012 tree1.Write("CPass1");
2017 TString stringSetup="";
2018 stringSetup+="1#QA.TPC#run#SummaryTPCQA/tpcQA#trending.root+"; //
2019 stringSetup+="1#Calib.TPC.CPass0#run#dcs#Calib.TPC.CPass0.root+"; //
2020 stringSetup+="1#Calib.TPC.CPass1#run#dcs#Calib.TPC.CPass1.root+"; //
2022 stringSetup+="1#CPass0#runnumber#CPass0#fproduction.root+"; //
2023 stringSetup+="1#CPass1#runnumber#CPass1#fproduction.root+"; //
2025 //Printf("stringSetup: %s", stringSetup.Data());
2026 AliXRDPROOFtoolkit::JoinTreesIndex("outAll.root","joinAll","run",stringSetup.Data(), 1);
2030 aliroot -b -q "mergeTree.C" > mergeTrees.log
2036 #all the final stuff goes in here for ease of use:
2047 if ! parseConfig ${configFile} "${extraOpts[@]}"; then return 1; fi
2049 #if which greadlink; then configFile=$(greadlink -f ${configFile}); fi
2051 #record the working directory provided by the batch system
2052 batchWorkingDirectory=${PWD}
2054 [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
2056 [[ ! -f ${configFile} ]] && echo "no config file ${configFile}!" && return
2058 [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
2060 #copy some useful stuff
2061 #and go to the commonOutputPath
2062 cp ${configFile} ${commonOutputPath}
2063 cd ${commonOutputPath}
2065 exec &> >(tee ${log})
2067 #summarize the global stuff
2068 echo "env script: ${alirootSource} ${alirootEnv}"
2069 echo "\$ALICE_ROOT=${ALICE_ROOT}"
2070 echo "commonOutputPath=${commonOutputPath}"
2072 #summarize the stacktraces
2075 print "frame/I:method/C:line/C:cpass/I:aliroot/I";
2080 if ($3 ~ /Ali*/) aliroot=1; else aliroot=0;
2082 if ($NF!="" && RT!="" && $3!="") print RT" "$3" "$NF" "0" "aliroot
2084 ' 000*/cpass0/*/stacktrace* 2>/dev/null > stacktrace.tree
2091 if ($3 ~ /Ali*/) aliroot=1; else aliroot=0;
2093 if ($NF!="" && RT!="" && $3!="") print RT" "$3" "$NF" "1" "aliroot
2095 ' 000*/cpass1/*/stacktrace* 2>/dev/null >> stacktrace.tree
2097 echo total numbers for the production:
2099 awk 'BEGIN {nFiles=0;nCore=0;}
2100 /^calibfile/ {nFiles++;}
2101 /core dumped/ {nCore++i;}
2102 END {print "cpass0 produced "nFiles" calib files, "nCore" core files";}' meta/cpass0.job*done 2>/dev/null
2103 awk 'BEGIN {nOK=0; nBAD=0; }
2104 /\/rec.log OK/ {nOK++;}
2105 /\/rec.log BAD/ {nBAD++;}
2106 /stderr BAD/ {if ($0 ~ /rec.log/){nBAD++;}}
2107 END {print "cpass0 reco: OK: "nOK"\tBAD: "nBAD;}' meta/cpass0.job*done 2>/dev/null
2108 awk 'BEGIN {nOK=0; nBAD=0; }
2109 /\/calib.log OK/ {nOK++;}
2110 /\/calib.log BAD/ {nBAD++;}
2111 END {print "cpass0 calib: OK: "nOK"\tBAD: "nBAD;}' meta/cpass0.job*done 2>/dev/null
2113 awk 'BEGIN {nOK=0; nBAD=0; }
2114 /merge.log OK/ {nOK++;}
2115 /merge.log BAD/ {nBAD++;}
2116 END {print "cpass0 merge: OK: "nOK"\tBAD: "nBAD;}' meta/merge.cpass0*done 2>/dev/null
2117 awk 'BEGIN {nOK=0; nBAD=0; }
2118 /ocdb.log OK/ {nOK++;}
2119 /ocdb.log BAD/ {nBAD++;}
2120 END {print "cpass0 OCDB: OK: "nOK"\tBAD: "nBAD;}' meta/merge.cpass0*done 2>/dev/null
2123 awk 'BEGIN {nFiles=0;nCore=0;}
2124 /^calibfile/ {nFiles++;}
2125 /core dumped/ {nCore++;}
2126 END {print "cpass1 produced "nFiles" calib files, "nCore" core files";}' meta/cpass1.job*done 2>/dev/null
2127 awk 'BEGIN {nOK=0; nBAD=0; }
2128 /\/rec.log OK/ {nOK++;}
2129 /\/rec.log BAD/ {nBAD++;}
2130 /stderr BAD/ {if ($0 ~ /rec.log/){nBAD++;}}
2131 END {print "cpass1 reco: OK: "nOK"\tBAD: "nBAD;}' meta/cpass1.job*done 2>/dev/null
2132 awk 'BEGIN {nOK=0; nBAD=0; }
2133 /\/calib.log OK/ {nOK++;}
2134 /\/calib.log BAD/ {nBAD++;}
2135 END {print "cpass1 calib: OK: "nOK"\tBAD: "nBAD;}' meta/cpass1.job*done 2>/dev/null
2137 awk 'BEGIN {nOK=0; nBAD=0; }
2138 /merge.log OK/ {nOK++;}
2139 /merge.log BAD/ {nBAD++;}
2140 END {print "cpass1 merge: OK: "nOK"\tBAD: "nBAD;}' meta/merge.cpass1*done 2>/dev/null
2141 awk 'BEGIN {nOK=0; nBAD=0; }
2142 /ocdb.log OK/ {nOK++;}
2143 /ocdb.log BAD/ {nBAD++;}
2144 END {print "cpass1 OCDB: OK: "nOK"\tBAD: "nBAD;}' meta/merge.cpass1*done 2>/dev/null
2148 /bin/ls -1 meta/merge.cpass0.run*.done | while read x
2150 dir=$(goPrintValues calibfile - ${x})
2151 runNumber=$(guessRunNumber ${dir})
2152 [[ -z ${runNumber} ]] && continue
2154 if $(/bin/ls meta/cpass0.job*.run${runNumber}.done &> /dev/null); then
2156 awk 'BEGIN {nOKrec=0;nBADrec=0;nOKcalib=0;nBADcalib=0;nOKstderr=0;nBADstderr=0;}
2157 /\/rec.log OK/ {nOKrec++;}
2158 /\/rec.log BAD/ {nBADrec++;}
2159 /stderr BAD/ {if ($0 ~ /rec.log/) {nBADrec++;} nBADstderr++;}
2160 /stderr OK/ {nOKstderr++;}
2161 /\/calib.log OK/ {nOKcalib++;}
2162 /\/calib.log BAD/ {nBADcalib++}
2163 END {print ""nOKrec" "nBADrec" "nOKstderr" "nBADstderr" "nOKcalib" "nBADcalib;}' meta/cpass0.job*.run${runNumber}.done 2>/dev/null
2167 if $(/bin/ls meta/cpass1.job*.run${runNumber}.done &>/dev/null); then
2169 awk 'BEGIN {nOKrec=0;nBADrec=0;nOKcalib=0;nBADcalib=0;nOKstderr=0;nBADstderr=0;nQAbarrelOK=0;nQAbarrelBAD=0;nQAouterOK=0;nQAouterBAD=0;}
2170 /\/rec.log OK/ {nOKrec++;}
2171 /\/rec.log BAD/ {nBADrec++;}
2172 /stderr BAD/ {if ($0 ~ /rec.log/) nBADrec++;nBADstderr++;}
2173 /stderr OK/ {nOKstderr++;}
2174 /\/calib.log OK/ {nOKcalib++;}
2175 /\/calib.log BAD/ {nBADcalib++}
2176 /\/qa_barrel.log OK/ {nQAbarrelOK++;}
2177 /\/qa_barrel.log BAD/ {nQAbarrelBAD++;}
2178 /\/qa_outer.log OK/ {nQAouterOK++;}
2179 /\/qa_outer.log BAD/ {nQAouterBAD++;}
2180 END {print ""nOKrec" "nBADrec" "nOKstderr" "nBADstderr" "nOKcalib" "nBADcalib" "nQAbarrelOK" "nQAbarrelBAD" "nQAouterOK" "nQAouterBAD;}' meta/cpass1.job*.run${runNumber}.done 2>/dev/null
2184 statusOCDBcpass0=$(awk '/ocdb.log/ {print $2} ' ${x} 2>/dev/null)
2185 statusOCDBcpass1=$(awk '/ocdb.log/ {print $2}' ${x/cpass0/cpass1} 2>/dev/null)
2186 statusQA=$(awk '/mergeMakeOCDB.log/ {print $2}' ${x/cpass0/cpass1} 2>/dev/null)
2188 printf "%s\t ocdb.log cpass0: %s\t ocdb.log cpass1: %s\tqa.log:%s\t| cpass0: rec:%s/%s stderr:%s/%s calib:%s/%s cpass1: rec:%s/%s stderr:%s/%s calib:%s/%s QAbarrel:%s/%s QAouter:%s/%s\n" ${runNumber} ${statusOCDBcpass0} ${statusOCDBcpass1} ${statusQA} ${statusCPass0[0]} ${statusCPass0[1]} ${statusCPass0[2]} ${statusCPass0[3]} ${statusCPass0[4]} ${statusCPass0[5]} ${statusCPass1[0]} ${statusCPass1[1]} ${statusCPass1[2]} ${statusCPass1[3]} ${statusCPass1[4]} ${statusCPass1[5]} ${statusCPass1[6]} ${statusCPass1[7]} ${statusCPass1[8]} ${statusCPass1[9]}
2191 #make lists with output files - QA, trending, filtering and calibration
2192 ### wait for the merging of all runs to be over ###
2194 goPrintValues qafile qa.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2196 goPrintValues calibfile calib.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2198 goPrintValues trendingfile trending.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2199 rm -f filtering.list
2200 goPrintValues filteredTree filtering.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2201 rm -f cpass0.dcsTree.list
2202 goPrintValues dcsTree cpass0.dcsTree.list ${commonOutputPath}/meta/merge.cpass0.run*.done &>/dev/null
2203 rm -f cpass1.dcsTree.list
2204 goPrintValues dcsTree cpass1.dcsTree.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2207 rm -f ${commonOutputPath}/trending_merged.root
2208 goMerge trending.list ${commonOutputPath}/trending.root ${configFile} "${extraOpts[@]}" &> mergeTrending.log
2210 goMakeSummaryTree ${commonOutputPath} 0
2211 goMakeSummaryTree ${commonOutputPath} 1
2213 goCreateQAplots "${PWD}/qa.list" "${productionID}" "QAplots" "${configFile}" "${extraOpts[@]}" filteringList="${PWD}/filtering.list" &>createQAplots.log
2215 #make a merged summary tree out of the QA trending, dcs trees and log summary trees
2216 goMakeMergedSummaryTree
2218 #if set, email the summary
2219 [[ -n ${MAILTO} ]] && cat ${log} | mail -s "benchmark ${productionID} done" ${MAILTO}
2226 if [[ $# -lt 1 ]] ; then
2229 #1. define vars/arrays
2230 DIR=${1} #use input or exec in current dir
2231 pass=${2-"0"} #pass from input
2232 outfile="summary_pass${pass}.tree"
2234 test -f ${outfile} && : >${outfile}
2235 errfile=${outfile/tree/err}
2236 test -f ${errfile} && : >${errfile}
2238 declare -a counterString=(TOFevents TOFtracks TPCevents TPCtracks TRDevents TRDtracks T0events SDDevents SDDtracks MeanVertexevents)
2239 Ncounter=${#counterString[@]}
2241 declare -a statusString=(TRDStatus TOFStatus TPCStatus T0Status MeanVertexStatus)
2242 Nstatus=${#statusString[@]}
2245 declare -a ratesString=(rec stderr calib qa_barrel qa_outer)
2246 Nrates=${#ratesString[@]}
2248 runs=( $(ls -1 ${DIR}/meta/merge.cpass0* | while read x; do guessRunNumber $x; done) )
2251 echo -n runnumber/I >>${outfile}
2252 echo -n :cpass${pass}status/I >>${outfile}
2253 echo -n :cpass${pass}QAstatus/I >>${outfile}
2254 for i in ${ratesString[@]}; do
2255 echo -n :${i}OK/I >>${outfile}
2256 echo -n :${i}BAD/I >>${outfile}
2259 for i in ${counterString[@]} ${statusString[@]} ; do
2260 echo -n :${i}/I >>${outfile}
2262 Ncolumns=$((2 + 2*Nrates + Ncounter + Nstatus))
2267 for runnumber in ${runs[@]} ; do
2271 filejob="${DIR}/meta/cpass${pass}.job*.run${runnumber}.done"
2272 filemerge="${DIR}/meta/merge.cpass${pass}.run${runnumber}.done"
2273 fileOCDB=$(grep /ocdb.log ${filemerge} | awk '{print $1}')
2274 if ! $(/bin/ls ${filemerge} &>/dev/null) ; then
2275 echo "${filemerge} does not exist!" >>${errfile}
2277 elif ! $(/bin/ls ${filejob} &>/dev/null) ; then
2278 echo "${filejob} does not exist!" >>${errfile}
2279 echo -n ${runnumber} >> ${outfile}
2280 for i in $(seq ${Ncolumns}) ; do
2281 echo -n "-1" >> ${outfile}
2286 echo -n ${runnumber} >> ${outfile}
2287 #pass0status= grep '/ocdb.log' ${filemerge} | cut -d' ' -f2 | tr OK x1 | tr BAD xx0 | tr -d 'x'
2288 passStatus=$(grep '/ocdb.log' ${filemerge} | grep OK | wc -l)
2289 echo -n " ${passStatus}" >> ${outfile}
2290 qaStatus=$(grep '/mergeMakeOCDB.log' ${filemerge} | grep OK | wc -l)
2291 echo -n " ${qaStatus}" >> ${outfile}
2295 for i in $(seq 0 $((${Nrates}-1))) ; do
2296 var1=$(grep "/${ratesString[${i}]}.log" ${filejob} | grep OK | wc -l)
2297 var2=$(grep "/${ratesString[${i}]}.log" ${filejob} | grep BAD | wc -l)
2299 if [[ ${ratesString[${i}]} == "stderr" ]] ; then
2300 var1=$(grep "stderr" ${filejob} | grep OK | wc -l)
2301 var2=$(grep "stderr" ${filejob} | grep "rec.log" | grep BAD | wc -l)
2303 echo -n " ${var1}" >> ${outfile}
2304 echo -n " ${var2}" >> ${outfile}
2307 if [[ -f ${fileOCDB} ]] ; then
2309 for i in $(seq 0 $((${Ncounter}-1))) ; do
2310 var1=$(grep Monalisa ${fileOCDB} | grep ${counterString[${i}]} | cut -f2)
2311 echo -n " ${var1:-"-1"}" >> ${outfile}
2315 for i in $(seq 0 $((${Nstatus}-1))) ; do
2316 var1=$(grep "calibration status=" ${fileOCDB} | grep ${statusString[${i}]/Status/} | cut -d'=' -f2)
2317 echo -n " ${var1:-"-1"}" >> ${outfile}
2335 defaultOCDB="raw://"
2337 #makeflowPath="/hera/alice/aux/cctools/bin"
2338 #makeflowOptions="-T wq -N alice -d all -C ali-copilot.cern.ch:9097"
2339 #makeflowOptions="-T wq -N alice -C ali-copilot.cern.ch:9097"
2341 #batchCommand="/usr/bin/qsub"
2342 batchFlags="-b y -cwd -l h_rt=24:0:0,h_rss=4G "
2343 baseOutputDirectory="$PWD/output"
2344 #alirootEnv="/cvmfs/alice.cern.ch/bin/alienv setenv AliRoot/v5-04-34-AN -c"
2345 #alirootEnv="/home/mkrzewic/alisoft/balice_master.sh"
2346 #trustedQAtrainMacro='/hera/alice/mkrzewic/gsisvn/Calibration/QAtrain_duo.C'
2347 reconstructInTemporaryDir=0
2348 recoTriggerOptions="\"\""
2349 percentProcessedFilesToContinue=100
2350 maxSecondsToWait=$(( 3600*24 ))
2353 postSetUpActionCPass0=""
2354 postSetUpActionCPass1=""
2356 runCPass0MergeMakeOCDB=1
2358 runCPass1MergeMakeOCDB=1
2360 filteringFactorHighPt=1e2
2361 filteringFactorV0s=1e1
2364 #dontRedirectStdOutToLog=1
2365 logToFinalDestination=1
2366 ALIROOT_FORCE_COREDUMP=1
2369 #first, source the config file
2370 if [ -f ${configFile} ]; then
2371 source ${configFile}
2373 echo "config file ${configFile} not found!"
2378 for opt in "${args[@]}"; do
2379 [[ "${opt}" =~ encodedSpaces=.* ]] && encodedSpaces=1 && echo "encodedSpaces!" && break
2382 #then, parse the options as they override the options from file
2383 for opt in "${args[@]}"; do
2384 [[ -z ${opt} ]] && continue
2385 [[ -n ${encodedSpaces} ]] && opt="$(decSpaces ${opt})"
2386 [[ "${opt}" =~ ^[[:space:]]*$ ]] && continue
2387 if [[ ! "${opt}" =~ .*=.* ]]; then
2388 echo "badly formatted option \"${opt}\" should be: option=value, stopping..."
2391 local var="${opt%%=*}"
2392 local value="${opt#*=}"
2393 echo "${var}=${value}"
2394 export ${var}="${value}"
2398 [[ -z ${alirootEnv} ]] && echo "alirootEnv not defined!" && return 1
2400 #export the aliroot function if defined to override normal behaviour
2401 [[ $(type -t aliroot) =~ "function" ]] && export -f aliroot
2409 if [[ -n ${useProfilingCommand} ]]; then
2410 valgrindLogFile="cpu.txt"
2411 [[ "${args}" =~ rec ]] && valgrindLogFile="cpu_rec.txt"
2412 [[ "${args}}" =~ Calib ]] && valgrindLogFile="cpu_calib.txt"
2413 [[ -n ${useProfilingCommand} ]] && useProfilingCommand="${useProfilingCommand} --log-file=${valgrindLogFile}"
2414 echo running ${useProfilingCommand} aliroot ${args}
2415 ${useProfilingCommand} aliroot ${args}
2417 #to prevent an infinite recursion use "command aliroot" to disable
2418 #aliases and functions
2419 echo running command aliroot ${args}
2420 command aliroot "$@"
2427 #guess the period from the path, pick the rightmost one
2432 legoTrainRunNumber=""
2435 local shortRunNumber=""
2437 declare -a path=( $1 )
2438 local dirDepth=$(( ${#path[*]}-1 ))
2440 #for ((x=${dirDepth};x>=0;x--)); do
2441 for ((x=0;x<=${dirDepth};x++)); do
2443 [[ $((x-1)) -ge 0 ]] && local fieldPrev=${path[$((x-1))]}
2444 local field=${path[${x}]}
2445 local fieldNext=${path[$((x+1))]}
2447 [[ ${field} =~ ^[0-9]*$ && ${fieldNext} =~ (.*\.zip$|.*\.root$) ]] && legoTrainRunNumber=${field}
2448 [[ -n ${legoTrainRunNumber} && -z ${pass} ]] && pass=${fieldPrev}
2449 [[ ${field} =~ ^LHC[0-9][0-9][a-z].*$ ]] && period=${field%_*}
2450 [[ ${field} =~ ^000[0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && runNumber=${field#000}
2451 [[ ${field} =~ ^[0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && shortRunNumber=${field}
2452 [[ ${field} =~ ^20[0-9][0-9]$ ]] && year=${field}
2453 [[ ${field} =~ ^(^sim$|^data$) ]] && dataType=${field}
2456 [[ -z ${legoTrainRunNumber} ]] && pass=${path[$((dirDepth-1))]}
2457 [[ "${dataType}" =~ ^sim$ ]] && pass="passMC" && runNumber=${shortRunNumber}
2459 #if [[ -z ${dataType} || -z ${year} || -z ${period} || -z ${runNumber}} || -z ${pass} ]];
2460 if [[ -z ${runNumber}} ]];
2471 #these functions encode strings to and from a space-less form
2472 #use when spaces are not well handled (e.g. in arguments to
2473 #commands in makeflow files, etc.
2474 encSpaces()(echo "${1// /@@@@}")
2475 decSpaces()(echo "${1//@@@@/ }")