PWGPP/benchmark/benchmark.sh

   1 #!/bin/bash
   2 #include benchmark.config
   3
   4 # this script runs the CPass0/CPass1 train
   5 # produced OCDB updates are local
   6
   7 main()
   8 {
   9   #run in proper mode depending on the selection
  10   if [[ $# -lt 1 ]]; then
  11     if [[ ! "${0}" =~ "bash" ]]; then
  12       echo "uses makeflow:"
  13       echo " ${0} \"run\" productionID inputList configFile [extraOpts]"
  14       echo "uses a batch system (SGE):"
  15       echo " ${0} \"submit\" productionID inputList configFile [extraOpts]"
  16       echo "extraOpts if non-empty override the config file, e.g.:"
  17       echo " ${0} submit test1 benchmark.list benchmark.config runNumber=169123 nEvents=10"
  18     fi
  19     return
  20   fi
  21
  22   #define some aliases - default is to call one of the functions directly
  23   runMode=${1}
  24   umask 0002
  25   shift
  26   case ${runMode} in
  27     "CPass0") goCPass0 "$@";;
  28     "CPass1") goCPass1 "$@";;
  29     "MakeLocalOCDBaccessConfig") goMakeLocalOCDBaccessConfig "$@";;
  30     "MergeCPass0") goMergeCPass0 "$@";;
  31     "MergeCPass1") goMergeCPass1 "$@";;
  32     "MakeFilteredTrees") goMakeFilteredTrees "$@";;
  33     "MakeSummary") goMakeSummary "$@";;
  34     "run") goSubmitMakeflow "$@";;
  35     "submit") goSubmitBatch "$@";;
  36     "test") goTest "$@";;
  37     "GenerateMakeflow") goGenerateMakeflow "$@";;
  38     "PrintValues") goPrintValues "$@";;
  39     "CreateQAplots") goCreateQAplots "$@";;
  40     "WaitForOutput") goWaitForOutput "$@";;
  41     "Merge") goMerge "$@";;
  42     *)
  43       eval ${runMode} $@
  44     ;;
  45   esac
  46 }
  47
  48 generateMC()
  49 {
  50   #generate one raw chunk in current directory
  51   SEED=${JOB_ID}${SGE_TASK_ID}
  52   export CONFIG_SEED=${SEED}
  53   runNumber=${1}
  54   OCDBpath=${2}
  55   nEventsim=${3}
  56   if [[ -n ${pretend} ]]; then
  57     touch galice.root
  58   else
  59     if [[ -f sim.C && -f Config.C ]] ; then
  60         time aliroot -b -q -x sim.C\(${runNumber},\"${OCDBpath}\",${nEventsim}\) >sim.log 2>&1
  61         mv syswatch.log simwatch.log
  62     fi
  63   fi
  64 }
  65
  66 goCPass0()
  67 {
  68   umask 0002
  69
  70   targetDirectory=${1}
  71   inputList=${2}
  72   nEvents=${3}
  73   ocdbPath=${4}
  74   configFile=${5}
  75   runNumber=${6}
  76   jobindex=${7}
  77   shift 7
  78   parseConfig ${configFile} $@
  79
  80   #use the jobindex only if set and non-negative
  81   if [[ -z ${jobindex} || ${jobindex} -lt 0 ]]; then
  82     [[ -n "${LSB_JOBINDEX}" ]] && jobindex=${LSB_JOBINDEX}
  83     [[ -n "${SGE_TASK_ID}" ]] && jobindex=${SGE_TASK_ID}
  84   fi
  85
  86   [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
  87   doneFile="${commonOutputPath}/meta/cpass0.job${jobindex}.run${runNumber}.done"
  88
  89   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
  90
  91   if [[ -n ${ALIROOT_FORCE_COREDUMP} ]]; then
  92     ulimit -c unlimited
  93     export ALIROOT_FORCE_COREDUMP
  94   fi
  95
  96   #the contents of this is stored in the tree and used later (e.g. AliAnalysisTaskPIDResponse)!
  97   #at the QA stage the pass number is guessed from the path stored here.
  98   #The Format is:
  99   #Packages= ;OutputDir= ;LPMPass= ;TriggerAlias= ;LPMRunNumber= ;LPMProductionType= ;LPMInteractionType= ;LPMProductionTag= ;LPMAnchorRun= ;LPMAnchorProduction= ;LPMAnchorYear=
 100   export PRODUCTION_METADATA="OutputDir=cpass0"
 101
 102   if [[ "${inputList}" =~ \.root$ ]]; then
 103     infile=${inputList}
 104   else
 105     infile=$(sed -ne "${jobindex}p" ${inputList} | egrep '\s*\w*/\w*')
 106   fi
 107   chunkName=${infile##*/}
 108
 109   outputDir=${targetDirectory}/${jobindex}_${chunkName%.*}
 110   mkdir -p ${outputDir}
 111   [[ ! -d ${outputDir} ]] && echo "cannot make ${outputDir}" && touch ${doneFile} && return 1
 112
 113   #runpath=${PWD}/rundir_cpass0_${runNumber}_${jobindex}
 114   runpath=${outputDir}
 115   [[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
 116   [[ ${reconstructInTemporaryDir} -eq 1 && -z ${TMPDIR} ]] && runpath=$(mktemp -d)
 117   mkdir -p ${runpath}
 118   [[ ! -d ${runpath} ]] && echo "cannot make runpath ${runpath}" && touch ${doneFile} && return 1
 119   cd ${runpath}
 120
 121   #runCPassX/C expects the raw chunk to be linked in the run dir
 122   #despite it being accessed by the full path
 123   ln -s ${infile} ${runpath}/${chunkName}
 124
 125   #####MC
 126   if [[ -n ${generateMC} ]]; then
 127     olddir=${PWD}
 128     outputDirMC=${commonOutputPath}/000${runNumber}/sim/${jobindex}
 129     simrunpath=${outputDirMC}
 130     [[ ${simulateInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && simrunpath=${TMPDIR}
 131     [[ ${simulateInTemporaryDir} -eq 1 && -z ${TMPDIR} ]] && simrunpath=$(mktemp -d)
 132     mkdir -p ${outputDirMC}
 133     mkdir -p ${simrunpath}
 134     cd ${simrunpath}
 135
 136     filesMC=(
 137               "${commonOutputPath}/sim.C"
 138               "${commonOutputPath}/rec.C"
 139               "${commonOutputPath}/Config.C"
 140               "${commonOutputPath}/OCDB_*.root"
 141     )
 142     for file in ${filesMC[*]}; do
 143       [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
 144     done
 145
 146     generateMC ${runNumber} ${ocdbPath} ${nEvents}
 147
 148     [[ ! "${simrunpath}" =~ "${outputDirMC}" ]] && mv * ${outputDirMC} #TODO check if it works
 149     cd ${olddir}
 150
 151     ln -s ${outputDirMC}/* ${runpath}/
 152
 153     inputList=${outputDirMC}/galice.root #TODO not valid outside shell !!!
 154     infile=""
 155   fi
 156   ######
 157
 158   [[ ! -f ${inputList} && -z ${pretend} ]] && echo "input file ${inputList} not found, exiting..." && touch ${doneFile} && return 1
 159
 160   logOutputDir=${runpath}
 161   [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir}
 162   [[ -z ${dontRedirectStdOutToLog} ]] && exec 1> ${logOutputDir}/stdout
 163   [[ -z ${dontRedirectStdOutToLog} ]] && exec 2> ${logOutputDir}/stderr
 164   echo "${0} $*"
 165
 166   echo "#####################"
 167   echo CPass0:
 168   echo JOB setup
 169   echo nEvents            ${nEvents}
 170   echo runNumber          ${runNumber}
 171   echo ocdbPath           ${ocdbPath}
 172   echo infile             ${infile}
 173   echo chunkName          ${chunkName}
 174   echo jobindex           ${jobindex}
 175   echo recoTriggerOptions ${recoTriggerOptions}
 176   echo targetDirectory    ${targetDirectory}
 177   echo commonOutputPath         ${commonOutputPath}
 178   echo doneFile      ${doneFile}
 179   echo runpath            ${runpath}
 180   echo outputDir          ${outputDir}
 181   echo ALICE_ROOT         ${ALICE_ROOT}
 182   echo PWD                ${PWD}
 183   echo "########## ###########"
 184
 185   alirootInfo > ALICE_ROOT.log
 186
 187   filesCPass0=(
 188                "${commonOutputPath}/runCPass0.sh"
 189                "${commonOutputPath}/recCPass0.C"
 190                "${commonOutputPath}/runCalibTrain.C"
 191                "${commonOutputPath}/localOCDBaccessConfig.C"
 192                "${commonOutputPath}/OCDB.root"
 193                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/runCPass0.sh"
 194                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/recCPass0.C"
 195                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/runCalibTrain.C"
 196   )
 197
 198   for file in ${filesCPass0[*]}; do
 199     [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
 200   done
 201
 202   echo "this directory (${PWD}) contents:"
 203   /bin/ls
 204   echo
 205   chmod u+x runCPass0.sh
 206
 207   sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh
 208
 209   if [[ -n ${postSetUpActionCPass0} ]]; then
 210     echo "running ${postSetUpActionCPass0}"
 211     eval ${postSetUpActionCPass0}
 212   fi
 213
 214   #run CPass0
 215   echo "${runpath}/runCPass0.sh ${infile} ${nEvents} ${runNumber} ${ocdbPath} ${recoTriggerOptions}"
 216   if [[ -n ${pretend} ]]; then
 217     touch AliESDs.root
 218     touch AliESDfriends.root
 219     touch AliESDfriends_v1.root
 220     touch rec.log
 221     touch calib.log
 222   else
 223     echo ./runCPass0.sh "${infile}" "${nEvents}" "${runNumber}" "${ocdbPath}" "${recoTriggerOptions}"
 224     ./runCPass0.sh "${infile}" "${nEvents}" "${runNumber}" "${ocdbPath}" "${recoTriggerOptions}"
 225   fi
 226
 227   #move stuff to final destination
 228   echo "this directory (${PWD}) contents:"
 229   /bin/ls
 230   echo
 231
 232   echo rm -f ./${chunkName}
 233   rm -f ./${chunkName}
 234   echo "cp --recursive ${runpath}/* ${outputDir}"
 235   cp -p --recursive ${runpath}/* ${outputDir}
 236   echo
 237
 238   #validate CPass0
 239   cd ${outputDir}
 240   touch ${doneFile}
 241   [[ -f ${outputDirMC}/galice.root ]] && echo "sim ${outputDirMC}/galice.root" >> ${doneFile}
 242   [[ -f AliESDfriends_v1.root ]] && echo "calibfile ${outputDir}/AliESDfriends_v1.root" >> ${doneFile}
 243   [[ -f AliESDs.root ]] && echo "esd ${outputDir}/AliESDs.root" >> ${doneFile}
 244   echo "dir ${outputDir}" >> ${doneFile}
 245   summarizeLogs >> ${doneFile}
 246
 247   [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
 248   return 0
 249 }
 250
 251 goCPass1()
 252 {
 253   umask 0002
 254
 255   targetDirectory=${1}
 256   inputList=${2}
 257   nEvents=${3}
 258   ocdbPath=${4}
 259   configFile=${5}
 260   runNumber=${6}
 261   jobindex=${7}
 262   shift 7
 263   parseConfig ${configFile} $@
 264
 265   #use the jobindex only if set and non-negative
 266   if [[ -z ${jobindex} || ${jobindex} -lt 0 ]]; then
 267     [[ -n "${LSB_JOBINDEX}" ]] && jobindex=${LSB_JOBINDEX}
 268     [[ -n "${SGE_TASK_ID}" ]] && jobindex=${SGE_TASK_ID}
 269   fi
 270
 271   [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
 272   doneFile="${commonOutputPath}/meta/cpass1.job${jobindex}.run${runNumber}.done"
 273
 274   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
 275
 276   if [[ -n ${ALIROOT_FORCE_COREDUMP} ]]; then
 277     ulimit -c unlimited
 278     export ALIROOT_FORCE_COREDUMP
 279   fi
 280
 281   #the contents of this is stored in the tree and used later (e.g. AliAnalysisTaskPIDResponse)!
 282   #at the QA stage the pass number is guessed from the path stored here.
 283   #The Format is:
 284   #Packages= ;OutputDir= ;LPMPass= ;TriggerAlias= ;LPMRunNumber= ;LPMProductionType= ;LPMInteractionType= ;LPMProductionTag= ;LPMAnchorRun= ;LPMAnchorProduction= ;LPMAnchorYear=
 285   export PRODUCTION_METADATA="OutputDir=cpass1"
 286
 287   [[ ! -f ${inputList} && -z ${pretend} ]] && echo "input file ${inputList} not found, exiting..." && touch ${doneFile} && return 1
 288   if [[ "${inputList}" =~ \.root$ ]]; then
 289     infile=${inputList}
 290   else
 291     infile=$(sed -ne "${jobindex}p" ${inputList} | egrep '\s*\w*/\w*')
 292   fi
 293   chunkName=${infile##*/}
 294
 295   outputDir=${targetDirectory}/${jobindex}_${chunkName%.*}
 296   mkdir -p ${outputDir}
 297   [[ ! -d ${outputDir} ]] && echo "cannot make ${outputDir}" && touch ${doneFile} && return 1
 298
 299   #runpath=${PWD}/rundir_cpass1_${runNumber}_${jobindex}
 300   runpath=${outputDir}
 301   [[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
 302   [[ ${reconstructInTemporaryDir} -eq 1 && -z ${TMPDIR} ]] && runpath=$(mktemp -d)
 303
 304   #MC
 305   if [[ "${infile}" =~ galice\.root ]]; then
 306     ln -s ${inputList%/*}/* ${runpath}
 307     infile=""
 308   fi
 309
 310   #init the running path
 311   mkdir -p ${runpath}
 312   [[ ! -d ${runpath} ]] && echo "cannot make runpath ${runpath}" && touch ${doneFile} && return 1
 313   cd ${runpath}
 314
 315   #this is needed for runCPass1.sh
 316   ln -s ${infile} ${runpath}/${chunkName}
 317
 318   logOutputDir=${runpath}
 319   [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir}
 320   [[ -z ${dontRedirectStdOutToLog} ]] && exec 1> ${logOutputDir}/stdout
 321   [[ -z ${dontRedirectStdOutToLog} ]] && exec 2> ${logOutputDir}/stderr
 322   echo "${0} $*"
 323
 324   echo "#####################"
 325   echo CPass1:
 326   echo JOB setup
 327   echo nEvents            ${nEvents}
 328   echo runNumber          ${runNumber}
 329   echo ocdbPath           ${ocdbPath}
 330   echo infile             ${infile}
 331   echo chunkName          ${chunkName}
 332   echo jobindex           ${jobindex}
 333   echo recoTriggerOptions ${recoTriggerOptions}
 334   echo targetDirectory    ${targetDirectory}
 335   echo commonOutputPath         ${commonOutputPath}
 336   echo doneFile      ${doneFile}
 337   echo runpath            ${runpath}
 338   echo outputDir          ${outputDir}
 339   echo ALICE_ROOT         ${ALICE_ROOT}
 340   echo PWD                ${PWD}
 341   echo "########## ###########"
 342
 343   alirootInfo > ALICE_ROOT.log
 344
 345   filesCPass1=(
 346                "${commonOutputPath}/runCPass1.sh"
 347                "${commonOutputPath}/recCPass1.C"
 348                "${commonOutputPath}/recCPass1_OuterDet.C"
 349                "${commonOutputPath}/runCalibTrain.C"
 350                "${commonOutputPath}/QAtrain_duo.C"
 351                "${commonOutputPath}/localOCDBaccessConfig.C"
 352                "${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz"
 353                "${commonOutputPath}/OCDB.root"
 354                "${trustedQAtrainMacro}"
 355                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/runCPass1.sh"
 356                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/recCPass1.C"
 357                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/recCPass1_OuterDet.C"
 358                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/runCalibTrain.C"
 359                "${ALICE_ROOT}/ANALYSIS/macros/QAtrain_duo.C"
 360   )
 361
 362   for file in ${filesCPass1[*]}; do
 363     [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
 364   done
 365
 366   echo "this directory (${PWD}) contents:"
 367   /bin/ls
 368   echo
 369
 370   sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh
 371
 372   if [[ -n ${postSetUpActionCPass1} ]]; then
 373     echo "running ${postSetUpActionCPass1}"
 374     eval ${postSetUpActionCPass1}
 375     echo
 376   fi
 377
 378   #configure local OCDB storage from CPass0 (creates the localOCDBaccessConfig.C script)
 379   if [[ -f cpass0.localOCDB.${runNumber}.tgz ]]; then
 380     echo goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz"
 381     goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz"
 382   else
 383     echo "WARNING: file cpass0.localOCDB.${runNumber}.tgz not found!"
 384   fi
 385
 386   if [[ ! $(/bin/ls -1 OCDB/*/*/*/*.root 2>/dev/null) ]]; then
 387     echo "cpass0 produced no calibration! exiting..."
 388     touch ${doneFile}
 389     return 1
 390   fi
 391
 392   #create the Barrel and OuterDet directories for CPass1 and link the local OCDB directory
 393   #there to make the localOCDBaccessConfig.C file work, since it may point to the OCDB
 394   #entries using a relative path, e.g. local://./OCDB
 395   mkdir Barrel OuterDet
 396   ln -s ../OCDB Barrel/OCDB
 397   ln -s ../OCDB OuterDet/OCDB
 398
 399   #run CPass1
 400   chmod u+x runCPass1.sh
 401   echo "${runpath}/runCPass1.sh ${infile} ${nEvents} ${runNumber} ${ocdbPath} ${recoTriggerOptions}"
 402   if [[ -n ${pretend} ]]; then
 403     touch AliESDs_Barrel.root
 404     touch AliESDfriends_Barrel.root
 405     touch AliESDfriends_v1.root
 406     touch QAresults_barrel.root
 407     touch EventStat_temp_barrel.root
 408     touch AODtpITS.root
 409     touch AliESDs_Outer.root
 410     touch AliESDfriends_Outer.root
 411     touch QAresults_outer.root
 412     touch EventStat_temp_outer.root
 413     touch rec.log
 414     touch calib.log
 415     touch qa.log
 416   else
 417     ./runCPass1.sh "${infile}" "${nEvents}" "${runNumber}" "${ocdbPath}" "${recoTriggerOptions}"
 418   fi
 419
 420   ##handle possible crashes in QA (happens often in trunk)
 421   ##rerun QA with a trusted aliroot version
 422   #if [[ $(validateLog qa_barrel.log) ]]; then
 423   #  echo "qa_barrel.log not validated!"
 424   #fi
 425   #if [[ ! -f QAresults_barrel.root && -f ${setupTrustedAliROOTenvInCurrentShell} || $(validateLog qa_barrel.log) ]]; then
 426   #  echo "WARNING: using trusted QA aliroot ${ALICE_ROOT}"
 427   #  source ${setupTrustedAliROOTenvInCurrentShell}
 428   #  cd Barrel
 429   #  rm QAresults_barrel.root
 430   #  rm EventStat_temp_barrel.root
 431   #  rm AODtpITS.root
 432   #  [[ ! -f AliESDs.root ]] && ln -s ../AliESDs_Barrel.root AliESDs.root
 433   #  [[ ! -f AliESDfriends.root ]] && ln -s ../AliESDfriends_Barrel.root AliESDfriends.root
 434   #  if [[ -n ${trustedQAtrainMacro} ]]; then
 435   #    eval "cp ${trustedQAtrainMacro} QAtrain_duo_trusted.C"
 436   #  fi
 437   #  echo executing aliroot -b -q "QAtrain_duo_trusted.C(\"_barrel\",${runNumber},\"wn.xml\",0,\"${ocdbPath}\")"
 438   #  time aliroot -b -q "QAtrain_duo.C(\"_barrel\",${runNumber},\"wn.xml\",0,\"${ocdbPath}\")" &> ../qa_barrel_trusted.log
 439   #  cd ../
 440   #fi
 441   [[ ! -f AliESDs_Barrel.root && -f Barrel/AliESDs.root ]] && mv Barrel/AliESDs.root AliESDs_Barrel.root
 442   [[ ! -f AliESDfriends_Barrel.root && -f Barrel/AliESDfriends.root ]] && mv Barrel/AliESDfriends.root AliESDfriends_Barrel.root
 443   [[ ! -f AliESDfriends_v1.root && -f Barrel/AliESDfriends_v1.root ]] && mv Barrel/AliESDfriends_v1.root .
 444   [[ ! -f QAresults_barrel.root && -f Barrel/QAresults_barrel.root ]] && mv Barrel/QAresults_barrel.root .
 445   [[ ! -f AliESDs_Outer.root && -f OuterDet/AliESDs.root ]] && mv OuterDet/AliESDs.root AliESDs_Outer.root
 446   [[ ! -f AliESDfriends_Outer.root && -f OuterDet/AliESDfriends.root ]] && mv OuterDet/AliESDfriends.root AliESDfriends_Outer.root
 447   [[ ! -f QAresults_outer.root && -f OuterDet/QAresults_outer.root ]] && mv OuterDet/QAresults_outer.root .
 448
 449   #move stuff to final destination
 450   echo "this directory (${PWD}) contents:"
 451   /bin/ls
 452   echo rm -f ./${chunkName}
 453   rm -f ./${chunkName}
 454   echo "cp --recursive ${runpath}/* ${outputDir}"
 455   cp -pf --recursive ${runpath}/* ${outputDir}
 456   echo
 457
 458   #validate CPass1
 459   cd ${outputDir}
 460   touch ${doneFile}
 461   [[ -f AliESDs_Barrel.root ]] && echo "esd ${outputDir}/AliESDs_Barrel.root" >> ${doneFile}
 462   [[ -f AliESDfriends_v1.root ]] && echo "calibfile ${outputDir}/AliESDfriends_v1.root" >> ${doneFile}
 463   [[ -f QAresults_Barrel.root ]] && echo "qafile ${outputDir}/QAresults_Barrel.root" >> ${doneFile}
 464   [[ -f QAresults_Outer.root ]] && echo "qafile ${outputDir}/QAresults_Outer.root" >> ${doneFile}
 465   echo "dir ${outputDir}" >> ${doneFile}
 466   summarizeLogs >> ${doneFile}
 467
 468   [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
 469   return 0
 470 }
 471
 472
 473 goMergeCPass0()
 474 {
 475   #
 476   # find the output files and merge them
 477   #
 478
 479   outputDir=${1}
 480   ocdbStorage=${2}
 481   configFile=${3}
 482   runNumber=${4}
 483   calibrationFilesToMerge=${5}  #can be a non-existent file, will then be produced on the fly
 484   shift 5
 485   parseConfig ${configFile} $@
 486
 487   [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
 488   doneFile="${commonOutputPath}/meta/merge.cpass0.run${runNumber}.done"
 489
 490   umask 0002
 491   ulimit -c unlimited
 492
 493   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
 494
 495   #runpath=${PWD}/rundir_cpass0_Merge_${runNumber}
 496   runpath=${outputDir}
 497   [[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
 498   [[ ${reconstructInTemporaryDir} -eq 1 && -z ${TMPDIR} ]] && runpath=$(mktemp -d)
 499
 500   mkdir -p ${runpath}
 501   [[ ! -d ${runpath} ]] && echo "not able to make the runpath ${runpath}" && touch ${doneFile} && return 1
 502   cd ${runpath}
 503
 504   logOutputDir=${runpath}
 505   [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir}
 506   [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/mergeMakeOCDB.log
 507   echo "${0} $*"
 508
 509   mergingScript="mergeMakeOCDB.byComponent.sh"
 510
 511   echo goMergeCPass0 SETUP:
 512   echo runNumber=${runNumber}
 513   echo outputDir=${outputDir}
 514   echo ocdbStorage=${ocdbStorage}
 515   echo calibrationFilesToMerge=${calibrationFilesToMerge}
 516   echo mergingScript=${mergingScript}
 517   echo commonOutputPath=${commonOutputPath}
 518   echo runpath=${runpath}
 519
 520   # copy files in case they are not already there
 521   filesMergeCPass0=(
 522                     "${commonOutputPath}/${calibrationFilesToMerge}"
 523                     "${commonOutputPath}/OCDB.root"
 524                     "${commonOutputPath}/localOCDBaccessConfig.C"
 525                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/mergeMakeOCDB.byComponent.sh"
 526                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/mergeByComponent.C"
 527                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/makeOCDB.C"
 528                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/merge.C"
 529                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/mergeMakeOCDB.sh"
 530   )
 531   for file in ${filesMergeCPass0[*]}; do
 532     [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
 533   done
 534
 535   sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh
 536
 537   alirootInfo > ALICE_ROOT.log
 538
 539   #
 540   echo "PWD"
 541   /bin/ls
 542   echo "PWD/.."
 543   /bin/ls ../
 544
 545
 546   #merge calibration
 547   chmod u+x ${mergingScript}
 548   mkdir -p ./OCDB
 549   if [[ ! -f ${calibrationFilesToMerge} ]]; then
 550     echo "/bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root > ${calibrationFilesToMerge}"
 551     /bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root 2>/dev/null > ${calibrationFilesToMerge}
 552   fi
 553
 554   echo "${mergingScript} ${calibrationFilesToMerge} ${runNumber} local://./OCDB ${ocdbStorage}"
 555   if [[ -n ${pretend} ]]; then
 556     touch CalibObjects.root
 557     touch ocdb.log
 558     touch merge.log
 559     mkdir -p ./OCDB/TPC/Calib/TimeGain/
 560     mkdir -p ./OCDB/TPC/Calib/TimeDrift/
 561     touch ./OCDB/TPC/Calib/TimeGain/someCalibObject_0-999999_cpass0.root
 562     touch ./OCDB/TPC/Calib/TimeDrift/otherCalibObject_0-999999_cpass0.root
 563   else
 564     ./${mergingScript} ${calibrationFilesToMerge} ${runNumber} "local://./OCDB" ${ocdbStorage}
 565   fi
 566
 567   ### produce the output
 568   #tar the produced OCDB for reuse
 569   tar czf ${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz ./OCDB
 570
 571   #produce the calib trees for expert QA (dcsTime.root)
 572   goMakeLocalOCDBaccessConfig ./OCDB
 573   echo aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")"
 574   aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")"
 575
 576   /bin/ls
 577
 578   #copy all to output dir
 579   cp -pf --recursive ${runpath}/* ${outputDir}
 580
 581   if [[ -n ${generateMC} ]]; then
 582     goPrintValues sim ${commonOutputPath}/meta/sim.run${runNumber}.list ${commonOutputPath}/meta/cpass0.job*.run${runNumber}.done
 583   fi
 584
 585   #validate merging cpass0
 586   cd ${outputDir}
 587   touch ${doneFile}
 588   [[ -f CalibObjects.root ]] && echo "calibfile ${outputDir}/CalibObjects.root" >> ${doneFile}
 589   [[ -f dcsTime.root ]] && echo "dcsTree ${outputDir}/dcsTime.root" >> ${doneFile}
 590   echo "dir ${outputDir}" >> ${doneFile}
 591   summarizeLogs >> ${doneFile}
 592
 593   [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
 594   return 0
 595 }
 596
 597 goMergeCPass1()
 598 {
 599   #
 600   # find the output files and merge them
 601   #
 602
 603   outputDir=${1}
 604   ocdbStorage=${2}
 605   configFile=${3}
 606   runNumber=${4}
 607   calibrationFilesToMerge=${5}
 608   qaFilesToMerge=${6}
 609   shift 6
 610   parseConfig ${configFile} $@
 611
 612   [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
 613   doneFile="${commonOutputPath}/meta/merge.cpass1.run${runNumber}.done"
 614
 615   umask 0002
 616   ulimit -c unlimited
 617
 618   #clean up first:
 619   rm -f ${outputDir}/*.log
 620   rm -f ${outputDir}/*.root
 621   rm -f ${outputDir}/*done
 622
 623   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
 624
 625   #runpath=${PWD}/rundir_cpass1_Merge_${runNumber}
 626   runpath=${outputDir}
 627   [[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
 628   [[ ${reconstructInTemporaryDir} -eq 1 && -z ${TMPDIR} ]] && runpath=$(mktemp -d)
 629
 630   mkdir -p ${runpath}
 631   [[ ! -d ${runpath} ]] && echo "not able to make the runpath ${runpath}" && touch ${doneFile} && return 1
 632   cd ${runpath}
 633
 634   logOutputDir=${runpath}
 635   [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir}
 636   [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/mergeMakeOCDB.log
 637   echo "${0} $*"
 638
 639   calibrationOutputFileName='AliESDfriends_v1.root'
 640   qaOutputFileName='QAresults*.root'
 641   mergingScript="mergeMakeOCDB.byComponent.sh"
 642   #important to have the string "Stage.txt" in the filename to trigger the merging
 643   #it has to be a list of directories containing the files
 644   qaMergedOutputFileName="QAresults_merged.root"
 645
 646   echo goMergeCPass1 SETUP:
 647   echo runNumber=${runNumber}
 648   echo outputDir=${outputDir}
 649   echo ocdbStorage=${ocdbStorage}
 650   echo calibrationFilesToMerge=$calibrationFilesToMerge
 651   echo qaFilesToMerge=$qaFilesToMerge
 652   echo calibrationOutputFileName=${calibrationOutputFileName}
 653   echo mergingScript=${mergingScript}
 654
 655   # copy files in case they are not already there
 656   filesMergeCPass1=(
 657                     "${commonOutputPath}/${calibrationFilesToMerge}"
 658                     "${commonOutputPath}/${qaFilesToMerge}"
 659                     "${commonOutputPath}/OCDB.root"
 660                     "${commonOutputPath}/localOCDBaccessConfig.C"
 661                     "${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz"
 662                     "${commonOutputPath}/QAtrain_duo.C"
 663                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/mergeMakeOCDB.byComponent.sh"
 664                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/mergeByComponent.C"
 665                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/makeOCDB.C"
 666                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/merge.C"
 667                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/mergeMakeOCDB.sh"
 668                     "${trustedQAtrainMacro}"
 669                     "${ALICE_ROOT}/ANALYSIS/macros/QAtrain_duo.C"
 670   )
 671   for file in ${filesMergeCPass1[*]}; do
 672     [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
 673   done
 674
 675   sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh
 676
 677   #configure local OCDB storage from CPass0 (creates the localOCDBaccessConfig.C script)
 678   if [[ -f cpass0.localOCDB.${runNumber}.tgz ]]; then
 679     echo goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz"
 680     goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz"
 681   else
 682     echo "WARNING: file cpass0.localOCDB.${runNumber}.tgz not found!"
 683   fi
 684
 685   alirootInfo > ALICE_ROOT.log
 686
 687   #
 688   /bin/ls
 689
 690   #merge calibration
 691   chmod u+x ${mergingScript}
 692   mkdir -p OCDB
 693   if [[ ! -f ${calibrationFilesToMerge} ]]; then
 694     echo "/bin/ls -1 ${outputDir}/*/${calibrationOutputFileName} > ${calibrationFilesToMerge}"
 695     /bin/ls -1 ${outputDir}/*/${calibrationOutputFileName} 2>/dev/null > ${calibrationFilesToMerge}
 696   fi
 697
 698   echo "${mergingScript} ${calibrationFilesToMerge} ${runNumber} local://./OCDB ${ocdbStorage}"
 699   if [[ -n ${pretend} ]]; then
 700     touch CalibObjects.root
 701     touch ocdb.log
 702     touch merge.log
 703     touch cpass1.localOCDB.${runNumber}.tgz
 704   else
 705     ./${mergingScript} ${calibrationFilesToMerge} ${runNumber} "local://./OCDB" ${ocdbStorage}
 706   fi
 707
 708   tar czf ${commonOutputPath}/meta/cpass1.localOCDB.${runNumber}.tgz ./OCDB
 709
 710   #merge QA
 711   [[ -n ${AliAnalysisTaskFilteredTree_fLowPtTrackDownscaligF} ]] && export AliAnalysisTaskFilteredTree_fLowPtTrackDownscaligF
 712   [[ -n ${AliAnalysisTaskFilteredTree_fLowPtV0DownscaligF} ]] && export AliAnalysisTaskFilteredTree_fLowPtV0DownscaligF
 713
 714   if [[ ! -f ${qaFilesToMerge} ]]; then
 715     #find the files, but only store the directories (QAtrain_duo.C requires this)
 716     echo "/bin/ls -1 ${outputDir}/*/${qaOutputFileName} | while read x; do echo ${x%/*}; done | sort | uniq > ${qaFilesToMerge}"
 717     eval "/bin/ls -1 ${outputDir}/*/${qaOutputFileName}" | while read x; do echo ${x%/*}; done | sort | uniq > ${qaFilesToMerge}
 718   fi
 719
 720   #do the merge
 721   #echo aliroot -l -b -q "merge.C(\"${qaFilesToMerge}\",\"\",kFALSE,\"${qaMergedOutputFileName}\")"
 722   echo aliroot -b -q "QAtrain_duo.C(\"_barrel\",${runNumber},\"${qaFilesToMerge}\",1,\"${ocdbStorage}\")"
 723   if [[ -n ${pretend} ]]; then
 724     touch ${qaMergedOutputFileName}
 725     touch merge.log
 726     touch trending.root
 727   else
 728     #aliroot -l -b -q "merge.C(\"${qaFilesToMerge}\",\"\",kFALSE,\"${qaMergedOutputFileName}\")"
 729     aliroot -b -q "QAtrain_duo.C(\"_barrel\",${runNumber},\"${qaFilesToMerge}\",1,\"${ocdbStorage}\")" > mergeQA.log
 730     mv QAresults_barrel.root ${qaMergedOutputFileName}
 731     mv trending_barrel.root trending.root
 732   fi
 733
 734   #produce the calib trees for expert QA
 735   echo aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")"
 736   aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")"
 737
 738   /bin/ls
 739
 740   #copy all to output dir
 741   cp -pf --recursive ${runpath}/* ${outputDir}
 742
 743   #validate merge cpass1
 744   cd ${outputDir}
 745   touch ${doneFile}
 746   [[ -f CalibObjects.root ]] && echo "calibfile ${outputDir}/CalibObjects.root" >> ${doneFile}
 747   [[ -f ${qaMergedOutputFileName} ]] && echo "qafile ${outputDir}/${qaMergedOutputFileName}" >> ${doneFile}
 748   [[ -f trending.root ]] && echo "trendingfile ${outputDir}/trending.root" >> ${doneFile}
 749   [[ -f dcsTime.root ]] && echo "dcsTree ${outputDir}/dcsTime.root" >> ${doneFile}
 750   echo "dir ${outputDir}" >> ${doneFile}
 751   summarizeLogs >>  ${doneFile}
 752
 753   [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
 754   return 0
 755 }
 756
 757 goMerge()
 758 {
 759   #generic root merge using CPass1 merge.C script
 760   inputList=${1}
 761   outputFile=${2}
 762   configFile=${3-"becnhmark.config"}
 763   shift 3
 764   parseConfig ${configFile} $@
 765
 766   [[ ! -f ${inputList} ]] && echo "inputList ${inputList} does not exist!" && return 1
 767   [[ ! -f ${configFile} ]] && echo "configFile ${configFile} does not exist!" && return 1
 768   umask 0002
 769   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
 770   rm -f ${outputFile}
 771   aliroot -b -q "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/merge.C(\"${inputList}\",\"\",kFALSE,\"${outputFile}\")" > merge_${inputList}.log
 772   return 0
 773 }
 774
 775 goSubmitMakeflow()
 776 {
 777   #run
 778   productionID=${1}
 779   inputFileList=${2}
 780   configFile=${3}
 781   shift 3
 782   extraOpts="$@"
 783   parseConfig ${configFile} ${extraOpts}
 784
 785   [[ -z ${configFile} ]] && configFile="benchmark.config"
 786   [[ ! -f ${configFile} ]] && echo "no config file found (${configFile})" && return 1
 787
 788   #create the directopry for the metadata
 789   mkdir meta
 790
 791   if [[ ! $(which makeflow &>/dev/null) && -n ${makeflowPath} ]]; then
 792     echo "setting the makflow path from the config: "
 793     echo "  export PATH=${makeflowPath}:${PATH}"
 794     export PATH=${makeflowPath}:${PATH}
 795   fi
 796
 797   #submit - use makeflow if available, fall back to old stuff when makeflow not there
 798   if which makeflow; then
 799
 800     goGenerateMakeflow ${productionID} ${inputFileList} ${configFile} "${extraOpts}" > benchmark.makeflow
 801
 802     makeflow ${makeflowOptions} benchmark.makeflow
 803     cd ../
 804   else
 805     echo "no makeflow!"
 806   fi
 807   return 0
 808 }
 809
 810 goGenerateMakeflow()
 811 {
 812   #generate the makeflow file
 813   productionID=${1}
 814   inputFileList=${2}
 815   configFile=${3}
 816   shift 3
 817   extraOpts="$@"
 818   parseConfig ${configFile} ${extraOpts}
 819
 820   [[ -z ${configFile} ]] && configFile="benchmark.config"
 821   [[ ! -f ${configFile} ]] && echo "no config file found (${configFile})" && return 1
 822
 823   commonOutputPath=${baseOutputDirectory}/${productionID}
 824
 825   #these files will be made a dependency - will be copied to the working dir of the jobs
 826   declare -a copyFiles
 827   inputFiles=(
 828               "OCDB.root"
 829               "localOCDBaccessConfig.C"
 830               "QAtrain_duo.C"
 831               "runCPass1.sh"
 832               "recCPass1.C"
 833               "recCPass1_OuterDet.C"
 834               "runCalibTrain.C"
 835               "runCPass0.sh"
 836               "recCPass0.C"
 837   )
 838   for file in ${inputFiles[*]}; do
 839     [[ -f ${file} ]] && copyFiles+=("${file}")
 840   done
 841
 842   #create the makeflow file
 843   [[ -n ${batchFlags} ]] && echo "BATCH_OPTIONS = ${batchFlags}"
 844   declare -a arr_cpass1_final
 845   listOfRuns[0]=${runNumber}
 846   [[ -z ${runNumber} ]] && listOfRuns=($(while read x; do guessRunNumber ${x}; done < ${inputFileList} | sort | uniq))
 847   runindex=0
 848   for runNumber in ${listOfRuns[*]}; do
 849     [[ -z ${runNumber} ]] && continue
 850     [[ ! ${runNumber} =~ ^[0-9]*[0-9]$ ]] && continue
 851
 852     unset arr_cpass0_outputs_onerun
 853     unset arr_cpass1_outputs_onerun
 854     declare -a arr_cpass0_outputs_onerun
 855     declare -a arr_cpass1_outputs_onerun
 856
 857     jobindex=0
 858     while read inputFile; do
 859       currentDefaultOCDB=${defaultOCDB}
 860       [[ -z ${autoOCDB} ]] && autoOCDB=1
 861       if [[ ${autoOCDB} -ne 0 ]]; then
 862         currentDefaultOCDB=$(setYear ${inputFile} ${defaultOCDB})
 863       fi
 864
 865       #CPass0
 866       arr_cpass0_outputs_onerun[${jobindex}]="meta/cpass0.job${jobindex}.run${runNumber}.done"
 867       echo "${arr_cpass0_outputs_onerun[${jobindex}]} : benchmark.sh ${configFile} ${copyFiles[@]}"
 868       echo " ${alirootEnv} ./benchmark.sh CPass0 ${commonOutputPath}/000${runNumber}/cpass0 ${inputFile} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} ${jobindex} ${extraOpts}"
 869       echo
 870
 871       #CPass1
 872       arr_cpass1_outputs_onerun[${jobindex}]="meta/cpass1.job${jobindex}.run${runNumber}.done"
 873       echo "${arr_cpass1_outputs_onerun[${jobindex}]} : benchmark.sh ${configFile} meta/cpass0.localOCDB.${runNumber}.tgz ${copyFiles[@]}"
 874       echo " ${alirootEnv} ./benchmark.sh CPass1 ${commonOutputPath}/000${runNumber}/cpass1 ${inputFile} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} ${jobindex} ${extraOpts}"
 875       echo
 876       ((jobindex++))
 877
 878     done< <(grep "/000${runNumber}/" ${inputFileList})
 879
 880     #CPass0 list of Calib files to merge
 881     arr_cpass0_calib_list[${runindex}]="meta/cpass0.calib.run${runNumber}.list"
 882     echo "${arr_cpass0_calib_list[${runindex}]} : benchmark.sh ${arr_cpass0_outputs_onerun[*]}"
 883     echo "  ./benchmark.sh PrintValues calibfile ${arr_cpass0_calib_list[${runindex}]} ${arr_cpass0_outputs_onerun[*]}"
 884     echo
 885
 886     #CPass0 merging
 887     arr_cpass0_final[${runindex}]="meta/merge.cpass0.run${runNumber}.done"
 888     echo "meta/cpass0.localOCDB.${runNumber}.tgz ${arr_cpass0_final[${runindex}]} : benchmark.sh ${configFile} ${arr_cpass0_calib_list[${runindex}]} ${copyFiles[@]}"
 889     echo " ${alirootEnv} ./benchmark.sh MergeCPass0 ${commonOutputPath}/000${runNumber}/cpass0 ${currentDefaultOCDB} ${configFile} ${runNumber} ${arr_cpass0_calib_list[${runindex}]} ${extraOpts}"
 890     echo
 891
 892     #CPass1 list of Calib/QA files to merge
 893     # the trick is to have the string "Stage.txt" in the file name of the list of directories with QA output to trigger
 894     # the production of the trending tree (only then the task->Finish() will be called in QAtrain_duo.C, on the grid
 895     # this corresponds to the last merging stage)
 896     arr_cpass1_calib_list[${runindex}]="meta/cpass1.calib.run${runNumber}.list"
 897     echo "${arr_cpass1_calib_list[${runindex}]} : benchmark.sh ${arr_cpass1_outputs_onerun[*]}"
 898     echo "  ./benchmark.sh PrintValues calibfile ${arr_cpass1_calib_list[${runindex}]} ${arr_cpass1_outputs_onerun[*]};"
 899     echo
 900     arr_cpass1_QA_list[${runindex}]="meta/cpass1.QA.run${runNumber}.lastMergingStage.txt.list"
 901     echo "${arr_cpass1_QA_list[${runindex}]}: benchmark.sh ${arr_cpass1_outputs_onerun[*]}"
 902     echo "  ./benchmark.sh PrintValues dir ${arr_cpass1_QA_list[${runindex}]} ${arr_cpass1_outputs_onerun[*]}"
 903     echo
 904
 905     #CPass1 merging
 906     arr_cpass1_final[${runindex}]="meta/merge.cpass1.run${runNumber}.done"
 907     echo "meta/cpass1.localOCDB.${runNumber}.tgz ${arr_cpass1_final[${runindex}]} :  benchmark.sh ${configFile} ${arr_cpass1_calib_list[${runindex}]} ${arr_cpass1_QA_list[${runindex}]} ${copyFiles[@]}"
 908     echo " ${alirootEnv} ./benchmark.sh MergeCPass1 ${commonOutputPath}/000${runNumber}/cpass1 ${currentDefaultOCDB} ${configFile} ${runNumber} ${arr_cpass1_calib_list[${runindex}]} ${arr_cpass1_QA_list[${runindex}]} ${extraOpts}"
 909     echo
 910
 911     arr_cpass1_ESD_list[${runindex}]="meta/cpass1.ESD.run${runNumber}.list"
 912     echo "${arr_cpass1_ESD_list[${runindex}]} : benchmark.sh ${arr_cpass1_outputs_onerun[*]}"
 913     echo "  ./benchmark.sh PrintValues esd ${arr_cpass1_ESD_list[${runindex}]} ${arr_cpass1_outputs_onerun[*]}"
 914
 915     if [ ${runESDfiltering} -eq 1 ]; then
 916       arr_cpass1_filtering[${runindex}]="meta/filtering.cpass1.run${runNumber}.done"
 917       echo "${arr_cpass1_filtering[${runindex}]} : benchmark.sh ${configFile} ${arr_cpass1_ESD_list[${runindex}]}"
 918       echo "  ${alirootEnv} ./benchmark.sh MakeFilteredTrees ${commonOutputPath}/000${runNumber}/cpass1 ${runNumber} ${arr_cpass1_ESD_list[${runindex}]} ${filteringFactorHighPt} ${filteringFactorV0s} ${currentDefaultOCDB} 1000000 0 10000000 0 ${configFile} AliESDs_Barrel.root ${extraOpts}"
 919     fi
 920
 921     ((runindex++))
 922   done
 923
 924   #Summary
 925   echo "summary.log : benchmark.sh ${configFile} ${arr_cpass1_final[*]} ${arr_cpass1_filtering[*]}"
 926   echo " LOCAL ./benchmark.sh MakeSummary ${configFile}"
 927   echo
 928
 929   return 0
 930 }
 931
 932 goPrintValues()
 933 {
 934   #print the values given the key from any number of files (store in output file on second argument)
 935   if [[ $# -lt 3 ]]; then
 936     echo "goPrintValues key outputFile inputFiles"
 937     echo "if outputFile is \"-\" dont redirect to a file"
 938     return
 939   fi
 940   key=${1}
 941   outputFile=${2}
 942   [[ ${outputFile} =~ "-" ]] && outputFile=""
 943   shift 2 #remove 2 first arguments from arg list to only pass the input files to awk
 944   awk -v key=${key} '$0 ~ key" " {print $2}' "$@" | tee ${outputFile}
 945   return 0
 946 }
 947
 948 goCreateQAplots()
 949 {
 950   umask 0002
 951   mergedQAfileList=${1}
 952   productionID=${2}
 953   outputDir=${3}
 954   configFile=${4}
 955   shift 4
 956   parseConfig ${configFile} ${@}
 957
 958   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
 959
 960   [[ -z ${logOutputDir} ]] && logOutputDir=${PWD}
 961   [[ -z ${dontRedirectStdOutToLog} ]] && exec 2>&1 > ${logOutputDir}/makeQAplots.log
 962   echo "${0} $*"
 963
 964   olddir=${PWD}
 965   mkdir -p ${outputDir}
 966   cd ${outputDir}
 967   [[ ! "${PWD}" = "${outputDir}" ]] && echo "cannot make ${outputDir}... exiting" && return 1
 968
 969   echo ${ALICE_ROOT}/PWGPP/QA/scripts/runQA.sh inputList=${mergedQAfileList}
 970   ${ALICE_ROOT}/PWGPP/QA/scripts/runQA.sh inputList="${mergedQAfileList}" inputListHighPtTrees="${filteringList}"
 971
 972   cd ${olddir}
 973   return 0
 974 }
 975
 976 goTest()
 977 {
 978   umask 0002
 979   exec 2>&1
 980   exec > >(tee test.log)
 981   echo "$@"
 982   echo something
 983   return 0
 984 }
 985
 986 alirootInfo()
 987 {
 988   umask 0002
 989   # save aliroot repository info
 990   [[ -z "${ALICE_ROOT}" ]] && return 1
 991
 992   echo "\${ALICE_ROOT}=${ALICE_ROOT}"
 993   echo "\${ROOTSYS}=${ROOTSYS}"
 994   echo "\${PATH}=${PATH}"
 995   echo "\${LD_LIBRARY_PATH}=${LD_LIBRARY_PATH}"
 996
 997   pushd ${PWD}
 998   cd ${ALICE_ROOT}
 999   echo
1000
1001   currentBranch=$(git rev-parse --abbrev-ref HEAD)
1002   git status
1003   echo ""
1004   echo ""
1005   git diff ${currentBranch}
1006   popd
1007   return 0
1008 }
1009
1010 setYear()
1011 {
1012   #set the year
1013   #  ${1} - year to be set
1014   #  ${2} - where to set the year
1015   year1=$(guessYear ${1})
1016   year2=$(guessYear ${2})
1017   local path=${2}
1018   [[ ${year1} -ne ${year2} && -n ${year2} && -n ${year1} ]] && path=${2/\/${year2}\//\/${year1}\/}
1019   echo ${path}
1020   return 0
1021 }
1022
1023 guessPeriod()
1024 {
1025   #guess the period from the path, pick the rightmost one
1026   local IFS="/"
1027   declare -a path=( ${1} )
1028   local dirDepth=${#path[*]}
1029   for ((x=${dirDepth}-1;x>=0;x--)); do
1030     local field=${path[${x}]}
1031     [[ ${field} =~ ^LHC[0-9][0-9][a-z]$ ]] && period=${field} && break
1032   done
1033   echo ${period}
1034   return 0
1035 }
1036
1037 guessYear()
1038 {
1039   #guess the year from the path, pick the rightmost one
1040   local IFS="/"
1041   declare -a path=( ${1} )
1042   local dirDepth=${#path[*]}
1043   for ((x=${dirDepth}-1;x>=0;x--)); do
1044     local field=${path[${x}]}
1045     [[ ${field} =~ ^20[0-9][0-9]$ ]] && year=${field} && break
1046   done
1047   echo ${year}
1048   return 0
1049 }
1050
1051 guessRunNumber()
1052 {
1053   #guess the run number from the path, pick the rightmost one
1054   #works for /path/foo/000123456/bar/...
1055   #and       /path/foo.run123456.bar
1056   local IFS="/."
1057   declare -a path=( ${1} )
1058   local dirDepth=${#path[*]}
1059   for ((x=${dirDepth}-1;x>=0;x--)); do
1060     local field=${path[${x}]}
1061     field=${field/run/000}
1062     [[ ${field} =~ ^000[0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && runNumber=${field#000} && break
1063   done
1064   echo ${runNumber}
1065   return 0
1066 }
1067
1068 validateLog()
1069 {
1070   log=${1}
1071   errorConditions=(
1072                     'There was a crash'
1073                     'floating'
1074                     'error while loading shared libraries'
1075                     'std::bad_alloc'
1076                     's_err_syswatch_'
1077                     'Thread [0-9]* (Thread'
1078                     'AliFatal'
1079                     'core dumped'
1080                     '\.C.*error:.*\.h: No such file'
1081   )
1082
1083   warningConditions=(
1084                      'This is serious !'
1085                      'rocVoltage out of range:'
1086   )
1087
1088   local logstatus=0
1089   local errorSummary=""
1090   local warningSummary=""
1091
1092   for ((i=0; i<${#errorConditions[@]};i++)); do
1093     local tmp=$(grep -m1 -e "${errorConditions[${i}]}" ${log})
1094     [[ -n ${tmp} ]] && tmp+=" : "
1095     errorSummary+=${tmp}
1096   done
1097
1098   for ((i=0; i<${#warningConditions[@]};i++)); do
1099     local tmp=$(grep -m1 -e "${warningConditions[${i}]}" ${log})
1100     [[ -n ${tmp} ]] && tmp+=" : "
1101     warningSummary+=${tmp}
1102   done
1103
1104   if [[ -n ${errorSummary} ]]; then
1105     echo "${errorSummary}"
1106     return 1
1107   fi
1108
1109   if [[ -n ${warningSummary} ]]; then
1110     echo "${warningSummary}"
1111     return 2
1112   fi
1113
1114   return 0
1115 }
1116
1117 summarizeLogs()
1118 {
1119   #print a summary of logs
1120   logFiles=(
1121             "*.log"
1122             "stdout"
1123             "stderr"
1124   )
1125
1126   #check logs
1127   local logstatus=0
1128   for log in ${logFiles[*]}; do
1129     finallog=${PWD%/}/${log}
1130     [[ ! -f ${log} ]] && continue
1131     errorSummary=$(validateLog ${log})
1132     validationStatus=$?
1133     if [[ ${validationStatus} -eq 0 ]]; then
1134       #in pretend mode randomly report an error in rec.log some cases
1135       if [[ -n ${pretend} && "${log}" == "rec.log" ]]; then
1136         [[ $(( ${RANDOM}%2 )) -ge 1 ]] && echo "${finallog} BAD random error" || echo "${finallog} OK"
1137       else
1138         echo "${finallog} OK"
1139       fi
1140     elif [[ ${validationStatus} -eq 1 ]]; then
1141       echo "${finallog} BAD ${errorSummary}"
1142     elif [[ ${validationStatus} -eq 2 ]]; then
1143       echo "${finallog} OK MWAH ${errorSummary}"
1144     fi
1145   done
1146
1147   #report core files
1148   while read x; do
1149     echo ${x}
1150     chmod 644 ${x}
1151     gdb --batch --quiet -ex "bt" -ex "quit" aliroot ${x} > stacktrace_${x//\//_}.log
1152   done < <(/bin/ls ${PWD}/*/core 2>/dev/null; /bin/ls ${PWD}/core 2>/dev/null)
1153
1154   return ${logstatus}
1155 }
1156
1157 spitOutLocalOCDBaccessConfig()
1158 {
1159   umask 0002
1160   #find ${1} -name "*root" | \
1161   /bin/ls -1 ${1}/*/*/*/*.root 2>/dev/null | \
1162   while read line
1163   do
1164     local tmp=${line#${1}}
1165     echo ${tmp%/*} | \
1166     awk -v ocdb=${1} '{print "  man->SetSpecificStorage(\""$1"\",\"local://"ocdb"\");"}'
1167   done
1168   return 0
1169 }
1170
1171 goMakeLocalOCDBaccessConfig()
1172 {
1173   umask 0002
1174   # make a script that sets the specific storages form all the root files produced by CPass0
1175   local localOCDBpathCPass0=${1}
1176   local OCDBpathPrefix=${2}
1177   [[ -z ${OCDBpathPrefix} ]] && OCDBpathPrefix="."
1178
1179   if [[ -f ${localOCDBpathCPass0} && ${localOCDBpathCPass0} =~ \.tgz$ ]]; then
1180     tar xzf ${localOCDBpathCPass0}
1181     local localOCDBpathCPass0="${OCDBpathPrefix}/OCDB"
1182   fi
1183
1184   echo
1185   echo creating the specific storage script
1186   echo   localOCDBaccessConfig.C
1187   echo   based on OCDB: ${localOCDBaccessConfig}
1188   echo
1189
1190   local tempLocalOCDB=""
1191   if [[ -f localOCDBaccessConfig.C ]]; then
1192     tempLocalOCDB=$(mktemp)
1193     echo "egrep "SetSpecificStorage" localOCDBaccessConfig.C > ${tempLocalOCDB}"
1194     egrep "SetSpecificStorage" localOCDBaccessConfig.C > ${tempLocalOCDB}
1195   fi
1196
1197   echo "localOCDBaccessConfig()"                               >  localOCDBaccessConfig.C
1198   echo "{"                                                     >> localOCDBaccessConfig.C
1199   echo "  AliCDBManager* man = AliCDBManager::Instance();"     >> localOCDBaccessConfig.C
1200   spitOutLocalOCDBaccessConfig ${localOCDBpathCPass0}|sort|uniq  >> localOCDBaccessConfig.C
1201   [[ -f "${tempLocalOCDB}" ]] && cat ${tempLocalOCDB}              >> localOCDBaccessConfig.C
1202   echo "}"                                                     >> localOCDBaccessConfig.C
1203
1204   [[ -f "${tempLocalOCDB}" ]] && rm -f ${tempLocalOCDB}
1205
1206   if ! grep SetSpecificStorage localOCDBaccessConfig.C; then
1207     echo
1208     echo "!!!!!!! CPass0 produced no OCDB entries"
1209     return 1
1210   fi
1211   return 0
1212 }
1213
1214 goMakeFilteredTrees()
1215 {
1216   outputDir=${1}
1217   runNumber=${2}
1218   #get path to input list
1219   inputListfiles=${3}
1220   #get scale number for tracks
1221   filterT=${4}
1222   #get scale number for V0s
1223   filterV=${5}
1224   #get OCDB path (optional)
1225   OCDBpath=${6}
1226   #get max number of files
1227   maxFiles=${7-"1000000"}
1228   #get offset of first file
1229   offsetFile=${8-"0"}
1230   #get max number of events
1231   maxEvents=${9-"30000000"}
1232   #get offset of first event
1233   offsetEvent=${10-"0"}
1234   configFile=${11-"benchmark.config"}
1235   esdFileName=${12-"AliESDs_Barrel.root"}
1236   shift 12
1237   parseConfig ${configFile} $@
1238
1239   commonOutputPath=${PWD}
1240   doneFile=${commonOutputPath}/meta/filtering.cpass1.run${runNumber}.done
1241
1242   runpath=${commonOutputPath}/rundir_filtering_run${runNumber}
1243
1244   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
1245
1246   mkdir -p ${outputDir}
1247   mkdir -p ${runpath}
1248
1249   cd ${runpath}
1250
1251   cat > filtering.log << EOF
1252   goMakeFilteredTrees config:
1253   runpath=${runpath}
1254   outputDir=${outputDir}
1255   commonOutputPath=${commonOutputPath}
1256   ALICE_ROOT=${ALICE_ROOT}
1257   PATH=${PATH}
1258   offsetEvent=$offsetEvent
1259   configFile=$configFile
1260   esdFileName=$esdFileName
1261 EOF
1262
1263   if [[ -z ${pretend} ]];then
1264     aliroot -l -b -q "${ALICE_ROOT}/PWGPP/macros/runFilteringTask.C(\"${inputListfiles}\",${filterT},${filterV},\"${OCDBpath}\",${maxFiles},${offsetFile},${maxEvents},${offsetEvent},\"${esdFileName}\")" &>> filtering.log
1265   else
1266     touch filtering.log FilterEvents_Trees.root
1267   fi
1268   pwd
1269   /bin/ls
1270   touch ${doneFile}
1271   summarizeLogs >>  ${doneFile}
1272   echo mv -f * ${outputDir}
1273   mv -f * ${outputDir}
1274   [[ -f ${outputDir}/FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFile}
1275   cd ${commonOutputPath}
1276   [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
1277   return 0
1278 }
1279
1280 submit()
1281 {
1282   umask 0002
1283   [[ $# -ne 6 ]] && echo "6 args needed, you supplied $#" && return 1
1284   JobID=${1}
1285   startID=${2}
1286   endID=${3}
1287   waitForJOBID=${4}
1288   command=${5}
1289   commandArgs=${6}
1290
1291   [[ -z ${waitForJOBID} ]] && waitForJOBID=0
1292
1293   newFarm=$(which qsub|grep "^/usr/bin/qsub")
1294
1295   batchSystem="SGE"
1296
1297   if [[ -z "${newFarm}" ]]
1298   then
1299     #old LSF
1300     # submit it (as job array)
1301     nFiles=$(( ${endID}-${startID}+1 ))
1302     while [ ${startID} -le ${nFiles}  ] ; do
1303       if [ $(expr ${nFiles} - ${startID}) -gt 999 ] ; then
1304         endID=$(expr ${startID} + 999)
1305       else
1306         endID=${nFiles}
1307       fi
1308     if [[ ${waitForJOBID} -eq 0 ]]; then
1309         echo ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}"
1310         ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}"
1311       else
1312         echo ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -w "ended(${waitForJOBID})" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}"
1313         ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -w "ended(${waitForJOBID})" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}"
1314       fi
1315       startID=$(expr ${endID} + 1)
1316     done
1317   else
1318     #new SGE farm
1319     if [[ ${waitForJOBID} =~ "000" ]]; then
1320       echo ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" ${commandArgs}
1321       ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" ${commandArgs}
1322     else
1323       echo ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -hold_jid "${waitForJOBID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" ${commandArgs}
1324       ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -hold_jid "${waitForJOBID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" ${commandArgs}
1325     fi
1326   fi
1327   return 0
1328 }
1329
1330 goSubmitBatch()
1331 {
1332   if [[ $# -lt 3 ]]; then
1333     echo "minimal use:"
1334     echo " ${0} submit fileList productionID configFile"
1335     return 0
1336   fi
1337
1338   productionID=${1}
1339   inputList=${2}
1340   configFile=${3:-"benchmark.config"}
1341   configFile=$(readlink -f ${configFile})
1342   shift 3
1343   extraOpts="$@"
1344   parseConfig ${configFile} ${extraOpts}
1345
1346   #redirect all output to submit.log
1347   echo "redirecting all output to ${PWD}/submit_${productionID//"/"/_}.log"
1348   exec 7>&1
1349   exec 1>submit_${productionID//"/"/_}.log 2>&1
1350
1351   umask 0002
1352   echo ${0}" submit $*"
1353   if [[ -z "${inputList}" || -z "${productionID}" ]]
1354   then
1355     echo
1356     echo " Usage: ${0} submit inputList productionID [configFile=benchmark.config]"
1357     echo
1358     return
1359   fi
1360
1361   # check if config file is there
1362   if [ ! -f ${configFile} ]; then
1363     echo "ERROR! Config File '${configFile}' not found" >&2
1364     return
1365   else
1366     echo "Using Config File: '${configFile}'"
1367   fi
1368
1369   [[ ! -f ${alirootEnv} ]] && echo "alirootEnv script ${alirootEnv} not found!..." && return 1
1370
1371   #move the script, config and some other stuff to ${commonOutputPath} first, then use them from there
1372   self=$(readlink -f "${0}")
1373   configPath=$(dirname ${configFile})
1374   export commonOutputPath=${baseOutputDirectory}/${productionID}
1375
1376   mkdir -p ${commonOutputPath}
1377   mkdir -p ${commonOutputPath}/logs
1378   mkdir -p ${commonOutputPath}/meta
1379
1380   cp ${self} ${commonOutputPath}
1381   cp ${configFile} ${commonOutputPath}
1382   cp ${inputList} ${commonOutputPath}
1383   self=${commonOutputPath}/${self##*/}
1384   chmod u+x ${self}
1385   configFile=${commonOutputPath}/${configFile##*/}
1386   inputList=${commonOutputPath}/${inputList##*/}
1387
1388   #convert to absolut pathnames
1389   inputList=$(readlink -f "${inputList}")
1390   #make list of runs
1391   if [[ -z ${runNumber} ]]; then
1392     listOfRuns=($(while read x; do guessRunNumber ${x}; done < ${inputList} | sort | uniq))
1393   else
1394     listOfRuns=${runNumber}
1395   fi
1396
1397   alirootSource=$(readlink -f "${alirootSource}")
1398
1399   echo ""
1400   echo "### BEGIN CONFIGURATION ###"
1401   echo ""
1402   echo "GENERAL:"
1403   echo ""
1404   echo "    productionID:    ${productionID}"
1405   echo "    batchCommand:    ${batchCommand}"
1406   echo "    batchFlags:      ${batchFlags}"
1407   echo "    alirootEnv:   ${alirootEnv}"
1408   ${alirootEnv} echo '    ALICE_ROOT:      ${ALICE_ROOT}'
1409   ${alirootEnv} echo '    ALIROOT_RELEASE: ${ALICE_RELEASE}'
1410   echo "    inputList:       ${inputList}"
1411   echo "    configPath:      ${configPath}"
1412   echo "    commonOutputPath:      ${commonOutputPath}"
1413   echo "    defaultOCDB:     ${defaultOCDB}"
1414   echo "      autoOCDB: ${autoOCDB}"
1415   echo "    recoTriggerOptions:   ${recoTriggerOptions}"
1416   echo "    runs:"
1417   echo "      ${listOfRuns[*]}"
1418   echo ""
1419   echo "THE TRAIN WILL RUN:"
1420
1421   if [ ${runCPass0reco} -eq 1 ]; then
1422     echo "    Pass0 - Recontruction"
1423   fi
1424
1425   if [ ${runCPass0MergeMakeOCDB} -eq 1 ]; then
1426     echo "    Pass0 - merging and OCDB export"
1427   fi
1428
1429   if [ ${runCPass1reco} -eq 1 ]; then
1430     echo "    Pass1 - Recontruction"
1431   fi
1432   if [ ${runCPass1MergeMakeOCDB} -eq 1 ]; then
1433     echo "    Pass1 - merging and OCDB export"
1434   fi
1435
1436   echo ""
1437   echo "LIMITS:"
1438   echo "    max. Events/Chunk:   ${nEvents}"
1439   echo "    max. Number of Chunks per Run:     ${nMaxChunks}"
1440   echo ""
1441   echo "### END CONFIGURATION ###"
1442   echo ""
1443
1444
1445   # check if input file is there
1446   if [ ! -f ${inputList} ]; then
1447     echo "ERROR! Input List '${inputList}' not found" >&2
1448     return
1449   fi
1450
1451   # define jobid (for dependent jobs)
1452   date=$(date +%Y_%m_%d_%H%M%S)
1453   #for each run we submit one jobarray:
1454   for runNumber in ${listOfRuns[*]}; do
1455
1456     [[ -z ${runNumber} ]] && continue
1457     [[ ! ${runNumber} =~ ^[0-9]*[0-9]$ ]] && continue
1458
1459     JOBpostfix="${productionID//"/"/_}_${runNumber}_${date}"
1460     JOBID1="p0_${JOBpostfix}"
1461     JOBID1wait="w0_${JOBpostfix}"
1462     JOBID2="m0_${JOBpostfix}"
1463     JOBID2wait="wm0_${JOBpostfix}"
1464     JOBID3="op0_${JOBpostfix}"
1465     JOBID3wait="wop0_${JOBpostfix}"
1466     JOBID4="p1_${JOBpostfix}"
1467     JOBID4wait="w1_${JOBpostfix}"
1468     JOBID5="m1_${JOBpostfix}"
1469     JOBID5wait="wm1_${JOBpostfix}"
1470     JOBID6="s1_${JOBpostfix}"
1471     JOBID6wait="ws1_${JOBpostfix}"
1472     JOBID7="QA_${JOBpostfix}"
1473     JOBmakeESDlistCPass1="lp1_${JOBpostfix}"
1474     JOBfilterESDcpass1="fp1_${JOBpostfix}"
1475     LASTJOB="000"
1476
1477     oneInputFile=$(egrep -m1 "${runNumber}\/" ${inputList})
1478
1479     currentDefaultOCDB=${defaultOCDB}
1480     [[ -z ${autoOCDB} ]] && autoOCDB=1
1481     if [[ ${autoOCDB} -ne 0 ]]; then
1482       currentDefaultOCDB=$(setYear ${oneInputFile} ${defaultOCDB})
1483     fi
1484
1485     echo "submitting run ${runNumber} with OCDB ${currentDefaultOCDB}"
1486
1487     ###############################################################################
1488     #run one chunk with valgrind:
1489     if [[ -n ${runValgrind} ]]; then
1490       [[ -z ${nEventsValgrind} ]] && nEventsValgrind=2
1491       [[ -z ${valgrindCommand} ]] && valgrindCommand="/usr/bin/valgrind --tool=callgrind --num-callers=40 -v --trace-children=yes"
1492       submit "valgrind" 1 1 000 "${alirootEnv} ${self}" "CPass0 ${commonOutputPath}/000${runNumber}/valgrind ${oneInputFile} ${nEventsValgrind} ${currentDefaultOCDB} ${configFile} ${runNumber} valgrind valgrindCommand=${valgrindCommand} ${extraOpts}"
1493     fi
1494
1495     ################################################################################
1496     ################################################################################
1497     # run the CPass0 if requested
1498
1499     if [ ${runCPass0reco} -eq 1 ]; then
1500
1501       echo
1502       echo "starting CPass0... for run ${runNumber}"
1503       echo
1504
1505       # create directory and copy all files that are needed
1506       targetDirectory="${commonOutputPath}/000${runNumber}/cpass0"
1507       mkdir -p ${targetDirectory}
1508
1509       filesCPass0=(
1510                     "${configPath}/runCPass0.sh"
1511                     "${configPath}/recCPass0.C"
1512                     "${configPath}/runCalibTrain.C"
1513                     "${configPath}/localOCDBaccessConfig.C"
1514                     "${configPath}/OCDB*.root"
1515                     "${configPath}/sim.C"
1516                     "${configPath}/rec.C"
1517                     "${configPath}/Config.C"
1518       )
1519       for file in ${filesCPass0[*]}; do
1520         [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
1521       done
1522
1523       localInputList=${targetDirectory}/${inputList##*/}
1524       [[ ! -f ${localInputList} ]] && egrep "\/000${runNumber}\/" ${inputList} > ${localInputList}
1525       # limit nFiles to nMaxChunks
1526       nFiles=$(wc -l < ${localInputList})
1527       [[ ${nFiles} -eq 0 ]] && echo "list contains ZERO files! exiting..." && return 1
1528       echo "raw files in list:    ${nFiles}"
1529       if [[ ${nMaxChunks} -gt 0 && ${nMaxChunks} -le ${nFiles} ]]; then
1530         nFiles=${nMaxChunks}
1531       fi
1532       echo "raw files to process: ${nFiles}"
1533       [[ -z "${percentProcessedFilesToContinue}" ]] && percentProcessedFilesToContinue=100
1534       if [[ ${percentProcessedFilesToContinue} -eq 100 ]]; then
1535         nFilesToWaitFor=${nFiles}
1536       else
1537         nFilesToWaitFor=$(( ${nFiles}-${nFiles}/(100/(100-${percentProcessedFilesToContinue})) ))
1538       fi
1539       echo "requested success rate is ${percentProcessedFilesToContinue}%"
1540       echo "merging will start after ${nFilesToWaitFor} jobs are done"
1541
1542       submit ${JOBID1} 1 ${nFiles} 000 "${alirootEnv} ${self}" "CPass0 ${targetDirectory} ${localInputList} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} -1 ${extraOpts}"
1543
1544       ## submit a monitoring job that will run until a certain number of jobs are done with reconstruction
1545       submit "${JOBID1wait}" 1 1 000 "${alirootEnv} ${self}" "WaitForOutput ${commonOutputPath} 'meta/cpass0.job*.run${runNumber}.done' ${nFilesToWaitFor} ${maxSecondsToWait} '-maxdepth 1'"
1546       LASTJOB=${JOBID1wait}
1547
1548     fi #end running CPass0
1549     ################################################################################
1550
1551
1552     ################################################################################
1553     # submit merging of CPass0, depends on the reconstruction
1554
1555     if [ ${runCPass0MergeMakeOCDB} -eq 1 ]; then
1556
1557       echo
1558       echo "submit CPass0 merging for run ${runNumber}"
1559       echo
1560
1561       targetDirectory="${commonOutputPath}/000${runNumber}/cpass0"
1562       mkdir -p ${targetDirectory}
1563
1564       #copy the scripts
1565       filesMergeCPass0=(
1566                         "${configPath}/OCDB.root"
1567                         "${configPath}/mergeMakeOCDB.byComponent.sh"
1568                         "${configPath}/mergeMakeOCDB.sh"
1569                         "${configPath}/localOCDBaccessConfig.C"
1570                         "${configPath}/mergeByComponent.C"
1571                         "${configPath}/makeOCDB.C"
1572                         "${configPath}/merge.C"
1573       )
1574       for file in ${filesMergeCPass0[*]}; do
1575         [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
1576       done
1577
1578       submit ${JOBID2} 1 1 "${LASTJOB}" "${alirootEnv} ${self}" "MergeCPass0 ${targetDirectory} ${currentDefaultOCDB} ${configFile} ${runNumber} cpass0.calib.run${runNumber}.list ${extraOpts}"
1579       LASTJOB=${JOBID2}
1580
1581       if [[ -n ${generateMC} ]]; then
1582         submit "mrl${JOBpostfix}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" "PrintValues sim ${commonOutputPath}/meta/sim.run${runNumber}.list ${commonOutputPath}/meta/cpass0.job*.run${runNumber}.done"
1583         LASTJOB="mrl${JOBpostfix}"
1584       fi
1585
1586       echo
1587     fi
1588     # end of merging CPass0
1589     ################################################################################
1590
1591     ################################################################################
1592     ################################################################################
1593     # run the CPass1 if requested
1594
1595     if [ ${runCPass1reco} -eq 1 ]; then
1596
1597       targetDirectory="${commonOutputPath}/000${runNumber}/cpass1"
1598       rm -f ${commonOutputPath}/meta/cpass1.job*.run${runNumber}.done
1599
1600       # safety feature: if we are re-running for any reason we want to delete the previous output first.
1601       [[ -d ${targetDirectory} ]] && rm -rf ${targetDirectory}/* && echo "removed old output at ${targetDirectory}/*"
1602
1603       echo
1604       echo "starting CPass1... for run ${runNumber}"
1605       echo
1606
1607       # create directory and copy all files that are needed
1608       mkdir -p ${targetDirectory}
1609
1610       filesCPass1=(
1611                     "${configPath}/runCPass1.sh"
1612                     "${configPath}/recCPass1.C"
1613                     "${configPath}/recCPass1_OuterDet.C"
1614                     "${configPath}/runCalibTrain.C"
1615                     "${configPath}/QAtrain_duo.C"
1616                     "${configPath}/localOCDBaccessConfig.C"
1617                     "${configPath}/OCDB.root"
1618       )
1619       for file in ${filesCPass1[*]}; do
1620         [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
1621       done
1622
1623       if [[ -n ${generateMC} ]]; then
1624         localInputList=${commonOutputPath}/meta/sim.run${runNumber}.list
1625       else
1626         localInputList=${targetDirectory}/${inputList##*/}
1627         [[ ! -f ${localInputList} ]] && egrep "\/000${runNumber}\/" ${inputList} > ${localInputList}
1628       fi
1629       # limit nFiles to nMaxChunks
1630       nFiles=$(wc -l < ${localInputList})
1631       [[ ${nFiles} -eq 0 ]] && echo "list contains ZERO files! continuing..." && continue
1632       echo "raw files in list:    ${nFiles}"
1633       if [[ ${nMaxChunks} -gt 0 && ${nMaxChunks} -le ${nFiles} ]]; then
1634         nFiles=${nMaxChunks}
1635       fi
1636       echo "raw files to process: ${nFiles}"
1637       [[ -z "${percentProcessedFilesToContinue}" ]] && percentProcessedFilesToContinue=100
1638       if [[ ${percentProcessedFilesToContinue} -eq 100 ]]; then
1639         nFilesToWaitFor=${nFiles}
1640       else
1641         nFilesToWaitFor=$(( ${nFiles}-${nFiles}/(100/(100-${percentProcessedFilesToContinue})) ))
1642       fi
1643       echo "requested success rate is ${percentProcessedFilesToContinue}%"
1644       echo "merging will start after ${nFilesToWaitFor} jobs are done"
1645
1646       submit ${JOBID4} 1 ${nFiles} "${LASTJOB}" "${alirootEnv} ${self}" "CPass1 ${targetDirectory} ${localInputList} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} -1 ${extraOpts}"
1647
1648       ################################################################################
1649       ## submit a monitoring job that will run until a certain number of jobs are done with reconstruction
1650       submit "${JOBID4wait}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" "WaitForOutput ${commonOutputPath} 'meta/cpass1.job*.run${runNumber}.done' ${nFilesToWaitFor} ${maxSecondsToWait} '-maxdepth 1'"
1651       LASTJOB=${JOBID4wait}
1652       ################################################################################
1653
1654       echo
1655     fi #end running CPass1
1656
1657     ################################################################################
1658     # submit merging of CPass1, depends on the reconstruction
1659     if [ ${runCPass1MergeMakeOCDB} -eq 1 ]; then
1660
1661       echo
1662       echo "submit CPass1 merging for run ${runNumber}"
1663       echo
1664
1665       targetDirectory="${commonOutputPath}/000${runNumber}/cpass1"
1666       rm -f ${commonOutputPath}/meta/merge.cpass1.run${runNumber}.done
1667       mkdir -p ${targetDirectory}
1668
1669       # copy files
1670       filesMergeCPass1=(
1671                         "${configPath}/OCDB.root"
1672                         "${configPath}/localOCDBaccessConfig.C"
1673                         "${configPath}/mergeMakeOCDB.byComponent.sh"
1674                         "${configPath}/mergeByComponent.C"
1675                         "${configPath}/makeOCDB.C"
1676                         "${configPath}/merge.C"
1677                         "${configPath}/mergeMakeOCDB.sh"
1678                         "${configPath}/QAtrain_duo.C"
1679       )
1680       for file in ${filesMergeCPass1[*]}; do
1681         [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
1682       done
1683
1684       submit "${JOBID5}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" "MergeCPass1 ${targetDirectory} ${currentDefaultOCDB} ${configFile} ${runNumber} cpass0.calib.run${runNumber}.list cpass1.QA.run${runNumber}.lastMergingStage.txt.list ${extraOpts}"
1685       LASTJOB=${JOBID5}
1686       echo
1687     fi
1688
1689     ##############################
1690     if [ ${runESDfiltering} -eq 1 ]; then
1691       rm -f ${commonOutputPath}/cpass1.ESD.run${runNumber}.list
1692       rm -f ${commonOutputPath}/meta/filtering.cpass1.run*.done
1693       echo
1694       echo submitting filtering for run ${runNumber}
1695       echo
1696       submit "${JOBmakeESDlistCPass1}" 1 1 "${LASTJOB}" "${self}" "PrintValues esd ${commonOutputPath}/meta/cpass1.ESD.run${runNumber}.list ${commonOutputPath}/meta/cpass1.job*.run${runNumber}.done "
1697       submit "${JOBfilterESDcpass1}" 1 1 "${JOBmakeESDlistCPass1}" "${alirootEnv} ${self}" "MakeFilteredTrees ${commonOutputPath}/000${runNumber}/cpass1 ${runNumber} ${commonOutputPath}/meta/cpass1.ESD.run${runNumber}.list ${filteringFactorHighPt} ${filteringFactorV0s} ${currentDefaultOCDB} 1000000 0 10000000 0 ${configFile} AliESDs_Barrel.root ${extraOpts}"
1698       LASTJOB=${JOBfilterESDcpass1}
1699     fi
1700
1701   done
1702
1703   #################################################################################
1704   #################################################################################
1705   if [ ${runESDfiltering} -eq 1 ]; then
1706     submit "${JOBID5wait}" 1 1 "${LASTJOB}" "${self}" "WaitForOutput ${commonOutputPath} 'meta/filtering.cpass1.run*.done' ${#listOfRuns[@]} ${maxSecondsToWait}"
1707   else
1708     submit "${JOBID5wait}" 1 1 "${LASTJOB}" "${self}" "WaitForOutput ${commonOutputPath} 'meta/merge.cpass1.run*.done' ${#listOfRuns[@]} ${maxSecondsToWait}"
1709   fi
1710   LASTJOB=${JOBID5wait}
1711
1712   #################################################################################
1713   echo
1714   echo "submit make a summary"
1715   echo
1716
1717   submit "${JOBID6}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" "MakeSummary ${configFile}"
1718   LASTJOB=${JOBID6}
1719   #################################################################################
1720
1721   #restore stdout
1722   exec 1>&7 7>&-
1723   echo "jobs submitted."
1724   return 0
1725 }
1726
1727 goWaitForOutput()
1728 {
1729   umask 0002
1730   [[ $# -lt 3 ]] && echo "goWaitForOutput() wrong number of arguments, exiting.." && return 1
1731   echo searchPath=${1}
1732   echo fileName=${2}
1733   echo numberOfFiles=${3}
1734   echo maxSecondsToWait=${4}
1735   searchPath=${1}
1736   fileName=${2}
1737   numberOfFiles=${3}
1738   maxSecondsToWait=${4}
1739   extraFindOptions=${5}
1740   echo "command to be executed: /bin/ls -1 ${searchPath}/${fileName} ${extraFindOptions}"
1741   [[ -z "${maxSecondsToWait}" ]] && maxSecondsToWait=$(( 3600*12 ))
1742   while true; do
1743     #n=$(find ${searchPath} ${extraFindOptions} -name "${fileName}" | wc -l)
1744     n=$(/bin/ls -1 ${searchPath}/${fileName} 2>/dev/null | wc -l)
1745     [[ ${n} -gt 0 ]] && echo "found ${n} X ${fileName}"
1746     [[ ${n} -ge ${numberOfFiles} ]] && break
1747     [[ ${SECONDS} -gt ${maxSecondsToWait} ]] && echo "timeout of ${maxSecondsToWait}s!" && break
1748     sleep 60
1749   done
1750   echo "DONE! exiting..."
1751   return 0
1752 }
1753
1754 mergeSysLogs()
1755 {
1756   outputFile=${1}
1757   shift
1758   inputFiles="$@"
1759   i=0
1760   if ! ls -1 ${inputFiles} &>/dev/null; then echo "the files dont exist!: ${inputFiles}"; return 1; fi
1761   while read x; do
1762     runNumber=$(guessRunNumber ${x})
1763     [[ -z ${runNumber} ]] && echo "run number cannot be guessed for ${x}" && continue
1764     awk -v run=${runNumber} -v i=${i} 'NR > 1 {print run" "$0} NR==1 && i==0 {print "run/I:"$0}' ${x}
1765     (( i++ ))
1766   done < <(ls -1 ${inputFiles}) > ${outputFile}
1767   return 0
1768 }
1769
1770 goMakeMergedSummaryTree()
1771 {
1772   # create list of calibration entries
1773   # takes no arguments, just run it in the base output
1774   # directory with the following files in the working directory
1775   #
1776   #  Calibration file lists:
1777   #       cpass0.dcsTree.list, cpass1.dcsTree.list
1778   #  QA trending root file:
1779   #       trending.root
1780   #
1781   #  Production infoo ascii files:
1782   #       summary_pass0.tree
1783   #       summary_pass1.tree
1784   #
1785
1786   [[ ! -f cpass0.dcsTree.list ]] && echo "no cpass0.dcsTree.list" && return 1
1787   [[ ! -f cpass1.dcsTree.list ]] && echo "no cpass1.dcsTree.list" && return 1
1788   [[ ! -f trending.root ]] && echo "no trending.root" && return 1
1789   [[ ! -f summary_pass0.tree ]] && echo "no summary_pass0.tree" && return 1
1790   [[ ! -f summary_pass1.tree ]] && echo "no summary_pass1.tree" && return 1
1791
1792   #first, dump the C macro to file
1793   cat << EOF > mergeTree.C
1794   //
1795   // Merge summary information
1796   // Following files are expected to be in the working directory
1797   //
1798   // Calibration file lists:
1799   //      cpass0.dcsTree.list, cpass1.dcsTree.list
1800   // QA trending root file:
1801   //      trending.root
1802   //
1803   // Production infoo ascii files:
1804   //      summary_pass0.tree
1805   //      summary_pass1.tree
1806   //
1807   void mergeTree(){
1808     //
1809     //
1810     //
1811     // Calibration values dump
1812     //
1813     AliXRDPROOFtoolkit::MakeTreeFromList("Calib.TPC.CPass0.root", "dcs","dcs","cpass0.dcsTree.list",1);
1814     AliXRDPROOFtoolkit::MakeTreeFromList("Calib.TPC.CPass1.root", "dcs","dcs","cpass1.dcsTree.list",1);
1815     //
1816     // Calibration status dump
1817     //
1818     TFile *fprod = TFile::Open("fproduction.root","recreate");
1819     TTree  tree0, tree1;
1820     tree0.ReadFile("summary_pass0.tree");
1821     tree1.ReadFile("summary_pass1.tree");
1822     tree0.Write("CPass0");
1823     tree1.Write("CPass1");
1824     fprod->Close();
1825     //
1826     //
1827     //
1828     TString stringSetup="";
1829     stringSetup+="1#QA.TPC#run#SummaryTPCQA/tpcQA#trending.root+";  //
1830     stringSetup+="1#Calib.TPC.CPass0#run#dcs#Calib.TPC.CPass0.root+";  //
1831     stringSetup+="1#Calib.TPC.CPass1#run#dcs#Calib.TPC.CPass1.root+";  //
1832     //
1833     stringSetup+="1#CPass0#runnumber#CPass0#fproduction.root+";  //
1834     stringSetup+="1#CPass1#runnumber#CPass1#fproduction.root+";  //
1835     //
1836     AliXRDPROOFtoolkit::JoinTreesIndex("outAll.root","joinAll","run",stringSetup.Data(), 1);
1837   }
1838 EOF
1839
1840   aliroot -b -q "mergeTree.C" > mergeTrees.log
1841   return $?
1842 }
1843
1844 goMakeSummary()
1845 {
1846   #all the final stuff goes in here for ease of use:
1847   # summary logs
1848   # qa plot making
1849   # final file lists
1850
1851   configFile=${1}
1852   shift 1
1853   extraOpts="$@"
1854   log="summary.log"
1855   productionID="qa"
1856   parseConfig ${configFile} ${extraOpts}
1857
1858   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
1859
1860   exec &> >(tee ${log})
1861
1862   [[ ! -f ${configFile} ]] && echo "no config file ${configFile}!" && return
1863
1864   [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
1865
1866   #summarize the global stuff
1867   echo "env script: ${alirootSource} ${alirootEnv}"
1868   echo "\$ALICE_ROOT=${ALICE_ROOT}"
1869   echo "commonOutputPath=${commonOutputPath}"
1870
1871   #summarize the stacktraces
1872   awk '
1873        BEGIN {
1874                print "frame/I:method/C:line/C:cpass/I:aliroot/I";
1875                RS="#[0-9]*";
1876                aliroot=0;
1877              }
1878              {
1879                if ($3 ~ /Ali*/) aliroot=1; else aliroot=0;
1880                gsub("#","",RT);
1881                if ($NF!="" && RT!="" && $3!="") print RT" "$3" "$NF" "0" "aliroot
1882              }
1883       ' 000*/cpass0/*/stacktrace* 2>/dev/null > stacktrace.tree
1884   awk '
1885        BEGIN {
1886                RS="#[0-9]*";
1887                aliroot=0;
1888              }
1889              {
1890                if ($3 ~ /Ali*/) aliroot=1; else aliroot=0;
1891                gsub("#","",RT);
1892                if ($NF!="" && RT!="" && $3!="") print RT" "$3" "$NF" "1" "aliroot
1893              }
1894       ' 000*/cpass1/*/stacktrace* 2>/dev/null >> stacktrace.tree
1895
1896   echo total numbers for the production:
1897   echo
1898   awk 'BEGIN {nFiles=0;nCore=0;}
1899   /^calibfile/ {nFiles++;}
1900   /core dumped/ {nCore++i;}
1901   END {print     "cpass0 produced "nFiles" calib files, "nCore" core files";}' meta/cpass0.job*done 2>/dev/null
1902   awk 'BEGIN {nOK=0; nBAD=0; }
1903   /\/rec.log OK/ {nOK++;}
1904   /\/rec.log BAD/ {nBAD++;}
1905   /stderr BAD/ {if ($0 ~ /rec.log/){nBAD++;}}
1906   END {print     "cpass0 reco:  OK: "nOK"\tBAD: "nBAD;}' meta/cpass0.job*done 2>/dev/null
1907   awk 'BEGIN {nOK=0; nBAD=0; }
1908   /\/calib.log OK/ {nOK++;}
1909   /\/calib.log BAD/ {nBAD++;}
1910   END {print "cpass0 calib: OK: "nOK"\tBAD: "nBAD;}' meta/cpass0.job*done 2>/dev/null
1911
1912   awk 'BEGIN {nOK=0; nBAD=0; }
1913   /merge.log OK/ {nOK++;}
1914   /merge.log BAD/ {nBAD++;}
1915   END {print "cpass0 merge: OK: "nOK"\tBAD: "nBAD;}' meta/merge.cpass0*done 2>/dev/null
1916   awk 'BEGIN {nOK=0; nBAD=0; }
1917   /ocdb.log OK/ {nOK++;}
1918   /ocdb.log BAD/ {nBAD++;}
1919   END {print   "cpass0 OCDB:  OK: "nOK"\tBAD: "nBAD;}' meta/merge.cpass0*done 2>/dev/null
1920
1921   echo
1922   awk 'BEGIN {nFiles=0;nCore=0;}
1923   /^calibfile/ {nFiles++;}
1924   /core dumped/ {nCore++;}
1925   END {print     "cpass1 produced "nFiles" calib files, "nCore" core files";}' meta/cpass1.job*done 2>/dev/null
1926   awk 'BEGIN {nOK=0; nBAD=0; }
1927   /\/rec.log OK/ {nOK++;}
1928   /\/rec.log BAD/ {nBAD++;}
1929   /stderr BAD/ {if ($0 ~ /rec.log/){nBAD++;}}
1930   END {print     "cpass1 reco:  OK: "nOK"\tBAD: "nBAD;}' meta/cpass1.job*done 2>/dev/null
1931   awk 'BEGIN {nOK=0; nBAD=0; }
1932   /\/calib.log OK/ {nOK++;}
1933   /\/calib.log BAD/ {nBAD++;}
1934   END {print "cpass1 calib: OK: "nOK"\tBAD: "nBAD;}' meta/cpass1.job*done 2>/dev/null
1935
1936   awk 'BEGIN {nOK=0; nBAD=0; }
1937   /merge.log OK/ {nOK++;}
1938   /merge.log BAD/ {nBAD++;}
1939   END {print "cpass1 merge: OK: "nOK"\tBAD: "nBAD;}' meta/merge.cpass1*done 2>/dev/null
1940   awk 'BEGIN {nOK=0; nBAD=0; }
1941   /ocdb.log OK/ {nOK++;}
1942   /ocdb.log BAD/ {nBAD++;}
1943   END {print   "cpass1 OCDB:  OK: "nOK"\tBAD: "nBAD;}' meta/merge.cpass1*done 2>/dev/null
1944
1945   echo
1946   echo per run stats:
1947   /bin/ls -1 meta/merge.cpass0.run*.done | while read x
1948 do
1949   dir=$(goPrintValues calibfile - ${x})
1950   runNumber=$(guessRunNumber ${dir})
1951   [[ -z ${runNumber} ]] && continue
1952
1953   if $(/bin/ls meta/cpass0.job*.run${runNumber}.done &> /dev/null); then
1954     statusCPass0=( $(
1955     awk 'BEGIN {nOKrec=0;nBADrec=0;nOKcalib=0;nBADcalib=0;nOKstderr=0;nBADstderr=0;}
1956     /\/rec.log OK/ {nOKrec++;}
1957     /\/rec.log BAD/ {nBADrec++;}
1958     /stderr BAD/ {if ($0 ~ /rec.log/) {nBADrec++;} nBADstderr++;}
1959     /stderr OK/ {nOKstderr++;}
1960     /\/calib.log OK/ {nOKcalib++;}
1961     /\/calib.log BAD/ {nBADcalib++}
1962     END {print ""nOKrec" "nBADrec" "nOKstderr" "nBADstderr" "nOKcalib" "nBADcalib;}' meta/cpass0.job*.run${runNumber}.done 2>/dev/null
1963     ) )
1964   fi
1965
1966   if $(/bin/ls meta/cpass1.job*.run${runNumber}.done &>/dev/null); then
1967     statusCPass1=( $(
1968     awk 'BEGIN {nOKrec=0;nBADrec=0;nOKcalib=0;nBADcalib=0;nOKstderr=0;nBADstderr=0;nQAbarrelOK=0;nQAbarrelBAD=0;nQAouterOK=0;nQAouterBAD=0;}
1969     /\/rec.log OK/ {nOKrec++;}
1970     /\/rec.log BAD/ {nBADrec++;}
1971     /stderr BAD/ {if ($0 ~ /rec.log/) nBADrec++;nBADstderr++;}
1972     /stderr OK/ {nOKstderr++;}
1973     /\/calib.log OK/ {nOKcalib++;}
1974     /\/calib.log BAD/ {nBADcalib++}
1975     /\/qa_barrel.log OK/ {nQAbarrelOK++;}
1976     /\/qa_barrel.log BAD/ {nQAbarrelBAD++;}
1977     /\/qa_outer.log OK/ {nQAouterOK++;}
1978     /\/qa_outer.log BAD/ {nQAouterBAD++;}
1979     END {print ""nOKrec" "nBADrec" "nOKstderr" "nBADstderr" "nOKcalib" "nBADcalib" "nQAbarrelOK" "nQAbarrelBAD" "nQAouterOK" "nQAouterBAD;}' meta/cpass1.job*.run${runNumber}.done 2>/dev/null
1980     ) )
1981   fi
1982
1983   statusOCDBcpass0=$(awk '/ocdb.log/ {print $2} ' ${x} 2>/dev/null)
1984   statusOCDBcpass1=$(awk '/ocdb.log/ {print $2}' ${x/cpass0/cpass1} 2>/dev/null)
1985   statusQA=$(awk '/mergeMakeOCDB.log/ {print $2}' ${x/cpass0/cpass1} 2>/dev/null)
1986
1987   printf "%s\t ocdb.log cpass0: %s\t ocdb.log cpass1: %s\tqa.log:%s\t| cpass0: rec:%s/%s stderr:%s/%s calib:%s/%s cpass1: rec:%s/%s stderr:%s/%s calib:%s/%s QAbarrel:%s/%s QAouter:%s/%s\n" ${runNumber} ${statusOCDBcpass0} ${statusOCDBcpass1} ${statusQA} ${statusCPass0[0]} ${statusCPass0[1]} ${statusCPass0[2]} ${statusCPass0[3]} ${statusCPass0[4]} ${statusCPass0[5]} ${statusCPass1[0]} ${statusCPass1[1]} ${statusCPass1[2]} ${statusCPass1[3]} ${statusCPass1[4]} ${statusCPass1[5]} ${statusCPass1[6]} ${statusCPass1[7]} ${statusCPass1[8]} ${statusCPass1[9]}
1988 done
1989
1990   #make lists with output files - QA, trending, filtering and calibration
1991   ### wait for the merging of all runs to be over ###
1992   rm -f qa.list
1993   goPrintValues qafile qa.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
1994   rm -f calib.list
1995   goPrintValues calibfile calib.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
1996   rm -f trending.list
1997   goPrintValues trendingfile trending.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
1998   rm -f filtering.list
1999   goPrintValues filteredTree filtering.list ${commonOutputPath}/meta/filtering.cpass1.run*.done &>/dev/null
2000   #/bin/ls ${commonOutputPath}/*/cpass0/dcs* > cpass0.dcsTree.list
2001   rm -f cpass0.dcsTree.list
2002   goPrintValues dcsTree cpass0.dcsTree.list ${commonOutputPath}/meta/merge.cpass0.run*.done &>/dev/null
2003   #/bin/ls ${commonOutputPath}/*/cpass1/dcs* > cpass1.dcsTree.list
2004   rm -f cpass1.dcsTree.list
2005   goPrintValues dcsTree cpass1.dcsTree.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2006
2007   #merge trending
2008   rm -f ${commonOutputPath}/trending_merged.root
2009   goMerge trending.list ${commonOutputPath}/trending.root ${configFile} ${extraOpts} &> mergeTrending.log
2010
2011   goMakeSummaryTree ${commonOutputPath} 0
2012   goMakeSummaryTree ${commonOutputPath} 1
2013
2014   goCreateQAplots "qa.list" "${productionID}" "QAplots" "${configFile}" ${extraOpts} filteringList="filtering.list" &>createQAplots.log
2015
2016   #make a merged summary tree out of the QA trending, dcs trees and log summary trees
2017   goMakeMergedSummaryTree
2018
2019   #if set, email the summary
2020   [[ -n ${MAILTO} ]] && cat ${log} | mail -s "benchmark ${productionID} done" ${MAILTO}
2021
2022   return 0
2023 }
2024
2025 goMakeSummaryTree()
2026 {
2027   if [[ $# -lt 1 ]] ; then
2028     return
2029   fi
2030   #1. define vars/arrays
2031   DIR=${1} #use input or exec in current dir
2032   pass=${2-"0"} #pass from input
2033   outfile="summary_pass${pass}.tree"
2034   Ncolumns=0
2035   test -f ${outfile} && : >${outfile}
2036   errfile=${outfile/tree/err}
2037   test -f ${errfile} && : >${errfile}
2038
2039   declare -a counterString=(TOFevents TOFtracks TPCevents TPCtracks TRDevents TRDtracks T0events SDDevents SDDtracks MeanVertexevents)
2040   Ncounter=${#counterString[@]}
2041
2042   declare -a statusString=(TRDStatus TOFStatus TPCStatus T0Status MeanVertexStatus)
2043   Nstatus=${#statusString[@]}
2044
2045
2046   declare -a ratesString=(rec stderr calib qa_barrel qa_outer)
2047   Nrates=${#ratesString[@]}
2048
2049   runs=( $(ls -1 ${DIR}/meta/merge.cpass0* | while read x; do guessRunNumber $x; done) )
2050   Nruns=${#runs[@]}
2051
2052   echo -n runnumber/I >>${outfile}
2053   echo -n :cpass${pass}status/I >>${outfile}
2054   echo -n :cpass${pass}QAstatus/I >>${outfile}
2055   for i in ${ratesString[@]}; do
2056     echo -n :${i}OK/I >>${outfile}
2057     echo -n :${i}BAD/I >>${outfile}
2058
2059   done
2060   for i in ${counterString[@]} ${statusString[@]} ; do
2061     echo -n :${i}/I >>${outfile}
2062   done
2063   Ncolumns=$((2 + 2*Nrates + Ncounter + Nstatus))
2064   echo >> ${outfile}
2065
2066   #2. loop runs
2067
2068   for runnumber in ${runs[@]} ; do
2069
2070
2071
2072     filejob="${DIR}/meta/cpass${pass}.job*.run${runnumber}.done"
2073     filemerge="${DIR}/meta/merge.cpass${pass}.run${runnumber}.done"
2074     fileOCDB=$(grep /ocdb.log ${filemerge} | awk '{print $1}')
2075     if ! $(/bin/ls ${filemerge} &>/dev/null) ; then
2076       echo "${filemerge} does not exist!" >>${errfile}
2077       continue
2078     elif ! $(/bin/ls ${filejob} &>/dev/null) ; then
2079       echo "${filejob} does not exist!" >>${errfile}
2080       echo -n ${runnumber} >> ${outfile}
2081       for i in $(seq ${Ncolumns}) ; do
2082         echo -n "-1" >> ${outfile}
2083       done
2084       echo >> ${outfile}
2085       continue
2086     fi
2087     echo -n ${runnumber} >> ${outfile}
2088     #pass0status= grep '/ocdb.log' ${filemerge} |  cut -d' ' -f2 | tr OK x1 | tr BAD xx0 | tr -d 'x'
2089     passStatus=$(grep '/ocdb.log' ${filemerge} | grep OK | wc -l)
2090     echo -n " ${passStatus}" >> ${outfile}
2091     qaStatus=$(grep '/mergeMakeOCDB.log' ${filemerge} | grep OK | wc -l)
2092     echo -n " ${qaStatus}" >> ${outfile}
2093
2094
2095     #fill OK/BAD rates
2096     for i in $(seq 0 $((${Nrates}-1))) ; do
2097       var1=$(grep "/${ratesString[${i}]}.log" ${filejob} | grep OK | wc -l)
2098       var2=$(grep "/${ratesString[${i}]}.log" ${filejob} | grep BAD | wc -l)
2099
2100       if [[ ${ratesString[${i}]} == "stderr" ]] ; then
2101         var1=$(grep "stderr" ${filejob} | grep OK | wc -l)
2102         var2=$(grep "stderr" ${filejob} | grep "rec.log" | grep BAD | wc -l)
2103       fi
2104       echo -n " ${var1}" >> ${outfile}
2105       echo -n " ${var2}" >> ${outfile}
2106     done
2107
2108     if [[ -f ${fileOCDB} ]] ; then
2109       #fill counter
2110       for i in $(seq 0 $((${Ncounter}-1))) ; do
2111         var1=$(grep Monalisa ${fileOCDB} | grep ${counterString[${i}]} | cut -f2)
2112         echo -n " ${var1:-"-1"}" >> ${outfile}
2113       done
2114
2115       #fill status
2116       for i in $(seq 0 $((${Nstatus}-1))) ; do
2117         var1=$(grep "calibration status=" ${fileOCDB} | grep ${statusString[${i}]/Status/} | cut -d'=' -f2)
2118         echo -n " ${var1:-"-1"}" >> ${outfile}
2119       done
2120     fi
2121     echo >> ${outfile}
2122   done
2123
2124   return 0
2125 }
2126
2127 parseConfig()
2128 {
2129   configFile=${1}
2130   shift
2131
2132   #first, source the config file
2133   if [ -f ${configFile} ]; then
2134     source ${configFile}
2135   else
2136     echo "config file ${configFile} not found!, skipping..."
2137   fi
2138
2139   #then, parse the options as theya override the optionf from file
2140   while [[ -n ${1} ]]; do
2141     var="${1#--}"
2142     echo "extraOpts: exporting ${var}"
2143     eval "export ${var}"
2144     shift
2145   done
2146
2147   #export the aliroot function if defined to override normal behaviour
2148   [[ $(type -t aliroot) =~ "function" ]] && export -f aliroot
2149   return 0
2150 }
2151
2152 aliroot()
2153 {
2154   args="$@"
2155   if [[ -n ${valgrindCommand} ]]; then
2156     valgrindLogFile="cpu.txt"
2157     [[ "${args}" =~ rec ]] && valgrindLogFile="cpu_rec.txt"
2158     [[ "${args}}" =~ Calib ]] && valgrindLogFile="cpu_calib.txt"
2159     [[ -n ${valgrindCommand} ]] && valgrindCommand="${valgrindCommand} --log-file=${valgrindLogFile}"
2160     echo running ${valgrindCommand} aliroot ${args}
2161     ${valgrindCommand} aliroot ${args}
2162   else
2163     #to prevent an infinite recursion use "command aliroot" to disable
2164     #aliases and functions
2165     echo running command aliroot ${args}
2166     command aliroot ${args}
2167   fi
2168   return 0
2169 }
2170
2171 guessRunData()
2172 {
2173   #guess the period from the path, pick the rightmost one
2174   period=""
2175   runNumber=""
2176   year=""
2177   pass=""
2178   legoTrainRunNumber=""
2179   dataType=""
2180
2181   local shortRunNumber=""
2182   local IFS="/"
2183   declare -a path=( $1 )
2184   local dirDepth=$(( ${#path[*]}-1 ))
2185   i=0
2186   #for ((x=${dirDepth};x>=0;x--)); do
2187   for ((x=0;x<=${dirDepth};x++)); do
2188
2189     [[ $((x-1)) -ge 0 ]] && local fieldPrev=${path[$((x-1))]}
2190     local field=${path[${x}]}
2191     local fieldNext=${path[$((x+1))]}
2192
2193     [[ ${field} =~ ^[0-9]*$ && ${fieldNext} =~ (.*\.zip$|.*\.root$) ]] && legoTrainRunNumber=${field}
2194     [[ -n ${legoTrainRunNumber} && -z ${pass} ]] && pass=${fieldPrev}
2195     [[ ${field} =~ ^LHC[0-9][0-9][a-z].*$ ]] && period=${field%_*}
2196     [[ ${field} =~ ^000[0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && runNumber=${field#000}
2197     [[ ${field} =~ ^[0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && shortRunNumber=${field}
2198     [[ ${field} =~ ^20[0-9][0-9]$ ]] && year=${field}
2199     [[ ${field} =~ ^(^sim$|^data$) ]] && dataType=${field}
2200     (( i++ ))
2201   done
2202   [[ -z ${legoTrainRunNumber} ]] && pass=${path[$((dirDepth-1))]}
2203   [[ "${dataType}" =~ ^sim$ ]] && pass="passMC" && runNumber=${shortRunNumber}
2204
2205   #if [[ -z ${dataType} || -z ${year} || -z ${period} || -z ${runNumber}} || -z ${pass} ]];
2206   if [[ -z ${runNumber}} ]];
2207   then
2208     #error condition
2209     return 1
2210   else
2211     #ALL OK
2212     return 0
2213   fi
2214   return 0
2215 }
2216
2217 main "$@"