PWGPP/benchmark/benchmark.sh

   1 #!/bin/bash
   2 #include benchmark.config
   3
   4 # this script runs the CPass0/CPass1 train
   5 # produced OCDB updates are local
   6
   7 main()
   8 {
   9   #run in proper mode depending on the selection
  10   if [[ $# -lt 1 ]]; then
  11     if [[ ! "${0}" =~ "bash" ]]; then
  12       echo "uses makeflow:"
  13       echo " ${0} \"run\" productionID inputList configFile [extraOpts]"
  14       echo "uses a batch system (SGE):"
  15       echo " ${0} \"submit\" productionID inputList configFile [extraOpts]"
  16       echo "extraOpts if non-empty override the config file, e.g.:"
  17       echo " ${0} submit benchmark.list test1 benchmark.config runNumber=169123 nEvents=10"
  18     fi
  19     return
  20   fi
  21
  22   #define some aliases - default is to call one of the functions directly
  23   runMode=${1}
  24   umask 0002
  25   shift
  26   case ${runMode} in
  27     "CPass0") goCPass0 "$@";;
  28     "CPass1") goCPass1 "$@";;
  29     "MakeLocalOCDBaccessConfig") goMakeLocalOCDBaccessConfig "$@";;
  30     "MergeCPass0") goMergeCPass0 "$@";;
  31     "MergeCPass1") goMergeCPass1 "$@";;
  32     "MakeFilteredTrees") goMakeFilteredTrees "$@";;
  33     "MakeSummary") goMakeSummary "$@";;
  34     "run") goSubmitMakeflow "$@";;
  35     "submit") goSubmitBatch "$@";;
  36     "test") goTest "$@";;
  37     "GenerateMakeflow") goGenerateMakeflow "$@";;
  38     "PrintValues") goPrintValues "$@";;
  39     "CreateQAplots") goCreateQAplots "$@";;
  40     "WaitForOutput") goWaitForOutput "$@";;
  41     "Merge") goMerge "$@";;
  42     *)
  43       eval ${runMode} $@
  44     ;;
  45   esac
  46 }
  47
  48 generateMC()
  49 {
  50   #generate one raw chunk in current directory
  51   SEED=${JOB_ID}${SGE_TASK_ID}
  52   export CONFIG_SEED=${SEED}
  53   runNumber=${1}
  54   OCDBpath=${2}
  55   nEventsim=${3}
  56   if [[ -n ${pretend} ]]; then
  57     touch galice.root
  58   else
  59     if [[ -f sim.C && -f Config.C ]] ; then
  60         time aliroot -b -q -x sim.C\(${runNumber},\"${OCDBpath}\",${nEventsim}\) >sim.log 2>&1
  61         mv syswatch.log simwatch.log
  62     fi
  63   fi
  64 }
  65
  66 goCPass0()
  67 {
  68   umask 0002
  69
  70   targetDirectory=${1}
  71   inputList=${2}
  72   nEvents=${3}
  73   ocdbPath=${4}
  74   configFile=${5}
  75   runNumber=${6}
  76   jobindex=${7}
  77   shift 7
  78   parseConfig ${configFile} $@
  79
  80   #use the jobindex only if set and non-negative
  81   if [[ -z ${jobindex} || ${jobindex} -lt 0 ]]; then
  82     [[ -n "${LSB_JOBINDEX}" ]] && jobindex=${LSB_JOBINDEX}
  83     [[ -n "${SGE_TASK_ID}" ]] && jobindex=${SGE_TASK_ID}
  84   fi
  85
  86   [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
  87   doneFile="${commonOutputPath}/meta/cpass0.job${jobindex}.run${runNumber}.done"
  88
  89   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
  90
  91   if [[ -n ${ALIROOT_FORCE_COREDUMP} ]]; then
  92     ulimit -c unlimited
  93     export ALIROOT_FORCE_COREDUMP
  94   fi
  95
  96   #the contents of this is stored in the tree and used later (e.g. AliAnalysisTaskPIDResponse)!
  97   #at the QA stage the pass number is guessed from the path stored here.
  98   #The Format is:
  99   #Packages= ;OutputDir= ;LPMPass= ;TriggerAlias= ;LPMRunNumber= ;LPMProductionType= ;LPMInteractionType= ;LPMProductionTag= ;LPMAnchorRun= ;LPMAnchorProduction= ;LPMAnchorYear=
 100   export PRODUCTION_METADATA="OutputDir=cpass0"
 101
 102   if [[ "${inputList}" =~ \.root$ ]]; then
 103     infile=${inputList}
 104   else
 105     infile=$(sed -ne "${jobindex}p" ${inputList} | egrep '\s*\w*/\w*')
 106   fi
 107   chunkName=${infile##*/}
 108
 109   outputDir=${targetDirectory}/${jobindex}_${chunkName%.*}
 110   mkdir -p ${outputDir}
 111   [[ ! -d ${outputDir} ]] && echo "cannot make ${outputDir}" && touch ${doneFile} && return 1
 112
 113   #runpath=${PWD}/rundir_cpass0_${runNumber}_${jobindex}
 114   runpath=${outputDir}
 115   [[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
 116   [[ ${reconstructInTemporaryDir} -eq 1 && -z ${TMPDIR} ]] && runpath=$(mktemp -d)
 117   mkdir -p ${runpath}
 118   [[ ! -d ${runpath} ]] && echo "cannot make runpath ${runpath}" && touch ${doneFile} && return 1
 119   cd ${runpath}
 120
 121   #runCPassX/C expects the raw chunk to be linked in the run dir
 122   #despite it being accessed by the full path
 123   ln -s ${infile} ${runpath}/${chunkName}
 124
 125   #####MC
 126   if [[ -n ${generateMC} ]]; then
 127     olddir=${PWD}
 128     outputDirMC=${commonOutputPath}/000${runNumber}/sim/${jobindex}
 129     simrunpath=${outputDirMC}
 130     [[ ${simulateInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && simrunpath=${TMPDIR}
 131     [[ ${simulateInTemporaryDir} -eq 1 && -z ${TMPDIR} ]] && simrunpath=$(mktemp -d)
 132     mkdir -p ${outputDirMC}
 133     mkdir -p ${simrunpath}
 134     cd ${simrunpath}
 135
 136     filesMC=(
 137               "${commonOutputPath}/sim.C"
 138               "${commonOutputPath}/rec.C"
 139               "${commonOutputPath}/Config.C"
 140               "${commonOutputPath}/OCDB_*.root"
 141     )
 142     for file in ${filesMC[*]}; do
 143       [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
 144     done
 145
 146     generateMC ${runNumber} ${ocdbPath} ${nEvents}
 147
 148     [[ ! "${simrunpath}" =~ "${outputDirMC}" ]] && mv * ${outputDirMC} #TODO check if it works
 149     cd ${olddir}
 150
 151     ln -s ${outputDirMC}/* ${runpath}/
 152
 153     inputList=${outputDirMC}/galice.root #TODO not valid outside shell !!!
 154     infile=""
 155   fi
 156   ######
 157
 158   [[ ! -f ${inputList} && -z ${pretend} ]] && echo "input file ${inputList} not found, exiting..." && touch ${doneFile} && return 1
 159
 160   logOutputDir=${runpath}
 161   [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir}
 162   [[ -z ${dontRedirectStdOutToLog} ]] && exec 1> ${logOutputDir}/stdout
 163   [[ -z ${dontRedirectStdOutToLog} ]] && exec 2> ${logOutputDir}/stderr
 164   echo "${0} $*"
 165
 166   echo "#####################"
 167   echo CPass0:
 168   echo JOB setup
 169   echo nEvents            ${nEvents}
 170   echo runNumber          ${runNumber}
 171   echo ocdbPath           ${ocdbPath}
 172   echo infile             ${infile}
 173   echo chunkName          ${chunkName}
 174   echo jobindex           ${jobindex}
 175   echo recoTriggerOptions ${recoTriggerOptions}
 176   echo targetDirectory    ${targetDirectory}
 177   echo commonOutputPath         ${commonOutputPath}
 178   echo doneFile      ${doneFile}
 179   echo runpath            ${runpath}
 180   echo outputDir          ${outputDir}
 181   echo ALICE_ROOT         ${ALICE_ROOT}
 182   echo PWD                ${PWD}
 183   echo "########## ###########"
 184
 185   alirootInfo > ALICE_ROOT.log
 186
 187   filesCPass0=(
 188                "${commonOutputPath}/runCPass0.sh"
 189                "${commonOutputPath}/recCPass0.C"
 190                "${commonOutputPath}/runCalibTrain.C"
 191                "${commonOutputPath}/localOCDBaccessConfig.C"
 192                "${commonOutputPath}/OCDB.root"
 193                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/runCPass0.sh"
 194                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/recCPass0.C"
 195                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/runCalibTrain.C"
 196   )
 197
 198   for file in ${filesCPass0[*]}; do
 199     [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
 200   done
 201
 202   echo "this directory (${PWD}) contents:"
 203   /bin/ls
 204   echo
 205   chmod u+x runCPass0.sh
 206
 207   sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh
 208
 209   if [[ -n ${postSetUpActionCPass0} ]]; then
 210     echo "running ${postSetUpActionCPass0}"
 211     eval ${postSetUpActionCPass0}
 212   fi
 213
 214   #run CPass0
 215   echo "${runpath}/runCPass0.sh ${infile} ${nEvents} ${runNumber} ${ocdbPath} ${recoTriggerOptions}"
 216   if [[ -n ${pretend} ]]; then
 217     touch AliESDs.root
 218     touch AliESDfriends.root
 219     touch AliESDfriends_v1.root
 220     touch rec.log
 221     touch calib.log
 222   else
 223     echo ./runCPass0.sh "${infile}" "${nEvents}" "${runNumber}" "${ocdbPath}" "${recoTriggerOptions}"
 224     ./runCPass0.sh "${infile}" "${nEvents}" "${runNumber}" "${ocdbPath}" "${recoTriggerOptions}"
 225   fi
 226
 227   #move stuff to final destination
 228   echo "this directory (${PWD}) contents:"
 229   /bin/ls
 230   echo
 231
 232   echo rm -f ./${chunkName}
 233   rm -f ./${chunkName}
 234   echo "cp --recursive ${runpath}/* ${outputDir}"
 235   cp -p --recursive ${runpath}/* ${outputDir}
 236   echo
 237
 238   #validate CPass0
 239   cd ${outputDir}
 240   touch ${doneFile}
 241   [[ -f ${outputDirMC}/galice.root ]] && echo "sim ${outputDirMC}/galice.root" >> ${doneFile}
 242   [[ -f AliESDfriends_v1.root ]] && echo "calibfile ${outputDir}/AliESDfriends_v1.root" >> ${doneFile}
 243   [[ -f AliESDs.root ]] && echo "esd ${outputDir}/AliESDs.root" >> ${doneFile}
 244   echo "dir ${outputDir}" >> ${doneFile}
 245   summarizeLogs >> ${doneFile}
 246
 247   [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
 248   return 0
 249 }
 250
 251 goCPass1()
 252 {
 253   umask 0002
 254
 255   targetDirectory=${1}
 256   inputList=${2}
 257   nEvents=${3}
 258   ocdbPath=${4}
 259   configFile=${5}
 260   runNumber=${6}
 261   jobindex=${7}
 262   shift 7
 263   parseConfig ${configFile} $@
 264
 265   #use the jobindex only if set and non-negative
 266   if [[ -z ${jobindex} || ${jobindex} -lt 0 ]]; then
 267     [[ -n "${LSB_JOBINDEX}" ]] && jobindex=${LSB_JOBINDEX}
 268     [[ -n "${SGE_TASK_ID}" ]] && jobindex=${SGE_TASK_ID}
 269   fi
 270
 271   [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
 272   doneFile="${commonOutputPath}/meta/cpass1.job${jobindex}.run${runNumber}.done"
 273
 274   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
 275
 276   if [[ -n ${ALIROOT_FORCE_COREDUMP} ]]; then
 277     ulimit -c unlimited
 278     export ALIROOT_FORCE_COREDUMP
 279   fi
 280
 281   #the contents of this is stored in the tree and used later (e.g. AliAnalysisTaskPIDResponse)!
 282   #at the QA stage the pass number is guessed from the path stored here.
 283   #The Format is:
 284   #Packages= ;OutputDir= ;LPMPass= ;TriggerAlias= ;LPMRunNumber= ;LPMProductionType= ;LPMInteractionType= ;LPMProductionTag= ;LPMAnchorRun= ;LPMAnchorProduction= ;LPMAnchorYear=
 285   export PRODUCTION_METADATA="OutputDir=cpass1"
 286
 287   [[ ! -f ${inputList} && -z ${pretend} ]] && echo "input file ${inputList} not found, exiting..." && touch ${doneFile} && return 1
 288   if [[ "${inputList}" =~ \.root$ ]]; then
 289     infile=${inputList}
 290   else
 291     infile=$(sed -ne "${jobindex}p" ${inputList} | egrep '\s*\w*/\w*')
 292   fi
 293   chunkName=${infile##*/}
 294
 295   outputDir=${targetDirectory}/${jobindex}_${chunkName%.*}
 296   mkdir -p ${outputDir}
 297   [[ ! -d ${outputDir} ]] && echo "cannot make ${outputDir}" && touch ${doneFile} && return 1
 298
 299   #runpath=${PWD}/rundir_cpass1_${runNumber}_${jobindex}
 300   runpath=${outputDir}
 301   [[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
 302   [[ ${reconstructInTemporaryDir} -eq 1 && -z ${TMPDIR} ]] && runpath=$(mktemp -d)
 303
 304   #MC
 305   if [[ "${infile}" =~ galice\.root ]]; then
 306     ln -s ${inputList%/*}/* ${runpath}
 307     infile=""
 308   fi
 309
 310   #init the running path
 311   mkdir -p ${runpath}
 312   [[ ! -d ${runpath} ]] && echo "cannot make runpath ${runpath}" && touch ${doneFile} && return 1
 313   cd ${runpath}
 314
 315   #this is needed for runCPass1.sh
 316   ln -s ${infile} ${runpath}/${chunkName}
 317
 318   logOutputDir=${runpath}
 319   [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir}
 320   [[ -z ${dontRedirectStdOutToLog} ]] && exec 1> ${logOutputDir}/stdout
 321   [[ -z ${dontRedirectStdOutToLog} ]] && exec 2> ${logOutputDir}/stderr
 322   echo "${0} $*"
 323
 324   echo "#####################"
 325   echo CPass1:
 326   echo JOB setup
 327   echo nEvents            ${nEvents}
 328   echo runNumber          ${runNumber}
 329   echo ocdbPath           ${ocdbPath}
 330   echo infile             ${infile}
 331   echo chunkName          ${chunkName}
 332   echo jobindex           ${jobindex}
 333   echo recoTriggerOptions ${recoTriggerOptions}
 334   echo targetDirectory    ${targetDirectory}
 335   echo commonOutputPath         ${commonOutputPath}
 336   echo doneFile      ${doneFile}
 337   echo runpath            ${runpath}
 338   echo outputDir          ${outputDir}
 339   echo ALICE_ROOT         ${ALICE_ROOT}
 340   echo PWD                ${PWD}
 341   echo "########## ###########"
 342
 343   alirootInfo > ALICE_ROOT.log
 344
 345   filesCPass1=(
 346                "${commonOutputPath}/runCPass1.sh"
 347                "${commonOutputPath}/recCPass1.C"
 348                "${commonOutputPath}/recCPass1_OuterDet.C"
 349                "${commonOutputPath}/runCalibTrain.C"
 350                "${commonOutputPath}/QAtrain_duo.C"
 351                "${commonOutputPath}/localOCDBaccessConfig.C"
 352                "${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz"
 353                "${commonOutputPath}/OCDB.root"
 354                "${trustedQAtrainMacro}"
 355                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/runCPass1.sh"
 356                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/recCPass1.C"
 357                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/recCPass1_OuterDet.C"
 358                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/runCalibTrain.C"
 359                "${ALICE_ROOT}/ANALYSIS/macros/QAtrain_duo.C"
 360   )
 361
 362   for file in ${filesCPass1[*]}; do
 363     [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
 364   done
 365
 366   echo "this directory (${PWD}) contents:"
 367   /bin/ls
 368   echo
 369
 370   sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh
 371
 372   if [[ -n ${postSetUpActionCPass1} ]]; then
 373     echo "running ${postSetUpActionCPass1}"
 374     eval ${postSetUpActionCPass1}
 375     echo
 376   fi
 377
 378   #configure local OCDB storage from CPass0 (creates the localOCDBaccessConfig.C script)
 379   if [[ -f cpass0.localOCDB.${runNumber}.tgz ]]; then
 380     echo goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz"
 381     goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz"
 382   else
 383     echo "WARNING: file cpass0.localOCDB.${runNumber}.tgz not found!"
 384   fi
 385
 386   if [[ ! $(/bin/ls -1 OCDB/*/*/*/*.root 2>/dev/null) ]]; then
 387     echo "cpass0 produced no calibration! exiting..."
 388     touch ${doneFile}
 389     return 1
 390   fi
 391
 392   #create the Barrel and OuterDet directories for CPass1 and link the local OCDB directory
 393   #there to make the localOCDBaccessConfig.C file work, since it may point to the OCDB
 394   #entries using a relative path, e.g. local://./OCDB
 395   mkdir Barrel OuterDet
 396   ln -s ../OCDB Barrel/OCDB
 397   ln -s ../OCDB OuterDet/OCDB
 398
 399   #run CPass1
 400   chmod u+x runCPass1.sh
 401   echo "${runpath}/runCPass1.sh ${infile} ${nEvents} ${runNumber} ${ocdbPath} ${recoTriggerOptions}"
 402   if [[ -n ${pretend} ]]; then
 403     touch AliESDs_Barrel.root
 404     touch AliESDfriends_Barrel.root
 405     touch AliESDfriends_v1.root
 406     touch QAresults_barrel.root
 407     touch EventStat_temp_barrel.root
 408     touch AODtpITS.root
 409     touch AliESDs_Outer.root
 410     touch AliESDfriends_Outer.root
 411     touch QAresults_outer.root
 412     touch EventStat_temp_outer.root
 413     touch rec.log
 414     touch calib.log
 415     touch qa.log
 416   else
 417     ./runCPass1.sh "${infile}" "${nEvents}" "${runNumber}" "${ocdbPath}" "${recoTriggerOptions}"
 418   fi
 419
 420   ##handle possible crashes in QA (happens often in trunk)
 421   ##rerun QA with a trusted aliroot version
 422   #if [[ $(validateLog qa_barrel.log) ]]; then
 423   #  echo "qa_barrel.log not validated!"
 424   #fi
 425   #if [[ ! -f QAresults_barrel.root && -f ${setupTrustedAliROOTenvInCurrentShell} || $(validateLog qa_barrel.log) ]]; then
 426   #  echo "WARNING: using trusted QA aliroot ${ALICE_ROOT}"
 427   #  source ${setupTrustedAliROOTenvInCurrentShell}
 428   #  cd Barrel
 429   #  rm QAresults_barrel.root
 430   #  rm EventStat_temp_barrel.root
 431   #  rm AODtpITS.root
 432   #  [[ ! -f AliESDs.root ]] && ln -s ../AliESDs_Barrel.root AliESDs.root
 433   #  [[ ! -f AliESDfriends.root ]] && ln -s ../AliESDfriends_Barrel.root AliESDfriends.root
 434   #  if [[ -n ${trustedQAtrainMacro} ]]; then
 435   #    eval "cp ${trustedQAtrainMacro} QAtrain_duo_trusted.C"
 436   #  fi
 437   #  echo executing aliroot -b -q "QAtrain_duo_trusted.C(\"_barrel\",${runNumber},\"wn.xml\",0,\"${ocdbPath}\")"
 438   #  time aliroot -b -q "QAtrain_duo.C(\"_barrel\",${runNumber},\"wn.xml\",0,\"${ocdbPath}\")" &> ../qa_barrel_trusted.log
 439   #  cd ../
 440   #fi
 441   [[ ! -f AliESDs_Barrel.root && -f Barrel/AliESDs.root ]] && mv Barrel/AliESDs.root AliESDs_Barrel.root
 442   [[ ! -f AliESDfriends_Barrel.root && -f Barrel/AliESDfriends.root ]] && mv Barrel/AliESDfriends.root AliESDfriends_Barrel.root
 443   [[ ! -f AliESDfriends_v1.root && -f Barrel/AliESDfriends_v1.root ]] && mv Barrel/AliESDfriends_v1.root .
 444   [[ ! -f QAresults_barrel.root && -f Barrel/QAresults_barrel.root ]] && mv Barrel/QAresults_barrel.root .
 445   [[ ! -f AliESDs_Outer.root && -f OuterDet/AliESDs.root ]] && mv OuterDet/AliESDs.root AliESDs_Outer.root
 446   [[ ! -f AliESDfriends_Outer.root && -f OuterDet/AliESDfriends.root ]] && mv OuterDet/AliESDfriends.root AliESDfriends_Outer.root
 447   [[ ! -f QAresults_outer.root && -f OuterDet/QAresults_outer.root ]] && mv OuterDet/QAresults_outer.root .
 448
 449   #move stuff to final destination
 450   echo "this directory (${PWD}) contents:"
 451   /bin/ls
 452   echo rm -f ./${chunkName}
 453   rm -f ./${chunkName}
 454   echo "cp --recursive ${runpath}/* ${outputDir}"
 455   cp -pf --recursive ${runpath}/* ${outputDir}
 456   echo
 457
 458   #validate CPass1
 459   cd ${outputDir}
 460   touch ${doneFile}
 461   [[ -f AliESDs_Barrel.root ]] && echo "esd ${outputDir}/AliESDs_Barrel.root" >> ${doneFile}
 462   [[ -f AliESDfriends_v1.root ]] && echo "calibfile ${outputDir}/AliESDfriends_v1.root" >> ${doneFile}
 463   [[ -f QAresults_Barrel.root ]] && echo "qafile ${outputDir}/QAresults_Barrel.root" >> ${doneFile}
 464   [[ -f QAresults_Outer.root ]] && echo "qafile ${outputDir}/QAresults_Outer.root" >> ${doneFile}
 465   echo "dir ${outputDir}" >> ${doneFile}
 466   summarizeLogs >> ${doneFile}
 467
 468   [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
 469   return 0
 470 }
 471
 472
 473 goMergeCPass0()
 474 {
 475   #
 476   # find the output files and merge them
 477   #
 478
 479   outputDir=${1}
 480   ocdbStorage=${2}
 481   configFile=${3}
 482   runNumber=${4}
 483   calibrationFilesToMerge=${5}  #can be a non-existent file, will then be produced on the fly
 484   shift 5
 485   parseConfig ${configFile} $@
 486
 487   [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
 488   doneFile="${commonOutputPath}/meta/merge.cpass0.run${runNumber}.done"
 489
 490   umask 0002
 491   ulimit -c unlimited
 492
 493   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
 494
 495   #runpath=${PWD}/rundir_cpass0_Merge_${runNumber}
 496   runpath=${outputDir}
 497   [[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
 498   [[ ${reconstructInTemporaryDir} -eq 1 && -z ${TMPDIR} ]] && runpath=$(mktemp -d)
 499
 500   mkdir -p ${runpath}
 501   [[ ! -d ${runpath} ]] && echo "not able to make the runpath ${runpath}" && touch ${doneFile} && return 1
 502   cd ${runpath}
 503
 504   logOutputDir=${runpath}
 505   [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir}
 506   [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/mergeMakeOCDB.log
 507   echo "${0} $*"
 508
 509   mergingScript="mergeMakeOCDB.byComponent.sh"
 510
 511   echo goMergeCPass0 SETUP:
 512   echo runNumber=${runNumber}
 513   echo outputDir=${outputDir}
 514   echo ocdbStorage=${ocdbStorage}
 515   echo calibrationFilesToMerge=${calibrationFilesToMerge}
 516   echo mergingScript=${mergingScript}
 517   echo commonOutputPath=${commonOutputPath}
 518   echo runpath=${runpath}
 519
 520   # copy files in case they are not already there
 521   filesMergeCPass0=(
 522                     "${commonOutputPath}/${calibrationFilesToMerge}"
 523                     "${commonOutputPath}/OCDB.root"
 524                     "${commonOutputPath}/localOCDBaccessConfig.C"
 525                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/mergeMakeOCDB.byComponent.sh"
 526                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/mergeByComponent.C"
 527                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/makeOCDB.C"
 528                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/merge.C"
 529                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/mergeMakeOCDB.sh"
 530   )
 531   for file in ${filesMergeCPass0[*]}; do
 532     [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
 533   done
 534
 535   sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh
 536
 537   alirootInfo > ALICE_ROOT.log
 538
 539   #
 540   echo "PWD"
 541   /bin/ls
 542   echo "PWD/.."
 543   /bin/ls ../
 544
 545
 546   #merge calibration
 547   chmod u+x ${mergingScript}
 548   mkdir -p ./OCDB
 549   if [[ ! -f ${calibrationFilesToMerge} ]]; then
 550     echo "/bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root > ${calibrationFilesToMerge}"
 551     /bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root 2>/dev/null > ${calibrationFilesToMerge}
 552   fi
 553
 554   echo "${mergingScript} ${calibrationFilesToMerge} ${runNumber} local://./OCDB ${ocdbStorage}"
 555   if [[ -n ${pretend} ]]; then
 556     touch CalibObjects.root
 557     touch ocdb.log
 558     touch merge.log
 559     mkdir -p ./OCDB/TPC/Calib/TimeGain/
 560     mkdir -p ./OCDB/TPC/Calib/TimeDrift/
 561     touch ./OCDB/TPC/Calib/TimeGain/someCalibObject_0-999999_cpass0.root
 562     touch ./OCDB/TPC/Calib/TimeDrift/otherCalibObject_0-999999_cpass0.root
 563   else
 564     ./${mergingScript} ${calibrationFilesToMerge} ${runNumber} "local://./OCDB" ${ocdbStorage}
 565   fi
 566
 567   ### produce the output
 568   #tar the produced OCDB for reuse
 569   tar czf ${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz ./OCDB
 570
 571   #produce the calib trees for expert QA (dcsTime.root)
 572   goMakeLocalOCDBaccessConfig ./OCDB
 573   echo aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")"
 574   aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")"
 575
 576   /bin/ls
 577
 578   #copy all to output dir
 579   cp -pf --recursive ${runpath}/* ${outputDir}
 580
 581   if [[ -n ${generateMC} ]]; then
 582     goPrintValues sim ${commonOutputPath}/meta/sim.run${runNumber}.list ${commonOutputPath}/meta/cpass0.job*.run${runNumber}.done
 583   fi
 584
 585   #validate merging cpass0
 586   cd ${outputDir}
 587   touch ${doneFile}
 588   [[ -f CalibObjects.root ]] && echo "calibfile ${outputDir}/CalibObjects.root" >> ${doneFile}
 589   [[ -f dcsTime.root ]] && echo "dcsTree ${outputDir}/dcsTime.root" >> ${doneFile}
 590   echo "dir ${outputDir}" >> ${doneFile}
 591   summarizeLogs >> ${doneFile}
 592
 593   [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
 594   return 0
 595 }
 596
 597 goMergeCPass1()
 598 {
 599   #
 600   # find the output files and merge them
 601   #
 602
 603   outputDir=${1}
 604   ocdbStorage=${2}
 605   configFile=${3}
 606   runNumber=${4}
 607   calibrationFilesToMerge=${5}
 608   qaFilesToMerge=${6}
 609   shift 6
 610   parseConfig ${configFile} $@
 611
 612   [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
 613   doneFile="${commonOutputPath}/meta/merge.cpass1.run${runNumber}.done"
 614
 615   umask 0002
 616   ulimit -c unlimited
 617
 618   #clean up first:
 619   rm -f ${outputDir}/*.log
 620   rm -f ${outputDir}/*.root
 621   rm -f ${outputDir}/*done
 622
 623   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
 624
 625   #runpath=${PWD}/rundir_cpass1_Merge_${runNumber}
 626   runpath=${outputDir}
 627   [[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
 628   [[ ${reconstructInTemporaryDir} -eq 1 && -z ${TMPDIR} ]] && runpath=$(mktemp -d)
 629
 630   mkdir -p ${runpath}
 631   [[ ! -d ${runpath} ]] && echo "not able to make the runpath ${runpath}" && touch ${doneFile} && return 1
 632   cd ${runpath}
 633
 634   logOutputDir=${runpath}
 635   [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir}
 636   [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/mergeMakeOCDB.log
 637   echo "${0} $*"
 638
 639   calibrationOutputFileName='AliESDfriends_v1.root'
 640   qaOutputFileName='QAresults*.root'
 641   mergingScript="mergeMakeOCDB.byComponent.sh"
 642   #important to have the string "Stage.txt" in the filename to trigger the merging
 643   #it has to be a list of directories containing the files
 644   qaMergedOutputFileName="QAresults_merged.root"
 645
 646   echo goMergeCPass1 SETUP:
 647   echo runNumber=${runNumber}
 648   echo outputDir=${outputDir}
 649   echo ocdbStorage=${ocdbStorage}
 650   echo calibrationFilesToMerge=$calibrationFilesToMerge
 651   echo qaFilesToMerge=$qaFilesToMerge
 652   echo calibrationOutputFileName=${calibrationOutputFileName}
 653   echo mergingScript=${mergingScript}
 654
 655   # copy files in case they are not already there
 656   filesMergeCPass1=(
 657                     "${commonOutputPath}/${calibrationFilesToMerge}"
 658                     "${commonOutputPath}/${qaFilesToMerge}"
 659                     "${commonOutputPath}/OCDB.root"
 660                     "${commonOutputPath}/localOCDBaccessConfig.C"
 661                     "${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz"
 662                     "${commonOutputPath}/QAtrain_duo.C"
 663                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/mergeMakeOCDB.byComponent.sh"
 664                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/mergeByComponent.C"
 665                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/makeOCDB.C"
 666                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/merge.C"
 667                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/mergeMakeOCDB.sh"
 668                     "${trustedQAtrainMacro}"
 669                     "${ALICE_ROOT}/ANALYSIS/macros/QAtrain_duo.C"
 670   )
 671   for file in ${filesMergeCPass1[*]}; do
 672     [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
 673   done
 674
 675   sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh
 676
 677   #configure local OCDB storage from CPass0 (creates the localOCDBaccessConfig.C script)
 678   if [[ -f cpass0.localOCDB.${runNumber}.tgz ]]; then
 679     echo goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz"
 680     goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz"
 681   else
 682     echo "WARNING: file cpass0.localOCDB.${runNumber}.tgz not found!"
 683   fi
 684
 685   alirootInfo > ALICE_ROOT.log
 686
 687   #
 688   /bin/ls
 689
 690   #merge calibration
 691   chmod u+x ${mergingScript}
 692   mkdir -p OCDB
 693   if [[ ! -f ${calibrationFilesToMerge} ]]; then
 694     echo "/bin/ls -1 ${outputDir}/*/${calibrationOutputFileName} > ${calibrationFilesToMerge}"
 695     /bin/ls -1 ${outputDir}/*/${calibrationOutputFileName} 2>/dev/null > ${calibrationFilesToMerge}
 696   fi
 697
 698   echo "${mergingScript} ${calibrationFilesToMerge} ${runNumber} local://./OCDB ${ocdbStorage}"
 699   if [[ -n ${pretend} ]]; then
 700     touch CalibObjects.root
 701     touch ocdb.log
 702     touch merge.log
 703     touch cpass1.localOCDB.${runNumber}.tgz
 704   else
 705     ./${mergingScript} ${calibrationFilesToMerge} ${runNumber} "local://./OCDB" ${ocdbStorage}
 706   fi
 707
 708   tar czf ${commonOutputPath}/meta/cpass1.localOCDB.${runNumber}.tgz ./OCDB
 709
 710   #merge QA
 711   [[ -n ${AliAnalysisTaskFilteredTree_fLowPtTrackDownscaligF} ]] && export AliAnalysisTaskFilteredTree_fLowPtTrackDownscaligF
 712   [[ -n ${AliAnalysisTaskFilteredTree_fLowPtV0DownscaligF} ]] && export AliAnalysisTaskFilteredTree_fLowPtV0DownscaligF
 713
 714   if [[ ! -f ${qaFilesToMerge} ]]; then
 715     #find the files, but only store the directories (QAtrain_duo.C requires this)
 716     echo "/bin/ls -1 ${outputDir}/*/${qaOutputFileName} | while read x; do echo ${x%/*}; done | sort | uniq > ${qaFilesToMerge}"
 717     eval "/bin/ls -1 ${outputDir}/*/${qaOutputFileName}" | while read x; do echo ${x%/*}; done | sort | uniq > ${qaFilesToMerge}
 718   fi
 719
 720   #do the merge
 721   #echo aliroot -l -b -q "merge.C(\"${qaFilesToMerge}\",\"\",kFALSE,\"${qaMergedOutputFileName}\")"
 722   echo aliroot -b -q "QAtrain_duo.C(\"_barrel\",${runNumber},\"${qaFilesToMerge}\",1,\"${ocdbStorage}\")"
 723   if [[ -n ${pretend} ]]; then
 724     touch ${qaMergedOutputFileName}
 725     touch merge.log
 726     touch trending.root
 727   else
 728     #aliroot -l -b -q "merge.C(\"${qaFilesToMerge}\",\"\",kFALSE,\"${qaMergedOutputFileName}\")"
 729     aliroot -b -q "QAtrain_duo.C(\"_barrel\",${runNumber},\"${qaFilesToMerge}\",1,\"${ocdbStorage}\")" > mergeQA.log
 730     mv QAresults_barrel.root ${qaMergedOutputFileName}
 731     mv trending_barrel.root trending.root
 732   fi
 733
 734   #produce the calib trees for expert QA
 735   echo aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")"
 736   aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")"
 737
 738   /bin/ls
 739
 740   #copy all to output dir
 741   cp -pf --recursive ${runpath}/* ${outputDir}
 742
 743   #validate merge cpass1
 744   cd ${outputDir}
 745   touch ${doneFile}
 746   [[ -f CalibObjects.root ]] && echo "calibfile ${outputDir}/CalibObjects.root" >> ${doneFile}
 747   [[ -f ${qaMergedOutputFileName} ]] && echo "qafile ${outputDir}/${qaMergedOutputFileName}" >> ${doneFile}
 748   [[ -f trending.root ]] && echo "trendingfile ${outputDir}/trending.root" >> ${doneFile}
 749   [[ -f dcsTime.root ]] && echo "dcsTree ${outputDir}/dcsTime.root" >> ${doneFile}
 750   echo "dir ${outputDir}" >> ${doneFile}
 751   summarizeLogs >>  ${doneFile}
 752
 753   [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
 754   return 0
 755 }
 756
 757 goMerge()
 758 {
 759   #generic root merge using CPass1 merge.C script
 760   inputList=${1}
 761   outputFile=${2}
 762   configFile=${3-"becnhmark.config"}
 763   shift 3
 764   parseConfig ${configFile} $@
 765
 766   [[ ! -f ${inputList} ]] && echo "inputList ${inputList} does not exist!" && return 1
 767   [[ ! -f ${configFile} ]] && echo "configFile ${configFile} does not exist!" && return 1
 768   umask 0002
 769   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
 770   rm -f ${outputFile}
 771   aliroot -b -q "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/merge.C(\"${inputList}\",\"\",kFALSE,\"${outputFile}\")" > merge_${inputList}.log
 772   return 0
 773 }
 774
 775 goSubmitMakeflow()
 776 {
 777   #run
 778   productionID=${1}
 779   inputFileList=${2}
 780   configFile=${3}
 781   shift 3
 782   extraOpts="$@"
 783   parseConfig ${configFile} ${extraOpts}
 784
 785   [[ -z ${configFile} ]] && configFile="benchmark.config"
 786   [[ ! -f ${configFile} ]] && echo "no config file found (${configFile})" && return 1
 787
 788   #create the directopry for the metadata
 789   mkdir meta
 790
 791   if [[ ! $(which makeflow &>/dev/null) && -n ${makeflowPath} ]]; then
 792     echo "setting the makflow path from the config: "
 793     echo "  export PATH=${makeflowPath}:${PATH}"
 794     export PATH=${makeflowPath}:${PATH}
 795   fi
 796
 797   #submit - use makeflow if available, fall back to old stuff when makeflow not there
 798   if which makeflow; then
 799
 800     goGenerateMakeflow ${productionID} ${inputFileList} ${configFile} "${extraOpts}" > benchmark.makeflow
 801
 802     makeflow ${makeflowOptions} benchmark.makeflow
 803     cd ../
 804   else
 805     echo "no makeflow!"
 806   fi
 807   return 0
 808 }
 809
 810 goGenerateMakeflow()
 811 {
 812   #generate the makeflow file
 813   productionID=${1}
 814   inputFileList=${2}
 815   configFile=${3}
 816   shift 3
 817   extraOpts="$@"
 818   parseConfig ${configFile} ${extraOpts}
 819
 820   [[ -z ${configFile} ]] && configFile="benchmark.config"
 821   [[ ! -f ${configFile} ]] && echo "no config file found (${configFile})" && return 1
 822
 823   commonOutputPath=${baseOutputDirectory}/${productionID}
 824
 825   #these files will be made a dependency - will be copied to the working dir of the jobs
 826   declare -a copyFiles
 827   inputFiles=(
 828               "OCDB.root"
 829               "localOCDBaccessConfig.C"
 830               "QAtrain_duo.C"
 831               "runCPass1.sh"
 832               "recCPass1.C"
 833               "recCPass1_OuterDet.C"
 834               "runCalibTrain.C"
 835               "runCPass0.sh"
 836               "recCPass0.C"
 837   )
 838   for file in ${inputFiles[*]}; do
 839     [[ -f ${file} ]] && copyFiles+=("${file}")
 840   done
 841
 842   #create the makeflow file
 843   [[ -n ${batchFlags} ]] && echo "BATCH_OPTIONS = ${batchFlags}"
 844   declare -a arr_cpass1_final
 845   listOfRuns[0]=${runNumber}
 846   [[ -z ${runNumber} ]] && listOfRuns=($(while read x; do guessRunNumber ${x}; done < ${inputFileList} | sort | uniq))
 847   runindex=0
 848   for runNumber in ${listOfRuns[*]}; do
 849     [[ -z ${runNumber} ]] && continue
 850     [[ ! ${runNumber} =~ ^[0-9]*[0-9]$ ]] && continue
 851
 852     unset arr_cpass0_outputs_onerun
 853     unset arr_cpass1_outputs_onerun
 854     declare -a arr_cpass0_outputs_onerun
 855     declare -a arr_cpass1_outputs_onerun
 856
 857     jobindex=0
 858     while read inputFile; do
 859       currentDefaultOCDB=${defaultOCDB}
 860       [[ -z ${autoOCDB} ]] && autoOCDB=1
 861       if [[ ${autoOCDB} -ne 0 ]]; then
 862         currentDefaultOCDB=$(setYear ${inputFile} ${defaultOCDB})
 863       fi
 864
 865       #CPass0
 866       arr_cpass0_outputs_onerun[${jobindex}]="meta/cpass0.job${jobindex}.run${runNumber}.done"
 867       echo "${arr_cpass0_outputs_onerun[${jobindex}]} : benchmark.sh ${configFile} ${copyFiles[@]}"
 868       echo " ${alirootEnv} ./benchmark.sh CPass0 ${commonOutputPath}/000${runNumber}/cpass0 ${inputFile} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} ${jobindex} ${extraOpts}"
 869       echo
 870
 871       #CPass1
 872       arr_cpass1_outputs_onerun[${jobindex}]="meta/cpass1.job${jobindex}.run${runNumber}.done"
 873       echo "${arr_cpass1_outputs_onerun[${jobindex}]} : benchmark.sh ${configFile} meta/cpass0.localOCDB.${runNumber}.tgz ${copyFiles[@]}"
 874       echo " ${alirootEnv} ./benchmark.sh CPass1 ${commonOutputPath}/000${runNumber}/cpass1 ${inputFile} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} ${jobindex} ${extraOpts}"
 875       echo
 876       ((jobindex++))
 877
 878     done< <(grep "/000${runNumber}/" ${inputFileList})
 879
 880     #CPass0 list of Calib files to merge
 881     arr_cpass0_calib_list[${runindex}]="meta/cpass0.calib.run${runNumber}.list"
 882     echo "${arr_cpass0_calib_list[${runindex}]} : benchmark.sh ${arr_cpass0_outputs_onerun[*]}"
 883     echo "  ./benchmark.sh PrintValues calibfile ${arr_cpass0_calib_list[${runindex}]} ${arr_cpass0_outputs_onerun[*]}"
 884     echo
 885
 886     #CPass0 merging
 887     arr_cpass0_final[${runindex}]="meta/merge.cpass0.run${runNumber}.done"
 888     echo "meta/cpass0.localOCDB.${runNumber}.tgz ${arr_cpass0_final[${runindex}]} : benchmark.sh ${configFile} ${arr_cpass0_calib_list[${runindex}]} ${copyFiles[@]}"
 889     echo " ${alirootEnv} ./benchmark.sh MergeCPass0 ${commonOutputPath}/000${runNumber}/cpass0 ${currentDefaultOCDB} ${configFile} ${runNumber} ${arr_cpass0_calib_list[${runindex}]} ${extraOpts}"
 890     echo
 891
 892     #CPass1 list of Calib/QA files to merge
 893     # the trick is to have the string "Stage.txt" in the file name of the list of directories with QA output to trigger
 894     # the production of the trending tree (only then the task->Finish() will be called in QAtrain_duo.C, on the grid
 895     # this corresponds to the last merging stage)
 896     arr_cpass1_calib_list[${runindex}]="meta/cpass1.calib.run${runNumber}.list"
 897     echo "${arr_cpass1_calib_list[${runindex}]} : benchmark.sh ${arr_cpass1_outputs_onerun[*]}"
 898     echo "  ./benchmark.sh PrintValues calibfile ${arr_cpass1_calib_list[${runindex}]} ${arr_cpass1_outputs_onerun[*]};"
 899     echo
 900     arr_cpass1_QA_list[${runindex}]="meta/cpass1.QA.run${runNumber}.lastMergingStage.txt.list"
 901     echo "${arr_cpass1_QA_list[${runindex}]}: benchmark.sh ${arr_cpass1_outputs_onerun[*]}"
 902     echo "  ./benchmark.sh PrintValues dir ${arr_cpass1_QA_list[${runindex}]} ${arr_cpass1_outputs_onerun[*]}"
 903     echo
 904
 905     #CPass1 merging
 906     arr_cpass1_final[${runindex}]="meta/merge.cpass1.run${runNumber}.done"
 907     echo "meta/cpass1.localOCDB.${runNumber}.tgz ${arr_cpass1_final[${runindex}]} :  benchmark.sh ${configFile} ${arr_cpass1_calib_list[${runindex}]} ${arr_cpass1_QA_list[${runindex}]} ${copyFiles[@]}"
 908     echo " ${alirootEnv} ./benchmark.sh MergeCPass1 ${commonOutputPath}/000${runNumber}/cpass1 ${currentDefaultOCDB} ${configFile} ${runNumber} ${arr_cpass1_calib_list[${runindex}]} ${arr_cpass1_QA_list[${runindex}]} ${extraOpts}"
 909     echo
 910
 911     arr_cpass1_ESD_list[${runindex}]="meta/cpass1.ESD.run${runNumber}.list"
 912     echo "${arr_cpass1_ESD_list[${runindex}]} : benchmark.sh ${arr_cpass1_outputs_onerun[*]}"
 913     echo "  ./benchmark.sh PrintValues esd ${arr_cpass1_ESD_list[${runindex}]} ${arr_cpass1_outputs_onerun[*]}"
 914
 915     if [ ${runESDfiltering} -eq 1 ]; then
 916       arr_cpass1_filtering[${runindex}]="meta/filtering.cpass1.run${runNumber}.done"
 917       echo "${arr_cpass1_filtering[${runindex}]} : benchmark.sh ${configFile} ${arr_cpass1_ESD_list[${runindex}]}"
 918       echo "  ${alirootEnv} ./benchmark.sh MakeFilteredTrees ${commonOutputPath}/000${runNumber}/cpass1 ${runNumber} ${arr_cpass1_ESD_list[${runindex}]} ${filteringFactorHighPt} ${filteringFactorV0s} ${currentDefaultOCDB} 1000000 0 10000000 0 ${configFile} AliESDs_Barrel.root ${extraOpts}"
 919     fi
 920
 921     ((runindex++))
 922   done
 923
 924   #Summary
 925   echo "summary.log : benchmark.sh ${configFile} ${arr_cpass1_final[*]} ${arr_cpass1_filtering[*]}"
 926   echo " LOCAL ./benchmark.sh MakeSummary ${configFile}"
 927   echo
 928
 929   return 0
 930 }
 931
 932 goPrintValues()
 933 {
 934   #print the values given the key from any number of files (store in output file on second argument)
 935   if [[ $# -lt 3 ]]; then
 936     echo "goPrintValues key outputFile inputFiles"
 937     echo "if outputFile is \"-\" dont redirect to a file"
 938     return
 939   fi
 940   key=${1}
 941   outputFile=${2}
 942   [[ ${outputFile} =~ "-" ]] && outputFile=""
 943   shift 2 #remove 2 first arguments from arg list to only pass the input files to awk
 944   awk -v key=${key} '$0 ~ key" " {print $2}' "$@" | tee ${outputFile}
 945   return 0
 946 }
 947
 948 goCreateQAplots()
 949 {
 950   umask 0002
 951   mergedQAfileList=${1}
 952   productionID=${2}
 953   outputDir=${3}
 954   configFile=${4}
 955   shift 4
 956   parseConfig ${configFile} ${@}
 957
 958   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
 959
 960   [[ -z ${logOutputDir} ]] && logOutputDir=${PWD}
 961   [[ -z ${dontRedirectStdOutToLog} ]] && exec 2>&1 > ${logOutputDir}/makeQAplots.log
 962   echo "${0} $*"
 963
 964   olddir=${PWD}
 965   mkdir -p ${outputDir}
 966   cd ${outputDir}
 967   [[ ! "${PWD}" = "${outputDir}" ]] && echo "cannot make ${outputDir}... exiting" && return 1
 968
 969   echo ${ALICE_ROOT}/PWGPP/QA/scripts/runQA.sh inputList=${mergedQAfileList}
 970   ${ALICE_ROOT}/PWGPP/QA/scripts/runQA.sh inputList=${mergedQAfileList}
 971
 972   cd ${olddir}
 973   return 0
 974 }
 975
 976 goTest()
 977 {
 978   umask 0002
 979   exec 2>&1
 980   exec > >(tee test.log)
 981   echo "$@"
 982   echo something
 983   return 0
 984 }
 985
 986 alirootInfo()
 987 {
 988   umask 0002
 989   # save aliroot repository info
 990   [[ -z "${ALICE_ROOT}" ]] && return 1
 991
 992   echo "\${ALICE_ROOT}=${ALICE_ROOT}"
 993   echo "\${ROOTSYS}=${ROOTSYS}"
 994   echo "\${PATH}=${PATH}"
 995   echo "\${LD_LIBRARY_PATH}=${LD_LIBRARY_PATH}"
 996
 997   pushd ${PWD}
 998   cd ${ALICE_ROOT}
 999   echo
1000
1001   currentBranch=$(git rev-parse --abbrev-ref HEAD)
1002   git status
1003   echo ""
1004   echo ""
1005   git diff ${currentBranch}
1006   popd
1007   return 0
1008 }
1009
1010 setYear()
1011 {
1012   #set the year
1013   #  ${1} - year to be set
1014   #  ${2} - where to set the year
1015   year1=$(guessYear ${1})
1016   year2=$(guessYear ${2})
1017   local path=${2}
1018   [[ ${year1} -ne ${year2} && -n ${year2} && -n ${year1} ]] && path=${2/\/${year2}\//\/${year1}\/}
1019   echo ${path}
1020   return 0
1021 }
1022
1023 guessPeriod()
1024 {
1025   #guess the period from the path, pick the rightmost one
1026   local IFS="/"
1027   declare -a path=( ${1} )
1028   local dirDepth=${#path[*]}
1029   for ((x=${dirDepth}-1;x>=0;x--)); do
1030     local field=${path[${x}]}
1031     [[ ${field} =~ ^LHC[0-9][0-9][a-z]$ ]] && period=${field} && break
1032   done
1033   echo ${period}
1034   return 0
1035 }
1036
1037 guessYear()
1038 {
1039   #guess the year from the path, pick the rightmost one
1040   local IFS="/"
1041   declare -a path=( ${1} )
1042   local dirDepth=${#path[*]}
1043   for ((x=${dirDepth}-1;x>=0;x--)); do
1044     local field=${path[${x}]}
1045     [[ ${field} =~ ^20[0-9][0-9]$ ]] && year=${field} && break
1046   done
1047   echo ${year}
1048   return 0
1049 }
1050
1051 guessRunNumber()
1052 {
1053   #guess the run number from the path, pick the rightmost one
1054   #works for /path/foo/000123456/bar/...
1055   #and       /path/foo.run123456.bar
1056   local IFS="/."
1057   declare -a path=( ${1} )
1058   local dirDepth=${#path[*]}
1059   for ((x=${dirDepth}-1;x>=0;x--)); do
1060     local field=${path[${x}]}
1061     field=${field/run/000}
1062     [[ ${field} =~ ^000[0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && runNumber=${field#000} && break
1063   done
1064   echo ${runNumber}
1065   return 0
1066 }
1067
1068 validateLog()
1069 {
1070   log=${1}
1071   errorConditions=(
1072                     'There was a crash'
1073                     'floating'
1074                     'error while loading shared libraries'
1075                     'std::bad_alloc'
1076                     's_err_syswatch_'
1077                     'Thread [0-9]* (Thread'
1078                     'AliFatal'
1079                     'core dumped'
1080                     '\.C.*error:.*\.h: No such file'
1081   )
1082
1083   warningConditions=(
1084                      'This is serious !'
1085                      'rocVoltage out of range:'
1086   )
1087
1088   local logstatus=0
1089   local errorSummary=""
1090   local warningSummary=""
1091
1092   for ((i=0; i<${#errorConditions[@]};i++)); do
1093     local tmp=$(grep -m1 -e "${errorConditions[${i}]}" ${log})
1094     [[ -n ${tmp} ]] && tmp+=" : "
1095     errorSummary+=${tmp}
1096   done
1097
1098   for ((i=0; i<${#warningConditions[@]};i++)); do
1099     local tmp=$(grep -m1 -e "${warningConditions[${i}]}" ${log})
1100     [[ -n ${tmp} ]] && tmp+=" : "
1101     warningSummary+=${tmp}
1102   done
1103
1104   if [[ -n ${errorSummary} ]]; then
1105     echo "${errorSummary}"
1106     return 1
1107   fi
1108
1109   if [[ -n ${warningSummary} ]]; then
1110     echo "${warningSummary}"
1111     return 2
1112   fi
1113
1114   return 0
1115 }
1116
1117 summarizeLogs()
1118 {
1119   #print a summary of logs
1120   logFiles=(
1121             "*.log"
1122             "stdout"
1123             "stderr"
1124   )
1125
1126   #check logs
1127   local logstatus=0
1128   for log in ${logFiles[*]}; do
1129     finallog=${PWD%/}/${log}
1130     [[ ! -f ${log} ]] && continue
1131     errorSummary=$(validateLog ${log})
1132     validationStatus=$?
1133     if [[ ${validationStatus} -eq 0 ]]; then
1134       #in pretend mode randomly report an error in rec.log some cases
1135       if [[ -n ${pretend} && "${log}" == "rec.log" ]]; then
1136         [[ $(( ${RANDOM}%2 )) -ge 1 ]] && echo "${finallog} BAD random error" || echo "${finallog} OK"
1137       else
1138         echo "${finallog} OK"
1139       fi
1140     elif [[ ${validationStatus} -eq 1 ]]; then
1141       echo "${finallog} BAD ${errorSummary}"
1142     elif [[ ${validationStatus} -eq 2 ]]; then
1143       echo "${finallog} OK MWAH ${errorSummary}"
1144     fi
1145   done
1146
1147   #report core files
1148   while read x; do
1149     echo ${x}
1150     chmod 644 ${x}
1151     gdb --batch --quiet -ex "bt" -ex "quit" aliroot ${x} > stacktrace_${x//\//_}.log
1152   done < <(/bin/ls ${PWD}/*/core 2>/dev/null; /bin/ls ${PWD}/core 2>/dev/null)
1153
1154   return ${logstatus}
1155 }
1156
1157 spitOutLocalOCDBaccessConfig()
1158 {
1159   umask 0002
1160   #find ${1} -name "*root" | \
1161   /bin/ls -1 ${1}/*/*/*/*.root 2>/dev/null | \
1162   while read line
1163   do
1164     local tmp=${line#${1}}
1165     echo ${tmp%/*} | \
1166     awk -v ocdb=${1} '{print "  man->SetSpecificStorage(\""$1"\",\"local://"ocdb"\");"}'
1167   done
1168   return 0
1169 }
1170
1171 goMakeLocalOCDBaccessConfig()
1172 {
1173   umask 0002
1174   # make a script that sets the specific storages form all the root files produced by CPass0
1175   local localOCDBpathCPass0=${1}
1176   local OCDBpathPrefix=${2}
1177   [[ -z ${OCDBpathPrefix} ]] && OCDBpathPrefix="."
1178
1179   if [[ -f ${localOCDBpathCPass0} && ${localOCDBpathCPass0} =~ \.tgz$ ]]; then
1180     tar xzf ${localOCDBpathCPass0}
1181     local localOCDBpathCPass0="${OCDBpathPrefix}/OCDB"
1182   fi
1183
1184   echo
1185   echo creating the specific storage script
1186   echo   localOCDBaccessConfig.C
1187   echo   based on OCDB: ${localOCDBaccessConfig}
1188   echo
1189
1190   local tempLocalOCDB=""
1191   if [[ -f localOCDBaccessConfig.C ]]; then
1192     tempLocalOCDB=$(mktemp)
1193     echo "egrep "SetSpecificStorage" localOCDBaccessConfig.C > ${tempLocalOCDB}"
1194     egrep "SetSpecificStorage" localOCDBaccessConfig.C > ${tempLocalOCDB}
1195   fi
1196
1197   echo "localOCDBaccessConfig()"                               >  localOCDBaccessConfig.C
1198   echo "{"                                                     >> localOCDBaccessConfig.C
1199   echo "  AliCDBManager* man = AliCDBManager::Instance();"     >> localOCDBaccessConfig.C
1200   spitOutLocalOCDBaccessConfig ${localOCDBpathCPass0}|sort|uniq  >> localOCDBaccessConfig.C
1201   [[ -f "${tempLocalOCDB}" ]] && cat ${tempLocalOCDB}              >> localOCDBaccessConfig.C
1202   echo "}"                                                     >> localOCDBaccessConfig.C
1203
1204   [[ -f "${tempLocalOCDB}" ]] && rm -f ${tempLocalOCDB}
1205
1206   if ! grep SetSpecificStorage localOCDBaccessConfig.C; then
1207     echo
1208     echo "!!!!!!! CPass0 produced no OCDB entries"
1209     return 1
1210   fi
1211   return 0
1212 }
1213
1214 goMakeFilteredTrees()
1215 {
1216   outputDir=${1}
1217   runNumber=${2}
1218   #get path to input list
1219   inputListfiles=${3}
1220   #get scale number for tracks
1221   filterT=${4}
1222   #get scale number for V0s
1223   filterV=${5}
1224   #get OCDB path (optional)
1225   OCDBpath=${6}
1226   #get max number of files
1227   maxFiles=${7-"1000000"}
1228   #get offset of first file
1229   offsetFile=${8-"0"}
1230   #get max number of events
1231   maxEvents=${9-"30000000"}
1232   #get offset of first event
1233   offsetEvent=${10-"0"}
1234   configFile=${11-"benchmark.config"}
1235   esdFileName=${12-"AliESDs_Barrel.root"}
1236   shift 12
1237   parseConfig ${configFile} $@
1238
1239   commonOutputPath=${PWD}
1240   doneFile=${commonOutputPath}/meta/filtering.cpass1.run${runNumber}.done
1241
1242   runpath=${commonOutputPath}/rundir_filtering_run${runNumber}
1243
1244   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
1245
1246   mkdir -p ${outputDir}
1247   mkdir -p ${runpath}
1248
1249   cd ${runpath}
1250
1251   cat > filtering.log << EOF
1252   goMakeFilteredTrees config:
1253   runpath=${runpath}
1254   outputDir=${outputDir}
1255   commonOutputPath=${commonOutputPath}
1256   ALICE_ROOT=${ALICE_ROOT}
1257   PATH=${PATH}
1258   offsetEvent=$offsetEvent
1259   configFile=$configFile
1260   esdFileName=$esdFileName
1261 EOF
1262
1263   if [[ -z ${pretend} ]];then
1264     aliroot -l -b -q "${ALICE_ROOT}/PWGPP/macros/runFilteringTask.C(\"${inputListfiles}\",${filterT},${filterV},\"${OCDBpath}\",${maxFiles},${offsetFile},${maxEvents},${offsetEvent},\"${esdFileName}\")" &>> filtering.log
1265   else
1266     touch filtering.log FilterEvents_Trees.root
1267   fi
1268   pwd
1269   /bin/ls
1270   touch ${doneFile}
1271   summarizeLogs >>  ${doneFile}
1272   echo mv -f * ${outputDir}
1273   mv -f * ${outputDir}
1274   [[ -f ${outputDir}/FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFile}
1275   cd ${commonOutputPath}
1276   [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
1277   return 0
1278 }
1279
1280 submit()
1281 {
1282   umask 0002
1283   [[ $# -ne 6 ]] && echo "6 args needed, you supplied $#" && return 1
1284   JobID=${1}
1285   startID=${2}
1286   endID=${3}
1287   waitForJOBID=${4}
1288   command=${5}
1289   commandArgs=${6}
1290
1291   [[ -z ${waitForJOBID} ]] && waitForJOBID=0
1292
1293   newFarm=$(which qsub|grep "^/usr/bin/qsub")
1294
1295   batchSystem="SGE"
1296
1297   if [[ -z "${newFarm}" ]]
1298   then
1299     #old LSF
1300     # submit it (as job array)
1301     nFiles=$(( ${endID}-${startID}+1 ))
1302     while [ ${startID} -le ${nFiles}  ] ; do
1303       if [ $(expr ${nFiles} - ${startID}) -gt 999 ] ; then
1304         endID=$(expr ${startID} + 999)
1305       else
1306         endID=${nFiles}
1307       fi
1308     if [[ ${waitForJOBID} -eq 0 ]]; then
1309         echo ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}"
1310         ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}"
1311       else
1312         echo ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -w "ended(${waitForJOBID})" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}"
1313         ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -w "ended(${waitForJOBID})" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}"
1314       fi
1315       startID=$(expr ${endID} + 1)
1316     done
1317   else
1318     #new SGE farm
1319     if [[ ${waitForJOBID} =~ "000" ]]; then
1320       echo ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" ${commandArgs}
1321       ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" ${commandArgs}
1322     else
1323       echo ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -hold_jid "${waitForJOBID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" ${commandArgs}
1324       ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -hold_jid "${waitForJOBID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" ${commandArgs}
1325     fi
1326   fi
1327   return 0
1328 }
1329
1330 goSubmitBatch()
1331 {
1332   if [[ $# -lt 3 ]]; then
1333     echo "minimal use:"
1334     echo " ${0} submit fileList productionID configFile"
1335     return 0
1336   fi
1337
1338   productionID=${1}
1339   inputList=${2}
1340   configFile=${3:-"benchmark.config"}
1341   configFile=$(readlink -f ${configFile})
1342   shift 3
1343   extraOpts="$@"
1344   parseConfig ${configFile} ${extraOpts}
1345
1346   #redirect all output to submit.log
1347   echo "redirecting all output to ${PWD}/submit_${productionID//"/"/_}.log"
1348   exec 7>&1
1349   exec 1>submit_${productionID//"/"/_}.log 2>&1
1350
1351   umask 0002
1352   echo ${0}" submit $*"
1353   if [[ -z "${inputList}" || -z "${productionID}" ]]
1354   then
1355     echo
1356     echo " Usage: ${0} submit inputList productionID [configFile=benchmark.config]"
1357     echo
1358     return
1359   fi
1360
1361   # check if config file is there
1362   if [ ! -f ${configFile} ]; then
1363     echo "ERROR! Config File '${configFile}' not found" >&2
1364     return
1365   else
1366     echo "Using Config File: '${configFile}'"
1367   fi
1368
1369   # and print the configuration
1370   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
1371
1372   #move the script, config and some other stuff to ${commonOutputPath} first, then use them from there
1373   self=$(readlink -f "${0}")
1374   configPath=$(dirname ${configFile})
1375   export commonOutputPath=${baseOutputDirectory}/${productionID}
1376
1377   mkdir -p ${commonOutputPath}
1378   mkdir -p ${commonOutputPath}/logs
1379   mkdir -p ${commonOutputPath}/meta
1380
1381   cp ${self} ${commonOutputPath}
1382   cp ${configFile} ${commonOutputPath}
1383   cp ${inputList} ${commonOutputPath}
1384   self=${commonOutputPath}/${self##*/}
1385   chmod u+x ${self}
1386   configFile=${commonOutputPath}/${configFile##*/}
1387   inputList=${commonOutputPath}/${inputList##*/}
1388
1389   #convert to absolut pathnames
1390   inputList=$(readlink -f "${inputList}")
1391   #make list of runs
1392   if [[ -z ${runNumber} ]]; then
1393     listOfRuns=($(while read x; do guessRunNumber ${x}; done < ${inputList} | sort | uniq))
1394   else
1395     listOfRuns=${runNumber}
1396   fi
1397
1398   alirootSource=$(readlink -f "${alirootSource}")
1399
1400   echo ""
1401   echo "### BEGIN CONFIGURATION ###"
1402   echo ""
1403   echo "GENERAL:"
1404   echo ""
1405   echo "    productionID:    ${productionID}"
1406   echo "    batchCommand:    ${batchCommand}"
1407   echo "    batchFlags:      ${batchFlags}"
1408   echo "    alirootSource:   ${alirootSource}"
1409   echo "    ALICE_ROOT:      ${ALICE_ROOT}"
1410   echo "    ALIROOT_RELEASE: ${ALICE_RELEASE}"
1411   echo "    inputList:       ${inputList}"
1412   echo "    configPath:      ${configPath}"
1413   echo "    commonOutputPath:      ${commonOutputPath}"
1414   echo "    defaultOCDB:     ${defaultOCDB}"
1415   echo "      autoOCDB: ${autoOCDB}"
1416   echo "    recoTriggerOptions:   ${recoTriggerOptions}"
1417   echo "    runs:"
1418   echo "      ${listOfRuns[*]}"
1419   echo ""
1420   echo "THE TRAIN WILL RUN:"
1421
1422   if [ ${runCPass0reco} -eq 1 ]; then
1423     echo "    Pass0 - Recontruction"
1424   fi
1425
1426   if [ ${runCPass0MergeMakeOCDB} -eq 1 ]; then
1427     echo "    Pass0 - merging and OCDB export"
1428   fi
1429
1430   if [ ${runCPass1reco} -eq 1 ]; then
1431     echo "    Pass1 - Recontruction"
1432   fi
1433   if [ ${runCPass1MergeMakeOCDB} -eq 1 ]; then
1434     echo "    Pass1 - merging and OCDB export"
1435   fi
1436
1437   echo ""
1438   echo "LIMITS:"
1439   echo "    max. Events/Chunk:   ${nEvents}"
1440   echo "    max. Number of Chunks per Run:     ${nMaxChunks}"
1441   echo ""
1442   echo "### END CONFIGURATION ###"
1443   echo ""
1444
1445
1446   # check if input file is there
1447   if [ ! -f ${inputList} ]; then
1448     echo "ERROR! Input List '${inputList}' not found" >&2
1449     return
1450   fi
1451
1452   # define jobid (for dependent jobs)
1453   date=$(date +%Y_%m_%d_%H%M%S)
1454   #for each run we submit one jobarray:
1455   for runNumber in ${listOfRuns[*]}; do
1456
1457     [[ -z ${runNumber} ]] && continue
1458     [[ ! ${runNumber} =~ ^[0-9]*[0-9]$ ]] && continue
1459
1460     JOBpostfix="${productionID//"/"/_}_${runNumber}_${date}"
1461     JOBID1="p0_${JOBpostfix}"
1462     JOBID1wait="w0_${JOBpostfix}"
1463     JOBID2="m0_${JOBpostfix}"
1464     JOBID2wait="wm0_${JOBpostfix}"
1465     JOBID3="op0_${JOBpostfix}"
1466     JOBID3wait="wop0_${JOBpostfix}"
1467     JOBID4="p1_${JOBpostfix}"
1468     JOBID4wait="w1_${JOBpostfix}"
1469     JOBID5="m1_${JOBpostfix}"
1470     JOBID5wait="wm1_${JOBpostfix}"
1471     JOBID6="s1_${JOBpostfix}"
1472     JOBID6wait="ws1_${JOBpostfix}"
1473     JOBID7="QA_${JOBpostfix}"
1474     JOBmakeESDlistCPass1="lp1_${JOBpostfix}"
1475     JOBfilterESDcpass1="fp1_${JOBpostfix}"
1476     LASTJOB="000"
1477
1478     oneInputFile=$(egrep -m1 "${runNumber}\/" ${inputList})
1479
1480     currentDefaultOCDB=${defaultOCDB}
1481     [[ -z ${autoOCDB} ]] && autoOCDB=1
1482     if [[ ${autoOCDB} -ne 0 ]]; then
1483       currentDefaultOCDB=$(setYear ${oneInputFile} ${defaultOCDB})
1484     fi
1485
1486     echo "submitting run ${runNumber} with OCDB ${currentDefaultOCDB}"
1487
1488     ###############################################################################
1489     #run one chunk with valgrind:
1490     if [[ -n ${runValgrind} ]]; then
1491       [[ -z ${nEventsValgrind} ]] && nEventsValgrind=2
1492       [[ -z ${valgrindCommand} ]] && valgrindCommand="/usr/bin/valgrind --tool=callgrind --num-callers=40 -v --trace-children=yes"
1493       submit "valgrind" 1 1 000 "${alirootEnv} ${self}" "CPass0 ${commonOutputPath}/000${runNumber}/valgrind ${oneInputFile} ${nEventsValgrind} ${currentDefaultOCDB} ${configFile} ${runNumber} valgrind valgrindCommand=${valgrindCommand} ${extraOpts}"
1494     fi
1495
1496     ################################################################################
1497     ################################################################################
1498     # run the CPass0 if requested
1499
1500     if [ ${runCPass0reco} -eq 1 ]; then
1501
1502       echo
1503       echo "starting CPass0... for run ${runNumber}"
1504       echo
1505
1506       # create directory and copy all files that are needed
1507       targetDirectory="${commonOutputPath}/000${runNumber}/cpass0"
1508       mkdir -p ${targetDirectory}
1509
1510       filesCPass0=(
1511                     "${configPath}/runCPass0.sh"
1512                     "${configPath}/recCPass0.C"
1513                     "${configPath}/runCalibTrain.C"
1514                     "${configPath}/localOCDBaccessConfig.C"
1515                     "${configPath}/OCDB*.root"
1516                     "${configPath}/sim.C"
1517                     "${configPath}/rec.C"
1518                     "${configPath}/Config.C"
1519       )
1520       for file in ${filesCPass0[*]}; do
1521         [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
1522       done
1523
1524       localInputList=${targetDirectory}/${inputList##*/}
1525       [[ ! -f ${localInputList} ]] && egrep "\/000${runNumber}\/" ${inputList} > ${localInputList}
1526       # limit nFiles to nMaxChunks
1527       nFiles=$(wc -l < ${localInputList})
1528       [[ ${nFiles} -eq 0 ]] && echo "list contains ZERO files! exiting..." && return 1
1529       echo "raw files in list:    ${nFiles}"
1530       if [[ ${nMaxChunks} -gt 0 && ${nMaxChunks} -le ${nFiles} ]]; then
1531         nFiles=${nMaxChunks}
1532       fi
1533       echo "raw files to process: ${nFiles}"
1534       [[ -z "${percentProcessedFilesToContinue}" ]] && percentProcessedFilesToContinue=100
1535       if [[ ${percentProcessedFilesToContinue} -eq 100 ]]; then
1536         nFilesToWaitFor=${nFiles}
1537       else
1538         nFilesToWaitFor=$(( ${nFiles}-${nFiles}/(100/(100-${percentProcessedFilesToContinue})) ))
1539       fi
1540       echo "requested success rate is ${percentProcessedFilesToContinue}%"
1541       echo "merging will start after ${nFilesToWaitFor} jobs are done"
1542
1543       submit ${JOBID1} 1 ${nFiles} 000 "${alirootEnv} ${self}" "CPass0 ${targetDirectory} ${localInputList} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} -1 ${extraOpts}"
1544
1545       ## submit a monitoring job that will run until a certain number of jobs are done with reconstruction
1546       submit "${JOBID1wait}" 1 1 000 "${alirootEnv} ${self}" "WaitForOutput ${commonOutputPath} 'meta/cpass0.job*.run${runNumber}.done' ${nFilesToWaitFor} ${maxSecondsToWait} '-maxdepth 1'"
1547       LASTJOB=${JOBID1wait}
1548
1549     fi #end running CPass0
1550     ################################################################################
1551
1552
1553     ################################################################################
1554     # submit merging of CPass0, depends on the reconstruction
1555
1556     if [ ${runCPass0MergeMakeOCDB} -eq 1 ]; then
1557
1558       echo
1559       echo "submit CPass0 merging for run ${runNumber}"
1560       echo
1561
1562       targetDirectory="${commonOutputPath}/000${runNumber}/cpass0"
1563       mkdir -p ${targetDirectory}
1564
1565       #copy the scripts
1566       filesMergeCPass0=(
1567                         "${configPath}/OCDB.root"
1568                         "${configPath}/mergeMakeOCDB.byComponent.sh"
1569                         "${configPath}/mergeMakeOCDB.sh"
1570                         "${configPath}/localOCDBaccessConfig.C"
1571                         "${configPath}/mergeByComponent.C"
1572                         "${configPath}/makeOCDB.C"
1573                         "${configPath}/merge.C"
1574       )
1575       for file in ${filesMergeCPass0[*]}; do
1576         [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
1577       done
1578
1579       submit ${JOBID2} 1 1 "${LASTJOB}" "${alirootEnv} ${self}" "MergeCPass0 ${targetDirectory} ${currentDefaultOCDB} ${configFile} ${runNumber} cpass0.calib.run${runNumber}.list ${extraOpts}"
1580       LASTJOB=${JOBID2}
1581
1582       if [[ -n ${generateMC} ]]; then
1583         submit "mrl${JOBpostfix}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" "PrintValues sim ${commonOutputPath}/meta/sim.run${runNumber}.list ${commonOutputPath}/meta/cpass0.job*.run${runNumber}.done"
1584         LASTJOB="mrl${JOBpostfix}"
1585       fi
1586
1587       echo
1588     fi
1589     # end of merging CPass0
1590     ################################################################################
1591
1592     ################################################################################
1593     ################################################################################
1594     # run the CPass1 if requested
1595
1596     if [ ${runCPass1reco} -eq 1 ]; then
1597
1598       targetDirectory="${commonOutputPath}/000${runNumber}/cpass1"
1599       rm -f ${commonOutputPath}/meta/cpass1.job*.run${runNumber}.done
1600
1601       # safety feature: if we are re-running for any reason we want to delete the previous output first.
1602       [[ -d ${targetDirectory} ]] && rm -rf ${targetDirectory}/* && echo "removed old output at ${targetDirectory}/*"
1603
1604       echo
1605       echo "starting CPass1... for run ${runNumber}"
1606       echo
1607
1608       # create directory and copy all files that are needed
1609       mkdir -p ${targetDirectory}
1610
1611       filesCPass1=(
1612                     "${configPath}/runCPass1.sh"
1613                     "${configPath}/recCPass1.C"
1614                     "${configPath}/recCPass1_OuterDet.C"
1615                     "${configPath}/runCalibTrain.C"
1616                     "${configPath}/QAtrain_duo.C"
1617                     "${configPath}/localOCDBaccessConfig.C"
1618                     "${configPath}/OCDB.root"
1619       )
1620       for file in ${filesCPass1[*]}; do
1621         [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
1622       done
1623
1624       if [[ -n ${generateMC} ]]; then
1625         localInputList=${commonOutputPath}/meta/sim.run${runNumber}.list
1626       else
1627         localInputList=${targetDirectory}/${inputList##*/}
1628         [[ ! -f ${localInputList} ]] && egrep "\/000${runNumber}\/" ${inputList} > ${localInputList}
1629       fi
1630       # limit nFiles to nMaxChunks
1631       nFiles=$(wc -l < ${localInputList})
1632       [[ ${nFiles} -eq 0 ]] && echo "list contains ZERO files! continuing..." && continue
1633       echo "raw files in list:    ${nFiles}"
1634       if [[ ${nMaxChunks} -gt 0 && ${nMaxChunks} -le ${nFiles} ]]; then
1635         nFiles=${nMaxChunks}
1636       fi
1637       echo "raw files to process: ${nFiles}"
1638       [[ -z "${percentProcessedFilesToContinue}" ]] && percentProcessedFilesToContinue=100
1639       if [[ ${percentProcessedFilesToContinue} -eq 100 ]]; then
1640         nFilesToWaitFor=${nFiles}
1641       else
1642         nFilesToWaitFor=$(( ${nFiles}-${nFiles}/(100/(100-${percentProcessedFilesToContinue})) ))
1643       fi
1644       echo "requested success rate is ${percentProcessedFilesToContinue}%"
1645       echo "merging will start after ${nFilesToWaitFor} jobs are done"
1646
1647       submit ${JOBID4} 1 ${nFiles} "${LASTJOB}" "${alirootEnv} ${self}" "CPass1 ${targetDirectory} ${localInputList} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} -1 ${extraOpts}"
1648
1649       ################################################################################
1650       ## submit a monitoring job that will run until a certain number of jobs are done with reconstruction
1651       submit "${JOBID4wait}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" "WaitForOutput ${commonOutputPath} 'meta/cpass1.job*.run${runNumber}.done' ${nFilesToWaitFor} ${maxSecondsToWait} '-maxdepth 1'"
1652       LASTJOB=${JOBID4wait}
1653       ################################################################################
1654
1655       echo
1656     fi #end running CPass1
1657
1658     ################################################################################
1659     # submit merging of CPass1, depends on the reconstruction
1660     if [ ${runCPass1MergeMakeOCDB} -eq 1 ]; then
1661
1662       echo
1663       echo "submit CPass1 merging for run ${runNumber}"
1664       echo
1665
1666       targetDirectory="${commonOutputPath}/000${runNumber}/cpass1"
1667       rm -f ${commonOutputPath}/meta/merge.cpass1.run${runNumber}.done
1668       mkdir -p ${targetDirectory}
1669
1670       # copy files
1671       filesMergeCPass1=(
1672                         "${configPath}/OCDB.root"
1673                         "${configPath}/localOCDBaccessConfig.C"
1674                         "${configPath}/mergeMakeOCDB.byComponent.sh"
1675                         "${configPath}/mergeByComponent.C"
1676                         "${configPath}/makeOCDB.C"
1677                         "${configPath}/merge.C"
1678                         "${configPath}/mergeMakeOCDB.sh"
1679                         "${configPath}/QAtrain_duo.C"
1680       )
1681       for file in ${filesMergeCPass1[*]}; do
1682         [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
1683       done
1684
1685       submit "${JOBID5}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" "MergeCPass1 ${targetDirectory} ${currentDefaultOCDB} ${configFile} ${runNumber} cpass0.calib.run${runNumber}.list cpass1.QA.run${runNumber}.lastMergingStage.txt.list ${extraOpts}"
1686       LASTJOB=${JOBID5}
1687       echo
1688     fi
1689
1690     ##############################
1691     if [ ${runESDfiltering} -eq 1 ]; then
1692       rm -f ${commonOutputPath}/cpass1.ESD.run${runNumber}.list
1693       rm -f ${commonOutputPath}/meta/filtering.cpass1.run*.done
1694       echo
1695       echo submitting filtering for run ${runNumber}
1696       echo
1697       submit "${JOBmakeESDlistCPass1}" 1 1 "${LASTJOB}" "${self}" "PrintValues esd ${commonOutputPath}/meta/cpass1.ESD.run${runNumber}.list ${commonOutputPath}/meta/cpass1.job*.run${runNumber}.done "
1698       submit "${JOBfilterESDcpass1}" 1 1 "${JOBmakeESDlistCPass1}" "${alirootEnv} ${self}" "MakeFilteredTrees ${commonOutputPath}/000${runNumber}/cpass1 ${runNumber} ${commonOutputPath}/meta/cpass1.ESD.run${runNumber}.list ${filteringFactorHighPt} ${filteringFactorV0s} ${currentDefaultOCDB} 1000000 0 10000000 0 ${configFile} AliESDs_Barrel.root ${extraOpts}"
1699       LASTJOB=${JOBfilterESDcpass1}
1700     fi
1701
1702   done
1703
1704   #################################################################################
1705   #################################################################################
1706   if [ ${runESDfiltering} -eq 1 ]; then
1707     submit "${JOBID5wait}" 1 1 "${LASTJOB}" "${self}" "WaitForOutput ${commonOutputPath} 'meta/filtering.cpass1.run*.done' ${#listOfRuns[@]} ${maxSecondsToWait}"
1708   else
1709     submit "${JOBID5wait}" 1 1 "${LASTJOB}" "${self}" "WaitForOutput ${commonOutputPath} 'meta/merge.cpass1.run*.done' ${#listOfRuns[@]} ${maxSecondsToWait}"
1710   fi
1711   LASTJOB=${JOBID5wait}
1712
1713   #################################################################################
1714   echo
1715   echo "submit make a summary"
1716   echo
1717
1718   submit "${JOBID6}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" "MakeSummary ${configFile}"
1719   LASTJOB=${JOBID6}
1720   #################################################################################
1721
1722   #restore stdout
1723   exec 1>&7 7>&-
1724   echo "jobs submitted."
1725   return 0
1726 }
1727
1728 goWaitForOutput()
1729 {
1730   umask 0002
1731   [[ $# -lt 3 ]] && echo "goWaitForOutput() wrong number of arguments, exiting.." && return 1
1732   echo searchPath=${1}
1733   echo fileName=${2}
1734   echo numberOfFiles=${3}
1735   echo maxSecondsToWait=${4}
1736   searchPath=${1}
1737   fileName=${2}
1738   numberOfFiles=${3}
1739   maxSecondsToWait=${4}
1740   extraFindOptions=${5}
1741   echo "command to be executed: /bin/ls -1 ${searchPath}/${fileName} ${extraFindOptions}"
1742   [[ -z "${maxSecondsToWait}" ]] && maxSecondsToWait=$(( 3600*12 ))
1743   while true; do
1744     #n=$(find ${searchPath} ${extraFindOptions} -name "${fileName}" | wc -l)
1745     n=$(/bin/ls -1 ${searchPath}/${fileName} 2>/dev/null | wc -l)
1746     [[ ${n} -gt 0 ]] && echo "found ${n} X ${fileName}"
1747     [[ ${n} -ge ${numberOfFiles} ]] && break
1748     [[ ${SECONDS} -gt ${maxSecondsToWait} ]] && echo "timeout of ${maxSecondsToWait}s!" && break
1749     sleep 60
1750   done
1751   echo "DONE! exiting..."
1752   return 0
1753 }
1754
1755 mergeSysLogs()
1756 {
1757   outputFile=${1}
1758   shift
1759   inputFiles="$@"
1760   i=0
1761   if ! ls -1 ${inputFiles} &>/dev/null; then echo "the files dont exist!: ${inputFiles}"; return 1; fi
1762   while read x; do
1763     runNumber=$(guessRunNumber ${x})
1764     [[ -z ${runNumber} ]] && echo "run number cannot be guessed for ${x}" && continue
1765     awk -v run=${runNumber} -v i=${i} 'NR > 1 {print run" "$0} NR==1 && i==0 {print "run/I:"$0}' ${x}
1766     (( i++ ))
1767   done < <(ls -1 ${inputFiles}) > ${outputFile}
1768   return 0
1769 }
1770
1771 goMakeMergedSummaryTree()
1772 {
1773   # create list of calibration entries
1774   # takes no arguments, just run it in the base output
1775   # directory with the following files in the working directory
1776   #
1777   #  Calibration file lists:
1778   #       cpass0.dcsTree.list, cpass1.dcsTree.list
1779   #  QA trending root file:
1780   #       trending.root
1781   #
1782   #  Production infoo ascii files:
1783   #       summary_pass0.tree
1784   #       summary_pass1.tree
1785   #
1786
1787   [[ ! -f cpass0.dcsTree.list ]] && echo "no cpass0.dcsTree.list" && return 1
1788   [[ ! -f cpass1.dcsTree.list ]] && echo "no cpass1.dcsTree.list" && return 1
1789   [[ ! -f trending.root ]] && echo "no trending.root" && return 1
1790   [[ ! -f summary_pass0.tree ]] && echo "no summary_pass0.tree" && return 1
1791   [[ ! -f summary_pass1.tree ]] && echo "no summary_pass1.tree" && return 1
1792
1793   #first, dump the C macro to file
1794   cat << EOF > mergeTree.C
1795   //
1796   // Merge summary information
1797   // Following files are expected to be in the working directory
1798   //
1799   // Calibration file lists:
1800   //      cpass0.dcsTree.list, cpass1.dcsTree.list
1801   // QA trending root file:
1802   //      trending.root
1803   //
1804   // Production infoo ascii files:
1805   //      summary_pass0.tree
1806   //      summary_pass1.tree
1807   //
1808   void mergeTree(){
1809     //
1810     //
1811     //
1812     // Calibration values dump
1813     //
1814     AliXRDPROOFtoolkit::MakeTreeFromList("Calib.TPC.CPass0.root", "dcs","dcs","cpass0.dcsTree.list",1);
1815     AliXRDPROOFtoolkit::MakeTreeFromList("Calib.TPC.CPass1.root", "dcs","dcs","cpass1.dcsTree.list",1);
1816     //
1817     // Calibration status dump
1818     //
1819     TFile *fprod = TFile::Open("fproduction.root","recreate");
1820     TTree  tree0, tree1;
1821     tree0.ReadFile("summary_pass0.tree");
1822     tree1.ReadFile("summary_pass1.tree");
1823     tree0.Write("CPass0");
1824     tree1.Write("CPass1");
1825     fprod->Close();
1826     //
1827     //
1828     //
1829     TString stringSetup="";
1830     stringSetup+="1#QA.TPC#run#SummaryTPCQA/tpcQA#trending.root+";  //
1831     stringSetup+="1#Calib.TPC.CPass0#run#dcs#Calib.TPC.CPass0.root+";  //
1832     stringSetup+="1#Calib.TPC.CPass1#run#dcs#Calib.TPC.CPass1.root+";  //
1833     //
1834     stringSetup+="1#CPass0#runnumber#CPass0#fproduction.root+";  //
1835     stringSetup+="1#CPass1#runnumber#CPass1#fproduction.root+";  //
1836     //
1837     AliXRDPROOFtoolkit::JoinTreesIndex("outAll.root","joinAll","run",stringSetup.Data(), 1);
1838   }
1839 EOF
1840
1841   aliroot -b -q "mergeTree.C" > mergeTrees.log
1842   return $?
1843 }
1844
1845 goMakeSummary()
1846 {
1847   #all the final stuff goes in here for ease of use:
1848   # summary logs
1849   # qa plot making
1850   # final file lists
1851
1852   configFile=${1}
1853   shift 1
1854   extraOpts="$@"
1855   log="summary.log"
1856   productionID="qa"
1857   parseConfig ${configFile} ${extraOpts}
1858
1859   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
1860
1861   exec &> >(tee ${log})
1862
1863   [[ ! -f ${configFile} ]] && echo "no config file ${configFile}!" && return
1864
1865   [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
1866
1867   #summarize the global stuff
1868   echo "env script: ${alirootSource} ${alirootEnv}"
1869   echo "\$ALICE_ROOT=${ALICE_ROOT}"
1870   echo "commonOutputPath=${commonOutputPath}"
1871
1872   #summarize the stacktraces
1873   awk '
1874        BEGIN {
1875                print "frame/I:method/C:line/C:cpass/I:aliroot/I";
1876                RS="#[0-9]*";
1877                aliroot=0;
1878              }
1879              {
1880                if ($3 ~ /Ali*/) aliroot=1; else aliroot=0;
1881                gsub("#","",RT);
1882                if ($NF!="" && RT!="" && $3!="") print RT" "$3" "$NF" "0" "aliroot
1883              }
1884       ' 000*/cpass0/*/stacktrace* 2>/dev/null > stacktrace.tree
1885   awk '
1886        BEGIN {
1887                RS="#[0-9]*";
1888                aliroot=0;
1889              }
1890              {
1891                if ($3 ~ /Ali*/) aliroot=1; else aliroot=0;
1892                gsub("#","",RT);
1893                if ($NF!="" && RT!="" && $3!="") print RT" "$3" "$NF" "1" "aliroot
1894              }
1895       ' 000*/cpass1/*/stacktrace* 2>/dev/null >> stacktrace.tree
1896
1897   echo total numbers for the production:
1898   echo
1899   awk 'BEGIN {nFiles=0;nCore=0;}
1900   /^calibfile/ {nFiles++;}
1901   /core dumped/ {nCore++i;}
1902   END {print     "cpass0 produced "nFiles" calib files, "nCore" core files";}' meta/cpass0.job*done 2>/dev/null
1903   awk 'BEGIN {nOK=0; nBAD=0; }
1904   /\/rec.log OK/ {nOK++;}
1905   /\/rec.log BAD/ {nBAD++;}
1906   /stderr BAD/ {if ($0 ~ /rec.log/){nBAD++;}}
1907   END {print     "cpass0 reco:  OK: "nOK"\tBAD: "nBAD;}' meta/cpass0.job*done 2>/dev/null
1908   awk 'BEGIN {nOK=0; nBAD=0; }
1909   /\/calib.log OK/ {nOK++;}
1910   /\/calib.log BAD/ {nBAD++;}
1911   END {print "cpass0 calib: OK: "nOK"\tBAD: "nBAD;}' meta/cpass0.job*done 2>/dev/null
1912
1913   awk 'BEGIN {nOK=0; nBAD=0; }
1914   /merge.log OK/ {nOK++;}
1915   /merge.log BAD/ {nBAD++;}
1916   END {print "cpass0 merge: OK: "nOK"\tBAD: "nBAD;}' meta/merge.cpass0*done 2>/dev/null
1917   awk 'BEGIN {nOK=0; nBAD=0; }
1918   /ocdb.log OK/ {nOK++;}
1919   /ocdb.log BAD/ {nBAD++;}
1920   END {print   "cpass0 OCDB:  OK: "nOK"\tBAD: "nBAD;}' meta/merge.cpass0*done 2>/dev/null
1921
1922   echo
1923   awk 'BEGIN {nFiles=0;nCore=0;}
1924   /^calibfile/ {nFiles++;}
1925   /core dumped/ {nCore++;}
1926   END {print     "cpass1 produced "nFiles" calib files, "nCore" core files";}' meta/cpass1.job*done 2>/dev/null
1927   awk 'BEGIN {nOK=0; nBAD=0; }
1928   /\/rec.log OK/ {nOK++;}
1929   /\/rec.log BAD/ {nBAD++;}
1930   /stderr BAD/ {if ($0 ~ /rec.log/){nBAD++;}}
1931   END {print     "cpass1 reco:  OK: "nOK"\tBAD: "nBAD;}' meta/cpass1.job*done 2>/dev/null
1932   awk 'BEGIN {nOK=0; nBAD=0; }
1933   /\/calib.log OK/ {nOK++;}
1934   /\/calib.log BAD/ {nBAD++;}
1935   END {print "cpass1 calib: OK: "nOK"\tBAD: "nBAD;}' meta/cpass1.job*done 2>/dev/null
1936
1937   awk 'BEGIN {nOK=0; nBAD=0; }
1938   /merge.log OK/ {nOK++;}
1939   /merge.log BAD/ {nBAD++;}
1940   END {print "cpass1 merge: OK: "nOK"\tBAD: "nBAD;}' meta/merge.cpass1*done 2>/dev/null
1941   awk 'BEGIN {nOK=0; nBAD=0; }
1942   /ocdb.log OK/ {nOK++;}
1943   /ocdb.log BAD/ {nBAD++;}
1944   END {print   "cpass1 OCDB:  OK: "nOK"\tBAD: "nBAD;}' meta/merge.cpass1*done 2>/dev/null
1945
1946   echo
1947   echo per run stats:
1948   /bin/ls -1 meta/merge.cpass0.run*.done | while read x
1949 do
1950   dir=$(goPrintValues calibfile - ${x})
1951   runNumber=$(guessRunNumber ${dir})
1952   [[ -z ${runNumber} ]] && continue
1953
1954   if $(/bin/ls meta/cpass0.job*.run${runNumber}.done &> /dev/null); then
1955     statusCPass0=( $(
1956     awk 'BEGIN {nOKrec=0;nBADrec=0;nOKcalib=0;nBADcalib=0;nOKstderr=0;nBADstderr=0;}
1957     /\/rec.log OK/ {nOKrec++;}
1958     /\/rec.log BAD/ {nBADrec++;}
1959     /stderr BAD/ {if ($0 ~ /rec.log/) {nBADrec++;} nBADstderr++;}
1960     /stderr OK/ {nOKstderr++;}
1961     /\/calib.log OK/ {nOKcalib++;}
1962     /\/calib.log BAD/ {nBADcalib++}
1963     END {print ""nOKrec" "nBADrec" "nOKstderr" "nBADstderr" "nOKcalib" "nBADcalib;}' meta/cpass0.job*.run${runNumber}.done 2>/dev/null
1964     ) )
1965   fi
1966
1967   if $(/bin/ls meta/cpass1.job*.run${runNumber}.done &>/dev/null); then
1968     statusCPass1=( $(
1969     awk 'BEGIN {nOKrec=0;nBADrec=0;nOKcalib=0;nBADcalib=0;nOKstderr=0;nBADstderr=0;nQAbarrelOK=0;nQAbarrelBAD=0;nQAouterOK=0;nQAouterBAD=0;}
1970     /\/rec.log OK/ {nOKrec++;}
1971     /\/rec.log BAD/ {nBADrec++;}
1972     /stderr BAD/ {if ($0 ~ /rec.log/) nBADrec++;nBADstderr++;}
1973     /stderr OK/ {nOKstderr++;}
1974     /\/calib.log OK/ {nOKcalib++;}
1975     /\/calib.log BAD/ {nBADcalib++}
1976     /\/qa_barrel.log OK/ {nQAbarrelOK++;}
1977     /\/qa_barrel.log BAD/ {nQAbarrelBAD++;}
1978     /\/qa_outer.log OK/ {nQAouterOK++;}
1979     /\/qa_outer.log BAD/ {nQAouterBAD++;}
1980     END {print ""nOKrec" "nBADrec" "nOKstderr" "nBADstderr" "nOKcalib" "nBADcalib" "nQAbarrelOK" "nQAbarrelBAD" "nQAouterOK" "nQAouterBAD;}' meta/cpass1.job*.run${runNumber}.done 2>/dev/null
1981     ) )
1982   fi
1983
1984   statusOCDBcpass0=$(awk '/ocdb.log/ {print $2} ' ${x} 2>/dev/null)
1985   statusOCDBcpass1=$(awk '/ocdb.log/ {print $2}' ${x/cpass0/cpass1} 2>/dev/null)
1986   statusQA=$(awk '/mergeMakeOCDB.log/ {print $2}' ${x/cpass0/cpass1} 2>/dev/null)
1987
1988   printf "%s\t ocdb.log cpass0: %s\t ocdb.log cpass1: %s\tqa.log:%s\t| cpass0: rec:%s/%s stderr:%s/%s calib:%s/%s cpass1: rec:%s/%s stderr:%s/%s calib:%s/%s QAbarrel:%s/%s QAouter:%s/%s\n" ${runNumber} ${statusOCDBcpass0} ${statusOCDBcpass1} ${statusQA} ${statusCPass0[0]} ${statusCPass0[1]} ${statusCPass0[2]} ${statusCPass0[3]} ${statusCPass0[4]} ${statusCPass0[5]} ${statusCPass1[0]} ${statusCPass1[1]} ${statusCPass1[2]} ${statusCPass1[3]} ${statusCPass1[4]} ${statusCPass1[5]} ${statusCPass1[6]} ${statusCPass1[7]} ${statusCPass1[8]} ${statusCPass1[9]}
1989 done
1990
1991   #make lists with output files - QA, trending, filtering and calibration
1992   ### wait for the merging of all runs to be over ###
1993   rm -f qa.list
1994   goPrintValues qafile qa.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
1995   rm -f calib.list
1996   goPrintValues calibfile calib.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
1997   rm -f trending.list
1998   goPrintValues trendingfile trending.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
1999   rm -f filtering.list
2000   goPrintValues filteredTree filtering.list ${commonOutputPath}/meta/filtering.cpass1.run*.done &>/dev/null
2001   #/bin/ls ${commonOutputPath}/*/cpass0/dcs* > cpass0.dcsTree.list
2002   rm -f cpass0.dcsTree.list
2003   goPrintValues dcsTree cpass0.dcsTree.list ${commonOutputPath}/meta/merge.cpass0.run*.done &>/dev/null
2004   #/bin/ls ${commonOutputPath}/*/cpass1/dcs* > cpass1.dcsTree.list
2005   rm -f cpass1.dcsTree.list
2006   goPrintValues dcsTree cpass1.dcsTree.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2007
2008   #merge trending
2009   rm -f ${commonOutputPath}/trending_merged.root
2010   goMerge trending.list ${commonOutputPath}/trending.root ${configFile} ${extraOpts} &> mergeTrending.log
2011
2012   goMakeSummaryTree ${commonOutputPath} 0
2013   goMakeSummaryTree ${commonOutputPath} 1
2014
2015   goCreateQAplots qa.list ${productionID} QAplots ${configFile} ${extraOpts} &>createQAplots.log
2016
2017   #make a merged summary tree out of the QA trending, dcs trees and log summary trees
2018   goMakeMergedSummaryTree
2019
2020   #if set, email the summary
2021   [[ -n ${MAILTO} ]] && cat ${log} | mail -s "benchmark ${productionID} done" ${MAILTO}
2022
2023   return 0
2024 }
2025
2026 goMakeSummaryTree()
2027 {
2028   if [[ $# -lt 1 ]] ; then
2029     return
2030   fi
2031   #1. define vars/arrays
2032   DIR=${1} #use input or exec in current dir
2033   pass=${2-"0"} #pass from input
2034   outfile="summary_pass${pass}.tree"
2035   Ncolumns=0
2036   test -f ${outfile} && : >${outfile}
2037   errfile=${outfile/tree/err}
2038   test -f ${errfile} && : >${errfile}
2039
2040   declare -a counterString=(TOFevents TOFtracks TPCevents TPCtracks TRDevents TRDtracks T0events SDDevents SDDtracks MeanVertexevents)
2041   Ncounter=${#counterString[@]}
2042
2043   declare -a statusString=(TRDStatus TOFStatus TPCStatus T0Status MeanVertexStatus)
2044   Nstatus=${#statusString[@]}
2045
2046
2047   declare -a ratesString=(rec stderr calib qa_barrel qa_outer)
2048   Nrates=${#ratesString[@]}
2049
2050   runs=( $(ls -1 ${DIR}/meta/merge.cpass0* | while read x; do guessRunNumber $x; done) )
2051   Nruns=${#runs[@]}
2052
2053   echo -n runnumber/I >>${outfile}
2054   echo -n :cpass${pass}status/I >>${outfile}
2055   echo -n :cpass${pass}QAstatus/I >>${outfile}
2056   for i in ${ratesString[@]}; do
2057     echo -n :${i}OK/I >>${outfile}
2058     echo -n :${i}BAD/I >>${outfile}
2059
2060   done
2061   for i in ${counterString[@]} ${statusString[@]} ; do
2062     echo -n :${i}/I >>${outfile}
2063   done
2064   Ncolumns=$((2 + 2*Nrates + Ncounter + Nstatus))
2065   echo >> ${outfile}
2066
2067   #2. loop runs
2068
2069   for runnumber in ${runs[@]} ; do
2070
2071
2072
2073     filejob="${DIR}/meta/cpass${pass}.job*.run${runnumber}.done"
2074     filemerge="${DIR}/meta/merge.cpass${pass}.run${runnumber}.done"
2075     fileOCDB=$(grep /ocdb.log ${filemerge} | awk '{print $1}')
2076     if ! $(/bin/ls ${filemerge} &>/dev/null) ; then
2077       echo "${filemerge} does not exist!" >>${errfile}
2078       continue
2079     elif ! $(/bin/ls ${filejob} &>/dev/null) ; then
2080       echo "${filejob} does not exist!" >>${errfile}
2081       echo -n ${runnumber} >> ${outfile}
2082       for i in $(seq ${Ncolumns}) ; do
2083         echo -n "-1" >> ${outfile}
2084       done
2085       echo >> ${outfile}
2086       continue
2087     fi
2088     echo -n ${runnumber} >> ${outfile}
2089     #pass0status= grep '/ocdb.log' ${filemerge} |  cut -d' ' -f2 | tr OK x1 | tr BAD xx0 | tr -d 'x'
2090     passStatus=$(grep '/ocdb.log' ${filemerge} | grep OK | wc -l)
2091     echo -n " ${passStatus}" >> ${outfile}
2092     qaStatus=$(grep '/mergeMakeOCDB.log' ${filemerge} | grep OK | wc -l)
2093     echo -n " ${qaStatus}" >> ${outfile}
2094
2095
2096     #fill OK/BAD rates
2097     for i in $(seq 0 $((${Nrates}-1))) ; do
2098       var1=$(grep "/${ratesString[${i}]}.log" ${filejob} | grep OK | wc -l)
2099       var2=$(grep "/${ratesString[${i}]}.log" ${filejob} | grep BAD | wc -l)
2100
2101       if [[ ${ratesString[${i}]} == "stderr" ]] ; then
2102         var1=$(grep "stderr" ${filejob} | grep OK | wc -l)
2103         var2=$(grep "stderr" ${filejob} | grep "rec.log" | grep BAD | wc -l)
2104       fi
2105       echo -n " ${var1}" >> ${outfile}
2106       echo -n " ${var2}" >> ${outfile}
2107     done
2108
2109     if [[ -f ${fileOCDB} ]] ; then
2110       #fill counter
2111       for i in $(seq 0 $((${Ncounter}-1))) ; do
2112         var1=$(grep Monalisa ${fileOCDB} | grep ${counterString[${i}]} | cut -f2)
2113         echo -n " ${var1:-"-1"}" >> ${outfile}
2114       done
2115
2116       #fill status
2117       for i in $(seq 0 $((${Nstatus}-1))) ; do
2118         var1=$(grep "calibration status=" ${fileOCDB} | grep ${statusString[${i}]/Status/} | cut -d'=' -f2)
2119         echo -n " ${var1:-"-1"}" >> ${outfile}
2120       done
2121     fi
2122     echo >> ${outfile}
2123   done
2124
2125   return 0
2126 }
2127
2128 parseConfig()
2129 {
2130   configFile=${1}
2131   shift
2132
2133   #first, source the config file
2134   if [ -f ${configFile} ]; then
2135     source ${configFile}
2136   else
2137     echo "config file ${configFile} not found!, skipping..."
2138   fi
2139
2140   #then, parse the options as theya override the optionf from file
2141   while [[ -n ${1} ]]; do
2142     var="${1#--}"
2143     echo "extraOpts: exporting ${var}"
2144     eval "export ${var}"
2145     shift
2146   done
2147
2148   #export the aliroot function if defined to override normal behaviour
2149   [[ $(type -t aliroot) =~ "function" ]] && export -f aliroot
2150   return 0
2151 }
2152
2153 aliroot()
2154 {
2155   args="$@"
2156   if [[ -n ${valgrindCommand} ]]; then
2157     valgrindLogFile="cpu.txt"
2158     [[ "${args}" =~ rec ]] && valgrindLogFile="cpu_rec.txt"
2159     [[ "${args}}" =~ Calib ]] && valgrindLogFile="cpu_calib.txt"
2160     [[ -n ${valgrindCommand} ]] && valgrindCommand="${valgrindCommand} --log-file=${valgrindLogFile}"
2161     echo running ${valgrindCommand} aliroot ${args}
2162     ${valgrindCommand} aliroot ${args}
2163   else
2164     #to prevent an infinite recursion use "command aliroot" to disable
2165     #aliases and functions
2166     echo running command aliroot ${args}
2167     command aliroot ${args}
2168   fi
2169   return 0
2170 }
2171
2172 guessRunData()
2173 {
2174   #guess the period from the path, pick the rightmost one
2175   period=""
2176   runNumber=""
2177   year=""
2178   pass=""
2179   legoTrainRunNumber=""
2180   dataType=""
2181
2182   local shortRunNumber=""
2183   local IFS="/"
2184   declare -a path=( $1 )
2185   local dirDepth=$(( ${#path[*]}-1 ))
2186   i=0
2187   #for ((x=${dirDepth};x>=0;x--)); do
2188   for ((x=0;x<=${dirDepth};x++)); do
2189
2190     [[ $((x-1)) -ge 0 ]] && local fieldPrev=${path[$((x-1))]}
2191     local field=${path[${x}]}
2192     local fieldNext=${path[$((x+1))]}
2193
2194     [[ ${field} =~ ^[0-9]*$ && ${fieldNext} =~ (.*\.zip$|.*\.root$) ]] && legoTrainRunNumber=${field}
2195     [[ -n ${legoTrainRunNumber} && -z ${pass} ]] && pass=${fieldPrev}
2196     [[ ${field} =~ ^LHC[0-9][0-9][a-z].*$ ]] && period=${field%_*}
2197     [[ ${field} =~ ^000[0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && runNumber=${field#000}
2198     [[ ${field} =~ ^[0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && shortRunNumber=${field}
2199     [[ ${field} =~ ^20[0-9][0-9]$ ]] && year=${field}
2200     [[ ${field} =~ ^(^sim$|^data$) ]] && dataType=${field}
2201     (( i++ ))
2202   done
2203   [[ -z ${legoTrainRunNumber} ]] && pass=${path[$((dirDepth-1))]}
2204   [[ "${dataType}" =~ ^sim$ ]] && pass="passMC" && runNumber=${shortRunNumber}
2205
2206   #if [[ -z ${dataType} || -z ${year} || -z ${period} || -z ${runNumber}} || -z ${pass} ]];
2207   if [[ -z ${runNumber}} ]];
2208   then
2209     #error condition
2210     return 1
2211   else
2212     #ALL OK
2213     return 0
2214   fi
2215   return 0
2216 }
2217
2218 main "$@"