PWGPP/benchmark/benchmark.sh

   1 #!/bin/bash
   2 #include benchmark.config
   3
   4 # this script runs the CPass0/CPass1 train
   5 # produced OCDB updates are local
   6
   7 main()
   8 {
   9   #run in proper mode depending on the selection
  10   if [[ $# -lt 1 ]]; then
  11     if [[ ! "${0}" =~ "bash" ]]; then
  12       echo "uses makeflow:"
  13       echo " ${0} \"run\" productionID inputList configFile [extraOpts]"
  14       echo "uses a batch system (SGE):"
  15       echo " ${0} \"submit\" productionID inputList configFile [extraOpts]"
  16       echo "extraOpts if non-empty override the config file, e.g.:"
  17       echo " ${0} submit test1 benchmark.list benchmark.config runNumber=169123 nEvents=10"
  18     fi
  19     return
  20   fi
  21
  22   #define some aliases - default is to call one of the functions directly
  23   runMode=${1}
  24   umask 0002
  25   shift
  26   case ${runMode} in
  27     "CPass0") goCPass0 "$@";;
  28     "CPass1") goCPass1 "$@";;
  29     "MakeLocalOCDBaccessConfig") goMakeLocalOCDBaccessConfig "$@";;
  30     "MergeCPass0") goMergeCPass0 "$@";;
  31     "MergeCPass1") goMergeCPass1 "$@";;
  32     "MakeFilteredTrees") goMakeFilteredTrees "$@";;
  33     "MakeSummary") goMakeSummary "$@";;
  34     "run") goSubmitMakeflow "$@";;
  35     "submit") goSubmitBatch "$@";;
  36     "test") goTest "$@";;
  37     "GenerateMakeflow") goGenerateMakeflow "$@";;
  38     "PrintValues") goPrintValues "$@";;
  39     "CreateQAplots") goCreateQAplots "$@";;
  40     "WaitForOutput") goWaitForOutput "$@";;
  41     "Merge") goMerge "$@";;
  42     *)
  43       ${runMode} "$@"
  44     ;;
  45   esac
  46 }
  47
  48 generateMC()
  49 {
  50   #generate one raw chunk in current directory
  51   SEED=${JOB_ID}${SGE_TASK_ID}
  52   export CONFIG_SEED=${SEED}
  53   runNumber=${1}
  54   OCDBpath=${2}
  55   nEventsim=${3}
  56   if [[ -n ${pretend} ]]; then
  57     touch galice.root
  58   else
  59     if [[ -f sim.C && -f Config.C ]] ; then
  60         time aliroot -b -q -x sim.C\(${runNumber},\"${OCDBpath}\",${nEventsim}\) >sim.log 2>&1
  61         mv syswatch.log simwatch.log
  62     fi
  63   fi
  64 }
  65
  66 goCPass0()
  67 {
  68   umask 0002
  69
  70   targetDirectory=${1}
  71   inputList=${2}
  72   nEvents=${3}
  73   ocdbPath=${4}
  74   configFile=${5}
  75   runNumber=${6}
  76   jobindex=${7}
  77   shift 7
  78   parseConfig ${configFile} "$@"
  79
  80   #use the jobindex only if set and non-negative
  81   if [[ -z ${jobindex} || ${jobindex} -lt 0 ]]; then
  82     [[ -n "${LSB_JOBINDEX}" ]] && jobindex=${LSB_JOBINDEX}
  83     [[ -n "${SGE_TASK_ID}" ]] && jobindex=${SGE_TASK_ID}
  84   fi
  85
  86   [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
  87   doneFile="${commonOutputPath}/meta/cpass0.job${jobindex}.run${runNumber}.done"
  88
  89   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
  90
  91   if [[ -n ${ALIROOT_FORCE_COREDUMP} ]]; then
  92     ulimit -c unlimited
  93     export ALIROOT_FORCE_COREDUMP
  94   fi
  95
  96   #the contents of this is stored in the tree and used later (e.g. AliAnalysisTaskPIDResponse)!
  97   #at the QA stage the pass number is guessed from the path stored here.
  98   #The Format is:
  99   #Packages= ;OutputDir= ;LPMPass= ;TriggerAlias= ;LPMRunNumber= ;LPMProductionType= ;LPMInteractionType= ;LPMProductionTag= ;LPMAnchorRun= ;LPMAnchorProduction= ;LPMAnchorYear=
 100   export PRODUCTION_METADATA="OutputDir=cpass0"
 101
 102   if [[ "${inputList}" =~ \.root$ ]]; then
 103     infile=${inputList}
 104   else
 105     infile=$(sed -ne "${jobindex}p" ${inputList} | egrep '\s*\w*/\w*')
 106   fi
 107   chunkName=${infile##*/}
 108
 109   outputDir=${targetDirectory}/${jobindex}_${chunkName%.*}
 110   mkdir -p ${outputDir}
 111   [[ ! -d ${outputDir} ]] && echo "cannot make ${outputDir}" && touch ${doneFile} && return 1
 112
 113   #runpath=${PWD}/rundir_cpass0_${runNumber}_${jobindex}
 114   runpath=${outputDir}
 115   [[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
 116   [[ ${reconstructInTemporaryDir} -eq 1 && -z ${TMPDIR} ]] && runpath=$(mktemp -d)
 117   mkdir -p ${runpath}
 118   [[ ! -d ${runpath} ]] && echo "cannot make runpath ${runpath}" && touch ${doneFile} && return 1
 119   cd ${runpath}
 120
 121   #runCPassX/C expects the raw chunk to be linked in the run dir
 122   #despite it being accessed by the full path
 123   ln -s ${infile} ${runpath}/${chunkName}
 124
 125   #####MC
 126   if [[ -n ${generateMC} ]]; then
 127     olddir=${PWD}
 128     outputDirMC=${commonOutputPath}/000${runNumber}/sim/${jobindex}
 129     simrunpath=${outputDirMC}
 130     [[ ${simulateInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && simrunpath=${TMPDIR}
 131     [[ ${simulateInTemporaryDir} -eq 1 && -z ${TMPDIR} ]] && simrunpath=$(mktemp -d)
 132     mkdir -p ${outputDirMC}
 133     mkdir -p ${simrunpath}
 134     cd ${simrunpath}
 135
 136     filesMC=(
 137               "${commonOutputPath}/sim.C"
 138               "${commonOutputPath}/rec.C"
 139               "${commonOutputPath}/Config.C"
 140               "${commonOutputPath}/OCDB_*.root"
 141     )
 142     for file in ${filesMC[*]}; do
 143       [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
 144     done
 145
 146     generateMC ${runNumber} ${ocdbPath} ${nEvents}
 147
 148     [[ ! "${simrunpath}" =~ "${outputDirMC}" ]] && mv * ${outputDirMC} #TODO check if it works
 149     cd ${olddir}
 150
 151     ln -s ${outputDirMC}/* ${runpath}/
 152
 153     inputList=${outputDirMC}/galice.root #TODO not valid outside shell !!!
 154     infile=""
 155   fi
 156   ######
 157
 158   [[ ! -f ${inputList} && -z ${pretend} ]] && echo "input file ${inputList} not found, exiting..." && touch ${doneFile} && return 1
 159
 160   logOutputDir=${runpath}
 161   [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir}
 162   [[ -z ${dontRedirectStdOutToLog} ]] && exec 1> ${logOutputDir}/stdout
 163   [[ -z ${dontRedirectStdOutToLog} ]] && exec 2> ${logOutputDir}/stderr
 164   echo "${0} $*"
 165
 166   echo "#####################"
 167   echo CPass0:
 168   echo JOB setup
 169   echo nEvents            ${nEvents}
 170   echo runNumber          ${runNumber}
 171   echo ocdbPath           ${ocdbPath}
 172   echo infile             ${infile}
 173   echo chunkName          ${chunkName}
 174   echo jobindex           ${jobindex}
 175   echo recoTriggerOptions ${recoTriggerOptions}
 176   echo targetDirectory    ${targetDirectory}
 177   echo commonOutputPath         ${commonOutputPath}
 178   echo doneFile      ${doneFile}
 179   echo runpath            ${runpath}
 180   echo outputDir          ${outputDir}
 181   echo ALICE_ROOT         ${ALICE_ROOT}
 182   echo PWD                ${PWD}
 183   echo "########## ###########"
 184
 185   alirootInfo > ALICE_ROOT.log
 186
 187   filesCPass0=(
 188                "${commonOutputPath}/runCPass0.sh"
 189                "${commonOutputPath}/recCPass0.C"
 190                "${commonOutputPath}/runCalibTrain.C"
 191                "${commonOutputPath}/localOCDBaccessConfig.C"
 192                "${commonOutputPath}/OCDB.root"
 193                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/runCPass0.sh"
 194                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/recCPass0.C"
 195                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/runCalibTrain.C"
 196   )
 197
 198   for file in ${filesCPass0[*]}; do
 199     [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
 200   done
 201
 202   echo "this directory (${PWD}) contents:"
 203   /bin/ls
 204   echo
 205   chmod u+x runCPass0.sh
 206
 207   sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh
 208
 209   if [[ -n ${postSetUpActionCPass0} ]]; then
 210     echo "running ${postSetUpActionCPass0}"
 211     eval ${postSetUpActionCPass0}
 212   fi
 213
 214   #run CPass0
 215   echo "${runpath}/runCPass0.sh ${infile} ${nEvents} ${runNumber} ${ocdbPath} ${recoTriggerOptions}"
 216   if [[ -n ${pretend} ]]; then
 217     touch AliESDs.root
 218     touch AliESDfriends.root
 219     touch AliESDfriends_v1.root
 220     touch rec.log
 221     touch calib.log
 222   else
 223     echo ./runCPass0.sh "${infile}" "${nEvents}" "${runNumber}" "${ocdbPath}" "${recoTriggerOptions}"
 224     ./runCPass0.sh "${infile}" "${nEvents}" "${runNumber}" "${ocdbPath}" "${recoTriggerOptions}"
 225   fi
 226
 227   #move stuff to final destination
 228   echo "this directory (${PWD}) contents:"
 229   /bin/ls
 230   echo
 231
 232   echo rm -f ./${chunkName}
 233   rm -f ./${chunkName}
 234   echo "cp --recursive ${runpath}/* ${outputDir}"
 235   cp -p --recursive ${runpath}/* ${outputDir}
 236   echo
 237
 238   #validate CPass0
 239   cd ${outputDir}
 240   touch ${doneFile}
 241   [[ -f ${outputDirMC}/galice.root ]] && echo "sim ${outputDirMC}/galice.root" >> ${doneFile}
 242   [[ -f AliESDfriends_v1.root ]] && echo "calibfile ${outputDir}/AliESDfriends_v1.root" >> ${doneFile}
 243   [[ -f AliESDs.root ]] && echo "esd ${outputDir}/AliESDs.root" >> ${doneFile}
 244   echo "dir ${outputDir}" >> ${doneFile}
 245   summarizeLogs >> ${doneFile}
 246
 247   [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
 248   return 0
 249 }
 250
 251 goCPass1()
 252 {
 253   umask 0002
 254
 255   targetDirectory=${1}
 256   inputList=${2}
 257   nEvents=${3}
 258   ocdbPath=${4}
 259   configFile=${5}
 260   runNumber=${6}
 261   jobindex=${7}
 262   shift 7
 263   extraOpts=("$@")
 264   parseConfig ${configFile} "$@"
 265
 266   #use the jobindex only if set and non-negative
 267   if [[ -z ${jobindex} || ${jobindex} -lt 0 ]]; then
 268     [[ -n "${LSB_JOBINDEX}" ]] && jobindex=${LSB_JOBINDEX}
 269     [[ -n "${SGE_TASK_ID}" ]] && jobindex=${SGE_TASK_ID}
 270   fi
 271
 272   [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
 273   doneFile="${commonOutputPath}/meta/cpass1.job${jobindex}.run${runNumber}.done"
 274
 275   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
 276
 277   if [[ -n ${ALIROOT_FORCE_COREDUMP} ]]; then
 278     ulimit -c unlimited
 279     export ALIROOT_FORCE_COREDUMP
 280   fi
 281
 282   #the contents of this is stored in the tree and used later (e.g. AliAnalysisTaskPIDResponse)!
 283   #at the QA stage the pass number is guessed from the path stored here.
 284   #The Format is:
 285   #Packages= ;OutputDir= ;LPMPass= ;TriggerAlias= ;LPMRunNumber= ;LPMProductionType= ;LPMInteractionType= ;LPMProductionTag= ;LPMAnchorRun= ;LPMAnchorProduction= ;LPMAnchorYear=
 286   export PRODUCTION_METADATA="OutputDir=cpass1"
 287
 288   [[ ! -f ${inputList} && -z ${pretend} ]] && echo "input file ${inputList} not found, exiting..." && touch ${doneFile} && return 1
 289   if [[ "${inputList}" =~ \.root$ ]]; then
 290     infile=${inputList}
 291   else
 292     infile=$(sed -ne "${jobindex}p" ${inputList} | egrep '\s*\w*/\w*')
 293   fi
 294   chunkName=${infile##*/}
 295
 296   outputDir=${targetDirectory}/${jobindex}_${chunkName%.*}
 297   mkdir -p ${outputDir}
 298   [[ ! -d ${outputDir} ]] && echo "cannot make ${outputDir}" && touch ${doneFile} && return 1
 299
 300   #runpath=${PWD}/rundir_cpass1_${runNumber}_${jobindex}
 301   runpath=${outputDir}
 302   [[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
 303   [[ ${reconstructInTemporaryDir} -eq 1 && -z ${TMPDIR} ]] && runpath=$(mktemp -d)
 304
 305   #MC
 306   if [[ "${infile}" =~ galice\.root ]]; then
 307     ln -s ${inputList%/*}/* ${runpath}
 308     infile=""
 309   fi
 310
 311   #init the running path
 312   mkdir -p ${runpath}
 313   [[ ! -d ${runpath} ]] && echo "cannot make runpath ${runpath}" && touch ${doneFile} && return 1
 314   cd ${runpath}
 315
 316   #this is needed for runCPass1.sh
 317   ln -s ${infile} ${runpath}/${chunkName}
 318
 319   logOutputDir=${runpath}
 320   [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir}
 321   [[ -z ${dontRedirectStdOutToLog} ]] && exec 1> ${logOutputDir}/stdout
 322   [[ -z ${dontRedirectStdOutToLog} ]] && exec 2> ${logOutputDir}/stderr
 323   echo "${0} $*"
 324
 325   echo "#####################"
 326   echo CPass1:
 327   echo JOB setup
 328   echo nEvents            ${nEvents}
 329   echo runNumber          ${runNumber}
 330   echo ocdbPath           ${ocdbPath}
 331   echo infile             ${infile}
 332   echo chunkName          ${chunkName}
 333   echo jobindex           ${jobindex}
 334   echo recoTriggerOptions ${recoTriggerOptions}
 335   echo targetDirectory    ${targetDirectory}
 336   echo commonOutputPath         ${commonOutputPath}
 337   echo doneFile      ${doneFile}
 338   echo runpath            ${runpath}
 339   echo outputDir          ${outputDir}
 340   echo ALICE_ROOT         ${ALICE_ROOT}
 341   echo PWD                ${PWD}
 342   echo "########## ###########"
 343
 344   alirootInfo > ALICE_ROOT.log
 345
 346   filesCPass1=(
 347                "${commonOutputPath}/runCPass1.sh"
 348                "${commonOutputPath}/recCPass1.C"
 349                "${commonOutputPath}/recCPass1_OuterDet.C"
 350                "${commonOutputPath}/runCalibTrain.C"
 351                "${commonOutputPath}/QAtrain_duo.C"
 352                "${commonOutputPath}/localOCDBaccessConfig.C"
 353                "${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz"
 354                "${commonOutputPath}/OCDB.root"
 355                "${trustedQAtrainMacro}"
 356                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/runCPass1.sh"
 357                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/recCPass1.C"
 358                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/recCPass1_OuterDet.C"
 359                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/runCalibTrain.C"
 360                "${ALICE_ROOT}/ANALYSIS/macros/QAtrain_duo.C"
 361   )
 362
 363   for file in ${filesCPass1[*]}; do
 364     [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
 365   done
 366
 367   echo "this directory (${PWD}) contents:"
 368   /bin/ls
 369   echo
 370
 371   sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh
 372
 373   if [[ -n ${postSetUpActionCPass1} ]]; then
 374     echo "running ${postSetUpActionCPass1}"
 375     eval ${postSetUpActionCPass1}
 376     echo
 377   fi
 378
 379   #configure local OCDB storage from CPass0 (creates the localOCDBaccessConfig.C script)
 380   if [[ -f cpass0.localOCDB.${runNumber}.tgz ]]; then
 381     echo goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz"
 382     goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz"
 383   else
 384     echo "WARNING: file cpass0.localOCDB.${runNumber}.tgz not found!"
 385   fi
 386
 387   if [[ ! $(/bin/ls -1 OCDB/*/*/*/*.root 2>/dev/null) ]]; then
 388     echo "cpass0 produced no calibration! exiting..."
 389     touch ${doneFile}
 390     return 1
 391   fi
 392
 393   #create the Barrel and OuterDet directories for CPass1 and link the local OCDB directory
 394   #there to make the localOCDBaccessConfig.C file work, since it may point to the OCDB
 395   #entries using a relative path, e.g. local://./OCDB
 396   mkdir Barrel OuterDet
 397   ln -s ../OCDB Barrel/OCDB
 398   ln -s ../OCDB OuterDet/OCDB
 399
 400   #setup the filtering
 401   #the following option enables the filtering task inside the QAtrain_duo.C
 402   [[ -n $runESDfiltering ]] && export QA_TaskFilteredTree=1
 403   #set the downscaling factors during the filtering fro expert QA (overrides the previous values)
 404   if [[ -n ${filteringFactorHighPt} ]]; then
 405     export AliAnalysisTaskFilteredTree_fLowPtTrackDownscaligF=${filteringFactorHighPt}
 406   fi
 407   if [[ -n ${filteringFactorV0s} ]]; then
 408     export AliAnalysisTaskFilteredTree_fLowPtV0DownscaligF=${filteringFactorV0s}
 409   fi
 410
 411   #run CPass1
 412   chmod u+x runCPass1.sh
 413   echo "${runpath}/runCPass1.sh ${infile} ${nEvents} ${runNumber} ${ocdbPath} ${recoTriggerOptions}"
 414   if [[ -n ${pretend} ]]; then
 415     touch AliESDs_Barrel.root
 416     touch AliESDfriends_Barrel.root
 417     touch AliESDfriends_v1.root
 418     touch QAresults_barrel.root
 419     touch EventStat_temp_barrel.root
 420     touch AODtpITS.root
 421     touch AliESDs_Outer.root
 422     touch AliESDfriends_Outer.root
 423     touch QAresults_outer.root
 424     touch EventStat_temp_outer.root
 425     touch rec.log
 426     touch calib.log
 427     touch qa.log
 428     touch filtering.log FilterEvents_Trees.root
 429   else
 430     ./runCPass1.sh "${infile}" "${nEvents}" "${runNumber}" "${ocdbPath}" "${recoTriggerOptions}"
 431
 432     [[ ! -f AliESDs_Barrel.root && -f Barrel/AliESDs.root ]] && mv Barrel/AliESDs.root AliESDs_Barrel.root
 433     [[ ! -f AliESDfriends_Barrel.root && -f Barrel/AliESDfriends.root ]] && mv Barrel/AliESDfriends.root AliESDfriends_Barrel.root
 434     [[ ! -f AliESDfriends_v1.root && -f Barrel/AliESDfriends_v1.root ]] && mv Barrel/AliESDfriends_v1.root .
 435     [[ ! -f QAresults_barrel.root && -f Barrel/QAresults_barrel.root ]] && mv Barrel/QAresults_barrel.root .
 436     [[ ! -f AliESDs_Outer.root && -f OuterDet/AliESDs.root ]] && mv OuterDet/AliESDs.root AliESDs_Outer.root
 437     [[ ! -f AliESDfriends_Outer.root && -f OuterDet/AliESDfriends.root ]] && mv OuterDet/AliESDfriends.root AliESDfriends_Outer.root
 438     [[ ! -f QAresults_outer.root && -f OuterDet/QAresults_outer.root ]] && mv OuterDet/QAresults_outer.root .
 439     [[ ! -f FilterEvents_Trees.root && -f Barrel/FilterEvents_Trees.root ]] && mv Barrel/FilterEvents_Trees.root .
 440
 441     #make the filtered tree (if requested and not already produced by QA
 442     [[ -f AliESDs_Barrel.root ]] && echo "AliESDs_Barrel.root" > filtered.list
 443     if [[ -n ${runESDfiltering} && ! -f FilterEvents_Trees.root ]]; then
 444       goMakeFilteredTrees ${PWD} ${runNumber} "filtered.list" ${filteringFactorHighPt} ${filteringFactorV0s} ${ocdbPath} 1000000 0 10000000 0 ${configFile} AliESDs_Barrel.root "${extraOpts[@]}"
 445     fi
 446
 447   fi
 448
 449   ##handle possible crashes in QA (happens often in trunk)
 450   ##rerun QA with a trusted aliroot version
 451   #if [[ $(validateLog qa_barrel.log) ]]; then
 452   #  echo "qa_barrel.log not validated!"
 453   #fi
 454   #if [[ ! -f QAresults_barrel.root && -f ${setupTrustedAliROOTenvInCurrentShell} || $(validateLog qa_barrel.log) ]]; then
 455   #  echo "WARNING: using trusted QA aliroot ${ALICE_ROOT}"
 456   #  source ${setupTrustedAliROOTenvInCurrentShell}
 457   #  cd Barrel
 458   #  rm QAresults_barrel.root
 459   #  rm EventStat_temp_barrel.root
 460   #  rm AODtpITS.root
 461   #  [[ ! -f AliESDs.root ]] && ln -s ../AliESDs_Barrel.root AliESDs.root
 462   #  [[ ! -f AliESDfriends.root ]] && ln -s ../AliESDfriends_Barrel.root AliESDfriends.root
 463   #  if [[ -n ${trustedQAtrainMacro} ]]; then
 464   #    eval "cp ${trustedQAtrainMacro} QAtrain_duo_trusted.C"
 465   #  fi
 466   #  echo executing aliroot -b -q "QAtrain_duo_trusted.C(\"_barrel\",${runNumber},\"wn.xml\",0,\"${ocdbPath}\")"
 467   #  time aliroot -b -q "QAtrain_duo.C(\"_barrel\",${runNumber},\"wn.xml\",0,\"${ocdbPath}\")" &> ../qa_barrel_trusted.log
 468   #  cd ../
 469   #fi
 470
 471   #move stuff to final destination
 472   echo "this directory (${PWD}) contents:"
 473   /bin/ls
 474   echo rm -f ./${chunkName}
 475   rm -f ./${chunkName}
 476   echo "cp --recursive ${runpath}/* ${outputDir}"
 477   cp -pf --recursive ${runpath}/* ${outputDir}
 478   echo
 479
 480   #validate CPass1
 481   cd ${outputDir}
 482   touch ${doneFile}
 483   [[ -f AliESDs_Barrel.root ]] && echo "esd ${outputDir}/AliESDs_Barrel.root" >> ${doneFile}
 484   [[ -f AliESDfriends_v1.root ]] && echo "calibfile ${outputDir}/AliESDfriends_v1.root" >> ${doneFile}
 485   [[ -f QAresults_Barrel.root ]] && echo "qafile ${outputDir}/QAresults_Barrel.root" >> ${doneFile}
 486   [[ -f QAresults_Outer.root ]] && echo "qafile ${outputDir}/QAresults_Outer.root" >> ${doneFile}
 487   [[ -f FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFile}
 488   echo "dir ${outputDir}" >> ${doneFile}
 489   summarizeLogs >> ${doneFile}
 490
 491   [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
 492   return 0
 493 }
 494
 495
 496 goMergeCPass0()
 497 {
 498   #
 499   # find the output files and merge them
 500   #
 501
 502   outputDir=${1}
 503   ocdbStorage=${2}
 504   configFile=${3}
 505   runNumber=${4}
 506   calibrationFilesToMerge=${5}  #can be a non-existent file, will then be produced on the fly
 507   shift 5
 508   parseConfig ${configFile} "$@"
 509
 510   [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
 511   doneFile="${commonOutputPath}/meta/merge.cpass0.run${runNumber}.done"
 512
 513   umask 0002
 514   ulimit -c unlimited
 515
 516   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
 517
 518   #runpath=${PWD}/rundir_cpass0_Merge_${runNumber}
 519   runpath=${outputDir}
 520   [[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
 521   [[ ${reconstructInTemporaryDir} -eq 1 && -z ${TMPDIR} ]] && runpath=$(mktemp -d)
 522
 523   mkdir -p ${runpath}
 524   [[ ! -d ${runpath} ]] && echo "not able to make the runpath ${runpath}" && touch ${doneFile} && return 1
 525   cd ${runpath}
 526
 527   logOutputDir=${runpath}
 528   [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir}
 529   [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/mergeMakeOCDB.log
 530   echo "${0} $*"
 531
 532   mergingScript="mergeMakeOCDB.byComponent.sh"
 533
 534   echo goMergeCPass0 SETUP:
 535   echo runNumber=${runNumber}
 536   echo outputDir=${outputDir}
 537   echo ocdbStorage=${ocdbStorage}
 538   echo calibrationFilesToMerge=${calibrationFilesToMerge}
 539   echo mergingScript=${mergingScript}
 540   echo commonOutputPath=${commonOutputPath}
 541   echo runpath=${runpath}
 542
 543   # copy files in case they are not already there
 544   filesMergeCPass0=(
 545                     "${commonOutputPath}/${calibrationFilesToMerge}"
 546                     "${commonOutputPath}/OCDB.root"
 547                     "${commonOutputPath}/localOCDBaccessConfig.C"
 548                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/mergeMakeOCDB.byComponent.sh"
 549                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/mergeByComponent.C"
 550                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/makeOCDB.C"
 551                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/merge.C"
 552                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/mergeMakeOCDB.sh"
 553   )
 554   for file in ${filesMergeCPass0[*]}; do
 555     [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
 556   done
 557
 558   sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh
 559
 560   alirootInfo > ALICE_ROOT.log
 561
 562   #
 563   echo "PWD"
 564   /bin/ls
 565   echo "PWD/.."
 566   /bin/ls ../
 567
 568
 569   #merge calibration
 570   chmod u+x ${mergingScript}
 571   mkdir -p ./OCDB
 572   if [[ ! -f ${calibrationFilesToMerge} ]]; then
 573     echo "/bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root > ${calibrationFilesToMerge}"
 574     /bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root 2>/dev/null > ${calibrationFilesToMerge}
 575   fi
 576
 577   echo "${mergingScript} ${calibrationFilesToMerge} ${runNumber} local://./OCDB ${ocdbStorage}"
 578   if [[ -n ${pretend} ]]; then
 579     touch CalibObjects.root
 580     touch ocdb.log
 581     touch merge.log
 582     mkdir -p ./OCDB/TPC/Calib/TimeGain/
 583     mkdir -p ./OCDB/TPC/Calib/TimeDrift/
 584     touch ./OCDB/TPC/Calib/TimeGain/someCalibObject_0-999999_cpass0.root
 585     touch ./OCDB/TPC/Calib/TimeDrift/otherCalibObject_0-999999_cpass0.root
 586   else
 587     ./${mergingScript} ${calibrationFilesToMerge} ${runNumber} "local://./OCDB" ${ocdbStorage}
 588   fi
 589
 590   ### produce the output
 591   #tar the produced OCDB for reuse
 592   tar czf ${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz ./OCDB
 593
 594   #produce the calib trees for expert QA (dcsTime.root)
 595   goMakeLocalOCDBaccessConfig ./OCDB
 596   echo aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")"
 597   aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")"
 598
 599   /bin/ls
 600
 601   #copy all to output dir
 602   cp -pf --recursive ${runpath}/* ${outputDir}
 603
 604   if [[ -n ${generateMC} ]]; then
 605     goPrintValues sim ${commonOutputPath}/meta/sim.run${runNumber}.list ${commonOutputPath}/meta/cpass0.job*.run${runNumber}.done
 606   fi
 607
 608   #validate merging cpass0
 609   cd ${outputDir}
 610   touch ${doneFile}
 611   [[ -f CalibObjects.root ]] && echo "calibfile ${outputDir}/CalibObjects.root" >> ${doneFile}
 612   [[ -f dcsTime.root ]] && echo "dcsTree ${outputDir}/dcsTime.root" >> ${doneFile}
 613   echo "dir ${outputDir}" >> ${doneFile}
 614   summarizeLogs >> ${doneFile}
 615
 616   [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
 617   return 0
 618 }
 619
 620 goMergeCPass1()
 621 {
 622   #
 623   # find the output files and merge them
 624   #
 625
 626   outputDir=${1}
 627   ocdbStorage=${2}
 628   configFile=${3}
 629   runNumber=${4}
 630   calibrationFilesToMerge=${5}
 631   qaFilesToMerge=${6}
 632   filteredFilesToMerge=${7}
 633   shift 7
 634   parseConfig ${configFile} "$@"
 635
 636   [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
 637   doneFile="${commonOutputPath}/meta/merge.cpass1.run${runNumber}.done"
 638
 639   umask 0002
 640   ulimit -c unlimited
 641
 642   #clean up first:
 643   rm -f ${outputDir}/*.log
 644   rm -f ${outputDir}/*.root
 645   rm -f ${outputDir}/*done
 646
 647   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
 648
 649   #runpath=${PWD}/rundir_cpass1_Merge_${runNumber}
 650   runpath=${outputDir}
 651   [[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
 652   [[ ${reconstructInTemporaryDir} -eq 1 && -z ${TMPDIR} ]] && runpath=$(mktemp -d)
 653
 654   mkdir -p ${runpath}
 655   [[ ! -d ${runpath} ]] && echo "not able to make the runpath ${runpath}" && touch ${doneFile} && return 1
 656   cd ${runpath}
 657
 658   logOutputDir=${runpath}
 659   [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir}
 660   [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/mergeMakeOCDB.log
 661   echo "${0} $*"
 662
 663   calibrationOutputFileName='AliESDfriends_v1.root'
 664   qaOutputFileName='QAresults*.root'
 665   mergingScript="mergeMakeOCDB.byComponent.sh"
 666   #important to have the string "Stage.txt" in the filename to trigger the merging
 667   #it has to be a list of directories containing the files
 668   qaMergedOutputFileName="QAresults_merged.root"
 669
 670   echo goMergeCPass1 SETUP:
 671   echo runNumber=${runNumber}
 672   echo outputDir=${outputDir}
 673   echo ocdbStorage=${ocdbStorage}
 674   echo calibrationFilesToMerge=$calibrationFilesToMerge
 675   echo qaFilesToMerge=$qaFilesToMerge
 676   echo calibrationOutputFileName=${calibrationOutputFileName}
 677   echo mergingScript=${mergingScript}
 678
 679   # copy files in case they are not already there
 680   filesMergeCPass1=(
 681                     "${commonOutputPath}/${calibrationFilesToMerge}"
 682                     "${commonOutputPath}/${qaFilesToMerge}"
 683                     "${commonOutputPath}/OCDB.root"
 684                     "${commonOutputPath}/localOCDBaccessConfig.C"
 685                     "${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz"
 686                     "${commonOutputPath}/QAtrain_duo.C"
 687                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/mergeMakeOCDB.byComponent.sh"
 688                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/mergeByComponent.C"
 689                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/makeOCDB.C"
 690                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/merge.C"
 691                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/mergeMakeOCDB.sh"
 692                     "${trustedQAtrainMacro}"
 693                     "${ALICE_ROOT}/ANALYSIS/macros/QAtrain_duo.C"
 694   )
 695   for file in ${filesMergeCPass1[*]}; do
 696     [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
 697   done
 698
 699   sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh
 700
 701   #configure local OCDB storage from CPass0 (creates the localOCDBaccessConfig.C script)
 702   if [[ -f cpass0.localOCDB.${runNumber}.tgz ]]; then
 703     echo goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz"
 704     goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz"
 705   else
 706     echo "WARNING: file cpass0.localOCDB.${runNumber}.tgz not found!"
 707   fi
 708
 709   alirootInfo > ALICE_ROOT.log
 710
 711   #
 712   /bin/ls
 713
 714   #merge calibration
 715   chmod u+x ${mergingScript}
 716   mkdir -p OCDB
 717
 718   #if not provided, create the lists of files to merge
 719   if [[ ! -f ${filteredFilesToMerge} ]]; then
 720     echo "/bin/ls -1 ${outputDir}/*/FilterEvents_Trees.root > ${filteredFilesToMerge}"
 721     /bin/ls -1 ${outputDir}/*/FilterEvents_Trees.root 2>/dev/null > ${filteredFilesToMerge}
 722   fi
 723   if [[ ! -f ${calibrationFilesToMerge} ]]; then
 724     echo "/bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root > ${calibrationFilesToMerge}"
 725     /bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root 2>/dev/null > ${calibrationFilesToMerge}
 726   fi
 727   if [[ ! -f ${qaFilesToMerge} ]]; then
 728     #find the files, but only store the directories (QAtrain_duo.C requires this)
 729     echo "/bin/ls -1 ${outputDir}/*/QAresults*.root | while read x; do echo ${x%/*}; done | sort | uniq > ${qaFilesToMerge}"
 730     /bin/ls -1 ${outputDir}/*/QAresults*.root | while read x; do echo ${x%/*}; done | sort | uniq > ${qaFilesToMerge}
 731   fi
 732
 733   echo "${mergingScript} ${calibrationFilesToMerge} ${runNumber} local://./OCDB ${ocdbStorage}"
 734   if [[ -n ${pretend} ]]; then
 735     touch CalibObjects.root
 736     touch ocdb.log
 737     touch merge.log
 738     touch cpass1.localOCDB.${runNumber}.tgz
 739   else
 740     ./${mergingScript} ${calibrationFilesToMerge} ${runNumber} "local://./OCDB" ${ocdbStorage}
 741   fi
 742
 743   tar czf ${commonOutputPath}/meta/cpass1.localOCDB.${runNumber}.tgz ./OCDB
 744
 745   #merge QA (and filtered trees)
 746   [[ -n ${AliAnalysisTaskFilteredTree_fLowPtTrackDownscaligF} ]] && export AliAnalysisTaskFilteredTree_fLowPtTrackDownscaligF
 747   [[ -n ${AliAnalysisTaskFilteredTree_fLowPtV0DownscaligF} ]] && export AliAnalysisTaskFilteredTree_fLowPtV0DownscaligF
 748
 749   #echo aliroot -l -b -q "merge.C(\"${qaFilesToMerge}\",\"\",kFALSE,\"${qaMergedOutputFileName}\")"
 750   echo aliroot -b -q "QAtrain_duo.C(\"_barrel\",${runNumber},\"${qaFilesToMerge}\",1,\"${ocdbStorage}\")"
 751   if [[ -n ${pretend} ]]; then
 752     touch ${qaMergedOutputFileName}
 753     touch merge.log
 754     touch trending.root
 755     touch FilterEvents_Trees.root
 756   else
 757     #aliroot -l -b -q "merge.C(\"${qaFilesToMerge}\",\"\",kFALSE,\"${qaMergedOutputFileName}\")"
 758     aliroot -b -q "QAtrain_duo.C(\"_barrel\",${runNumber},\"${qaFilesToMerge}\",1,\"${ocdbStorage}\")" > mergeQA.log
 759     mv QAresults_barrel.root ${qaMergedOutputFileName}
 760     mv trending_barrel.root trending.root
 761
 762     #merge filtered trees
 763     echo aliroot -l -b -q "merge.C(\"${qaFilesToMerge}\",\"\",kFALSE,\"${qaMergedOutputFileName}\")"
 764     aliroot -l -b -q "merge.C(\"${filteredFilesToMerge}\",\"\",kFALSE,\"FilterEvents_Trees.root\")"
 765   fi
 766
 767   #produce the calib trees for expert QA
 768   echo aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")"
 769   aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")"
 770
 771   /bin/ls
 772
 773   #copy all to output dir
 774   cp -pf --recursive ${runpath}/* ${outputDir}
 775
 776   #validate merge cpass1
 777   cd ${outputDir}
 778   touch ${doneFile}
 779   [[ -f CalibObjects.root ]] && echo "calibfile ${outputDir}/CalibObjects.root" >> ${doneFile}
 780   [[ -f ${qaMergedOutputFileName} ]] && echo "qafile ${outputDir}/${qaMergedOutputFileName}" >> ${doneFile}
 781   [[ -f trending.root ]] && echo "trendingfile ${outputDir}/trending.root" >> ${doneFile}
 782   [[ -f dcsTime.root ]] && echo "dcsTree ${outputDir}/dcsTime.root" >> ${doneFile}
 783   echo "dir ${outputDir}" >> ${doneFile}
 784   summarizeLogs >>  ${doneFile}
 785
 786   [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
 787   return 0
 788 }
 789
 790 goMerge()
 791 {
 792   #generic root merge using CPass1 merge.C script
 793   inputList=${1}
 794   outputFile=${2}
 795   configFile=${3-"becnhmark.config"}
 796   shift 3
 797   parseConfig ${configFile} "$@"
 798
 799   [[ ! -f ${inputList} ]] && echo "inputList ${inputList} does not exist!" && return 1
 800   [[ ! -f ${configFile} ]] && echo "configFile ${configFile} does not exist!" && return 1
 801   umask 0002
 802   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
 803   rm -f ${outputFile}
 804   aliroot -b -q "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/merge.C(\"${inputList}\",\"\",kFALSE,\"${outputFile}\")" > merge_${inputList}.log
 805   return 0
 806 }
 807
 808 goSubmitMakeflow()
 809 {
 810   #run
 811   productionID=${1}
 812   inputFileList=${2}
 813   configFile=${3}
 814   shift 3
 815   extraOpts=("$@")
 816   parseConfig ${configFile} "${extraOpts[@]}"
 817
 818   [[ -z ${configFile} ]] && configFile="benchmark.config"
 819   [[ ! -f ${configFile} ]] && echo "no config file found (${configFile})" && return 1
 820
 821   #create the directopry for the metadata
 822   mkdir meta
 823
 824   if [[ ! $(which makeflow &>/dev/null) && -n ${makeflowPath} ]]; then
 825     echo "setting the makflow path from the config: "
 826     echo "  export PATH=${makeflowPath}:${PATH}"
 827     export PATH=${makeflowPath}:${PATH}
 828   fi
 829
 830   #submit - use makeflow if available, fall back to old stuff when makeflow not there
 831   if which makeflow; then
 832
 833     goGenerateMakeflow ${productionID} ${inputFileList} ${configFile} "${extraOpts[@]}" > benchmark.makeflow
 834
 835     makeflow ${makeflowOptions} benchmark.makeflow
 836     cd ../
 837   else
 838     echo "no makeflow!"
 839   fi
 840   return 0
 841 }
 842
 843 goGenerateMakeflow()
 844 {
 845   #generate the makeflow file
 846   productionID=${1}
 847   inputFileList=${2}
 848   configFile=${3}
 849   shift 3
 850   extraOpts=("$@")
 851   parseConfig ${configFile} "${extraOpts[@]}"
 852
 853   [[ -z ${configFile} ]] && configFile="benchmark.config"
 854   [[ ! -f ${configFile} ]] && echo "no config file found (${configFile})" && return 1
 855
 856   commonOutputPath=${baseOutputDirectory}/${productionID}
 857
 858   #these files will be made a dependency - will be copied to the working dir of the jobs
 859   declare -a copyFiles
 860   inputFiles=(
 861               "OCDB.root"
 862               "localOCDBaccessConfig.C"
 863               "QAtrain_duo.C"
 864               "runCPass1.sh"
 865               "recCPass1.C"
 866               "recCPass1_OuterDet.C"
 867               "runCalibTrain.C"
 868               "runCPass0.sh"
 869               "recCPass0.C"
 870   )
 871   for file in ${inputFiles[*]}; do
 872     [[ -f ${file} ]] && copyFiles+=("${file}")
 873   done
 874
 875   #create the makeflow file
 876   [[ -n ${batchFlags} ]] && echo "BATCH_OPTIONS = ${batchFlags}"
 877   declare -A arr_cpass0_merged arr_cpass1_merged
 878   declare -A arr_cpass0_calib_list arr_cpass1_calib_list
 879   declare -A arr_cpass1_QA_list arr_cpass1_ESD_list arr_cpass1_filtered_list
 880   declare -A listOfRuns
 881   [[ -n ${runNumber} ]] && listOfRuns[${runNumber}]=1
 882   while read x; do listOfRuns[$(guessRunNumber ${x})]=1; done < ${inputFileList}
 883   for runNumber in "${!listOfRuns[@]}"; do
 884     [[ -z ${runNumber} ]] && continue
 885     [[ ! ${runNumber} =~ ^[0-9]*[0-9]$ ]] && continue
 886
 887     unset arr_cpass0_outputs
 888     unset arr_cpass1_outputs
 889     declare -a arr_cpass0_outputs
 890     declare -a arr_cpass1_outputs
 891
 892     jobindex=0
 893     while read inputFile; do
 894       currentDefaultOCDB=${defaultOCDB}
 895       [[ -z ${autoOCDB} ]] && autoOCDB=1
 896       if [[ ${autoOCDB} -ne 0 ]]; then
 897         currentDefaultOCDB=$(setYear ${inputFile} ${defaultOCDB})
 898       fi
 899
 900       #CPass0
 901       arr_cpass0_outputs[${jobindex}]="meta/cpass0.job${jobindex}.run${runNumber}.done"
 902       echo "${arr_cpass0_outputs[${jobindex}]} : benchmark.sh ${configFile} ${copyFiles[@]}"
 903       echo -n " ${alirootEnv} ./benchmark.sh CPass0 ${commonOutputPath}/000${runNumber}/cpass0 ${inputFile} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} ${jobindex}"" "
 904       for extraOption in "${extraOpts[@]}"; do echo -n \"${extraOption}\"" "; done; echo
 905       echo
 906
 907       #CPass1
 908       arr_cpass1_outputs[${jobindex}]="meta/cpass1.job${jobindex}.run${runNumber}.done"
 909       echo "${arr_cpass1_outputs[${jobindex}]} : benchmark.sh ${configFile} meta/cpass0.localOCDB.${runNumber}.tgz ${copyFiles[@]}"
 910       echo -n " ${alirootEnv} ./benchmark.sh CPass1 ${commonOutputPath}/000${runNumber}/cpass1 ${inputFile} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} ${jobindex}"" "
 911       for extraOption in "${extraOpts[@]}"; do echo -n \"${extraOption}\"" "; done; echo
 912       echo
 913       ((jobindex++))
 914
 915     done< <(grep "/000${runNumber}/" ${inputFileList})
 916
 917     #CPass0 list of Calib files to merge
 918     arr_cpass0_calib_list[${runNumber}]="meta/cpass0.calib.run${runNumber}.list"
 919     echo "${arr_cpass0_calib_list[${runNumber}]} : benchmark.sh ${arr_cpass0_outputs[*]}"
 920     echo "  ./benchmark.sh PrintValues calibfile ${arr_cpass0_calib_list[${runNumber}]} ${arr_cpass0_outputs[*]}"
 921     echo
 922
 923     #CPass0 merging
 924     arr_cpass0_merged[${runNumber}]="meta/merge.cpass0.run${runNumber}.done"
 925     echo "meta/cpass0.localOCDB.${runNumber}.tgz ${arr_cpass0_merged[${runNumber}]} : benchmark.sh ${configFile} ${arr_cpass0_calib_list[${runNumber}]} ${copyFiles[@]}"
 926     echo -n " ${alirootEnv} ./benchmark.sh MergeCPass0 ${commonOutputPath}/000${runNumber}/cpass0 ${currentDefaultOCDB} ${configFile} ${runNumber} ${arr_cpass0_calib_list[${runNumber}]}"" "
 927     for extraOption in "${extraOpts[@]}"; do echo -n \"${extraOption}\"" "; done; echo
 928     echo
 929
 930     #CPass1 list of Calib/QA files to merge
 931     # the trick is to have the string "Stage.txt" in the file name of the list of directories with QA output to trigger
 932     # the production of the trending tree (only then the task->Finish() will be called in QAtrain_duo.C, on the grid
 933     # this corresponds to the last merging stage)
 934     arr_cpass1_calib_list[${runNumber}]="meta/cpass1.calib.run${runNumber}.list"
 935     echo "${arr_cpass1_calib_list[${runNumber}]} : benchmark.sh ${arr_cpass1_outputs[*]}"
 936     echo "  ./benchmark.sh PrintValues calibfile ${arr_cpass1_calib_list[${runNumber}]} ${arr_cpass1_outputs[*]};"
 937     echo
 938     arr_cpass1_QA_list[${runNumber}]="meta/cpass1.QA.run${runNumber}.lastMergingStage.txt.list"
 939     echo "${arr_cpass1_QA_list[${runNumber}]}: benchmark.sh ${arr_cpass1_outputs[*]}"
 940     echo "  ./benchmark.sh PrintValues dir ${arr_cpass1_QA_list[${runNumber}]} ${arr_cpass1_outputs[*]}"
 941     echo
 942
 943     #CPass1 output file lists
 944     arr_cpass1_ESD_list[${runNumber}]="meta/cpass1.ESD.run${runNumber}.list"
 945     echo "${arr_cpass1_ESD_list[${runNumber}]} : benchmark.sh ${arr_cpass1_outputs[*]}"
 946     echo "  ./benchmark.sh PrintValues esd ${arr_cpass1_ESD_list[${runNumber}]} ${arr_cpass1_outputs[*]}"
 947     arr_cpass1_filtered_list[${runNumber}]="meta/cpass1.filtered.run${runNumber}.list"
 948     echo "${arr_cpass1_filtered_list[${runNumber}]} : benchmark.sh ${arr_cpass1_outputs[*]}"
 949     echo "  ./benchmark.sh PrintValues filteredTree ${arr_cpass1_filtered_list[${runNumber}]} ${arr_cpass1_outputs[*]}"
 950
 951     #CPass1 merging
 952     arr_cpass1_merged[${runNumber}]="meta/merge.cpass1.run${runNumber}.done"
 953     echo "meta/cpass1.localOCDB.${runNumber}.tgz ${arr_cpass1_merged[${runNumber}]} :  benchmark.sh ${configFile} ${arr_cpass1_calib_list[${runNumber}]} ${arr_cpass1_QA_list[${runNumber}]} ${copyFiles[@]}"
 954     echo -n " ${alirootEnv} ./benchmark.sh MergeCPass1 ${commonOutputPath}/000${runNumber}/cpass1 ${currentDefaultOCDB} ${configFile} ${runNumber} ${arr_cpass1_calib_list[${runNumber}]} ${arr_cpass1_QA_list[${runNumber}]} ${arr_cpass1_filtered_list[${runNumber}]}"" "
 955     for extraOption in "${extraOpts[@]}"; do echo -n \"${extraOption}\"" "; done; echo
 956     echo
 957
 958   done #runs
 959
 960   #Summary
 961   echo "summary.log : benchmark.sh ${configFile} ${arr_cpass1_merged[*]}"
 962   echo " LOCAL ./benchmark.sh MakeSummary ${configFile}"
 963   echo
 964
 965   return 0
 966 }
 967
 968 goPrintValues()
 969 {
 970   #print the values given the key from any number of files (store in output file on second argument)
 971   if [[ $# -lt 3 ]]; then
 972     echo "goPrintValues key outputFile inputFiles"
 973     echo "if outputFile is \"-\" dont redirect to a file"
 974     return
 975   fi
 976   key=${1}
 977   outputFile=${2}
 978   [[ ${outputFile} =~ "-" ]] && outputFile=""
 979   shift 2 #remove 2 first arguments from arg list to only pass the input files to awk
 980   awk -v key=${key} '$0 ~ key" " {print $2}' "$@" | tee ${outputFile}
 981   return 0
 982 }
 983
 984 goCreateQAplots()
 985 {
 986   umask 0002
 987   mergedQAfileList=${1}
 988   productionID=${2}
 989   outputDir=${3}
 990   configFile=${4}
 991   shift 4
 992   parseConfig ${configFile} ${@}
 993
 994   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
 995
 996   [[ -z ${logOutputDir} ]] && logOutputDir=${PWD}
 997   [[ -z ${dontRedirectStdOutToLog} ]] && exec 2>&1 > ${logOutputDir}/makeQAplots.log
 998   echo "${0} $*"
 999
1000   olddir=${PWD}
1001   mkdir -p ${outputDir}
1002   cd ${outputDir}
1003   [[ ! "${PWD}" = "${outputDir}" ]] && echo "cannot make ${outputDir}... exiting" && return 1
1004
1005   echo ${ALICE_ROOT}/PWGPP/QA/scripts/runQA.sh inputList=${mergedQAfileList}
1006   ${ALICE_ROOT}/PWGPP/QA/scripts/runQA.sh inputList="${mergedQAfileList}" inputListHighPtTrees="${filteringList}"
1007
1008   cd ${olddir}
1009   return 0
1010 }
1011
1012 goTest()
1013 {
1014   umask 0002
1015   exec 2>&1
1016   exec > >(tee test.log)
1017   echo "$@"
1018   echo something
1019   return 0
1020 }
1021
1022 alirootInfo()
1023 {
1024   umask 0002
1025   # save aliroot repository info
1026   [[ -z "${ALICE_ROOT}" ]] && return 1
1027
1028   echo "\${ALICE_ROOT}=${ALICE_ROOT}"
1029   echo "\${ROOTSYS}=${ROOTSYS}"
1030   echo "\${PATH}=${PATH}"
1031   echo "\${LD_LIBRARY_PATH}=${LD_LIBRARY_PATH}"
1032
1033   pushd ${PWD}
1034   cd ${ALICE_ROOT}
1035   echo
1036
1037   currentBranch=$(git rev-parse --abbrev-ref HEAD)
1038   git status
1039   echo ""
1040   echo ""
1041   git diff ${currentBranch}
1042   popd
1043   return 0
1044 }
1045
1046 setYear()
1047 {
1048   #set the year
1049   #  ${1} - year to be set
1050   #  ${2} - where to set the year
1051   year1=$(guessYear ${1})
1052   year2=$(guessYear ${2})
1053   local path=${2}
1054   [[ ${year1} -ne ${year2} && -n ${year2} && -n ${year1} ]] && path=${2/\/${year2}\//\/${year1}\/}
1055   echo ${path}
1056   return 0
1057 }
1058
1059 guessPeriod()
1060 {
1061   #guess the period from the path, pick the rightmost one
1062   local IFS="/"
1063   declare -a path=( ${1} )
1064   local dirDepth=${#path[*]}
1065   for ((x=${dirDepth}-1;x>=0;x--)); do
1066     local field=${path[${x}]}
1067     [[ ${field} =~ ^LHC[0-9][0-9][a-z]$ ]] && period=${field} && break
1068   done
1069   echo ${period}
1070   return 0
1071 }
1072
1073 guessYear()
1074 {
1075   #guess the year from the path, pick the rightmost one
1076   local IFS="/"
1077   declare -a path=( ${1} )
1078   local dirDepth=${#path[*]}
1079   for ((x=${dirDepth}-1;x>=0;x--)); do
1080     local field=${path[${x}]}
1081     [[ ${field} =~ ^20[0-9][0-9]$ ]] && year=${field} && break
1082   done
1083   echo ${year}
1084   return 0
1085 }
1086
1087 guessRunNumber()
1088 {
1089   #guess the run number from the path, pick the rightmost one
1090   #works for /path/foo/000123456/bar/...
1091   #and       /path/foo.run123456.bar
1092   local IFS="/."
1093   declare -a path=( ${1} )
1094   local dirDepth=${#path[*]}
1095   for ((x=${dirDepth}-1;x>=0;x--)); do
1096     local field=${path[${x}]}
1097     field=${field/run/000}
1098     [[ ${field} =~ ^000[0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && runNumber=${field#000} && break
1099   done
1100   echo ${runNumber}
1101   return 0
1102 }
1103
1104 validateLog()
1105 {
1106   log=${1}
1107   errorConditions=(
1108                     'There was a crash'
1109                     'floating'
1110                     'error while loading shared libraries'
1111                     'std::bad_alloc'
1112                     's_err_syswatch_'
1113                     'Thread [0-9]* (Thread'
1114                     'AliFatal'
1115                     'core dumped'
1116                     '\.C.*error:.*\.h: No such file'
1117   )
1118
1119   warningConditions=(
1120                      'This is serious !'
1121                      'rocVoltage out of range:'
1122   )
1123
1124   local logstatus=0
1125   local errorSummary=""
1126   local warningSummary=""
1127
1128   for ((i=0; i<${#errorConditions[@]};i++)); do
1129     local tmp=$(grep -m1 -e "${errorConditions[${i}]}" ${log})
1130     [[ -n ${tmp} ]] && tmp+=" : "
1131     errorSummary+=${tmp}
1132   done
1133
1134   for ((i=0; i<${#warningConditions[@]};i++)); do
1135     local tmp=$(grep -m1 -e "${warningConditions[${i}]}" ${log})
1136     [[ -n ${tmp} ]] && tmp+=" : "
1137     warningSummary+=${tmp}
1138   done
1139
1140   if [[ -n ${errorSummary} ]]; then
1141     echo "${errorSummary}"
1142     return 1
1143   fi
1144
1145   if [[ -n ${warningSummary} ]]; then
1146     echo "${warningSummary}"
1147     return 2
1148   fi
1149
1150   return 0
1151 }
1152
1153 summarizeLogs()
1154 {
1155   #print a summary of logs
1156   logFiles=(
1157             "*.log"
1158             "stdout"
1159             "stderr"
1160   )
1161
1162   #check logs
1163   local logstatus=0
1164   for log in ${logFiles[*]}; do
1165     finallog=${PWD%/}/${log}
1166     [[ ! -f ${log} ]] && continue
1167     errorSummary=$(validateLog ${log})
1168     validationStatus=$?
1169     if [[ ${validationStatus} -eq 0 ]]; then
1170       #in pretend mode randomly report an error in rec.log some cases
1171       if [[ -n ${pretend} && "${log}" == "rec.log" ]]; then
1172         [[ $(( ${RANDOM}%2 )) -ge 1 ]] && echo "${finallog} BAD random error" || echo "${finallog} OK"
1173       else
1174         echo "${finallog} OK"
1175       fi
1176     elif [[ ${validationStatus} -eq 1 ]]; then
1177       echo "${finallog} BAD ${errorSummary}"
1178     elif [[ ${validationStatus} -eq 2 ]]; then
1179       echo "${finallog} OK MWAH ${errorSummary}"
1180     fi
1181   done
1182
1183   #report core files
1184   while read x; do
1185     echo ${x}
1186     chmod 644 ${x}
1187     gdb --batch --quiet -ex "bt" -ex "quit" aliroot ${x} > stacktrace_${x//\//_}.log
1188   done < <(/bin/ls ${PWD}/*/core 2>/dev/null; /bin/ls ${PWD}/core 2>/dev/null)
1189
1190   return ${logstatus}
1191 }
1192
1193 spitOutLocalOCDBaccessConfig()
1194 {
1195   umask 0002
1196   #find ${1} -name "*root" | \
1197   /bin/ls -1 ${1}/*/*/*/*.root 2>/dev/null | \
1198   while read line
1199   do
1200     local tmp=${line#${1}}
1201     echo ${tmp%/*} | \
1202     awk -v ocdb=${1} '{print "  man->SetSpecificStorage(\""$1"\",\"local://"ocdb"\");"}'
1203   done
1204   return 0
1205 }
1206
1207 goMakeLocalOCDBaccessConfig()
1208 {
1209   umask 0002
1210   # make a script that sets the specific storages form all the root files produced by CPass0
1211   local localOCDBpathCPass0=${1}
1212   local OCDBpathPrefix=${2}
1213   [[ -z ${OCDBpathPrefix} ]] && OCDBpathPrefix="."
1214
1215   if [[ -f ${localOCDBpathCPass0} && ${localOCDBpathCPass0} =~ \.tgz$ ]]; then
1216     tar xzf ${localOCDBpathCPass0}
1217     local localOCDBpathCPass0="${OCDBpathPrefix}/OCDB"
1218   fi
1219
1220   echo
1221   echo creating the specific storage script
1222   echo   localOCDBaccessConfig.C
1223   echo   based on OCDB: ${localOCDBaccessConfig}
1224   echo
1225
1226   local tempLocalOCDB=""
1227   if [[ -f localOCDBaccessConfig.C ]]; then
1228     tempLocalOCDB=$(mktemp)
1229     echo "egrep "SetSpecificStorage" localOCDBaccessConfig.C > ${tempLocalOCDB}"
1230     egrep "SetSpecificStorage" localOCDBaccessConfig.C > ${tempLocalOCDB}
1231   fi
1232
1233   echo "localOCDBaccessConfig()"                               >  localOCDBaccessConfig.C
1234   echo "{"                                                     >> localOCDBaccessConfig.C
1235   echo "  AliCDBManager* man = AliCDBManager::Instance();"     >> localOCDBaccessConfig.C
1236   spitOutLocalOCDBaccessConfig ${localOCDBpathCPass0}|sort|uniq  >> localOCDBaccessConfig.C
1237   [[ -f "${tempLocalOCDB}" ]] && cat ${tempLocalOCDB}              >> localOCDBaccessConfig.C
1238   echo "}"                                                     >> localOCDBaccessConfig.C
1239
1240   [[ -f "${tempLocalOCDB}" ]] && rm -f ${tempLocalOCDB}
1241
1242   if ! grep SetSpecificStorage localOCDBaccessConfig.C; then
1243     echo
1244     echo "!!!!!!! CPass0 produced no OCDB entries"
1245     return 1
1246   fi
1247   return 0
1248 }
1249
1250 goMakeFilteredTrees()
1251 {
1252   outputDir=${1}
1253   runNumber=${2}
1254   #get path to input list
1255   inputListfiles=${3}
1256   #get scale number for tracks
1257   filterT=${4}
1258   #get scale number for V0s
1259   filterV=${5}
1260   #get OCDB path (optional)
1261   OCDBpath=${6}
1262   #get max number of files
1263   maxFiles=${7-"1000000"}
1264   #get offset of first file
1265   offsetFile=${8-"0"}
1266   #get max number of events
1267   maxEvents=${9-"30000000"}
1268   #get offset of first event
1269   offsetEvent=${10-"0"}
1270   configFile=${11-"benchmark.config"}
1271   esdFileName=${12-"AliESDs_Barrel.root"}
1272   shift 12
1273   parseConfig ${configFile} "$@"
1274
1275   commonOutputPath=${PWD}
1276   doneFile=${commonOutputPath}/meta/filtering.cpass1.run${runNumber}.done
1277
1278   runpath=${outputDir}
1279   [[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
1280   [[ ${reconstructInTemporaryDir} -eq 1 && -z ${TMPDIR} ]] && runpath=$(mktemp -d)
1281
1282
1283   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
1284
1285   mkdir -p ${outputDir}
1286   mkdir -p ${runpath}
1287
1288   cd ${runpath}
1289
1290   cat > filtering.log << EOF
1291   goMakeFilteredTrees config:
1292   runpath=${runpath}
1293   outputDir=${outputDir}
1294   commonOutputPath=${commonOutputPath}
1295   ALICE_ROOT=${ALICE_ROOT}
1296   PATH=${PATH}
1297   offsetEvent=$offsetEvent
1298   configFile=$configFile
1299   esdFileName=$esdFileName
1300 EOF
1301
1302   if [[ -z ${pretend} ]];then
1303     aliroot -l -b -q "${ALICE_ROOT}/PWGPP/macros/runFilteringTask.C(\"${inputListfiles}\",${filterT},${filterV},\"${OCDBpath}\",${maxFiles},${offsetFile},${maxEvents},${offsetEvent},\"${esdFileName}\")" &>> filtering.log
1304   else
1305     touch filtering.log FilterEvents_Trees.root
1306   fi
1307   pwd
1308   /bin/ls
1309   touch ${doneFile}
1310   summarizeLogs >>  ${doneFile}
1311   echo mv -f * ${outputDir}
1312   mv -f * ${outputDir}
1313   [[ -f ${outputDir}/FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFile}
1314   cd ${commonOutputPath}
1315   [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
1316   return 0
1317 }
1318
1319 submit()
1320 {
1321   umask 0002
1322   [[ $# -lt 5 ]] && echo "at least 5 args needed, you supplied $#" && return 1
1323   JobID=${1}
1324   startID=${2}
1325   endID=${3}
1326   waitForJOBID=${4}
1327   command=${5}
1328   shift 5
1329   commandArgs=("$@")
1330
1331   #add quote strings around the extra arguments
1332   for ((i=0; i<${#commandArgs[@]}; i++)); do
1333     commandArgs[i]=\"${commandArgs[i]}\"
1334   done
1335
1336   [[ -z ${waitForJOBID} ]] && waitForJOBID=0
1337
1338   newFarm=$(which qsub|grep "^/usr/bin/qsub")
1339
1340   batchSystem="SGE"
1341
1342   if [[ -z "${newFarm}" ]]
1343   then
1344     #old LSF
1345     # submit it (as job array)
1346     nFiles=$(( ${endID}-${startID}+1 ))
1347     while [ ${startID} -le ${nFiles}  ] ; do
1348       if [ $(expr ${nFiles} - ${startID}) -gt 999 ] ; then
1349         endID=$(expr ${startID} + 999)
1350       else
1351         endID=${nFiles}
1352       fi
1353     if [[ ${waitForJOBID} -eq 0 ]]; then
1354         echo ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}"
1355         ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}"
1356       else
1357         echo ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -w "ended(${waitForJOBID})" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}"
1358         ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -w "ended(${waitForJOBID})" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}"
1359       fi
1360       startID=$(expr ${endID} + 1)
1361     done
1362   else
1363     #new SGE farm
1364     if [[ ${waitForJOBID} =~ "000" ]]; then
1365       echo ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}"
1366       ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}"
1367     else
1368       echo ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -hold_jid "${waitForJOBID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}"
1369       ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -hold_jid "${waitForJOBID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}"
1370     fi
1371   fi
1372   return 0
1373 }
1374
1375 goSubmitBatch()
1376 {
1377   if [[ $# -lt 3 ]]; then
1378     echo "minimal use:"
1379     echo " ${0} submit fileList productionID configFile"
1380     return 0
1381   fi
1382
1383   productionID=${1}
1384   inputList=${2}
1385   configFile=${3:-"benchmark.config"}
1386   configFile=$(readlink -f ${configFile})
1387   shift 3
1388   extraOpts=("$@")
1389   parseConfig ${configFile} "${extraOpts[@]}"
1390
1391   #redirect all output to submit.log
1392   echo "redirecting all output to ${PWD}/submit_${productionID//"/"/_}.log"
1393   exec 7>&1
1394   exec 1>submit_${productionID//"/"/_}.log 2>&1
1395
1396   umask 0002
1397   echo ${0}" submit $*"
1398   if [[ -z "${inputList}" || -z "${productionID}" ]]
1399   then
1400     echo
1401     echo " Usage: ${0} submit inputList productionID [configFile=benchmark.config]"
1402     echo
1403     return
1404   fi
1405
1406   # check if config file is there
1407   if [ ! -f ${configFile} ]; then
1408     echo "ERROR! Config File '${configFile}' not found" >&2
1409     return
1410   else
1411     echo "Using Config File: '${configFile}'"
1412   fi
1413
1414   [[ ! -f ${alirootEnv} ]] && echo "alirootEnv script ${alirootEnv} not found!..." && return 1
1415
1416   #move the script, config and some other stuff to ${commonOutputPath} first, then use them from there
1417   self=$(readlink -f "${0}")
1418   configPath=$(dirname ${configFile})
1419   export commonOutputPath=${baseOutputDirectory}/${productionID}
1420
1421   mkdir -p ${commonOutputPath}
1422   mkdir -p ${commonOutputPath}/logs
1423   mkdir -p ${commonOutputPath}/meta
1424
1425   cp ${self} ${commonOutputPath}
1426   cp ${configFile} ${commonOutputPath}
1427   cp ${inputList} ${commonOutputPath}
1428   self=${commonOutputPath}/${self##*/}
1429   chmod u+x ${self}
1430   configFile=${commonOutputPath}/${configFile##*/}
1431   inputList=${commonOutputPath}/${inputList##*/}
1432
1433   #convert to absolut pathnames
1434   inputList=$(readlink -f "${inputList}")
1435   #make list of runs
1436   if [[ -z ${runNumber} ]]; then
1437     listOfRuns=($(while read x; do guessRunNumber ${x}; done < ${inputList} | sort | uniq))
1438   else
1439     listOfRuns=${runNumber}
1440   fi
1441
1442   alirootSource=$(readlink -f "${alirootSource}")
1443
1444   echo ""
1445   echo "### BEGIN CONFIGURATION ###"
1446   echo ""
1447   echo "GENERAL:"
1448   echo ""
1449   echo "    productionID:    ${productionID}"
1450   echo "    batchCommand:    ${batchCommand}"
1451   echo "    batchFlags:      ${batchFlags}"
1452   echo "    alirootEnv:   ${alirootEnv}"
1453   ${alirootEnv} echo '    ALICE_ROOT:      ${ALICE_ROOT}'
1454   ${alirootEnv} echo '    ALIROOT_RELEASE: ${ALICE_RELEASE}'
1455   echo "    inputList:       ${inputList}"
1456   echo "    configPath:      ${configPath}"
1457   echo "    commonOutputPath:      ${commonOutputPath}"
1458   echo "    defaultOCDB:     ${defaultOCDB}"
1459   echo "      autoOCDB: ${autoOCDB}"
1460   echo "    recoTriggerOptions:   ${recoTriggerOptions}"
1461   echo "    runs:"
1462   echo "      ${listOfRuns[*]}"
1463   echo ""
1464   echo "THE TRAIN WILL RUN:"
1465
1466   if [ ${runCPass0reco} -eq 1 ]; then
1467     echo "    Pass0 - Recontruction"
1468   fi
1469
1470   if [ ${runCPass0MergeMakeOCDB} -eq 1 ]; then
1471     echo "    Pass0 - merging and OCDB export"
1472   fi
1473
1474   if [ ${runCPass1reco} -eq 1 ]; then
1475     echo "    Pass1 - Recontruction"
1476   fi
1477   if [ ${runCPass1MergeMakeOCDB} -eq 1 ]; then
1478     echo "    Pass1 - merging and OCDB export"
1479   fi
1480
1481   echo ""
1482   echo "LIMITS:"
1483   echo "    max. Events/Chunk:   ${nEvents}"
1484   echo "    max. Number of Chunks per Run:     ${nMaxChunks}"
1485   echo ""
1486   echo "### END CONFIGURATION ###"
1487   echo ""
1488
1489
1490   # check if input file is there
1491   if [ ! -f ${inputList} ]; then
1492     echo "ERROR! Input List '${inputList}' not found" >&2
1493     return
1494   fi
1495
1496   # define jobid (for dependent jobs)
1497   date=$(date +%Y_%m_%d_%H%M%S)
1498   #for each run we submit one jobarray:
1499   for runNumber in ${listOfRuns[*]}; do
1500
1501     [[ -z ${runNumber} ]] && continue
1502     [[ ! ${runNumber} =~ ^[0-9]*[0-9]$ ]] && continue
1503
1504     JOBpostfix="${productionID//"/"/_}_${runNumber}_${date}"
1505     JOBID1="p0_${JOBpostfix}"
1506     JOBID1wait="w0_${JOBpostfix}"
1507     JOBID2="m0_${JOBpostfix}"
1508     JOBID2wait="wm0_${JOBpostfix}"
1509     JOBID3="op0_${JOBpostfix}"
1510     JOBID3wait="wop0_${JOBpostfix}"
1511     JOBID4="p1_${JOBpostfix}"
1512     JOBID4wait="w1_${JOBpostfix}"
1513     JOBID5="m1_${JOBpostfix}"
1514     JOBID5wait="wm1_${JOBpostfix}"
1515     JOBID6="s1_${JOBpostfix}"
1516     JOBID6wait="ws1_${JOBpostfix}"
1517     JOBID7="QA_${JOBpostfix}"
1518     JOBmakeESDlistCPass1="lp1_${JOBpostfix}"
1519     JOBfilterESDcpass1="fp1_${JOBpostfix}"
1520     LASTJOB="000"
1521
1522     oneInputFile=$(egrep -m1 "${runNumber}\/" ${inputList})
1523
1524     currentDefaultOCDB=${defaultOCDB}
1525     [[ -z ${autoOCDB} ]] && autoOCDB=1
1526     if [[ ${autoOCDB} -ne 0 ]]; then
1527       currentDefaultOCDB=$(setYear ${oneInputFile} ${defaultOCDB})
1528     fi
1529
1530     echo "submitting run ${runNumber} with OCDB ${currentDefaultOCDB}"
1531
1532     ###############################################################################
1533     #run one chunk with valgrind:
1534     if [[ -n ${runValgrind} ]]; then
1535       [[ -z ${nEventsValgrind} ]] && nEventsValgrind=2
1536       [[ -z ${valgrindCommand} ]] && valgrindCommand="/usr/bin/valgrind --tool=callgrind --num-callers=40 -v --trace-children=yes"
1537       submit "valgrind" 1 1 000 "${alirootEnv} ${self}" CPass0 ${commonOutputPath}/000${runNumber}/valgrind ${oneInputFile} ${nEventsValgrind} ${currentDefaultOCDB} ${configFile} ${runNumber} valgrind valgrindCommand=${valgrindCommand} "${extraOpts[@]}"
1538     fi
1539
1540     ################################################################################
1541     ################################################################################
1542     # run the CPass0 if requested
1543
1544     if [ ${runCPass0reco} -eq 1 ]; then
1545
1546       echo
1547       echo "starting CPass0... for run ${runNumber}"
1548       echo
1549
1550       # create directory and copy all files that are needed
1551       targetDirectory="${commonOutputPath}/000${runNumber}/cpass0"
1552       mkdir -p ${targetDirectory}
1553
1554       filesCPass0=(
1555                     "${configPath}/runCPass0.sh"
1556                     "${configPath}/recCPass0.C"
1557                     "${configPath}/runCalibTrain.C"
1558                     "${configPath}/localOCDBaccessConfig.C"
1559                     "${configPath}/OCDB*.root"
1560                     "${configPath}/sim.C"
1561                     "${configPath}/rec.C"
1562                     "${configPath}/Config.C"
1563       )
1564       for file in ${filesCPass0[*]}; do
1565         [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
1566       done
1567
1568       localInputList=${targetDirectory}/${inputList##*/}
1569       [[ ! -f ${localInputList} ]] && egrep "\/000${runNumber}\/" ${inputList} > ${localInputList}
1570       # limit nFiles to nMaxChunks
1571       nFiles=$(wc -l < ${localInputList})
1572       [[ ${nFiles} -eq 0 ]] && echo "list contains ZERO files! exiting..." && return 1
1573       echo "raw files in list:    ${nFiles}"
1574       if [[ ${nMaxChunks} -gt 0 && ${nMaxChunks} -le ${nFiles} ]]; then
1575         nFiles=${nMaxChunks}
1576       fi
1577       echo "raw files to process: ${nFiles}"
1578       [[ -z "${percentProcessedFilesToContinue}" ]] && percentProcessedFilesToContinue=100
1579       if [[ ${percentProcessedFilesToContinue} -eq 100 ]]; then
1580         nFilesToWaitFor=${nFiles}
1581       else
1582         nFilesToWaitFor=$(( ${nFiles}-${nFiles}/(100/(100-${percentProcessedFilesToContinue})) ))
1583       fi
1584       echo "requested success rate is ${percentProcessedFilesToContinue}%"
1585       echo "merging will start after ${nFilesToWaitFor} jobs are done"
1586
1587       submit ${JOBID1} 1 ${nFiles} 000 "${alirootEnv} ${self}" CPass0 ${targetDirectory} ${localInputList} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} -1 "${extraOpts[@]}"
1588
1589       ## submit a monitoring job that will run until a certain number of jobs are done with reconstruction
1590       submit "${JOBID1wait}" 1 1 000 "${alirootEnv} ${self}" WaitForOutput ${commonOutputPath} "meta/cpass0.job*.run${runNumber}.done" ${nFilesToWaitFor} ${maxSecondsToWait} '-maxdepth 1'
1591       LASTJOB=${JOBID1wait}
1592
1593     fi #end running CPass0
1594     ################################################################################
1595
1596
1597     ################################################################################
1598     # submit merging of CPass0, depends on the reconstruction
1599
1600     if [ ${runCPass0MergeMakeOCDB} -eq 1 ]; then
1601
1602       echo
1603       echo "submit CPass0 merging for run ${runNumber}"
1604       echo
1605
1606       targetDirectory="${commonOutputPath}/000${runNumber}/cpass0"
1607       mkdir -p ${targetDirectory}
1608
1609       #copy the scripts
1610       filesMergeCPass0=(
1611                         "${configPath}/OCDB.root"
1612                         "${configPath}/mergeMakeOCDB.byComponent.sh"
1613                         "${configPath}/mergeMakeOCDB.sh"
1614                         "${configPath}/localOCDBaccessConfig.C"
1615                         "${configPath}/mergeByComponent.C"
1616                         "${configPath}/makeOCDB.C"
1617                         "${configPath}/merge.C"
1618       )
1619       for file in ${filesMergeCPass0[*]}; do
1620         [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
1621       done
1622
1623       submit ${JOBID2} 1 1 "${LASTJOB}" "${alirootEnv} ${self}" MergeCPass0 ${targetDirectory} ${currentDefaultOCDB} ${configFile} ${runNumber} cpass0.calib.run${runNumber}.list "${extraOpts[@]}"
1624       LASTJOB=${JOBID2}
1625
1626       if [[ -n ${generateMC} ]]; then
1627         submit "mrl${JOBpostfix}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" PrintValues sim ${commonOutputPath}/meta/sim.run${runNumber}.list ${commonOutputPath}/meta/cpass0.job*.run${runNumber}.done
1628         LASTJOB="mrl${JOBpostfix}"
1629       fi
1630
1631       echo
1632     fi
1633     # end of merging CPass0
1634     ################################################################################
1635
1636     ################################################################################
1637     ################################################################################
1638     # run the CPass1 if requested
1639
1640     if [ ${runCPass1reco} -eq 1 ]; then
1641
1642       targetDirectory="${commonOutputPath}/000${runNumber}/cpass1"
1643       rm -f ${commonOutputPath}/meta/cpass1.job*.run${runNumber}.done
1644
1645       # safety feature: if we are re-running for any reason we want to delete the previous output first.
1646       [[ -d ${targetDirectory} ]] && rm -rf ${targetDirectory}/* && echo "removed old output at ${targetDirectory}/*"
1647
1648       echo
1649       echo "starting CPass1... for run ${runNumber}"
1650       echo
1651
1652       # create directory and copy all files that are needed
1653       mkdir -p ${targetDirectory}
1654
1655       filesCPass1=(
1656                     "${configPath}/runCPass1.sh"
1657                     "${configPath}/recCPass1.C"
1658                     "${configPath}/recCPass1_OuterDet.C"
1659                     "${configPath}/runCalibTrain.C"
1660                     "${configPath}/QAtrain_duo.C"
1661                     "${configPath}/localOCDBaccessConfig.C"
1662                     "${configPath}/OCDB.root"
1663       )
1664       for file in ${filesCPass1[*]}; do
1665         [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
1666       done
1667
1668       if [[ -n ${generateMC} ]]; then
1669         localInputList=${commonOutputPath}/meta/sim.run${runNumber}.list
1670       else
1671         localInputList=${targetDirectory}/${inputList##*/}
1672         [[ ! -f ${localInputList} ]] && egrep "\/000${runNumber}\/" ${inputList} > ${localInputList}
1673       fi
1674       # limit nFiles to nMaxChunks
1675       nFiles=$(wc -l < ${localInputList})
1676       [[ ${nFiles} -eq 0 ]] && echo "list contains ZERO files! continuing..." && continue
1677       echo "raw files in list:    ${nFiles}"
1678       if [[ ${nMaxChunks} -gt 0 && ${nMaxChunks} -le ${nFiles} ]]; then
1679         nFiles=${nMaxChunks}
1680       fi
1681       echo "raw files to process: ${nFiles}"
1682       [[ -z "${percentProcessedFilesToContinue}" ]] && percentProcessedFilesToContinue=100
1683       if [[ ${percentProcessedFilesToContinue} -eq 100 ]]; then
1684         nFilesToWaitFor=${nFiles}
1685       else
1686         nFilesToWaitFor=$(( ${nFiles}-${nFiles}/(100/(100-${percentProcessedFilesToContinue})) ))
1687       fi
1688       echo "requested success rate is ${percentProcessedFilesToContinue}%"
1689       echo "merging will start after ${nFilesToWaitFor} jobs are done"
1690
1691       submit ${JOBID4} 1 ${nFiles} "${LASTJOB}" "${alirootEnv} ${self}" CPass1 ${targetDirectory} ${localInputList} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} -1 "${extraOpts[@]}"
1692
1693       ################################################################################
1694       ## submit a monitoring job that will run until a certain number of jobs are done with reconstruction
1695       submit "${JOBID4wait}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" WaitForOutput ${commonOutputPath} "meta/cpass1.job*.run${runNumber}.done" ${nFilesToWaitFor} ${maxSecondsToWait} '-maxdepth 1'
1696       LASTJOB=${JOBID4wait}
1697       ################################################################################
1698
1699       echo
1700     fi #end running CPass1
1701
1702     ################################################################################
1703     # submit merging of CPass1, depends on the reconstruction
1704     if [ ${runCPass1MergeMakeOCDB} -eq 1 ]; then
1705
1706       echo
1707       echo "submit CPass1 merging for run ${runNumber}"
1708       echo
1709
1710       targetDirectory="${commonOutputPath}/000${runNumber}/cpass1"
1711       rm -f ${commonOutputPath}/meta/merge.cpass1.run${runNumber}.done
1712       mkdir -p ${targetDirectory}
1713
1714       # copy files
1715       filesMergeCPass1=(
1716                         "${configPath}/OCDB.root"
1717                         "${configPath}/localOCDBaccessConfig.C"
1718                         "${configPath}/mergeMakeOCDB.byComponent.sh"
1719                         "${configPath}/mergeByComponent.C"
1720                         "${configPath}/makeOCDB.C"
1721                         "${configPath}/merge.C"
1722                         "${configPath}/mergeMakeOCDB.sh"
1723                         "${configPath}/QAtrain_duo.C"
1724       )
1725       for file in ${filesMergeCPass1[*]}; do
1726         [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
1727       done
1728
1729       submit "${JOBID5}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" MergeCPass1 ${targetDirectory} ${currentDefaultOCDB} ${configFile} ${runNumber} cpass1.calib.run${runNumber}.list cpass1.QA.run${runNumber}.lastMergingStage.txt.list cpass1.filtered.run${runNumber}.list "${extraOpts[@]}"
1730       LASTJOB=${JOBID5}
1731       echo
1732     fi
1733
1734     ###############################
1735     #if [ ${runESDfiltering} -eq 1 ]; then
1736     #  rm -f ${commonOutputPath}/cpass1.ESD.run${runNumber}.list
1737     #  rm -f ${commonOutputPath}/meta/filtering.cpass1.run*.done
1738     #  echo
1739     #  echo submitting filtering for run ${runNumber}
1740     #  echo
1741     #  submit "${JOBmakeESDlistCPass1}" 1 1 "${LASTJOB}" "${self}" PrintValues esd ${commonOutputPath}/meta/cpass1.ESD.run${runNumber}.list ${commonOutputPath}/meta/cpass1.job*.run${runNumber}.done
1742     #  submit "${JOBfilterESDcpass1}" 1 1 "${JOBmakeESDlistCPass1}" "${alirootEnv} ${self}" MakeFilteredTrees ${commonOutputPath}/000${runNumber}/cpass1 ${runNumber} ${commonOutputPath}/meta/cpass1.ESD.run${runNumber}.list ${filteringFactorHighPt} ${filteringFactorV0s} ${currentDefaultOCDB} 1000000 0 10000000 0 ${configFile} AliESDs_Barrel.root "${extraOpts[@]}"
1743     #  LASTJOB=${JOBfilterESDcpass1}
1744     #fi
1745
1746   done
1747
1748   #################################################################################
1749   #################################################################################
1750   #if [ ${runESDfiltering} -eq 1 ]; then
1751   #  submit "${JOBID5wait}" 1 1 "${LASTJOB}" "${self}" WaitForOutput ${commonOutputPath} "meta/filtering.cpass1.run*.done" "${#listOfRuns[@]}" ${maxSecondsToWait}
1752   #else
1753     submit "${JOBID5wait}" 1 1 "${LASTJOB}" "${self}" WaitForOutput ${commonOutputPath} "meta/merge.cpass1.run*.done" ${#listOfRuns[@]} ${maxSecondsToWait}
1754   #fi
1755   LASTJOB=${JOBID5wait}
1756
1757   #################################################################################
1758   echo
1759   echo "submit make a summary"
1760   echo
1761
1762   submit "${JOBID6}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" MakeSummary ${configFile}
1763   LASTJOB=${JOBID6}
1764   #################################################################################
1765
1766   #restore stdout
1767   exec 1>&7 7>&-
1768   echo "jobs submitted."
1769   return 0
1770 }
1771
1772 goWaitForOutput()
1773 {
1774   umask 0002
1775   [[ $# -lt 3 ]] && echo "goWaitForOutput() wrong number of arguments, exiting.." && return 1
1776   echo searchPath=${1}
1777   echo fileName=${2}
1778   echo numberOfFiles=${3}
1779   echo maxSecondsToWait=${4}
1780   searchPath=${1}
1781   fileName=${2}
1782   numberOfFiles=${3}
1783   maxSecondsToWait=${4}
1784   extraFindOptions=${5}
1785   echo "command to be executed: /bin/ls -1 ${searchPath}/${fileName} ${extraFindOptions}"
1786   [[ -z "${maxSecondsToWait}" ]] && maxSecondsToWait=$(( 3600*12 ))
1787   while true; do
1788     #n=$(find ${searchPath} ${extraFindOptions} -name "${fileName}" | wc -l)
1789     n=$(/bin/ls -1 ${searchPath}/${fileName} 2>/dev/null | wc -l)
1790     [[ ${n} -gt 0 ]] && echo "found ${n} X ${fileName}"
1791     [[ ${n} -ge ${numberOfFiles} ]] && break
1792     [[ ${SECONDS} -gt ${maxSecondsToWait} ]] && echo "timeout of ${maxSecondsToWait}s!" && break
1793     sleep 60
1794   done
1795   echo "DONE! exiting..."
1796   return 0
1797 }
1798
1799 mergeSysLogs()
1800 {
1801   outputFile=${1}
1802   shift
1803   inputFiles="$@"
1804   i=0
1805   if ! ls -1 ${inputFiles} &>/dev/null; then echo "the files dont exist!: ${inputFiles}"; return 1; fi
1806   while read x; do
1807     runNumber=$(guessRunNumber ${x})
1808     [[ -z ${runNumber} ]] && echo "run number cannot be guessed for ${x}" && continue
1809     awk -v run=${runNumber} -v i=${i} 'NR > 1 {print run" "$0} NR==1 && i==0 {print "run/I:"$0}' ${x}
1810     (( i++ ))
1811   done < <(ls -1 ${inputFiles}) > ${outputFile}
1812   return 0
1813 }
1814
1815 goMakeMergedSummaryTree()
1816 {
1817   # create list of calibration entries
1818   # takes no arguments, just run it in the base output
1819   # directory with the following files in the working directory
1820   #
1821   #  Calibration file lists:
1822   #       cpass0.dcsTree.list, cpass1.dcsTree.list
1823   #  QA trending root file:
1824   #       trending.root
1825   #
1826   #  Production infoo ascii files:
1827   #       summary_pass0.tree
1828   #       summary_pass1.tree
1829   #
1830
1831   [[ ! -f cpass0.dcsTree.list ]] && echo "no cpass0.dcsTree.list" && return 1
1832   [[ ! -f cpass1.dcsTree.list ]] && echo "no cpass1.dcsTree.list" && return 1
1833   [[ ! -f trending.root ]] && echo "no trending.root" && return 1
1834   [[ ! -f summary_pass0.tree ]] && echo "no summary_pass0.tree" && return 1
1835   [[ ! -f summary_pass1.tree ]] && echo "no summary_pass1.tree" && return 1
1836
1837   #first, dump the C macro to file
1838   cat << EOF > mergeTree.C
1839   //
1840   // Merge summary information
1841   // Following files are expected to be in the working directory
1842   //
1843   // Calibration file lists:
1844   //      cpass0.dcsTree.list, cpass1.dcsTree.list
1845   // QA trending root file:
1846   //      trending.root
1847   //
1848   // Production infoo ascii files:
1849   //      summary_pass0.tree
1850   //      summary_pass1.tree
1851   //
1852   void mergeTree(){
1853     //
1854     //
1855     //
1856     // Calibration values dump
1857     //
1858     AliXRDPROOFtoolkit::MakeTreeFromList("Calib.TPC.CPass0.root", "dcs","dcs","cpass0.dcsTree.list",1);
1859     AliXRDPROOFtoolkit::MakeTreeFromList("Calib.TPC.CPass1.root", "dcs","dcs","cpass1.dcsTree.list",1);
1860     //
1861     // Calibration status dump
1862     //
1863     TFile *fprod = TFile::Open("fproduction.root","recreate");
1864     TTree  tree0, tree1;
1865     tree0.ReadFile("summary_pass0.tree");
1866     tree1.ReadFile("summary_pass1.tree");
1867     tree0.Write("CPass0");
1868     tree1.Write("CPass1");
1869     fprod->Close();
1870     //
1871     //
1872     //
1873     TString stringSetup="";
1874     stringSetup+="1#QA.TPC#run#SummaryTPCQA/tpcQA#trending.root+";  //
1875     stringSetup+="1#Calib.TPC.CPass0#run#dcs#Calib.TPC.CPass0.root+";  //
1876     stringSetup+="1#Calib.TPC.CPass1#run#dcs#Calib.TPC.CPass1.root+";  //
1877     //
1878     stringSetup+="1#CPass0#runnumber#CPass0#fproduction.root+";  //
1879     stringSetup+="1#CPass1#runnumber#CPass1#fproduction.root+";  //
1880     //
1881     AliXRDPROOFtoolkit::JoinTreesIndex("outAll.root","joinAll","run",stringSetup.Data(), 1);
1882   }
1883 EOF
1884
1885   aliroot -b -q "mergeTree.C" > mergeTrees.log
1886   return $?
1887 }
1888
1889 goMakeSummary()
1890 {
1891   #all the final stuff goes in here for ease of use:
1892   # summary logs
1893   # qa plot making
1894   # final file lists
1895   #some defaults:
1896   log="summary.log"
1897   productionID="qa"
1898
1899   configFile=${1}
1900   shift 1
1901   extraOpts=("$@")
1902   parseConfig ${configFile} "${extraOpts[@]}"
1903
1904   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
1905
1906   exec &> >(tee ${log})
1907
1908   [[ ! -f ${configFile} ]] && echo "no config file ${configFile}!" && return
1909
1910   [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
1911
1912   #summarize the global stuff
1913   echo "env script: ${alirootSource} ${alirootEnv}"
1914   echo "\$ALICE_ROOT=${ALICE_ROOT}"
1915   echo "commonOutputPath=${commonOutputPath}"
1916
1917   #summarize the stacktraces
1918   awk '
1919        BEGIN {
1920                print "frame/I:method/C:line/C:cpass/I:aliroot/I";
1921                RS="#[0-9]*";
1922                aliroot=0;
1923              }
1924              {
1925                if ($3 ~ /Ali*/) aliroot=1; else aliroot=0;
1926                gsub("#","",RT);
1927                if ($NF!="" && RT!="" && $3!="") print RT" "$3" "$NF" "0" "aliroot
1928              }
1929       ' 000*/cpass0/*/stacktrace* 2>/dev/null > stacktrace.tree
1930   awk '
1931        BEGIN {
1932                RS="#[0-9]*";
1933                aliroot=0;
1934              }
1935              {
1936                if ($3 ~ /Ali*/) aliroot=1; else aliroot=0;
1937                gsub("#","",RT);
1938                if ($NF!="" && RT!="" && $3!="") print RT" "$3" "$NF" "1" "aliroot
1939              }
1940       ' 000*/cpass1/*/stacktrace* 2>/dev/null >> stacktrace.tree
1941
1942   echo total numbers for the production:
1943   echo
1944   awk 'BEGIN {nFiles=0;nCore=0;}
1945   /^calibfile/ {nFiles++;}
1946   /core dumped/ {nCore++i;}
1947   END {print     "cpass0 produced "nFiles" calib files, "nCore" core files";}' meta/cpass0.job*done 2>/dev/null
1948   awk 'BEGIN {nOK=0; nBAD=0; }
1949   /\/rec.log OK/ {nOK++;}
1950   /\/rec.log BAD/ {nBAD++;}
1951   /stderr BAD/ {if ($0 ~ /rec.log/){nBAD++;}}
1952   END {print     "cpass0 reco:  OK: "nOK"\tBAD: "nBAD;}' meta/cpass0.job*done 2>/dev/null
1953   awk 'BEGIN {nOK=0; nBAD=0; }
1954   /\/calib.log OK/ {nOK++;}
1955   /\/calib.log BAD/ {nBAD++;}
1956   END {print "cpass0 calib: OK: "nOK"\tBAD: "nBAD;}' meta/cpass0.job*done 2>/dev/null
1957
1958   awk 'BEGIN {nOK=0; nBAD=0; }
1959   /merge.log OK/ {nOK++;}
1960   /merge.log BAD/ {nBAD++;}
1961   END {print "cpass0 merge: OK: "nOK"\tBAD: "nBAD;}' meta/merge.cpass0*done 2>/dev/null
1962   awk 'BEGIN {nOK=0; nBAD=0; }
1963   /ocdb.log OK/ {nOK++;}
1964   /ocdb.log BAD/ {nBAD++;}
1965   END {print   "cpass0 OCDB:  OK: "nOK"\tBAD: "nBAD;}' meta/merge.cpass0*done 2>/dev/null
1966
1967   echo
1968   awk 'BEGIN {nFiles=0;nCore=0;}
1969   /^calibfile/ {nFiles++;}
1970   /core dumped/ {nCore++;}
1971   END {print     "cpass1 produced "nFiles" calib files, "nCore" core files";}' meta/cpass1.job*done 2>/dev/null
1972   awk 'BEGIN {nOK=0; nBAD=0; }
1973   /\/rec.log OK/ {nOK++;}
1974   /\/rec.log BAD/ {nBAD++;}
1975   /stderr BAD/ {if ($0 ~ /rec.log/){nBAD++;}}
1976   END {print     "cpass1 reco:  OK: "nOK"\tBAD: "nBAD;}' meta/cpass1.job*done 2>/dev/null
1977   awk 'BEGIN {nOK=0; nBAD=0; }
1978   /\/calib.log OK/ {nOK++;}
1979   /\/calib.log BAD/ {nBAD++;}
1980   END {print "cpass1 calib: OK: "nOK"\tBAD: "nBAD;}' meta/cpass1.job*done 2>/dev/null
1981
1982   awk 'BEGIN {nOK=0; nBAD=0; }
1983   /merge.log OK/ {nOK++;}
1984   /merge.log BAD/ {nBAD++;}
1985   END {print "cpass1 merge: OK: "nOK"\tBAD: "nBAD;}' meta/merge.cpass1*done 2>/dev/null
1986   awk 'BEGIN {nOK=0; nBAD=0; }
1987   /ocdb.log OK/ {nOK++;}
1988   /ocdb.log BAD/ {nBAD++;}
1989   END {print   "cpass1 OCDB:  OK: "nOK"\tBAD: "nBAD;}' meta/merge.cpass1*done 2>/dev/null
1990
1991   echo
1992   echo per run stats:
1993   /bin/ls -1 meta/merge.cpass0.run*.done | while read x
1994 do
1995   dir=$(goPrintValues calibfile - ${x})
1996   runNumber=$(guessRunNumber ${dir})
1997   [[ -z ${runNumber} ]] && continue
1998
1999   if $(/bin/ls meta/cpass0.job*.run${runNumber}.done &> /dev/null); then
2000     statusCPass0=( $(
2001     awk 'BEGIN {nOKrec=0;nBADrec=0;nOKcalib=0;nBADcalib=0;nOKstderr=0;nBADstderr=0;}
2002     /\/rec.log OK/ {nOKrec++;}
2003     /\/rec.log BAD/ {nBADrec++;}
2004     /stderr BAD/ {if ($0 ~ /rec.log/) {nBADrec++;} nBADstderr++;}
2005     /stderr OK/ {nOKstderr++;}
2006     /\/calib.log OK/ {nOKcalib++;}
2007     /\/calib.log BAD/ {nBADcalib++}
2008     END {print ""nOKrec" "nBADrec" "nOKstderr" "nBADstderr" "nOKcalib" "nBADcalib;}' meta/cpass0.job*.run${runNumber}.done 2>/dev/null
2009     ) )
2010   fi
2011
2012   if $(/bin/ls meta/cpass1.job*.run${runNumber}.done &>/dev/null); then
2013     statusCPass1=( $(
2014     awk 'BEGIN {nOKrec=0;nBADrec=0;nOKcalib=0;nBADcalib=0;nOKstderr=0;nBADstderr=0;nQAbarrelOK=0;nQAbarrelBAD=0;nQAouterOK=0;nQAouterBAD=0;}
2015     /\/rec.log OK/ {nOKrec++;}
2016     /\/rec.log BAD/ {nBADrec++;}
2017     /stderr BAD/ {if ($0 ~ /rec.log/) nBADrec++;nBADstderr++;}
2018     /stderr OK/ {nOKstderr++;}
2019     /\/calib.log OK/ {nOKcalib++;}
2020     /\/calib.log BAD/ {nBADcalib++}
2021     /\/qa_barrel.log OK/ {nQAbarrelOK++;}
2022     /\/qa_barrel.log BAD/ {nQAbarrelBAD++;}
2023     /\/qa_outer.log OK/ {nQAouterOK++;}
2024     /\/qa_outer.log BAD/ {nQAouterBAD++;}
2025     END {print ""nOKrec" "nBADrec" "nOKstderr" "nBADstderr" "nOKcalib" "nBADcalib" "nQAbarrelOK" "nQAbarrelBAD" "nQAouterOK" "nQAouterBAD;}' meta/cpass1.job*.run${runNumber}.done 2>/dev/null
2026     ) )
2027   fi
2028
2029   statusOCDBcpass0=$(awk '/ocdb.log/ {print $2} ' ${x} 2>/dev/null)
2030   statusOCDBcpass1=$(awk '/ocdb.log/ {print $2}' ${x/cpass0/cpass1} 2>/dev/null)
2031   statusQA=$(awk '/mergeMakeOCDB.log/ {print $2}' ${x/cpass0/cpass1} 2>/dev/null)
2032
2033   printf "%s\t ocdb.log cpass0: %s\t ocdb.log cpass1: %s\tqa.log:%s\t| cpass0: rec:%s/%s stderr:%s/%s calib:%s/%s cpass1: rec:%s/%s stderr:%s/%s calib:%s/%s QAbarrel:%s/%s QAouter:%s/%s\n" ${runNumber} ${statusOCDBcpass0} ${statusOCDBcpass1} ${statusQA} ${statusCPass0[0]} ${statusCPass0[1]} ${statusCPass0[2]} ${statusCPass0[3]} ${statusCPass0[4]} ${statusCPass0[5]} ${statusCPass1[0]} ${statusCPass1[1]} ${statusCPass1[2]} ${statusCPass1[3]} ${statusCPass1[4]} ${statusCPass1[5]} ${statusCPass1[6]} ${statusCPass1[7]} ${statusCPass1[8]} ${statusCPass1[9]}
2034 done
2035
2036   #make lists with output files - QA, trending, filtering and calibration
2037   ### wait for the merging of all runs to be over ###
2038   rm -f qa.list
2039   goPrintValues qafile qa.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2040   rm -f calib.list
2041   goPrintValues calibfile calib.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2042   rm -f trending.list
2043   goPrintValues trendingfile trending.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2044   rm -f filtering.list
2045   goPrintValues filteredTree filtering.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2046   #/bin/ls ${commonOutputPath}/*/cpass0/dcs* > cpass0.dcsTree.list
2047   rm -f cpass0.dcsTree.list
2048   goPrintValues dcsTree cpass0.dcsTree.list ${commonOutputPath}/meta/merge.cpass0.run*.done &>/dev/null
2049   #/bin/ls ${commonOutputPath}/*/cpass1/dcs* > cpass1.dcsTree.list
2050   rm -f cpass1.dcsTree.list
2051   goPrintValues dcsTree cpass1.dcsTree.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2052
2053   #merge trending
2054   rm -f ${commonOutputPath}/trending_merged.root
2055   goMerge trending.list ${commonOutputPath}/trending.root ${configFile} "${extraOpts[@]}" &> mergeTrending.log
2056
2057   goMakeSummaryTree ${commonOutputPath} 0
2058   goMakeSummaryTree ${commonOutputPath} 1
2059
2060   goCreateQAplots "qa.list" "${productionID}" "QAplots" "${configFile}" "${extraOpts[@]}" filteringList="filtering.list" &>createQAplots.log
2061
2062   #make a merged summary tree out of the QA trending, dcs trees and log summary trees
2063   goMakeMergedSummaryTree
2064
2065   #if set, email the summary
2066   [[ -n ${MAILTO} ]] && cat ${log} | mail -s "benchmark ${productionID} done" ${MAILTO}
2067
2068   return 0
2069 }
2070
2071 goMakeSummaryTree()
2072 {
2073   if [[ $# -lt 1 ]] ; then
2074     return
2075   fi
2076   #1. define vars/arrays
2077   DIR=${1} #use input or exec in current dir
2078   pass=${2-"0"} #pass from input
2079   outfile="summary_pass${pass}.tree"
2080   Ncolumns=0
2081   test -f ${outfile} && : >${outfile}
2082   errfile=${outfile/tree/err}
2083   test -f ${errfile} && : >${errfile}
2084
2085   declare -a counterString=(TOFevents TOFtracks TPCevents TPCtracks TRDevents TRDtracks T0events SDDevents SDDtracks MeanVertexevents)
2086   Ncounter=${#counterString[@]}
2087
2088   declare -a statusString=(TRDStatus TOFStatus TPCStatus T0Status MeanVertexStatus)
2089   Nstatus=${#statusString[@]}
2090
2091
2092   declare -a ratesString=(rec stderr calib qa_barrel qa_outer)
2093   Nrates=${#ratesString[@]}
2094
2095   runs=( $(ls -1 ${DIR}/meta/merge.cpass0* | while read x; do guessRunNumber $x; done) )
2096   Nruns=${#runs[@]}
2097
2098   echo -n runnumber/I >>${outfile}
2099   echo -n :cpass${pass}status/I >>${outfile}
2100   echo -n :cpass${pass}QAstatus/I >>${outfile}
2101   for i in ${ratesString[@]}; do
2102     echo -n :${i}OK/I >>${outfile}
2103     echo -n :${i}BAD/I >>${outfile}
2104
2105   done
2106   for i in ${counterString[@]} ${statusString[@]} ; do
2107     echo -n :${i}/I >>${outfile}
2108   done
2109   Ncolumns=$((2 + 2*Nrates + Ncounter + Nstatus))
2110   echo >> ${outfile}
2111
2112   #2. loop runs
2113
2114   for runnumber in ${runs[@]} ; do
2115
2116
2117
2118     filejob="${DIR}/meta/cpass${pass}.job*.run${runnumber}.done"
2119     filemerge="${DIR}/meta/merge.cpass${pass}.run${runnumber}.done"
2120     fileOCDB=$(grep /ocdb.log ${filemerge} | awk '{print $1}')
2121     if ! $(/bin/ls ${filemerge} &>/dev/null) ; then
2122       echo "${filemerge} does not exist!" >>${errfile}
2123       continue
2124     elif ! $(/bin/ls ${filejob} &>/dev/null) ; then
2125       echo "${filejob} does not exist!" >>${errfile}
2126       echo -n ${runnumber} >> ${outfile}
2127       for i in $(seq ${Ncolumns}) ; do
2128         echo -n "-1" >> ${outfile}
2129       done
2130       echo >> ${outfile}
2131       continue
2132     fi
2133     echo -n ${runnumber} >> ${outfile}
2134     #pass0status= grep '/ocdb.log' ${filemerge} |  cut -d' ' -f2 | tr OK x1 | tr BAD xx0 | tr -d 'x'
2135     passStatus=$(grep '/ocdb.log' ${filemerge} | grep OK | wc -l)
2136     echo -n " ${passStatus}" >> ${outfile}
2137     qaStatus=$(grep '/mergeMakeOCDB.log' ${filemerge} | grep OK | wc -l)
2138     echo -n " ${qaStatus}" >> ${outfile}
2139
2140
2141     #fill OK/BAD rates
2142     for i in $(seq 0 $((${Nrates}-1))) ; do
2143       var1=$(grep "/${ratesString[${i}]}.log" ${filejob} | grep OK | wc -l)
2144       var2=$(grep "/${ratesString[${i}]}.log" ${filejob} | grep BAD | wc -l)
2145
2146       if [[ ${ratesString[${i}]} == "stderr" ]] ; then
2147         var1=$(grep "stderr" ${filejob} | grep OK | wc -l)
2148         var2=$(grep "stderr" ${filejob} | grep "rec.log" | grep BAD | wc -l)
2149       fi
2150       echo -n " ${var1}" >> ${outfile}
2151       echo -n " ${var2}" >> ${outfile}
2152     done
2153
2154     if [[ -f ${fileOCDB} ]] ; then
2155       #fill counter
2156       for i in $(seq 0 $((${Ncounter}-1))) ; do
2157         var1=$(grep Monalisa ${fileOCDB} | grep ${counterString[${i}]} | cut -f2)
2158         echo -n " ${var1:-"-1"}" >> ${outfile}
2159       done
2160
2161       #fill status
2162       for i in $(seq 0 $((${Nstatus}-1))) ; do
2163         var1=$(grep "calibration status=" ${fileOCDB} | grep ${statusString[${i}]/Status/} | cut -d'=' -f2)
2164         echo -n " ${var1:-"-1"}" >> ${outfile}
2165       done
2166     fi
2167     echo >> ${outfile}
2168   done
2169
2170   return 0
2171 }
2172
2173 parseConfig()
2174 {
2175   configFile=${1}
2176   shift
2177   args=("$@")
2178
2179   #some defaults
2180   filteringFactorHighPt=1e2
2181   filteringFactorV0s=1e1
2182
2183   #first, source the config file
2184   if [ -f ${configFile} ]; then
2185     source ${configFile}
2186   else
2187     echo "config file ${configFile} not found!, skipping..."
2188   fi
2189
2190   #then, parse the options as theya override the optionf from file
2191   for ((i=0;i<${#args[@]};i++)) ;do
2192     local var="${args[i]%%=*}"
2193     local value="${args[i]#*=}"
2194     echo exporting ${var}="${value}"
2195     export ${var}="${value}"
2196     shift
2197   done
2198
2199   #export the aliroot function if defined to override normal behaviour
2200   [[ $(type -t aliroot) =~ "function" ]] && export -f aliroot
2201   return 0
2202 }
2203
2204 aliroot()
2205 {
2206   args="$@"
2207   if [[ -n ${valgrindCommand} ]]; then
2208     valgrindLogFile="cpu.txt"
2209     [[ "${args}" =~ rec ]] && valgrindLogFile="cpu_rec.txt"
2210     [[ "${args}}" =~ Calib ]] && valgrindLogFile="cpu_calib.txt"
2211     [[ -n ${valgrindCommand} ]] && valgrindCommand="${valgrindCommand} --log-file=${valgrindLogFile}"
2212     echo running ${valgrindCommand} aliroot ${args}
2213     ${valgrindCommand} aliroot ${args}
2214   else
2215     #to prevent an infinite recursion use "command aliroot" to disable
2216     #aliases and functions
2217     echo running command aliroot ${args}
2218     command aliroot "$@"
2219   fi
2220   return 0
2221 }
2222
2223 guessRunData()
2224 {
2225   #guess the period from the path, pick the rightmost one
2226   period=""
2227   runNumber=""
2228   year=""
2229   pass=""
2230   legoTrainRunNumber=""
2231   dataType=""
2232
2233   local shortRunNumber=""
2234   local IFS="/"
2235   declare -a path=( $1 )
2236   local dirDepth=$(( ${#path[*]}-1 ))
2237   i=0
2238   #for ((x=${dirDepth};x>=0;x--)); do
2239   for ((x=0;x<=${dirDepth};x++)); do
2240
2241     [[ $((x-1)) -ge 0 ]] && local fieldPrev=${path[$((x-1))]}
2242     local field=${path[${x}]}
2243     local fieldNext=${path[$((x+1))]}
2244
2245     [[ ${field} =~ ^[0-9]*$ && ${fieldNext} =~ (.*\.zip$|.*\.root$) ]] && legoTrainRunNumber=${field}
2246     [[ -n ${legoTrainRunNumber} && -z ${pass} ]] && pass=${fieldPrev}
2247     [[ ${field} =~ ^LHC[0-9][0-9][a-z].*$ ]] && period=${field%_*}
2248     [[ ${field} =~ ^000[0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && runNumber=${field#000}
2249     [[ ${field} =~ ^[0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && shortRunNumber=${field}
2250     [[ ${field} =~ ^20[0-9][0-9]$ ]] && year=${field}
2251     [[ ${field} =~ ^(^sim$|^data$) ]] && dataType=${field}
2252     (( i++ ))
2253   done
2254   [[ -z ${legoTrainRunNumber} ]] && pass=${path[$((dirDepth-1))]}
2255   [[ "${dataType}" =~ ^sim$ ]] && pass="passMC" && runNumber=${shortRunNumber}
2256
2257   #if [[ -z ${dataType} || -z ${year} || -z ${period} || -z ${runNumber}} || -z ${pass} ]];
2258   if [[ -z ${runNumber}} ]];
2259   then
2260     #error condition
2261     return 1
2262   else
2263     #ALL OK
2264     return 0
2265   fi
2266   return 0
2267 }
2268
2269 main "$@"