PWGPP/benchmark/benchmark.sh

   1 #!/bin/bash
   2 #include benchmark.config
   3
   4 # this script runs the CPass0/CPass1 train
   5 # produced OCDB updates are local
   6
   7 main()
   8 {
   9   #run in proper mode depending on the selection
  10   if [[ $# -lt 1 ]]; then
  11     if [[ ! "${0}" =~ "bash" ]]; then
  12       echo "uses makeflow:"
  13       echo " ${0} \"run\" productionID inputList configFile [extraOpts]"
  14       echo "uses a batch system (SGE):"
  15       echo " ${0} \"submit\" productionID inputList configFile [extraOpts]"
  16       echo "extraOpts if non-empty override the config file, e.g.:"
  17       echo " ${0} submit benchmark.list test1 benchmark.config runNumber=169123 nEvents=10"
  18     fi
  19     return
  20   fi
  21
  22   #define some aliases - default is to call one of the functions directly
  23   runMode=${1}
  24   umask 0002
  25   shift
  26   case ${runMode} in
  27     "CPass0") goCPass0 "$@";;
  28     "CPass1") goCPass1 "$@";;
  29     "MakeLocalOCDBaccessConfig") goMakeLocalOCDBaccessConfig "$@";;
  30     "MergeCPass0") goMergeCPass0 "$@";;
  31     "MergeCPass1") goMergeCPass1 "$@";;
  32     "MakeFilteredTrees") goMakeFilteredTrees "$@";;
  33     "MakeSummary") goMakeSummary "$@";;
  34     "run") goSubmitMakeflow "$@";;
  35     "submit") goSubmitBatch "$@";;
  36     "test") goTest "$@";;
  37     "GenerateMakeflow") goGenerateMakeflow "$@";;
  38     "PrintValues") goPrintValues "$@";;
  39     "CreateQAplots") goCreateQAplots "$@";;
  40     "WaitForOutput") goWaitForOutput "$@";;
  41     "Merge") goMerge "$@";;
  42     *)
  43       eval ${runMode} $@
  44     ;;
  45   esac
  46 }
  47
  48 generateMC()
  49 {
  50   #generate one raw chunk in current directory
  51   SEED=${JOB_ID}${SGE_TASK_ID}
  52   export CONFIG_SEED=${SEED}
  53   runNumber=${1}
  54   OCDBpath=${2}
  55   nEventsim=${3}
  56   if [[ -n ${pretend} ]]; then
  57     touch galice.root
  58   else
  59     if [[ -f sim.C && -f Config.C ]] ; then
  60         time aliroot -b -q -x sim.C\(${runNumber},\"${OCDBpath}\",${nEventsim}\) >sim.log 2>&1
  61         mv syswatch.log simwatch.log
  62     fi
  63   fi
  64 }
  65
  66 goCPass0()
  67 {
  68   umask 0002
  69
  70   targetDirectory=${1}
  71   inputList=${2}
  72   nEvents=${3}
  73   ocdbPath=${4}
  74   configFile=${5}
  75   runNumber=${6}
  76   jobindex=${7}
  77   shift 7
  78   parseConfig ${configFile} $@
  79
  80   #use the jobindex only if set and non-negative
  81   if [[ -z ${jobindex} || ${jobindex} -lt 0 ]]; then
  82     [[ -n "${LSB_JOBINDEX}" ]] && jobindex=${LSB_JOBINDEX}
  83     [[ -n "${SGE_TASK_ID}" ]] && jobindex=${SGE_TASK_ID}
  84   fi
  85
  86   [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
  87   doneFile="${commonOutputPath}/meta/cpass0.job${jobindex}.run${runNumber}.done"
  88
  89   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
  90
  91   if [[ -n ${ALIROOT_FORCE_COREDUMP} ]]; then
  92     ulimit -c unlimited
  93     export ALIROOT_FORCE_COREDUMP
  94   fi
  95
  96   #the contents of this is stored in the tree and used later (e.g. AliAnalysisTaskPIDResponse)!
  97   #at the QA stage the pass number is guessed from the path stored here.
  98   #The Format is:
  99   #Packages= ;OutputDir= ;LPMPass= ;TriggerAlias= ;LPMRunNumber= ;LPMProductionType= ;LPMInteractionType= ;LPMProductionTag= ;LPMAnchorRun= ;LPMAnchorProduction= ;LPMAnchorYear=
 100   export PRODUCTION_METADATA="OutputDir=cpass0"
 101
 102   #runCPassX/C expects the raw chunk to be linked in the run dir
 103   #despite it being accessed by the full path
 104   chunkName=${infile##*/}
 105   ln -s ${infile} ${runpath}/${chunkName}
 106
 107   outputDir=${targetDirectory}/${jobindex}_${chunkName%.*}
 108   mkdir -p ${outputDir}
 109   [[ ! -d ${outputDir} ]] && echo "cannot make ${outputDir}" && touch ${doneFile} && return 1
 110
 111   #runpath=${PWD}/rundir_cpass0_${runNumber}_${jobindex}
 112   runpath=${outputDir}
 113   [[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
 114   [[ ${reconstructInTemporaryDir} -eq 1 && -z ${TMPDIR} ]] && runpath=$(mktemp -d)
 115
 116   mkdir -p ${runpath}
 117   [[ ! -d ${runpath} ]] && echo "cannot make runpath ${runpath}" && touch ${doneFile} && return 1
 118   cd ${runpath}
 119
 120   if [[ "${inputList}" =~ \.root$ ]]; then
 121     infile=${inputList}
 122   else
 123     infile=$(sed -ne "${jobindex}p" ${inputList} | egrep '\s*\w*/\w*')
 124   fi
 125
 126   #####MC
 127   if [[ -n ${generateMC} ]]; then
 128     olddir=${PWD}
 129     outputDirMC=${commonOutputPath}/000${runNumber}/sim/${jobindex}
 130     simrunpath=${outputDirMC}
 131     [[ ${simulateInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && simrunpath=${TMPDIR}
 132     [[ ${simulateInTemporaryDir} -eq 1 && -z ${TMPDIR} ]] && simrunpath=$(mktemp -d)
 133     mkdir -p ${outputDirMC}
 134     mkdir -p ${simrunpath}
 135     cd ${simrunpath}
 136
 137     filesMC=(
 138               "${commonOutputPath}/sim.C"
 139               "${commonOutputPath}/rec.C"
 140               "${commonOutputPath}/Config.C"
 141               "${commonOutputPath}/OCDB_*.root"
 142     )
 143     for file in ${filesMC[*]}; do
 144       [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
 145     done
 146
 147     generateMC ${runNumber} ${ocdbPath} ${nEvents}
 148
 149     [[ ! "${simrunpath}" =~ "${outputDirMC}" ]] && mv * ${outputDirMC} #TODO check if it works
 150     cd ${olddir}
 151
 152     ln -s ${outputDirMC}/* ${runpath}/
 153
 154     inputList=${outputDirMC}/galice.root #TODO not valid outside shell !!!
 155     infile=""
 156   fi
 157   ######
 158
 159   [[ ! -f ${inputList} && -z ${pretend} ]] && echo "input file ${inputList} not found, exiting..." && touch ${doneFile} && return 1
 160
 161   logOutputDir=${runpath}
 162   [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir}
 163   [[ -z ${dontRedirectStdOutToLog} ]] && exec 1> ${logOutputDir}/stdout
 164   [[ -z ${dontRedirectStdOutToLog} ]] && exec 2> ${logOutputDir}/stderr
 165   echo "${0} $*"
 166
 167   echo "#####################"
 168   echo CPass0:
 169   echo JOB setup
 170   echo nEvents            ${nEvents}
 171   echo runNumber          ${runNumber}
 172   echo ocdbPath           ${ocdbPath}
 173   echo infile             ${infile}
 174   echo chunkName          ${chunkName}
 175   echo jobindex           ${jobindex}
 176   echo recoTriggerOptions ${recoTriggerOptions}
 177   echo targetDirectory    ${targetDirectory}
 178   echo commonOutputPath         ${commonOutputPath}
 179   echo doneFile      ${doneFile}
 180   echo runpath            ${runpath}
 181   echo outputDir          ${outputDir}
 182   echo ALICE_ROOT         ${ALICE_ROOT}
 183   echo PWD                ${PWD}
 184   echo "########## ###########"
 185
 186   alirootInfo > ALICE_ROOT.log
 187
 188   filesCPass0=(
 189                "${commonOutputPath}/runCPass0.sh"
 190                "${commonOutputPath}/recCPass0.C"
 191                "${commonOutputPath}/runCalibTrain.C"
 192                "${commonOutputPath}/localOCDBaccessConfig.C"
 193                "${commonOutputPath}/OCDB.root"
 194                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/runCPass0.sh"
 195                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/recCPass0.C"
 196                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/runCalibTrain.C"
 197   )
 198
 199   for file in ${filesCPass0[*]}; do
 200     [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
 201   done
 202
 203   echo "this directory (${PWD}) contents:"
 204   /bin/ls
 205   echo
 206   chmod u+x runCPass0.sh
 207
 208   sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh
 209
 210   if [[ -n ${postSetUpActionCPass0} ]]; then
 211     echo "running ${postSetUpActionCPass0}"
 212     eval ${postSetUpActionCPass0}
 213   fi
 214
 215   #run CPass0
 216   echo "${runpath}/runCPass0.sh ${infile} ${nEvents} ${runNumber} ${ocdbPath} ${recoTriggerOptions}"
 217   if [[ -n ${pretend} ]]; then
 218     touch AliESDs.root
 219     touch AliESDfriends.root
 220     touch AliESDfriends_v1.root
 221     touch rec.log
 222     touch calib.log
 223   else
 224     echo ./runCPass0.sh "${infile}" "${nEvents}" "${runNumber}" "${ocdbPath}" "${recoTriggerOptions}"
 225     ./runCPass0.sh "${infile}" "${nEvents}" "${runNumber}" "${ocdbPath}" "${recoTriggerOptions}"
 226   fi
 227
 228   #move stuff to final destination
 229   echo "this directory (${PWD}) contents:"
 230   /bin/ls
 231   echo
 232
 233   echo rm -f ./${chunkName}
 234   rm -f ./${chunkName}
 235   echo "cp --recursive ${runpath}/* ${outputDir}"
 236   cp -p --recursive ${runpath}/* ${outputDir}
 237   echo
 238
 239   #validate CPass0
 240   cd ${outputDir}
 241   touch ${doneFile}
 242   [[ -f ${outputDirMC}/galice.root ]] && echo "sim ${outputDirMC}/galice.root" >> ${doneFile}
 243   [[ -f AliESDfriends_v1.root ]] && echo "calibfile ${outputDir}/AliESDfriends_v1.root" >> ${doneFile}
 244   [[ -f AliESDs.root ]] && echo "esd ${outputDir}/AliESDs.root" >> ${doneFile}
 245   echo "dir ${outputDir}" >> ${doneFile}
 246   summarizeLogs >> ${doneFile}
 247
 248   [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
 249 }
 250
 251 goCPass1()
 252 {
 253   umask 0002
 254
 255   targetDirectory=${1}
 256   inputList=${2}
 257   nEvents=${3}
 258   ocdbPath=${4}
 259   configFile=${5}
 260   runNumber=${6}
 261   jobindex=${7}
 262   shift 7
 263   parseConfig ${configFile} $@
 264
 265   #use the jobindex only if set and non-negative
 266   if [[ -z ${jobindex} || ${jobindex} -lt 0 ]]; then
 267     [[ -n "${LSB_JOBINDEX}" ]] && jobindex=${LSB_JOBINDEX}
 268     [[ -n "${SGE_TASK_ID}" ]] && jobindex=${SGE_TASK_ID}
 269   fi
 270
 271   [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
 272   doneFile="${commonOutputPath}/meta/cpass1.job${jobindex}.run${runNumber}.done"
 273
 274   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
 275
 276   if [[ -n ${ALIROOT_FORCE_COREDUMP} ]]; then
 277     ulimit -c unlimited
 278     export ALIROOT_FORCE_COREDUMP
 279   fi
 280
 281   #the contents of this is stored in the tree and used later (e.g. AliAnalysisTaskPIDResponse)!
 282   #at the QA stage the pass number is guessed from the path stored here.
 283   #The Format is:
 284   #Packages= ;OutputDir= ;LPMPass= ;TriggerAlias= ;LPMRunNumber= ;LPMProductionType= ;LPMInteractionType= ;LPMProductionTag= ;LPMAnchorRun= ;LPMAnchorProduction= ;LPMAnchorYear=
 285   export PRODUCTION_METADATA="OutputDir=cpass1"
 286
 287   [[ ! -f ${inputList} && -z ${pretend} ]] && echo "input file ${inputList} not found, exiting..." && touch ${doneFile} && return 1
 288   if [[ "${inputList}" =~ \.root$ ]]; then
 289     infile=${inputList}
 290   else
 291     infile=$(sed -ne "${jobindex}p" ${inputList} | egrep '\s*\w*/\w*')
 292   fi
 293
 294   chunkName=${infile##*/}
 295   outputDir=${targetDirectory}/${jobindex}_${chunkName%.*}
 296   mkdir -p ${outputDir}
 297   [[ ! -d ${outputDir} ]] && echo "cannot make ${outputDir}" && touch ${doneFile} && return 1
 298
 299   #runpath=${PWD}/rundir_cpass1_${runNumber}_${jobindex}
 300   runpath=${outputDir}
 301   [[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
 302   [[ ${reconstructInTemporaryDir} -eq 1 && -z ${TMPDIR} ]] && runpath=$(mktemp -d)
 303
 304   #MC
 305   if [[ "${infile}" =~ galice\.root ]]; then
 306     ln -s ${inputList%/*}/* ${runpath}
 307     infile=""
 308   fi
 309
 310   #init the running path
 311   mkdir -p ${runpath}
 312   [[ ! -d ${runpath} ]] && echo "cannot make runpath ${runpath}" && touch ${doneFile} && return 1
 313   cd ${runpath}
 314
 315   #this is needed for runCPass1.sh
 316   ln -s ${infile} ${runpath}/${chunkName}
 317
 318   logOutputDir=${runpath}
 319   [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir}
 320   [[ -z ${dontRedirectStdOutToLog} ]] && exec 1> ${logOutputDir}/stdout
 321   [[ -z ${dontRedirectStdOutToLog} ]] && exec 2> ${logOutputDir}/stderr
 322   echo "${0} $*"
 323
 324   echo "#####################"
 325   echo CPass1:
 326   echo JOB setup
 327   echo nEvents            ${nEvents}
 328   echo runNumber          ${runNumber}
 329   echo ocdbPath           ${ocdbPath}
 330   echo infile             ${infile}
 331   echo chunkName          ${chunkName}
 332   echo jobindex           ${jobindex}
 333   echo recoTriggerOptions ${recoTriggerOptions}
 334   echo targetDirectory    ${targetDirectory}
 335   echo commonOutputPath         ${commonOutputPath}
 336   echo doneFile      ${doneFile}
 337   echo runpath            ${runpath}
 338   echo outputDir          ${outputDir}
 339   echo ALICE_ROOT         ${ALICE_ROOT}
 340   echo PWD                ${PWD}
 341   echo "########## ###########"
 342
 343   alirootInfo > ALICE_ROOT.log
 344
 345   filesCPass1=(
 346                "${commonOutputPath}/runCPass1.sh"
 347                "${commonOutputPath}/recCPass1.C"
 348                "${commonOutputPath}/recCPass1_OuterDet.C"
 349                "${commonOutputPath}/runCalibTrain.C"
 350                "${commonOutputPath}/QAtrain_duo.C"
 351                "${commonOutputPath}/localOCDBaccessConfig.C"
 352                "${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz"
 353                "${commonOutputPath}/OCDB.root"
 354                "${trustedQAtrainMacro}"
 355                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/runCPass1.sh"
 356                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/recCPass1.C"
 357                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/recCPass1_OuterDet.C"
 358                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/runCalibTrain.C"
 359                "${ALICE_ROOT}/ANALYSIS/macros/QAtrain_duo.C"
 360   )
 361
 362   for file in ${filesCPass1[*]}; do
 363     [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
 364   done
 365
 366   echo "this directory (${PWD}) contents:"
 367   /bin/ls
 368   echo
 369
 370   sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh
 371
 372   if [[ -n ${postSetUpActionCPass1} ]]; then
 373     echo "running ${postSetUpActionCPass1}"
 374     eval ${postSetUpActionCPass1}
 375     echo
 376   fi
 377
 378   #configure local OCDB storage from CPass0 (creates the localOCDBaccessConfig.C script)
 379   if [[ -f cpass0.localOCDB.${runNumber}.tgz ]]; then
 380     echo goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz"
 381     goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz"
 382   else
 383     echo "WARNING: file cpass0.localOCDB.${runNumber}.tgz not found!"
 384   fi
 385
 386   if [[ ! $(/bin/ls -1 OCDB/*/*/*/*.root 2>/dev/null) ]]; then
 387     echo "cpass0 produced no calibration! exiting..."
 388     touch ${doneFile}
 389     return 1
 390   fi
 391
 392   #create the Barrel and OuterDet directories for CPass1 and link the local OCDB directory
 393   #there to make the localOCDBaccessConfig.C file work, since it may point to the OCDB
 394   #entries using a relative path, e.g. local://./OCDB
 395   mkdir Barrel OuterDet
 396   ln -s ../OCDB Barrel/OCDB
 397   ln -s ../OCDB OuterDet/OCDB
 398
 399   #run CPass1
 400   chmod u+x runCPass1.sh
 401   echo "${runpath}/runCPass1.sh ${infile} ${nEvents} ${runNumber} ${ocdbPath} ${recoTriggerOptions}"
 402   if [[ -n ${pretend} ]]; then
 403     touch AliESDs_Barrel.root
 404     touch AliESDfriends_Barrel.root
 405     touch AliESDfriends_v1.root
 406     touch QAresults_barrel.root
 407     touch EventStat_temp_barrel.root
 408     touch AODtpITS.root
 409     touch AliESDs_Outer.root
 410     touch AliESDfriends_Outer.root
 411     touch QAresults_outer.root
 412     touch EventStat_temp_outer.root
 413     touch rec.log
 414     touch calib.log
 415     touch qa.log
 416   else
 417     ./runCPass1.sh "${infile}" "${nEvents}" "${runNumber}" "${ocdbPath}" "${recoTriggerOptions}"
 418   fi
 419
 420   ##handle possible crashes in QA (happens often in trunk)
 421   ##rerun QA with a trusted aliroot version
 422   #if [[ $(validateLog qa_barrel.log) ]]; then
 423   #  echo "qa_barrel.log not validated!"
 424   #fi
 425   #if [[ ! -f QAresults_barrel.root && -f ${setupTrustedAliROOTenvInCurrentShell} || $(validateLog qa_barrel.log) ]]; then
 426   #  echo "WARNING: using trusted QA aliroot ${ALICE_ROOT}"
 427   #  source ${setupTrustedAliROOTenvInCurrentShell}
 428   #  cd Barrel
 429   #  rm QAresults_barrel.root
 430   #  rm EventStat_temp_barrel.root
 431   #  rm AODtpITS.root
 432   #  [[ ! -f AliESDs.root ]] && ln -s ../AliESDs_Barrel.root AliESDs.root
 433   #  [[ ! -f AliESDfriends.root ]] && ln -s ../AliESDfriends_Barrel.root AliESDfriends.root
 434   #  if [[ -n ${trustedQAtrainMacro} ]]; then
 435   #    eval "cp ${trustedQAtrainMacro} QAtrain_duo_trusted.C"
 436   #  fi
 437   #  echo executing aliroot -b -q "QAtrain_duo_trusted.C(\"_barrel\",${runNumber},\"wn.xml\",0,\"${ocdbPath}\")"
 438   #  time aliroot -b -q "QAtrain_duo.C(\"_barrel\",${runNumber},\"wn.xml\",0,\"${ocdbPath}\")" &> ../qa_barrel_trusted.log
 439   #  cd ../
 440   #fi
 441   [[ ! -f AliESDs_Barrel.root && -f Barrel/AliESDs.root ]] && mv Barrel/AliESDs.root AliESDs_Barrel.root
 442   [[ ! -f AliESDfriends_Barrel.root && -f Barrel/AliESDfriends.root ]] && mv Barrel/AliESDfriends.root AliESDfriends_Barrel.root
 443   [[ ! -f AliESDfriends_v1.root && -f Barrel/AliESDfriends_v1.root ]] && mv Barrel/AliESDfriends_v1.root .
 444   [[ ! -f QAresults_barrel.root && -f Barrel/QAresults_barrel.root ]] && mv Barrel/QAresults_barrel.root .
 445   [[ ! -f AliESDs_Outer.root && -f OuterDet/AliESDs.root ]] && mv OuterDet/AliESDs.root AliESDs_Outer.root
 446   [[ ! -f AliESDfriends_Outer.root && -f OuterDet/AliESDfriends.root ]] && mv OuterDet/AliESDfriends.root AliESDfriends_Outer.root
 447   [[ ! -f QAresults_outer.root && -f OuterDet/QAresults_outer.root ]] && mv OuterDet/QAresults_outer.root .
 448
 449   #move stuff to final destination
 450   echo "this directory (${PWD}) contents:"
 451   /bin/ls
 452   echo rm -f ./${chunkName}
 453   rm -f ./${chunkName}
 454   echo "cp --recursive ${runpath}/* ${outputDir}"
 455   cp -pf --recursive ${runpath}/* ${outputDir}
 456   echo
 457
 458   #validate CPass1
 459   cd ${outputDir}
 460   touch ${doneFile}
 461   [[ -f AliESDs_Barrel.root ]] && echo "esd ${outputDir}/AliESDs_Barrel.root" >> ${doneFile}
 462   [[ -f AliESDfriends_v1.root ]] && echo "calibfile ${outputDir}/AliESDfriends_v1.root" >> ${doneFile}
 463   [[ -f QAresults_Barrel.root ]] && echo "qafile ${outputDir}/QAresults_Barrel.root" >> ${doneFile}
 464   [[ -f QAresults_Outer.root ]] && echo "qafile ${outputDir}/QAresults_Outer.root" >> ${doneFile}
 465   echo "dir ${outputDir}" >> ${doneFile}
 466   summarizeLogs >> ${doneFile}
 467
 468   [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
 469 }
 470
 471
 472 goMergeCPass0()
 473 {
 474   #
 475   # find the output files and merge them
 476   #
 477
 478   outputDir=${1}
 479   ocdbStorage=${2}
 480   configFile=${3}
 481   runNumber=${4}
 482   calibrationFilesToMerge=${5}  #can be a non-existent file, will then be produced on the fly
 483   shift 5
 484   parseConfig ${configFile} $@
 485
 486   [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
 487   doneFile="${commonOutputPath}/meta/merge.cpass0.run${runNumber}.done"
 488
 489   umask 0002
 490   ulimit -c unlimited
 491
 492   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
 493
 494   #runpath=${PWD}/rundir_cpass0_Merge_${runNumber}
 495   runpath=${outputDir}
 496   [[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
 497   [[ ${reconstructInTemporaryDir} -eq 1 && -z ${TMPDIR} ]] && runpath=$(mktemp -d)
 498
 499   mkdir -p ${runpath}
 500   [[ ! -d ${runpath} ]] && echo "not able to make the runpath ${runpath}" && touch ${doneFile} && return 1
 501   cd ${runpath}
 502
 503   logOutputDir=${runpath}
 504   [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir}
 505   [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/mergeMakeOCDB.log
 506   echo "${0} $*"
 507
 508   mergingScript="mergeMakeOCDB.byComponent.sh"
 509
 510   echo goMergeCPass0 SETUP:
 511   echo runNumber=${runNumber}
 512   echo outputDir=${outputDir}
 513   echo ocdbStorage=${ocdbStorage}
 514   echo calibrationFilesToMerge=${calibrationFilesToMerge}
 515   echo mergingScript=${mergingScript}
 516   echo commonOutputPath=${commonOutputPath}
 517   echo runpath=${runpath}
 518
 519   # copy files in case they are not already there
 520   filesMergeCPass0=(
 521                     "${commonOutputPath}/${calibrationFilesToMerge}"
 522                     "${commonOutputPath}/OCDB.root"
 523                     "${commonOutputPath}/localOCDBaccessConfig.C"
 524                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/mergeMakeOCDB.byComponent.sh"
 525                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/mergeByComponent.C"
 526                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/makeOCDB.C"
 527                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/merge.C"
 528                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/mergeMakeOCDB.sh"
 529   )
 530   for file in ${filesMergeCPass0[*]}; do
 531     [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
 532   done
 533
 534   sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh
 535
 536   alirootInfo > ALICE_ROOT.log
 537
 538   #
 539   echo "PWD"
 540   /bin/ls
 541   echo "PWD/.."
 542   /bin/ls ../
 543
 544
 545   #merge calibration
 546   chmod u+x ${mergingScript}
 547   mkdir -p ./OCDB
 548   if [[ ! -f ${calibrationFilesToMerge} ]]; then
 549     echo "/bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root > ${calibrationFilesToMerge}"
 550     /bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root 2>/dev/null > ${calibrationFilesToMerge}
 551   fi
 552
 553   echo "${mergingScript} ${calibrationFilesToMerge} ${runNumber} local://./OCDB ${ocdbStorage}"
 554   if [[ -n ${pretend} ]]; then
 555     touch CalibObjects.root
 556     touch ocdb.log
 557     touch merge.log
 558     mkdir -p ./OCDB/TPC/Calib/TimeGain/
 559     mkdir -p ./OCDB/TPC/Calib/TimeDrift/
 560     touch ./OCDB/TPC/Calib/TimeGain/someCalibObject_0-999999_cpass0.root
 561     touch ./OCDB/TPC/Calib/TimeDrift/otherCalibObject_0-999999_cpass0.root
 562   else
 563     ./${mergingScript} ${calibrationFilesToMerge} ${runNumber} "local://./OCDB" ${ocdbStorage}
 564   fi
 565
 566   ### produce the output
 567   #tar the produced OCDB for reuse
 568   tar czf ${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz ./OCDB
 569
 570   #produce the calib trees for expert QA (dcsTime.root)
 571   goMakeLocalOCDBaccessConfig ./OCDB
 572   echo aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")"
 573   aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")"
 574
 575   /bin/ls
 576
 577   #copy all to output dir
 578   cp -pf --recursive ${runpath}/* ${outputDir}
 579
 580   if [[ -n ${generateMC} ]]; then
 581     goPrintValues sim ${commonOutputPath}/meta/sim.run${runNumber}.list ${commonOutputPath}/meta/cpass0.job*.run${runNumber}.done
 582   fi
 583
 584   #validate merging cpass0
 585   cd ${outputDir}
 586   touch ${doneFile}
 587   [[ -f CalibObjects.root ]] && echo "calibfile ${outputDir}/CalibObjects.root" >> ${doneFile}
 588   [[ -f dcsTime.root ]] && echo "dcsTree ${outputDir}/dcsTime.root" >> ${doneFile}
 589   echo "dir ${outputDir}" >> ${doneFile}
 590   summarizeLogs >> ${doneFile}
 591
 592   [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
 593 }
 594
 595 goMergeCPass1()
 596 {
 597   #
 598   # find the output files and merge them
 599   #
 600
 601   outputDir=${1}
 602   ocdbStorage=${2}
 603   configFile=${3}
 604   runNumber=${4}
 605   calibrationFilesToMerge=${5}
 606   qaFilesToMerge=${6}
 607   shift 6
 608   parseConfig ${configFile} $@
 609
 610   [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
 611   doneFile="${commonOutputPath}/meta/merge.cpass1.run${runNumber}.done"
 612
 613   umask 0002
 614   ulimit -c unlimited
 615
 616   #clean up first:
 617   rm -f ${outputDir}/*.log
 618   rm -f ${outputDir}/*.root
 619   rm -f ${outputDir}/*done
 620
 621   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
 622
 623   #runpath=${PWD}/rundir_cpass1_Merge_${runNumber}
 624   runpath=${outputDir}
 625   [[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
 626   [[ ${reconstructInTemporaryDir} -eq 1 && -z ${TMPDIR} ]] && runpath=$(mktemp -d)
 627
 628   mkdir -p ${runpath}
 629   [[ ! -d ${runpath} ]] && echo "not able to make the runpath ${runpath}" && touch ${doneFile} && return 1
 630   cd ${runpath}
 631
 632   logOutputDir=${runpath}
 633   [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir}
 634   [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/mergeMakeOCDB.log
 635   echo "${0} $*"
 636
 637   calibrationOutputFileName='AliESDfriends_v1.root'
 638   qaOutputFileName='QAresults*.root'
 639   mergingScript="mergeMakeOCDB.byComponent.sh"
 640   #important to have the string "Stage.txt" in the filename to trigger the merging
 641   #it has to be a list of directories containing the files
 642   qaMergedOutputFileName="QAresults_merged.root"
 643
 644   echo goMergeCPass1 SETUP:
 645   echo runNumber=${runNumber}
 646   echo outputDir=${outputDir}
 647   echo ocdbStorage=${ocdbStorage}
 648   echo calibrationFilesToMerge=$calibrationFilesToMerge
 649   echo qaFilesToMerge=$qaFilesToMerge
 650   echo calibrationOutputFileName=${calibrationOutputFileName}
 651   echo mergingScript=${mergingScript}
 652
 653   # copy files in case they are not already there
 654   filesMergeCPass1=(
 655                     "${commonOutputPath}/${calibrationFilesToMerge}"
 656                     "${commonOutputPath}/${qaFilesToMerge}"
 657                     "${commonOutputPath}/OCDB.root"
 658                     "${commonOutputPath}/localOCDBaccessConfig.C"
 659                     "${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz"
 660                     "${commonOutputPath}/QAtrain_duo.C"
 661                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/mergeMakeOCDB.byComponent.sh"
 662                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/mergeByComponent.C"
 663                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/makeOCDB.C"
 664                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/merge.C"
 665                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/mergeMakeOCDB.sh"
 666                     "${trustedQAtrainMacro}"
 667                     "${ALICE_ROOT}/ANALYSIS/macros/QAtrain_duo.C"
 668   )
 669   for file in ${filesMergeCPass1[*]}; do
 670     [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
 671   done
 672
 673   sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh
 674
 675   #configure local OCDB storage from CPass0 (creates the localOCDBaccessConfig.C script)
 676   if [[ -f cpass0.localOCDB.${runNumber}.tgz ]]; then
 677     echo goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz"
 678     goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz"
 679   else
 680     echo "WARNING: file cpass0.localOCDB.${runNumber}.tgz not found!"
 681   fi
 682
 683   alirootInfo > ALICE_ROOT.log
 684
 685   #
 686   /bin/ls
 687
 688   #merge calibration
 689   chmod u+x ${mergingScript}
 690   mkdir -p OCDB
 691   if [[ ! -f ${calibrationFilesToMerge} ]]; then
 692     echo "/bin/ls -1 ${outputDir}/*/${calibrationOutputFileName} > ${calibrationFilesToMerge}"
 693     /bin/ls -1 ${outputDir}/*/${calibrationOutputFileName} 2>/dev/null > ${calibrationFilesToMerge}
 694   fi
 695
 696   echo "${mergingScript} ${calibrationFilesToMerge} ${runNumber} local://./OCDB ${ocdbStorage}"
 697   if [[ -n ${pretend} ]]; then
 698     touch CalibObjects.root
 699     touch ocdb.log
 700     touch merge.log
 701     touch cpass1.localOCDB.${runNumber}.tgz
 702   else
 703     ./${mergingScript} ${calibrationFilesToMerge} ${runNumber} "local://./OCDB" ${ocdbStorage}
 704   fi
 705
 706   tar czf ${commonOutputPath}/meta/cpass1.localOCDB.${runNumber}.tgz ./OCDB
 707
 708   #merge QA
 709   [[ -n ${AliAnalysisTaskFilteredTree_fLowPtTrackDownscaligF} ]] && export AliAnalysisTaskFilteredTree_fLowPtTrackDownscaligF
 710   [[ -n ${AliAnalysisTaskFilteredTree_fLowPtV0DownscaligF} ]] && export AliAnalysisTaskFilteredTree_fLowPtV0DownscaligF
 711
 712   if [[ ! -f ${qaFilesToMerge} ]]; then
 713     #find the files, but only store the directories (QAtrain_duo.C requires this)
 714     echo "/bin/ls -1 ${outputDir}/*/${qaOutputFileName} | while read x; do echo ${x%/*}; done | sort | uniq > ${qaFilesToMerge}"
 715     eval "/bin/ls -1 ${outputDir}/*/${qaOutputFileName}" | while read x; do echo ${x%/*}; done | sort | uniq > ${qaFilesToMerge}
 716   fi
 717
 718   #do the merge
 719   #echo aliroot -l -b -q "merge.C(\"${qaFilesToMerge}\",\"\",kFALSE,\"${qaMergedOutputFileName}\")"
 720   echo aliroot -b -q "QAtrain_duo.C(\"_barrel\",${runNumber},\"${qaFilesToMerge}\",1,\"${ocdbStorage}\")"
 721   if [[ -n ${pretend} ]]; then
 722     touch ${qaMergedOutputFileName}
 723     touch merge.log
 724     touch trending.root
 725   else
 726     #aliroot -l -b -q "merge.C(\"${qaFilesToMerge}\",\"\",kFALSE,\"${qaMergedOutputFileName}\")"
 727     aliroot -b -q "QAtrain_duo.C(\"_barrel\",${runNumber},\"${qaFilesToMerge}\",1,\"${ocdbStorage}\")" > mergeQA.log
 728     mv QAresults_barrel.root ${qaMergedOutputFileName}
 729     mv trending_barrel.root trending.root
 730   fi
 731
 732   #produce the calib trees for expert QA
 733   echo aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")"
 734   aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")"
 735
 736   /bin/ls
 737
 738   #copy all to output dir
 739   cp -pf --recursive ${runpath}/* ${outputDir}
 740
 741   #validate merge cpass1
 742   cd ${outputDir}
 743   touch ${doneFile}
 744   [[ -f CalibObjects.root ]] && echo "calibfile ${outputDir}/CalibObjects.root" >> ${doneFile}
 745   [[ -f ${qaMergedOutputFileName} ]] && echo "qafile ${outputDir}/${qaMergedOutputFileName}" >> ${doneFile}
 746   [[ -f trending.root ]] && echo "trendingfile ${outputDir}/trending.root" >> ${doneFile}
 747   [[ -f dcsTime.root ]] && echo "dcsTree ${outputDir}/dcsTime.root" >> ${doneFile}
 748   echo "dir ${outputDir}" >> ${doneFile}
 749   summarizeLogs >>  ${doneFile}
 750
 751   [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
 752 }
 753
 754 goMerge()
 755 {
 756   #generic root merge using CPass1 merge.C script
 757   inputList=${1}
 758   outputFile=${2}
 759   configFile=${3-"becnhmark.config"}
 760   shift 3
 761   parseConfig ${configFile} $@
 762
 763   [[ ! -f ${inputList} ]] && echo "inputList ${inputList} does not exist!" && return 1
 764   [[ ! -f ${configFile} ]] && echo "configFile ${configFile} does not exist!" && return 1
 765   umask 0002
 766   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
 767   rm -f ${outputFile}
 768   aliroot -b -q "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/merge.C(\"${inputList}\",\"\",kFALSE,\"${outputFile}\")" > merge_${inputList}.log
 769 }
 770
 771 goSubmitMakeflow()
 772 {
 773   #run
 774   productionID=${1}
 775   inputFileList=${2}
 776   configFile=${3}
 777   shift 3
 778   extraOpts="$@"
 779   parseConfig ${configFile} ${extraOpts}
 780
 781   [[ -z ${configFile} ]] && configFile="benchmark.config"
 782   [[ ! -f ${configFile} ]] && echo "no config file found (${configFile})" && return 1
 783
 784   #create the directopry for the metadata
 785   mkdir meta
 786
 787   if [[ ! $(which makeflow &>/dev/null) && -n ${makeflowPath} ]]; then
 788     echo "setting the makflow path from the config: "
 789     echo "  export PATH=${makeflowPath}:${PATH}"
 790     export PATH=${makeflowPath}:${PATH}
 791   fi
 792
 793   #submit - use makeflow if available, fall back to old stuff when makeflow not there
 794   if which makeflow; then
 795
 796     goGenerateMakeflow ${productionID} ${inputFileList} ${configFile} "${extraOpts}" > benchmark.makeflow
 797
 798     makeflow ${makeflowOptions} benchmark.makeflow
 799     cd ../
 800   else
 801     echo "no makeflow!"
 802   fi
 803 }
 804
 805 goGenerateMakeflow()
 806 {
 807   #generate the makeflow file
 808   productionID=${1}
 809   inputFileList=${2}
 810   configFile=${3}
 811   shift 3
 812   extraOpts="$@"
 813   parseConfig ${configFile} ${extraOpts}
 814
 815   [[ -z ${configFile} ]] && configFile="benchmark.config"
 816   [[ ! -f ${configFile} ]] && echo "no config file found (${configFile})" && return 1
 817
 818   commonOutputPath=${baseOutputDirectory}/${productionID}
 819
 820   #these files will be made a dependency - will be copied to the working dir of the jobs
 821   declare -a copyFiles
 822   inputFiles=(
 823               "OCDB.root"
 824               "localOCDBaccessConfig.C"
 825               "QAtrain_duo.C"
 826               "runCPass1.sh"
 827               "recCPass1.C"
 828               "recCPass1_OuterDet.C"
 829               "runCalibTrain.C"
 830               "runCPass0.sh"
 831               "recCPass0.C"
 832   )
 833   for file in ${inputFiles[*]}; do
 834     [[ -f ${file} ]] && copyFiles+=("${file}")
 835   done
 836
 837   #create the makeflow file
 838   [[ -n ${batchFlags} ]] && echo "BATCH_OPTIONS = ${batchFlags}"
 839   declare -a arr_cpass1_final
 840   listOfRuns[0]=${runNumber}
 841   [[ -z ${runNumber} ]] && listOfRuns=($(while read x; do guessRunNumber ${x}; done < ${inputFileList} | sort | uniq))
 842   runindex=0
 843   for runNumber in ${listOfRuns[*]}; do
 844     [[ -z ${runNumber} ]] && continue
 845     [[ ! ${runNumber} =~ ^[0-9]*[0-9]$ ]] && continue
 846
 847     unset arr_cpass0_outputs_onerun
 848     unset arr_cpass1_outputs_onerun
 849     declare -a arr_cpass0_outputs_onerun
 850     declare -a arr_cpass1_outputs_onerun
 851
 852     jobindex=0
 853     while read inputFile; do
 854       currentDefaultOCDB=${defaultOCDB}
 855       [[ -z ${autoOCDB} ]] && autoOCDB=1
 856       if [[ ${autoOCDB} -ne 0 ]]; then
 857         currentDefaultOCDB=$(setYear ${inputFile} ${defaultOCDB})
 858       fi
 859
 860       #CPass0
 861       arr_cpass0_outputs_onerun[${jobindex}]="meta/cpass0.job${jobindex}.run${runNumber}.done"
 862       echo "${arr_cpass0_outputs_onerun[${jobindex}]} : benchmark.sh ${configFile} ${copyFiles[@]}"
 863       echo " ${alirootEnv} ./benchmark.sh CPass0 ${commonOutputPath}/000${runNumber}/cpass0 ${inputFile} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} ${jobindex} ${extraOpts}"
 864       echo
 865
 866       #CPass1
 867       arr_cpass1_outputs_onerun[${jobindex}]="meta/cpass1.job${jobindex}.run${runNumber}.done"
 868       echo "${arr_cpass1_outputs_onerun[${jobindex}]} : benchmark.sh ${configFile} meta/cpass0.localOCDB.${runNumber}.tgz ${copyFiles[@]}"
 869       echo " ${alirootEnv} ./benchmark.sh CPass1 ${commonOutputPath}/000${runNumber}/cpass1 ${inputFile} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} ${jobindex} ${extraOpts}"
 870       echo
 871       ((jobindex++))
 872
 873     done< <(grep "/000${runNumber}/" ${inputFileList})
 874
 875     #CPass0 list of Calib files to merge
 876     arr_cpass0_calib_list[${runindex}]="meta/cpass0.calib.run${runNumber}.list"
 877     echo "${arr_cpass0_calib_list[${runindex}]} : benchmark.sh ${arr_cpass0_outputs_onerun[*]}"
 878     echo "  ./benchmark.sh PrintValues calibfile ${arr_cpass0_calib_list[${runindex}]} ${arr_cpass0_outputs_onerun[*]}"
 879     echo
 880
 881     #CPass0 merging
 882     arr_cpass0_final[${runindex}]="meta/merge.cpass0.run${runNumber}.done"
 883     echo "meta/cpass0.localOCDB.${runNumber}.tgz ${arr_cpass0_final[${runindex}]} : benchmark.sh ${configFile} ${arr_cpass0_calib_list[${runindex}]} ${copyFiles[@]}"
 884     echo " ${alirootEnv} ./benchmark.sh MergeCPass0 ${commonOutputPath}/000${runNumber}/cpass0 ${currentDefaultOCDB} ${configFile} ${runNumber} ${arr_cpass0_calib_list[${runindex}]} ${extraOpts}"
 885     echo
 886
 887     #CPass1 list of Calib/QA files to merge
 888     # the trick is to have the string "Stage.txt" in the file name of the list of directories with QA output to trigger
 889     # the production of the trending tree (only then the task->Finish() will be called in QAtrain_duo.C, on the grid
 890     # this corresponds to the last merging stage)
 891     arr_cpass1_calib_list[${runindex}]="meta/cpass1.calib.run${runNumber}.list"
 892     echo "${arr_cpass1_calib_list[${runindex}]} : benchmark.sh ${arr_cpass1_outputs_onerun[*]}"
 893     echo "  ./benchmark.sh PrintValues calibfile ${arr_cpass1_calib_list[${runindex}]} ${arr_cpass1_outputs_onerun[*]};"
 894     echo
 895     arr_cpass1_QA_list[${runindex}]="meta/cpass1.QA.run${runNumber}.lastMergingStage.txt.list"
 896     echo "${arr_cpass1_QA_list[${runindex}]}: benchmark.sh ${arr_cpass1_outputs_onerun[*]}"
 897     echo "  ./benchmark.sh PrintValues dir ${arr_cpass1_QA_list[${runindex}]} ${arr_cpass1_outputs_onerun[*]}"
 898     echo
 899
 900     #CPass1 merging
 901     arr_cpass1_final[${runindex}]="meta/merge.cpass1.run${runNumber}.done"
 902     echo "meta/cpass1.localOCDB.${runNumber}.tgz ${arr_cpass1_final[${runindex}]} :  benchmark.sh ${configFile} ${arr_cpass1_calib_list[${runindex}]} ${arr_cpass1_QA_list[${runindex}]} ${copyFiles[@]}"
 903     echo " ${alirootEnv} ./benchmark.sh MergeCPass1 ${commonOutputPath}/000${runNumber}/cpass1 ${currentDefaultOCDB} ${configFile} ${runNumber} ${arr_cpass1_calib_list[${runindex}]} ${arr_cpass1_QA_list[${runindex}]} ${extraOpts}"
 904     echo
 905
 906     arr_cpass1_ESD_list[${runindex}]="meta/cpass1.ESD.run${runNumber}.list"
 907     echo "${arr_cpass1_ESD_list[${runindex}]} : benchmark.sh ${arr_cpass1_outputs_onerun[*]}"
 908     echo "  ./benchmark.sh PrintValues esd ${arr_cpass1_ESD_list[${runindex}]} ${arr_cpass1_outputs_onerun[*]}"
 909
 910     if [ ${runESDfiltering} -eq 1 ]; then
 911       arr_cpass1_filtering[${runindex}]="meta/filtering.cpass1.run${runNumber}.done"
 912       echo "${arr_cpass1_filtering[${runindex}]} : benchmark.sh ${configFile} ${arr_cpass1_ESD_list[${runindex}]}"
 913       echo "  ${alirootEnv} ./benchmark.sh MakeFilteredTrees ${commonOutputPath}/000${runNumber}/cpass1 ${runNumber} ${arr_cpass1_ESD_list[${runindex}]} ${filteringFactorHighPt} ${filteringFactorV0s} ${currentDefaultOCDB} 1000000 0 10000000 0 ${configFile} AliESDs_Barrel.root ${extraOpts}"
 914     fi
 915
 916     ((runindex++))
 917   done
 918
 919   #Summary
 920   echo "summary.log : benchmark.sh ${configFile} ${arr_cpass1_final[*]} ${arr_cpass1_filtering[*]}"
 921   echo " LOCAL ./benchmark.sh MakeSummary ${configFile}"
 922   echo
 923
 924 }
 925
 926 goPrintValues()
 927 {
 928   #print the values given the key from any number of files (store in output file on second argument)
 929   if [[ $# -lt 3 ]]; then
 930     echo "goPrintValues key outputFile inputFiles"
 931     echo "if outputFile is \"-\" dont redirect to a file"
 932     return
 933   fi
 934   key=${1}
 935   outputFile=${2}
 936   [[ ${outputFile} =~ "-" ]] && outputFile=""
 937   shift 2 #remove 2 first arguments from arg list to only pass the input files to awk
 938   awk -v key=${key} '$0 ~ key" " {print $2}' "$@" | tee ${outputFile}
 939 }
 940
 941 goCreateQAplots()
 942 {
 943   umask 0002
 944   mergedQAfileList=${1}
 945   productionID=${2}
 946   outputDir=${3}
 947   configFile=${4}
 948   shift 4
 949   parseConfig ${configFile} ${@}
 950
 951   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
 952
 953   [[ -z ${logOutputDir} ]] && logOutputDir=${PWD}
 954   [[ -z ${dontRedirectStdOutToLog} ]] && exec 2>&1 > ${logOutputDir}/makeQAplots.log
 955   echo "${0} $*"
 956
 957   olddir=${PWD}
 958   mkdir -p ${outputDir}
 959   cd ${outputDir}
 960   [[ ! "${PWD}" = "${outputDir}" ]] && echo "cannot make ${outputDir}... exiting" && return 1
 961
 962   echo ${ALICE_ROOT}/PWGPP/QA/scripts/runQA.sh inputList=${mergedQAfileList}
 963   ${ALICE_ROOT}/PWGPP/QA/scripts/runQA.sh inputList=${mergedQAfileList}
 964
 965   cd ${olddir}
 966 }
 967
 968 goTest()
 969 {
 970   umask 0002
 971   exec 2>&1
 972   exec > >(tee test.log)
 973   echo "$@"
 974   echo something
 975 }
 976
 977 alirootInfo()
 978 {
 979   umask 0002
 980   # save aliroot repository info
 981   [[ -z "${ALICE_ROOT}" ]] && return 1
 982
 983   echo "\${ALICE_ROOT}=${ALICE_ROOT}"
 984   echo "\${ROOTSYS}=${ROOTSYS}"
 985   echo "\${PATH}=${PATH}"
 986   echo "\${LD_LIBRARY_PATH}=${LD_LIBRARY_PATH}"
 987
 988   pushd ${PWD}
 989   cd ${ALICE_ROOT}
 990   echo
 991
 992   currentBranch=$(git rev-parse --abbrev-ref HEAD)
 993   git status
 994   echo ""
 995   echo ""
 996   git diff ${currentBranch}
 997   popd
 998 }
 999
1000 setYear()
1001 {
1002   #set the year
1003   #  ${1} - year to be set
1004   #  ${2} - where to set the year
1005   year1=$(guessYear ${1})
1006   year2=$(guessYear ${2})
1007   local path=${2}
1008   [[ ${year1} -ne ${year2} && -n ${year2} && -n ${year1} ]] && path=${2/\/${year2}\//\/${year1}\/}
1009   echo ${path}
1010 }
1011
1012 guessPeriod()
1013 {
1014   #guess the period from the path, pick the rightmost one
1015   local IFS="/"
1016   declare -a path=( ${1} )
1017   local dirDepth=${#path[*]}
1018   for ((x=${dirDepth}-1;x>=0;x--)); do
1019     local field=${path[${x}]}
1020     [[ ${field} =~ ^LHC[0-9][0-9][a-z]$ ]] && period=${field} && break
1021   done
1022   echo ${period}
1023 }
1024
1025 guessYear()
1026 {
1027   #guess the year from the path, pick the rightmost one
1028   local IFS="/"
1029   declare -a path=( ${1} )
1030   local dirDepth=${#path[*]}
1031   for ((x=${dirDepth}-1;x>=0;x--)); do
1032     local field=${path[${x}]}
1033     [[ ${field} =~ ^20[0-9][0-9]$ ]] && year=${field} && break
1034   done
1035   echo ${year}
1036 }
1037
1038 guessRunNumber()
1039 {
1040   #guess the run number from the path, pick the rightmost one
1041   #works for /path/foo/000123456/bar/...
1042   #and       /path/foo.run123456.bar
1043   local IFS="/."
1044   declare -a path=( ${1} )
1045   local dirDepth=${#path[*]}
1046   for ((x=${dirDepth}-1;x>=0;x--)); do
1047     local field=${path[${x}]}
1048     field=${field/run/000}
1049     [[ ${field} =~ ^000[0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && runNumber=${field#000} && break
1050   done
1051   echo ${runNumber}
1052 }
1053
1054 validateLog()
1055 {
1056   log=${1}
1057   errorConditions=(
1058                     'There was a crash'
1059                     'floating'
1060                     'error while loading shared libraries'
1061                     'std::bad_alloc'
1062                     's_err_syswatch_'
1063                     'Thread [0-9]* (Thread'
1064                     'AliFatal'
1065                     'core dumped'
1066                     '\.C.*error:.*\.h: No such file'
1067   )
1068
1069   warningConditions=(
1070                      'This is serious !'
1071                      'rocVoltage out of range:'
1072   )
1073
1074   local logstatus=0
1075   local errorSummary=""
1076   local warningSummary=""
1077
1078   for ((i=0; i<${#errorConditions[@]};i++)); do
1079     local tmp=$(grep -m1 -e "${errorConditions[${i}]}" ${log})
1080     [[ -n ${tmp} ]] && tmp+=" : "
1081     errorSummary+=${tmp}
1082   done
1083
1084   for ((i=0; i<${#warningConditions[@]};i++)); do
1085     local tmp=$(grep -m1 -e "${warningConditions[${i}]}" ${log})
1086     [[ -n ${tmp} ]] && tmp+=" : "
1087     warningSummary+=${tmp}
1088   done
1089
1090   if [[ -n ${errorSummary} ]]; then
1091     echo "${errorSummary}"
1092     return 1
1093   fi
1094
1095   if [[ -n ${warningSummary} ]]; then
1096     echo "${warningSummary}"
1097     return 2
1098   fi
1099
1100   return 0
1101 }
1102
1103 summarizeLogs()
1104 {
1105   #print a summary of logs
1106   logFiles=(
1107             "*.log"
1108             "stdout"
1109             "stderr"
1110   )
1111
1112   #check logs
1113   local logstatus=0
1114   for log in ${logFiles[*]}; do
1115     finallog=${PWD%/}/${log}
1116     [[ ! -f ${log} ]] && continue
1117     errorSummary=$(validateLog ${log})
1118     validationStatus=$?
1119     if [[ ${validationStatus} -eq 0 ]]; then
1120       #in pretend mode randomly report an error in rec.log some cases
1121       if [[ -n ${pretend} && "${log}" == "rec.log" ]]; then
1122         [[ $(( ${RANDOM}%2 )) -ge 1 ]] && echo "${finallog} BAD random error" || echo "${finallog} OK"
1123       else
1124         echo "${finallog} OK"
1125       fi
1126     elif [[ ${validationStatus} -eq 1 ]]; then
1127       echo "${finallog} BAD ${errorSummary}"
1128     elif [[ ${validationStatus} -eq 2 ]]; then
1129       echo "${finallog} OK MWAH ${errorSummary}"
1130     fi
1131   done
1132
1133   #report core files
1134   while read x; do
1135     echo ${x}
1136     chmod 644 ${x}
1137     gdb --batch --quiet -ex "bt" -ex "quit" aliroot ${x} > stacktrace_${x//\//_}.log
1138   done < <(/bin/ls ${PWD}/*/core 2>/dev/null; /bin/ls ${PWD}/core 2>/dev/null)
1139
1140   return ${logstatus}
1141 }
1142
1143 spitOutLocalOCDBaccessConfig()
1144 {
1145   umask 0002
1146   #find ${1} -name "*root" | \
1147   /bin/ls -1 ${1}/*/*/*/*.root 2>/dev/null | \
1148   while read line
1149   do
1150     local tmp=${line#${1}}
1151     echo ${tmp%/*} | \
1152     awk -v ocdb=${1} '{print "  man->SetSpecificStorage(\""$1"\",\"local://"ocdb"\");"}'
1153   done
1154 }
1155
1156 goMakeLocalOCDBaccessConfig()
1157 {
1158   umask 0002
1159   # make a script that sets the specific storages form all the root files produced by CPass0
1160   local localOCDBpathCPass0=${1}
1161   local OCDBpathPrefix=${2}
1162   [[ -z ${OCDBpathPrefix} ]] && OCDBpathPrefix="."
1163
1164   if [[ -f ${localOCDBpathCPass0} && ${localOCDBpathCPass0} =~ \.tgz$ ]]; then
1165     tar xzf ${localOCDBpathCPass0}
1166     local localOCDBpathCPass0="${OCDBpathPrefix}/OCDB"
1167   fi
1168
1169   echo
1170   echo creating the specific storage script
1171   echo   localOCDBaccessConfig.C
1172   echo   based on OCDB: ${localOCDBaccessConfig}
1173   echo
1174
1175   local tempLocalOCDB=""
1176   if [[ -f localOCDBaccessConfig.C ]]; then
1177     tempLocalOCDB=$(mktemp)
1178     echo "egrep "SetSpecificStorage" localOCDBaccessConfig.C > ${tempLocalOCDB}"
1179     egrep "SetSpecificStorage" localOCDBaccessConfig.C > ${tempLocalOCDB}
1180   fi
1181
1182   echo "localOCDBaccessConfig()"                               >  localOCDBaccessConfig.C
1183   echo "{"                                                     >> localOCDBaccessConfig.C
1184   echo "  AliCDBManager* man = AliCDBManager::Instance();"     >> localOCDBaccessConfig.C
1185   spitOutLocalOCDBaccessConfig ${localOCDBpathCPass0}|sort|uniq  >> localOCDBaccessConfig.C
1186   [[ -f "${tempLocalOCDB}" ]] && cat ${tempLocalOCDB}              >> localOCDBaccessConfig.C
1187   echo "}"                                                     >> localOCDBaccessConfig.C
1188
1189   [[ -f "${tempLocalOCDB}" ]] && rm -f ${tempLocalOCDB}
1190
1191   if ! grep SetSpecificStorage localOCDBaccessConfig.C; then
1192     echo
1193     echo "!!!!!!! CPass0 produced no OCDB entries"
1194     return 1
1195   fi
1196 }
1197
1198 goMakeFilteredTrees()
1199 {
1200   outputDir=${1}
1201   runNumber=${2}
1202   #get path to input list
1203   inputListfiles=${3}
1204   #get scale number for tracks
1205   filterT=${4}
1206   #get scale number for V0s
1207   filterV=${5}
1208   #get OCDB path (optional)
1209   OCDBpath=${6}
1210   #get max number of files
1211   maxFiles=${7-"1000000"}
1212   #get offset of first file
1213   offsetFile=${8-"0"}
1214   #get max number of events
1215   maxEvents=${9-"30000000"}
1216   #get offset of first event
1217   offsetEvent=${10-"0"}
1218   configFile=${11-"benchmark.config"}
1219   esdFileName=${12-"AliESDs_Barrel.root"}
1220   shift 12
1221   parseConfig ${configFile} $@
1222
1223   commonOutputPath=${PWD}
1224   doneFile=${commonOutputPath}/meta/filtering.cpass1.run${runNumber}.done
1225
1226   runpath=${commonOutputPath}/rundir_filtering_run${runNumber}
1227
1228   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
1229
1230   mkdir -p ${outputDir}
1231   mkdir -p ${runpath}
1232
1233   cd ${runpath}
1234
1235   cat > filtering.log << EOF
1236   goMakeFilteredTrees config:
1237   runpath=${runpath}
1238   outputDir=${outputDir}
1239   commonOutputPath=${commonOutputPath}
1240   ALICE_ROOT=${ALICE_ROOT}
1241   PATH=${PATH}
1242   offsetEvent=$offsetEvent
1243   configFile=$configFile
1244   esdFileName=$esdFileName
1245 EOF
1246
1247   if [[ -z ${pretend} ]];then
1248     aliroot -l -b -q "${ALICE_ROOT}/PWGPP/macros/runFilteringTask.C(\"${inputListfiles}\",${filterT},${filterV},\"${OCDBpath}\",${maxFiles},${offsetFile},${maxEvents},${offsetEvent},\"${esdFileName}\")" &>> filtering.log
1249   else
1250     touch filtering.log FilterEvents_Trees.root
1251   fi
1252   pwd
1253   /bin/ls
1254   touch ${doneFile}
1255   summarizeLogs >>  ${doneFile}
1256   echo mv -f * ${outputDir}
1257   mv -f * ${outputDir}
1258   [[ -f ${outputDir}/FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFile}
1259   cd ${commonOutputPath}
1260   [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
1261 }
1262
1263 submit()
1264 {
1265   umask 0002
1266   [[ $# -ne 6 ]] && echo "6 args needed, you supplied $#" && return 1
1267   JobID=${1}
1268   startID=${2}
1269   endID=${3}
1270   waitForJOBID=${4}
1271   command=${5}
1272   commandArgs=${6}
1273
1274   [[ -z ${waitForJOBID} ]] && waitForJOBID=0
1275
1276   newFarm=$(which qsub|grep "^/usr/bin/qsub")
1277
1278   batchSystem="SGE"
1279
1280   if [[ -z "${newFarm}" ]]
1281   then
1282     #old LSF
1283     # submit it (as job array)
1284     nFiles=$(( ${endID}-${startID}+1 ))
1285     while [ ${startID} -le ${nFiles}  ] ; do
1286       if [ $(expr ${nFiles} - ${startID}) -gt 999 ] ; then
1287         endID=$(expr ${startID} + 999)
1288       else
1289         endID=${nFiles}
1290       fi
1291     if [[ ${waitForJOBID} -eq 0 ]]; then
1292         echo ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}"
1293         ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}"
1294       else
1295         echo ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -w "ended(${waitForJOBID})" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}"
1296         ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -w "ended(${waitForJOBID})" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}"
1297       fi
1298       startID=$(expr ${endID} + 1)
1299     done
1300   else
1301     #new SGE farm
1302     if [[ ${waitForJOBID} =~ "000" ]]; then
1303       echo ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" ${commandArgs}
1304       ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" ${commandArgs}
1305     else
1306       echo ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -hold_jid "${waitForJOBID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" ${commandArgs}
1307       ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -hold_jid "${waitForJOBID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" ${commandArgs}
1308     fi
1309   fi
1310 }
1311
1312 goSubmitBatch()
1313 {
1314   if [[ $# -lt 3 ]]; then
1315     echo "minimal use:"
1316     echo " ${0} submit fileList productionID configFile"
1317     return 0
1318   fi
1319
1320   productionID=${1}
1321   inputList=${2}
1322   configFile=${3:-"benchmark.config"}
1323   configFile=$(readlink -f ${configFile})
1324   shift 3
1325   extraOpts="$@"
1326   parseConfig ${configFile} ${extraOpts}
1327
1328   #redirect all output to submit.log
1329   echo "redirecting all output to ${PWD}/submit_${productionID//"/"/_}.log"
1330   exec 7>&1
1331   exec 1>submit_${productionID//"/"/_}.log 2>&1
1332
1333   umask 0002
1334   echo ${0}" submit $*"
1335   if [[ -z "${inputList}" || -z "${productionID}" ]]
1336   then
1337     echo
1338     echo " Usage: ${0} submit inputList productionID [configFile=benchmark.config]"
1339     echo
1340     return
1341   fi
1342
1343   # check if config file is there
1344   if [ ! -f ${configFile} ]; then
1345     echo "ERROR! Config File '${configFile}' not found" >&2
1346     return
1347   else
1348     echo "Using Config File: '${configFile}'"
1349   fi
1350
1351   # and print the configuration
1352   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
1353
1354   #move the script, config and some other stuff to ${commonOutputPath} first, then use them from there
1355   self=$(readlink -f "${0}")
1356   configPath=$(dirname ${configFile})
1357   export commonOutputPath=${baseOutputDirectory}/${productionID}
1358
1359   mkdir -p ${commonOutputPath}
1360   mkdir -p ${commonOutputPath}/logs
1361   mkdir -p ${commonOutputPath}/meta
1362
1363   cp ${self} ${commonOutputPath}
1364   cp ${configFile} ${commonOutputPath}
1365   cp ${inputList} ${commonOutputPath}
1366   self=${commonOutputPath}/${self##*/}
1367   chmod u+x ${self}
1368   configFile=${commonOutputPath}/${configFile##*/}
1369   inputList=${commonOutputPath}/${inputList##*/}
1370
1371   #convert to absolut pathnames
1372   inputList=$(readlink -f "${inputList}")
1373   #make list of runs
1374   if [[ -z ${runNumber} ]]; then
1375     listOfRuns=($(while read x; do guessRunNumber ${x}; done < ${inputList} | sort | uniq))
1376   else
1377     listOfRuns=${runNumber}
1378   fi
1379
1380   alirootSource=$(readlink -f "${alirootSource}")
1381
1382   echo ""
1383   echo "### BEGIN CONFIGURATION ###"
1384   echo ""
1385   echo "GENERAL:"
1386   echo ""
1387   echo "    productionID:    ${productionID}"
1388   echo "    batchCommand:    ${batchCommand}"
1389   echo "    batchFlags:      ${batchFlags}"
1390   echo "    alirootSource:   ${alirootSource}"
1391   echo "    ALICE_ROOT:      ${ALICE_ROOT}"
1392   echo "    ALIROOT_RELEASE: ${ALICE_RELEASE}"
1393   echo "    inputList:       ${inputList}"
1394   echo "    configPath:      ${configPath}"
1395   echo "    commonOutputPath:      ${commonOutputPath}"
1396   echo "    defaultOCDB:     ${defaultOCDB}"
1397   echo "      autoOCDB: ${autoOCDB}"
1398   echo "    recoTriggerOptions:   ${recoTriggerOptions}"
1399   echo "    runs:"
1400   echo "      ${listOfRuns[*]}"
1401   echo ""
1402   echo "THE TRAIN WILL RUN:"
1403
1404   if [ ${runCPass0reco} -eq 1 ]; then
1405     echo "    Pass0 - Recontruction"
1406   fi
1407
1408   if [ ${runCPass0MergeMakeOCDB} -eq 1 ]; then
1409     echo "    Pass0 - merging and OCDB export"
1410   fi
1411
1412   if [ ${runCPass1reco} -eq 1 ]; then
1413     echo "    Pass1 - Recontruction"
1414   fi
1415   if [ ${runCPass1MergeMakeOCDB} -eq 1 ]; then
1416     echo "    Pass1 - merging and OCDB export"
1417   fi
1418
1419   echo ""
1420   echo "LIMITS:"
1421   echo "    max. Events/Chunk:   ${nEvents}"
1422   echo "    max. Number of Chunks per Run:     ${nMaxChunks}"
1423   echo ""
1424   echo "### END CONFIGURATION ###"
1425   echo ""
1426
1427
1428   # check if input file is there
1429   if [ ! -f ${inputList} ]; then
1430     echo "ERROR! Input List '${inputList}' not found" >&2
1431     return
1432   fi
1433
1434   # define jobid (for dependent jobs)
1435   date=$(date +%Y_%m_%d_%H%M%S)
1436   #for each run we submit one jobarray:
1437   for runNumber in ${listOfRuns[*]}; do
1438
1439     [[ -z ${runNumber} ]] && continue
1440     [[ ! ${runNumber} =~ ^[0-9]*[0-9]$ ]] && continue
1441
1442     JOBpostfix="${productionID//"/"/_}_${runNumber}_${date}"
1443     JOBID1="p0_${JOBpostfix}"
1444     JOBID1wait="w0_${JOBpostfix}"
1445     JOBID2="m0_${JOBpostfix}"
1446     JOBID2wait="wm0_${JOBpostfix}"
1447     JOBID3="op0_${JOBpostfix}"
1448     JOBID3wait="wop0_${JOBpostfix}"
1449     JOBID4="p1_${JOBpostfix}"
1450     JOBID4wait="w1_${JOBpostfix}"
1451     JOBID5="m1_${JOBpostfix}"
1452     JOBID5wait="wm1_${JOBpostfix}"
1453     JOBID6="s1_${JOBpostfix}"
1454     JOBID6wait="ws1_${JOBpostfix}"
1455     JOBID7="QA_${JOBpostfix}"
1456     JOBmakeESDlistCPass1="lp1_${JOBpostfix}"
1457     JOBfilterESDcpass1="fp1_${JOBpostfix}"
1458     LASTJOB="000"
1459
1460     oneInputFile=$(egrep -m1 "${runNumber}\/" ${inputList})
1461
1462     currentDefaultOCDB=${defaultOCDB}
1463     [[ -z ${autoOCDB} ]] && autoOCDB=1
1464     if [[ ${autoOCDB} -ne 0 ]]; then
1465       currentDefaultOCDB=$(setYear ${oneInputFile} ${defaultOCDB})
1466     fi
1467
1468     echo "submitting run ${runNumber} with OCDB ${currentDefaultOCDB}"
1469
1470     ###############################################################################
1471     #run one chunk with valgrind:
1472     if [[ -n ${runValgrind} ]]; then
1473       [[ -z ${nEventsValgrind} ]] && nEventsValgrind=2
1474       [[ -z ${valgrindCommand} ]] && valgrindCommand="/usr/bin/valgrind --tool=callgrind --num-callers=40 -v --trace-children=yes"
1475       submit "valgrind" 1 1 000 "${alirootEnv} ${self}" "CPass0 ${commonOutputPath}/000${runNumber}/valgrind ${oneInputFile} ${nEventsValgrind} ${currentDefaultOCDB} ${configFile} ${runNumber} valgrind valgrindCommand=${valgrindCommand} ${extraOpts}"
1476     fi
1477
1478     ################################################################################
1479     ################################################################################
1480     # run the CPass0 if requested
1481
1482     if [ ${runCPass0reco} -eq 1 ]; then
1483
1484       echo
1485       echo "starting CPass0... for run ${runNumber}"
1486       echo
1487
1488       # create directory and copy all files that are needed
1489       targetDirectory="${commonOutputPath}/000${runNumber}/cpass0"
1490       mkdir -p ${targetDirectory}
1491
1492       filesCPass0=(
1493                     "${configPath}/runCPass0.sh"
1494                     "${configPath}/recCPass0.C"
1495                     "${configPath}/runCalibTrain.C"
1496                     "${configPath}/localOCDBaccessConfig.C"
1497                     "${configPath}/OCDB*.root"
1498                     "${configPath}/sim.C"
1499                     "${configPath}/rec.C"
1500                     "${configPath}/Config.C"
1501       )
1502       for file in ${filesCPass0[*]}; do
1503         [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
1504       done
1505
1506       localInputList=${targetDirectory}/${inputList##*/}
1507       [[ ! -f ${localInputList} ]] && egrep "\/000${runNumber}\/" ${inputList} > ${localInputList}
1508       # limit nFiles to nMaxChunks
1509       nFiles=$(wc -l < ${localInputList})
1510       [[ ${nFiles} -eq 0 ]] && echo "list contains ZERO files! exiting..." && return 1
1511       echo "raw files in list:    ${nFiles}"
1512       if [[ ${nMaxChunks} -gt 0 && ${nMaxChunks} -le ${nFiles} ]]; then
1513         nFiles=${nMaxChunks}
1514       fi
1515       echo "raw files to process: ${nFiles}"
1516       [[ -z "${percentProcessedFilesToContinue}" ]] && percentProcessedFilesToContinue=100
1517       if [[ ${percentProcessedFilesToContinue} -eq 100 ]]; then
1518         nFilesToWaitFor=${nFiles}
1519       else
1520         nFilesToWaitFor=$(( ${nFiles}-${nFiles}/(100/(100-${percentProcessedFilesToContinue})) ))
1521       fi
1522       echo "requested success rate is ${percentProcessedFilesToContinue}%"
1523       echo "merging will start after ${nFilesToWaitFor} jobs are done"
1524
1525       submit ${JOBID1} 1 ${nFiles} 000 "${alirootEnv} ${self}" "CPass0 ${targetDirectory} ${localInputList} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} -1 ${extraOpts}"
1526
1527       ## submit a monitoring job that will run until a certain number of jobs are done with reconstruction
1528       submit "${JOBID1wait}" 1 1 000 "${alirootEnv} ${self}" "WaitForOutput ${commonOutputPath} 'meta/cpass0.job*.run${runNumber}.done' ${nFilesToWaitFor} ${maxSecondsToWait} '-maxdepth 1'"
1529       LASTJOB=${JOBID1wait}
1530
1531     fi #end running CPass0
1532     ################################################################################
1533
1534
1535     ################################################################################
1536     # submit merging of CPass0, depends on the reconstruction
1537
1538     if [ ${runCPass0MergeMakeOCDB} -eq 1 ]; then
1539
1540       echo
1541       echo "submit CPass0 merging for run ${runNumber}"
1542       echo
1543
1544       targetDirectory="${commonOutputPath}/000${runNumber}/cpass0"
1545       mkdir -p ${targetDirectory}
1546
1547       #copy the scripts
1548       filesMergeCPass0=(
1549                         "${configPath}/OCDB.root"
1550                         "${configPath}/mergeMakeOCDB.byComponent.sh"
1551                         "${configPath}/mergeMakeOCDB.sh"
1552                         "${configPath}/localOCDBaccessConfig.C"
1553                         "${configPath}/mergeByComponent.C"
1554                         "${configPath}/makeOCDB.C"
1555                         "${configPath}/merge.C"
1556       )
1557       for file in ${filesMergeCPass0[*]}; do
1558         [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
1559       done
1560
1561       submit ${JOBID2} 1 1 "${LASTJOB}" "${alirootEnv} ${self}" "MergeCPass0 ${targetDirectory} ${currentDefaultOCDB} ${configFile} ${runNumber} cpass0.calib.run${runNumber}.list ${extraOpts}"
1562       LASTJOB=${JOBID2}
1563
1564       if [[ -n ${generateMC} ]]; then
1565         submit "mrl${JOBpostfix}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" "PrintValues sim ${commonOutputPath}/meta/sim.run${runNumber}.list ${commonOutputPath}/meta/cpass0.job*.run${runNumber}.done"
1566         LASTJOB="mrl${JOBpostfix}"
1567       fi
1568
1569       echo
1570     fi
1571     # end of merging CPass0
1572     ################################################################################
1573
1574     ################################################################################
1575     ################################################################################
1576     # run the CPass1 if requested
1577
1578     if [ ${runCPass1reco} -eq 1 ]; then
1579
1580       targetDirectory="${commonOutputPath}/000${runNumber}/cpass1"
1581       rm -f ${commonOutputPath}/meta/cpass1.job*.run${runNumber}.done
1582
1583       # safety feature: if we are re-running for any reason we want to delete the previous output first.
1584       [[ -d ${targetDirectory} ]] && rm -rf ${targetDirectory}/* && echo "removed old output at ${targetDirectory}/*"
1585
1586       echo
1587       echo "starting CPass1... for run ${runNumber}"
1588       echo
1589
1590       # create directory and copy all files that are needed
1591       mkdir -p ${targetDirectory}
1592
1593       filesCPass1=(
1594                     "${configPath}/runCPass1.sh"
1595                     "${configPath}/recCPass1.C"
1596                     "${configPath}/recCPass1_OuterDet.C"
1597                     "${configPath}/runCalibTrain.C"
1598                     "${configPath}/QAtrain_duo.C"
1599                     "${configPath}/localOCDBaccessConfig.C"
1600                     "${configPath}/OCDB.root"
1601       )
1602       for file in ${filesCPass1[*]}; do
1603         [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
1604       done
1605
1606       if [[ -n ${generateMC} ]]; then
1607         localInputList=${commonOutputPath}/meta/sim.run${runNumber}.list
1608       else
1609         localInputList=${targetDirectory}/${inputList##*/}
1610         [[ ! -f ${localInputList} ]] && egrep "\/000${runNumber}\/" ${inputList} > ${localInputList}
1611       fi
1612       # limit nFiles to nMaxChunks
1613       nFiles=$(wc -l < ${localInputList})
1614       [[ ${nFiles} -eq 0 ]] && echo "list contains ZERO files! continuing..." && continue
1615       echo "raw files in list:    ${nFiles}"
1616       if [[ ${nMaxChunks} -gt 0 && ${nMaxChunks} -le ${nFiles} ]]; then
1617         nFiles=${nMaxChunks}
1618       fi
1619       echo "raw files to process: ${nFiles}"
1620       [[ -z "${percentProcessedFilesToContinue}" ]] && percentProcessedFilesToContinue=100
1621       if [[ ${percentProcessedFilesToContinue} -eq 100 ]]; then
1622         nFilesToWaitFor=${nFiles}
1623       else
1624         nFilesToWaitFor=$(( ${nFiles}-${nFiles}/(100/(100-${percentProcessedFilesToContinue})) ))
1625       fi
1626       echo "requested success rate is ${percentProcessedFilesToContinue}%"
1627       echo "merging will start after ${nFilesToWaitFor} jobs are done"
1628
1629       submit ${JOBID4} 1 ${nFiles} "${LASTJOB}" "${alirootEnv} ${self}" "CPass1 ${targetDirectory} ${localInputList} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} -1 ${extraOpts}"
1630
1631       ################################################################################
1632       ## submit a monitoring job that will run until a certain number of jobs are done with reconstruction
1633       submit "${JOBID4wait}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" "WaitForOutput ${commonOutputPath} 'meta/cpass1.job*.run${runNumber}.done' ${nFilesToWaitFor} ${maxSecondsToWait} '-maxdepth 1'"
1634       LASTJOB=${JOBID4wait}
1635       ################################################################################
1636
1637       echo
1638     fi #end running CPass1
1639
1640     ################################################################################
1641     # submit merging of CPass1, depends on the reconstruction
1642     if [ ${runCPass1MergeMakeOCDB} -eq 1 ]; then
1643
1644       echo
1645       echo "submit CPass1 merging for run ${runNumber}"
1646       echo
1647
1648       targetDirectory="${commonOutputPath}/000${runNumber}/cpass1"
1649       rm -f ${commonOutputPath}/meta/merge.cpass1.run${runNumber}.done
1650       mkdir -p ${targetDirectory}
1651
1652       # copy files
1653       filesMergeCPass1=(
1654                         "${configPath}/OCDB.root"
1655                         "${configPath}/localOCDBaccessConfig.C"
1656                         "${configPath}/mergeMakeOCDB.byComponent.sh"
1657                         "${configPath}/mergeByComponent.C"
1658                         "${configPath}/makeOCDB.C"
1659                         "${configPath}/merge.C"
1660                         "${configPath}/mergeMakeOCDB.sh"
1661                         "${configPath}/QAtrain_duo.C"
1662       )
1663       for file in ${filesMergeCPass1[*]}; do
1664         [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
1665       done
1666
1667       submit "${JOBID5}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" "MergeCPass1 ${targetDirectory} ${currentDefaultOCDB} ${configFile} ${runNumber} cpass0.calib.run${runNumber}.list cpass1.QA.run${runNumber}.lastMergingStage.txt.list ${extraOpts}"
1668       LASTJOB=${JOBID5}
1669       echo
1670     fi
1671
1672     ##############################
1673     if [ ${runESDfiltering} -eq 1 ]; then
1674       rm -f ${commonOutputPath}/cpass1.ESD.run${runNumber}.list
1675       rm -f ${commonOutputPath}/meta/filtering.cpass1.run*.done
1676       echo
1677       echo submitting filtering for run ${runNumber}
1678       echo
1679       submit "${JOBmakeESDlistCPass1}" 1 1 "${LASTJOB}" "${self}" "PrintValues esd ${commonOutputPath}/meta/cpass1.ESD.run${runNumber}.list ${commonOutputPath}/meta/cpass1.job*.run${runNumber}.done "
1680       submit "${JOBfilterESDcpass1}" 1 1 "${JOBmakeESDlistCPass1}" "${alirootEnv} ${self}" "MakeFilteredTrees ${commonOutputPath}/000${runNumber}/cpass1 ${runNumber} ${commonOutputPath}/meta/cpass1.ESD.run${runNumber}.list ${filteringFactorHighPt} ${filteringFactorV0s} ${currentDefaultOCDB} 1000000 0 10000000 0 ${configFile} AliESDs_Barrel.root ${extraOpts}"
1681       LASTJOB=${JOBfilterESDcpass1}
1682     fi
1683
1684   done
1685
1686   #################################################################################
1687   #################################################################################
1688   if [ ${runESDfiltering} -eq 1 ]; then
1689     submit "${JOBID5wait}" 1 1 "${LASTJOB}" "${self}" "WaitForOutput ${commonOutputPath} 'meta/filtering.cpass1.run*.done' ${#listOfRuns[@]} ${maxSecondsToWait}"
1690   else
1691     submit "${JOBID5wait}" 1 1 "${LASTJOB}" "${self}" "WaitForOutput ${commonOutputPath} 'meta/merge.cpass1.run*.done' ${#listOfRuns[@]} ${maxSecondsToWait}"
1692   fi
1693   LASTJOB=${JOBID5wait}
1694
1695   [[ -z ${runMakeSummary} ]] && runMakeSummary=0
1696   if [ ${runMakeSummary} -eq 1 ]; then
1697     echo
1698     echo "submit make a summary"
1699     echo
1700
1701     submit "${JOBID6}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" "MakeSummary ${configFile}"
1702     LASTJOB=${JOBID6}
1703   fi
1704   #################################################################################
1705
1706   #restore stdout
1707   exec 1>&7 7>&-
1708   echo "jobs submitted."
1709 }
1710
1711 goWaitForOutput()
1712 {
1713   umask 0002
1714   [[ $# -lt 3 ]] && echo "goWaitForOutput() wrong number of arguments, exiting.." && return 1
1715   echo searchPath=${1}
1716   echo fileName=${2}
1717   echo numberOfFiles=${3}
1718   echo maxSecondsToWait=${4}
1719   searchPath=${1}
1720   fileName=${2}
1721   numberOfFiles=${3}
1722   maxSecondsToWait=${4}
1723   extraFindOptions=${5}
1724   echo "command to be executed: /bin/ls -1 ${searchPath}/${fileName} ${extraFindOptions}"
1725   [[ -z "${maxSecondsToWait}" ]] && maxSecondsToWait=$(( 3600*12 ))
1726   while true; do
1727     #n=$(find ${searchPath} ${extraFindOptions} -name "${fileName}" | wc -l)
1728     n=$(/bin/ls -1 ${searchPath}/${fileName} 2>/dev/null | wc -l)
1729     [[ ${n} -gt 0 ]] && echo "found ${n} X ${fileName}"
1730     [[ ${n} -ge ${numberOfFiles} ]] && break
1731     [[ ${SECONDS} -gt ${maxSecondsToWait} ]] && echo "timeout of ${maxSecondsToWait}s!" && break
1732     sleep 60
1733   done
1734   echo "DONE! exiting..."
1735 }
1736
1737 mergeSysLogs()
1738 {
1739   outputFile=${1}
1740   shift
1741   inputFiles="$@"
1742   i=0
1743   if ! ls -1 ${inputFiles} &>/dev/null; then echo "the files dont exist!: ${inputFiles}"; return 1; fi
1744   while read x; do
1745     runNumber=$(guessRunNumber ${x})
1746     [[ -z ${runNumber} ]] && echo "run number cannot be guessed for ${x}" && continue
1747     awk -v run=${runNumber} -v i=${i} 'NR > 1 {print run" "$0} NR==1 && i==0 {print "run/I:"$0}' ${x}
1748     (( i++ ))
1749   done < <(ls -1 ${inputFiles}) > ${outputFile}
1750 }
1751
1752 goMakeMergedSummaryTree()
1753 {
1754   # create list of calibration entries
1755   # takes no arguments, just run it in the base output
1756   # directory with the following files in the working directory
1757   #
1758   #  Calibration file lists:
1759   #       cpass0.dcsTree.list, cpass1.dcsTree.list
1760   #  QA trending root file:
1761   #       trending.root
1762   #
1763   #  Production infoo ascii files:
1764   #       summary_pass0.tree
1765   #       summary_pass1.tree
1766   #
1767
1768   [[ ! -f cpass0.dcsTree.list ]] && echo "no cpass0.dcsTree.list" && return 1
1769   [[ ! -f cpass1.dcsTree.list ]] && echo "no cpass1.dcsTree.list" && return 1
1770   [[ ! -f trending.root ]] && echo "no trending.root" && return 1
1771   [[ ! -f summary_pass0.tree ]] && echo "no summary_pass0.tree" && return 1
1772   [[ ! -f summary_pass1.tree ]] && echo "no summary_pass1.tree" && return 1
1773
1774   #first, dump the C macro to file
1775   cat << EOF > mergeTree.C
1776   //
1777   // Merge summary information
1778   // Following files are expected to be in the working directory
1779   //
1780   // Calibration file lists:
1781   //      cpass0.dcsTree.list, cpass1.dcsTree.list
1782   // QA trending root file:
1783   //      trending.root
1784   //
1785   // Production infoo ascii files:
1786   //      summary_pass0.tree
1787   //      summary_pass1.tree
1788   //
1789   void mergeTree(){
1790     //
1791     //
1792     //
1793     // Calibration values dump
1794     //
1795     AliXRDPROOFtoolkit::MakeTreeFromList("Calib.TPC.CPass0.root", "dcs","dcs","cpass0.dcsTree.list",1);
1796     AliXRDPROOFtoolkit::MakeTreeFromList("Calib.TPC.CPass1.root", "dcs","dcs","cpass1.dcsTree.list",1);
1797     //
1798     // Calibration status dump
1799     //
1800     TFile *fprod = TFile::Open("fproduction.root","recreate");
1801     TTree  tree0, tree1;
1802     tree0.ReadFile("summary_pass0.tree");
1803     tree1.ReadFile("summary_pass1.tree");
1804     tree0.Write("CPass0");
1805     tree1.Write("CPass1");
1806     fprod->Close();
1807     //
1808     //
1809     //
1810     TString stringSetup="";
1811     stringSetup+="1#QA.TPC#run#SummaryTPCQA/tpcQA#trending.root+";  //
1812     stringSetup+="1#Calib.TPC.CPass0#run#dcs#Calib.TPC.CPass0.root+";  //
1813     stringSetup+="1#Calib.TPC.CPass1#run#dcs#Calib.TPC.CPass1.root+";  //
1814     //
1815     stringSetup+="1#CPass0#runnumber#CPass0#fproduction.root+";  //
1816     stringSetup+="1#CPass1#runnumber#CPass1#fproduction.root+";  //
1817     //
1818     AliXRDPROOFtoolkit::JoinTreesIndex("outAll.root","joinAll","run",stringSetup.Data(), 1);
1819   }
1820 EOF
1821
1822   aliroot -b -q "mergeTree.C" > mergeTrees.log
1823   return $?
1824 }
1825
1826 goMakeSummary()
1827 {
1828   #all the final stuff goes in here for ease of use:
1829   # summary logs
1830   # qa plot making
1831   # final file lists
1832
1833   configFile=${1}
1834   shift 1
1835   extraOpts="$@"
1836   log="summary.log"
1837   productionID="qa"
1838   parseConfig ${configFile} ${extraOpts}
1839
1840   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
1841
1842   exec &> >(tee ${log})
1843
1844   [[ ! -f ${configFile} ]] && echo "no config file ${configFile}!" && return
1845
1846   [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
1847
1848   #summarize the global stuff
1849   echo "env script: ${alirootSource} ${alirootEnv}"
1850   echo "\$ALICE_ROOT=${ALICE_ROOT}"
1851   echo "commonOutputPath=${commonOutputPath}"
1852
1853   #summarize the stacktraces
1854   awk '
1855        BEGIN {
1856                print "frame/I:method/C:line/C:cpass/I:aliroot/I";
1857                RS="#[0-9]*";
1858                aliroot=0;
1859              }
1860              {
1861                if ($3 ~ /Ali*/) aliroot=1; else aliroot=0;
1862                gsub("#","",RT);
1863                if ($NF!="" && RT!="" && $3!="") print RT" "$3" "$NF" "0" "aliroot
1864              }
1865       ' 000*/cpass0/*/stacktrace* 2>/dev/null > stacktrace.tree
1866   awk '
1867        BEGIN {
1868                RS="#[0-9]*";
1869                aliroot=0;
1870              }
1871              {
1872                if ($3 ~ /Ali*/) aliroot=1; else aliroot=0;
1873                gsub("#","",RT);
1874                if ($NF!="" && RT!="" && $3!="") print RT" "$3" "$NF" "1" "aliroot
1875              }
1876       ' 000*/cpass1/*/stacktrace* 2>/dev/null >> stacktrace.tree
1877
1878   echo total numbers for the production:
1879   echo
1880   awk 'BEGIN {nFiles=0;nCore=0;}
1881   /^calibfile/ {nFiles++;}
1882   /core dumped/ {nCore++i;}
1883   END {print     "cpass0 produced "nFiles" calib files, "nCore" core files";}' meta/cpass0.job*done 2>/dev/null
1884   awk 'BEGIN {nOK=0; nBAD=0; }
1885   /\/rec.log OK/ {nOK++;}
1886   /\/rec.log BAD/ {nBAD++;}
1887   /stderr BAD/ {if ($0 ~ /rec.log/){nBAD++;}}
1888   END {print     "cpass0 reco:  OK: "nOK"\tBAD: "nBAD;}' meta/cpass0.job*done 2>/dev/null
1889   awk 'BEGIN {nOK=0; nBAD=0; }
1890   /\/calib.log OK/ {nOK++;}
1891   /\/calib.log BAD/ {nBAD++;}
1892   END {print "cpass0 calib: OK: "nOK"\tBAD: "nBAD;}' meta/cpass0.job*done 2>/dev/null
1893
1894   awk 'BEGIN {nOK=0; nBAD=0; }
1895   /merge.log OK/ {nOK++;}
1896   /merge.log BAD/ {nBAD++;}
1897   END {print "cpass0 merge: OK: "nOK"\tBAD: "nBAD;}' meta/merge.cpass0*done 2>/dev/null
1898   awk 'BEGIN {nOK=0; nBAD=0; }
1899   /ocdb.log OK/ {nOK++;}
1900   /ocdb.log BAD/ {nBAD++;}
1901   END {print   "cpass0 OCDB:  OK: "nOK"\tBAD: "nBAD;}' meta/merge.cpass0*done 2>/dev/null
1902
1903   echo
1904   awk 'BEGIN {nFiles=0;nCore=0;}
1905   /^calibfile/ {nFiles++;}
1906   /core dumped/ {nCore++;}
1907   END {print     "cpass1 produced "nFiles" calib files, "nCore" core files";}' meta/cpass1.job*done 2>/dev/null
1908   awk 'BEGIN {nOK=0; nBAD=0; }
1909   /\/rec.log OK/ {nOK++;}
1910   /\/rec.log BAD/ {nBAD++;}
1911   /stderr BAD/ {if ($0 ~ /rec.log/){nBAD++;}}
1912   END {print     "cpass1 reco:  OK: "nOK"\tBAD: "nBAD;}' meta/cpass1.job*done 2>/dev/null
1913   awk 'BEGIN {nOK=0; nBAD=0; }
1914   /\/calib.log OK/ {nOK++;}
1915   /\/calib.log BAD/ {nBAD++;}
1916   END {print "cpass1 calib: OK: "nOK"\tBAD: "nBAD;}' meta/cpass1.job*done 2>/dev/null
1917
1918   awk 'BEGIN {nOK=0; nBAD=0; }
1919   /merge.log OK/ {nOK++;}
1920   /merge.log BAD/ {nBAD++;}
1921   END {print "cpass1 merge: OK: "nOK"\tBAD: "nBAD;}' meta/merge.cpass1*done 2>/dev/null
1922   awk 'BEGIN {nOK=0; nBAD=0; }
1923   /ocdb.log OK/ {nOK++;}
1924   /ocdb.log BAD/ {nBAD++;}
1925   END {print   "cpass1 OCDB:  OK: "nOK"\tBAD: "nBAD;}' meta/merge.cpass1*done 2>/dev/null
1926
1927   echo
1928   echo per run stats:
1929   /bin/ls -1 meta/merge.cpass0.run*.done | while read x
1930 do
1931   dir=$(goPrintValues calibfile - ${x})
1932   runNumber=$(guessRunNumber ${dir})
1933   [[ -z ${runNumber} ]] && continue
1934
1935   if $(/bin/ls meta/cpass0.job*.run${runNumber}.done &> /dev/null); then
1936     statusCPass0=( $(
1937     awk 'BEGIN {nOKrec=0;nBADrec=0;nOKcalib=0;nBADcalib=0;nOKstderr=0;nBADstderr=0;}
1938     /\/rec.log OK/ {nOKrec++;}
1939     /\/rec.log BAD/ {nBADrec++;}
1940     /stderr BAD/ {if ($0 ~ /rec.log/) {nBADrec++;} nBADstderr++;}
1941     /stderr OK/ {nOKstderr++;}
1942     /\/calib.log OK/ {nOKcalib++;}
1943     /\/calib.log BAD/ {nBADcalib++}
1944     END {print ""nOKrec" "nBADrec" "nOKstderr" "nBADstderr" "nOKcalib" "nBADcalib;}' meta/cpass0.job*.run${runNumber}.done 2>/dev/null
1945     ) )
1946   fi
1947
1948   if $(/bin/ls meta/cpass1.job*.run${runNumber}.done &>/dev/null); then
1949     statusCPass1=( $(
1950     awk 'BEGIN {nOKrec=0;nBADrec=0;nOKcalib=0;nBADcalib=0;nOKstderr=0;nBADstderr=0;nQAbarrelOK=0;nQAbarrelBAD=0;nQAouterOK=0;nQAouterBAD=0;}
1951     /\/rec.log OK/ {nOKrec++;}
1952     /\/rec.log BAD/ {nBADrec++;}
1953     /stderr BAD/ {if ($0 ~ /rec.log/) nBADrec++;nBADstderr++;}
1954     /stderr OK/ {nOKstderr++;}
1955     /\/calib.log OK/ {nOKcalib++;}
1956     /\/calib.log BAD/ {nBADcalib++}
1957     /\/qa_barrel.log OK/ {nQAbarrelOK++;}
1958     /\/qa_barrel.log BAD/ {nQAbarrelBAD++;}
1959     /\/qa_outer.log OK/ {nQAouterOK++;}
1960     /\/qa_outer.log BAD/ {nQAouterBAD++;}
1961     END {print ""nOKrec" "nBADrec" "nOKstderr" "nBADstderr" "nOKcalib" "nBADcalib" "nQAbarrelOK" "nQAbarrelBAD" "nQAouterOK" "nQAouterBAD;}' meta/cpass1.job*.run${runNumber}.done 2>/dev/null
1962     ) )
1963   fi
1964
1965   statusOCDBcpass0=$(awk '/ocdb.log/ {print $2} ' ${x} 2>/dev/null)
1966   statusOCDBcpass1=$(awk '/ocdb.log/ {print $2}' ${x/cpass0/cpass1} 2>/dev/null)
1967   statusQA=$(awk '/mergeMakeOCDB.log/ {print $2}' ${x/cpass0/cpass1} 2>/dev/null)
1968
1969   printf "%s\t ocdb.log cpass0: %s\t ocdb.log cpass1: %s\tqa.log:%s\t| cpass0: rec:%s/%s stderr:%s/%s calib:%s/%s cpass1: rec:%s/%s stderr:%s/%s calib:%s/%s QAbarrel:%s/%s QAouter:%s/%s\n" ${runNumber} ${statusOCDBcpass0} ${statusOCDBcpass1} ${statusQA} ${statusCPass0[0]} ${statusCPass0[1]} ${statusCPass0[2]} ${statusCPass0[3]} ${statusCPass0[4]} ${statusCPass0[5]} ${statusCPass1[0]} ${statusCPass1[1]} ${statusCPass1[2]} ${statusCPass1[3]} ${statusCPass1[4]} ${statusCPass1[5]} ${statusCPass1[6]} ${statusCPass1[7]} ${statusCPass1[8]} ${statusCPass1[9]}
1970 done
1971
1972   #make lists with output files - QA, trending, filtering and calibration
1973   ### wait for the merging of all runs to be over ###
1974   rm -f qa.list
1975   goPrintValues qafile qa.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
1976   rm -f calib.list
1977   goPrintValues calibfile calib.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
1978   rm -f trending.list
1979   goPrintValues trendingfile trending.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
1980   rm -f filtering.list
1981   goPrintValues filteredTree filtering.list ${commonOutputPath}/meta/filtering.cpass1.run*.done &>/dev/null
1982   #/bin/ls ${commonOutputPath}/*/cpass0/dcs* > cpass0.dcsTree.list
1983   rm -f cpass0.dcsTree.list
1984   goPrintValues dcsTree cpass0.dcsTree.list ${commonOutputPath}/meta/merge.cpass0.run*.done &>/dev/null
1985   #/bin/ls ${commonOutputPath}/*/cpass1/dcs* > cpass1.dcsTree.list
1986   rm -f cpass1.dcsTree.list
1987   goPrintValues dcsTree cpass1.dcsTree.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
1988
1989   #merge trending
1990   rm -f ${commonOutputPath}/trending_merged.root
1991   goMerge trending.list ${commonOutputPath}/trending.root ${configFile} ${extraOpts} &> mergeTrending.log
1992
1993   goMakeSummaryTree ${commonOutputPath} 0
1994   goMakeSummaryTree ${commonOutputPath} 1
1995
1996   goCreateQAplots qa.list ${productionID} QAplots ${configFile} ${extraOpts} &>createQAplots.log
1997
1998   #make a merged summary tree out of the QA trending, dcs trees and log summary trees
1999   goMakeMergedSummaryTree
2000
2001   #if set, email the summary
2002   [[ -n ${mailSummaryTo} ]] && cat ${log} | mail -s "benchmark ${productionID} done" ${mailSummaryTo}
2003
2004   return 0
2005 }
2006
2007 goMakeSummaryTree()
2008 {
2009   if [[ $# -lt 1 ]] ; then
2010     return
2011   fi
2012   #1. define vars/arrays
2013   DIR=${1} #use input or exec in current dir
2014   pass=${2-"0"} #pass from input
2015   outfile="summary_pass${pass}.tree"
2016   Ncolumns=0
2017   test -f ${outfile} && : >${outfile}
2018   errfile=${outfile/tree/err}
2019   test -f ${errfile} && : >${errfile}
2020
2021   declare -a counterString=(TOFevents TOFtracks TPCevents TPCtracks TRDevents TRDtracks T0events SDDevents SDDtracks MeanVertexevents)
2022   Ncounter=${#counterString[@]}
2023
2024   declare -a statusString=(TRDStatus TOFStatus TPCStatus T0Status MeanVertexStatus)
2025   Nstatus=${#statusString[@]}
2026
2027
2028   declare -a ratesString=(rec stderr calib qa_barrel qa_outer)
2029   Nrates=${#ratesString[@]}
2030
2031   runs=( $(ls -1 ${DIR}/meta/merge.cpass0* | while read x; do guessRunNumber $x; done) )
2032   Nruns=${#runs[@]}
2033
2034   echo -n runnumber/I >>${outfile}
2035   echo -n :cpass${pass}status/I >>${outfile}
2036   echo -n :cpass${pass}QAstatus/I >>${outfile}
2037   for i in ${ratesString[@]}; do
2038     echo -n :${i}OK/I >>${outfile}
2039     echo -n :${i}BAD/I >>${outfile}
2040
2041   done
2042   for i in ${counterString[@]} ${statusString[@]} ; do
2043     echo -n :${i}/I >>${outfile}
2044   done
2045   Ncolumns=$((2 + 2*Nrates + Ncounter + Nstatus))
2046   echo >> ${outfile}
2047
2048   #2. loop runs
2049
2050   for runnumber in ${runs[@]} ; do
2051
2052
2053
2054     filejob="${DIR}/meta/cpass${pass}.job*.run${runnumber}.done"
2055     filemerge="${DIR}/meta/merge.cpass${pass}.run${runnumber}.done"
2056     fileOCDB=$(grep /ocdb.log ${filemerge} | awk '{print $1}')
2057     if ! $(/bin/ls ${filemerge} &>/dev/null) ; then
2058       echo "${filemerge} does not exist!" >>${errfile}
2059       continue
2060     elif ! $(/bin/ls ${filejob} &>/dev/null) ; then
2061       echo "${filejob} does not exist!" >>${errfile}
2062       echo -n ${runnumber} >> ${outfile}
2063       for i in $(seq ${Ncolumns}) ; do
2064         echo -n "-1" >> ${outfile}
2065       done
2066       echo >> ${outfile}
2067       continue
2068     fi
2069     echo -n ${runnumber} >> ${outfile}
2070     #pass0status= grep '/ocdb.log' ${filemerge} |  cut -d' ' -f2 | tr OK x1 | tr BAD xx0 | tr -d 'x'
2071     passStatus=$(grep '/ocdb.log' ${filemerge} | grep OK | wc -l)
2072     echo -n " ${passStatus}" >> ${outfile}
2073     qaStatus=$(grep '/mergeMakeOCDB.log' ${filemerge} | grep OK | wc -l)
2074     echo -n " ${qaStatus}" >> ${outfile}
2075
2076
2077     #fill OK/BAD rates
2078     for i in $(seq 0 $((${Nrates}-1))) ; do
2079       var1=$(grep "/${ratesString[${i}]}.log" ${filejob} | grep OK | wc -l)
2080       var2=$(grep "/${ratesString[${i}]}.log" ${filejob} | grep BAD | wc -l)
2081
2082       if [[ ${ratesString[${i}]} == "stderr" ]] ; then
2083         var1=$(grep "stderr" ${filejob} | grep OK | wc -l)
2084         var2=$(grep "stderr" ${filejob} | grep "rec.log" | grep BAD | wc -l)
2085       fi
2086       echo -n " ${var1}" >> ${outfile}
2087       echo -n " ${var2}" >> ${outfile}
2088     done
2089
2090     if [[ -f ${fileOCDB} ]] ; then
2091       #fill counter
2092       for i in $(seq 0 $((${Ncounter}-1))) ; do
2093         var1=$(grep Monalisa ${fileOCDB} | grep ${counterString[${i}]} | cut -f2)
2094         echo -n " ${var1:-"-1"}" >> ${outfile}
2095       done
2096
2097       #fill status
2098       for i in $(seq 0 $((${Nstatus}-1))) ; do
2099         var1=$(grep "calibration status=" ${fileOCDB} | grep ${statusString[${i}]/Status/} | cut -d'=' -f2)
2100         echo -n " ${var1:-"-1"}" >> ${outfile}
2101       done
2102     fi
2103     echo >> ${outfile}
2104   done
2105
2106 }
2107
2108 parseConfig()
2109 {
2110   configFile=${1}
2111   shift
2112
2113   #first, source the config file
2114   if [ -f ${configFile} ]; then
2115     source ${configFile}
2116   else
2117     echo "config file ${configFile} not found!, skipping..."
2118   fi
2119
2120   #then, parse the options as theya override the optionf from file
2121   while [[ -n ${1} ]]; do
2122     var="${1#--}"
2123     echo "extraOpts: exporting ${var}"
2124     eval "export ${var}"
2125     shift
2126   done
2127
2128   #export the aliroot function if defined to override normal behaviour
2129   [[ $(type -t aliroot) =~ "function" ]] && export -f aliroot
2130 }
2131
2132 aliroot()
2133 {
2134   args="$@"
2135   if [[ -n ${valgrindCommand} ]]; then
2136     valgrindLogFile="cpu.txt"
2137     [[ "${args}" =~ rec ]] && valgrindLogFile="cpu_rec.txt"
2138     [[ "${args}}" =~ Calib ]] && valgrindLogFile="cpu_calib.txt"
2139     [[ -n ${valgrindCommand} ]] && valgrindCommand="${valgrindCommand} --log-file=${valgrindLogFile}"
2140     echo running ${valgrindCommand} aliroot ${args}
2141     ${valgrindCommand} aliroot ${args}
2142   else
2143     #to prevent an infinite recursion use "command aliroot" to disable
2144     #aliases and functions
2145     echo running command aliroot ${args}
2146     command aliroot ${args}
2147   fi
2148 }
2149
2150 guessRunData()
2151 {
2152   #guess the period from the path, pick the rightmost one
2153   period=""
2154   runNumber=""
2155   year=""
2156   pass=""
2157   legoTrainRunNumber=""
2158   dataType=""
2159
2160   local shortRunNumber=""
2161   local IFS="/"
2162   declare -a path=( $1 )
2163   local dirDepth=$(( ${#path[*]}-1 ))
2164   i=0
2165   #for ((x=${dirDepth};x>=0;x--)); do
2166   for ((x=0;x<=${dirDepth};x++)); do
2167
2168     [[ $((x-1)) -ge 0 ]] && local fieldPrev=${path[$((x-1))]}
2169     local field=${path[${x}]}
2170     local fieldNext=${path[$((x+1))]}
2171
2172     [[ ${field} =~ ^[0-9]*$ && ${fieldNext} =~ (.*\.zip$|.*\.root$) ]] && legoTrainRunNumber=${field}
2173     [[ -n ${legoTrainRunNumber} && -z ${pass} ]] && pass=${fieldPrev}
2174     [[ ${field} =~ ^LHC[0-9][0-9][a-z].*$ ]] && period=${field%_*}
2175     [[ ${field} =~ ^000[0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && runNumber=${field#000}
2176     [[ ${field} =~ ^[0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && shortRunNumber=${field}
2177     [[ ${field} =~ ^20[0-9][0-9]$ ]] && year=${field}
2178     [[ ${field} =~ ^(^sim$|^data$) ]] && dataType=${field}
2179     (( i++ ))
2180   done
2181   [[ -z ${legoTrainRunNumber} ]] && pass=${path[$((dirDepth-1))]}
2182   [[ "${dataType}" =~ ^sim$ ]] && pass="passMC" && runNumber=${shortRunNumber}
2183
2184   #if [[ -z ${dataType} || -z ${year} || -z ${period} || -z ${runNumber}} || -z ${pass} ]];
2185   if [[ -z ${runNumber}} ]];
2186   then
2187     #error condition
2188     return 1
2189   else
2190     #ALL OK
2191     return 0
2192   fi
2193 }
2194
2195 main "$@"