PWGPP/benchmark/benchmark.sh

   1 #!/bin/bash
   2 #include benchmark.config
   3
   4 # this script runs the CPass0/CPass1 train
   5 # produced OCDB updates are local
   6
   7 main()
   8 {
   9   #run in proper mode depending on the selection
  10   if [[ $# -lt 1 ]]; then
  11     if [[ ! "${0}" =~ "bash" ]]; then
  12       echo "uses makeflow:"
  13       echo " ${0} \"run\" productionID inputList configFile [extraOpts]"
  14       echo "uses a batch system (SGE):"
  15       echo " ${0} \"submit\" productionID inputList configFile [extraOpts]"
  16       echo "extraOpts if non-empty override the config file, e.g.:"
  17       echo " ${0} submit test1 benchmark.list benchmark.config runNumber=169123 nEvents=10"
  18     fi
  19     return
  20   fi
  21
  22   #define some aliases - default is to call one of the functions directly
  23   runMode=${1}
  24   umask 0002
  25   shift
  26   case ${runMode} in
  27     "CPass0") goCPass0 "$@";;
  28     "CPass1") goCPass1 "$@";;
  29     "MakeLocalOCDBaccessConfig") goMakeLocalOCDBaccessConfig "$@";;
  30     "MergeCPass0") goMergeCPass0 "$@";;
  31     "MergeCPass1") goMergeCPass1 "$@";;
  32     "MakeFilteredTrees") goMakeFilteredTrees "$@";;
  33     "MakeSummary") goMakeSummary "$@";;
  34     "run") goSubmitMakeflow "$@";;
  35     "submit") goSubmitBatch "$@";;
  36     "test") goTest "$@";;
  37     "GenerateMakeflow") goGenerateMakeflow "$@";;
  38     "PrintValues") goPrintValues "$@";;
  39     "CreateQAplots") goCreateQAplots "$@";;
  40     "WaitForOutput") goWaitForOutput "$@";;
  41     "Merge") goMerge "$@";;
  42     *)
  43       ${runMode} "$@"
  44     ;;
  45   esac
  46   return 0
  47 }
  48
  49 generateMC()
  50 {
  51   #generate one raw chunk in current directory
  52   SEED=${JOB_ID}${SGE_TASK_ID}
  53   export CONFIG_SEED=${SEED}
  54   runNumber=${1}
  55   OCDBpath=${2}
  56   nEventsim=${3}
  57   if [[ -n ${pretend} ]]; then
  58     touch galice.root
  59   else
  60     if [[ -f sim.C && -f Config.C ]] ; then
  61         time aliroot -b -q -x sim.C\(${runNumber},\"${OCDBpath}\",${nEventsim}\) >sim.log 2>&1
  62         mv syswatch.log simwatch.log
  63     fi
  64   fi
  65 }
  66
  67 goCPass0()
  68 {
  69   umask 0002
  70
  71   targetDirectory=${1}
  72   inputList=${2}
  73   nEvents=${3}
  74   ocdbPath=${4}
  75   configFile=${5}
  76   runNumber=${6}
  77   jobindex=${7}
  78   shift 7
  79   if ! parseConfig ${configFile} "$@"; then return 1; fi
  80
  81   #record the working directory provided by the batch system
  82   batchWorkingDirectory=${PWD}
  83
  84   #use the jobindex only if set and non-negative
  85   if [[ -z ${jobindex} || ${jobindex} -lt 0 ]]; then
  86     [[ -n "${LSB_JOBINDEX}" ]] && jobindex=${LSB_JOBINDEX}
  87     [[ -n "${SGE_TASK_ID}" ]] && jobindex=${SGE_TASK_ID}
  88   fi
  89
  90   [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
  91   doneFile="${commonOutputPath}/meta/cpass0.job${jobindex}.run${runNumber}.done"
  92
  93   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
  94
  95   if [[ -n ${ALIROOT_FORCE_COREDUMP} ]]; then
  96     ulimit -c unlimited
  97     export ALIROOT_FORCE_COREDUMP
  98   fi
  99
 100   #the contents of this is stored in the tree and used later (e.g. AliAnalysisTaskPIDResponse)!
 101   #at the QA stage the pass number is guessed from the path stored here.
 102   #The Format is:
 103   #Packages= ;OutputDir= ;LPMPass= ;TriggerAlias= ;LPMRunNumber= ;LPMProductionType= ;LPMInteractionType= ;LPMProductionTag= ;LPMAnchorRun= ;LPMAnchorProduction= ;LPMAnchorYear=
 104   export PRODUCTION_METADATA="OutputDir=cpass0"
 105
 106   if [[ "${inputList}" =~ \.root$ ]]; then
 107     infile=${inputList}
 108   else
 109     infile=$(sed -ne "${jobindex}p" ${inputList} | egrep '\s*\w*/\w*')
 110   fi
 111   chunkName=${infile##*/}
 112
 113   outputDir=${targetDirectory}/${jobindex}_${chunkName%.*}
 114   mkdir -p ${outputDir}
 115   [[ ! -d ${outputDir} ]] && echo "cannot make ${outputDir}" && touch ${doneFile} && return 1
 116
 117   #runpath=${PWD}/rundir_cpass0_${runNumber}_${jobindex}
 118   runpath=${outputDir}
 119   [[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
 120   [[ ${reconstructInTemporaryDir} -eq 1 && -z ${TMPDIR} ]] && runpath=$(mktemp -d)
 121   mkdir -p ${runpath}
 122   [[ ! -d ${runpath} ]] && echo "cannot make runpath ${runpath}" && touch ${doneFile} && return 1
 123   cd ${runpath}
 124   [[ ! ${PWD} =~ ${runpath} ]] && echo "PWD=$PWD is not the runpath=${runpath}" && touch ${doneFile} && return 1
 125
 126   #runCPassX/C expects the raw chunk to be linked in the run dir
 127   #despite it being accessed by the full path
 128   ln -s ${infile} ${runpath}/${chunkName}
 129
 130   #####MC
 131   if [[ -n ${generateMC} ]]; then
 132     olddir=${PWD}
 133     outputDirMC=${commonOutputPath}/000${runNumber}/sim/${jobindex}
 134     simrunpath=${outputDirMC}
 135     [[ ${simulateInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && simrunpath=${TMPDIR}
 136     [[ ${simulateInTemporaryDir} -eq 1 && -z ${TMPDIR} ]] && simrunpath=$(mktemp -d)
 137     mkdir -p ${outputDirMC}
 138     mkdir -p ${simrunpath}
 139     cd ${simrunpath}
 140
 141     filesMC=(
 142               "${batchWorkingDirectory}/sim.C"
 143               "${batchWorkingDirectory}/rec.C"
 144               "${batchWorkingDirectory}/Config.C"
 145               "${batchWorkingDirectory}/OCDB_*.root"
 146     )
 147     for file in ${filesMC[*]}; do
 148       [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
 149     done
 150
 151     generateMC ${runNumber} ${ocdbPath} ${nEvents}
 152
 153     [[ ! "${simrunpath}" =~ "${outputDirMC}" ]] && mv * ${outputDirMC} #TODO check if it works
 154     cd ${olddir}
 155
 156     ln -s ${outputDirMC}/* ${runpath}/
 157
 158     inputList=${outputDirMC}/galice.root #TODO not valid outside shell !!!
 159     infile=""
 160   fi
 161   ######
 162
 163   [[ ! -f ${inputList} && -z ${pretend} ]] && echo "input file ${inputList} not found, exiting..." && touch ${doneFile} && return 1
 164
 165   logOutputDir=${runpath}
 166   [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir}
 167   [[ -z ${dontRedirectStdOutToLog} ]] && exec 1> ${logOutputDir}/stdout
 168   [[ -z ${dontRedirectStdOutToLog} ]] && exec 2> ${logOutputDir}/stderr
 169   echo "${0} $*"
 170
 171   echo "#####################"
 172   echo CPass0:
 173   echo JOB setup
 174   echo nEvents            ${nEvents}
 175   echo runNumber          ${runNumber}
 176   echo ocdbPath           ${ocdbPath}
 177   echo infile             ${infile}
 178   echo chunkName          ${chunkName}
 179   echo jobindex           ${jobindex}
 180   echo recoTriggerOptions ${recoTriggerOptions}
 181   echo targetDirectory    ${targetDirectory}
 182   echo commonOutputPath         ${commonOutputPath}
 183   echo doneFile      ${doneFile}
 184   echo runpath            ${runpath}
 185   echo outputDir          ${outputDir}
 186   echo ALICE_ROOT         ${ALICE_ROOT}
 187   echo PWD                ${PWD}
 188   echo "########## ###########"
 189
 190   alirootInfo > ALICE_ROOT.log
 191
 192   filesCPass0=(
 193                "${batchWorkingDirectory}/runCPass0.sh"
 194                "${batchWorkingDirectory}/recCPass0.C"
 195                "${batchWorkingDirectory}/runCalibTrain.C"
 196                "${batchWorkingDirectory}/localOCDBaccessConfig.C"
 197                "${batchWorkingDirectory}/OCDB.root"
 198                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/runCPass0.sh"
 199                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/recCPass0.C"
 200                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/runCalibTrain.C"
 201   )
 202
 203   for file in ${filesCPass0[*]}; do
 204     [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
 205     [[ ${file##*/} =~ .*\.sh ]] && chmod +x ${file##*/}
 206   done
 207
 208   echo "this directory (${PWD}) contents:"
 209   /bin/ls
 210   echo
 211   chmod u+x runCPass0.sh
 212
 213   sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh
 214
 215   if [[ -n ${postSetUpActionCPass0} ]]; then
 216     echo "running ${postSetUpActionCPass0}"
 217     eval ${postSetUpActionCPass0}
 218   fi
 219
 220   #run CPass0
 221   echo "${runpath}/runCPass0.sh ${infile} ${nEvents} ${runNumber} ${ocdbPath} ${recoTriggerOptions}"
 222   if [[ -n ${pretend} ]]; then
 223     touch AliESDs.root
 224     touch AliESDfriends.root
 225     touch AliESDfriends_v1.root
 226     touch rec.log
 227     touch calib.log
 228   else
 229     echo ./runCPass0.sh "${infile}" "${nEvents}" "${runNumber}" "${ocdbPath}" "${recoTriggerOptions}"
 230     ./runCPass0.sh "${infile}" "${nEvents}" "${runNumber}" "${ocdbPath}" "${recoTriggerOptions}"
 231   fi
 232
 233   #move stuff to final destination
 234   echo "this directory (${PWD}) contents:"
 235   /bin/ls
 236   echo
 237
 238   echo rm -f ./${chunkName}
 239   rm -f ./${chunkName}
 240   echo "cp --recursive ${runpath}/* ${outputDir}"
 241   cp -p --recursive ${runpath}/* ${outputDir}
 242   echo
 243
 244   #validate CPass0
 245   cd ${outputDir}
 246   touch ${doneFile}
 247   echo "dir ${outputDir}" >> ${doneFile}
 248   if summarizeLogs >> ${doneFile}; then
 249     [[ -f ${outputDirMC}/galice.root ]] && echo "sim ${outputDirMC}/galice.root" >> ${doneFile}
 250     [[ -f AliESDfriends_v1.root ]] && echo "calibfile ${outputDir}/AliESDfriends_v1.root" >> ${doneFile}
 251     [[ -f AliESDs.root ]] && echo "esd ${outputDir}/AliESDs.root" >> ${doneFile}
 252   fi
 253
 254   [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
 255   return 0
 256 }
 257
 258 goCPass1()
 259 {
 260   umask 0002
 261
 262   targetDirectory=${1}
 263   inputList=${2}
 264   nEvents=${3}
 265   ocdbPath=${4}
 266   configFile=${5}
 267   runNumber=${6}
 268   jobindex=${7}
 269   shift 7
 270   extraOpts=("$@")
 271   if ! parseConfig ${configFile} "$@"; then return 1; fi
 272
 273   #record the working directory provided by the batch system
 274   batchWorkingDirectory=${PWD}
 275
 276   #use the jobindex only if set and non-negative
 277   if [[ -z ${jobindex} || ${jobindex} -lt 0 ]]; then
 278     [[ -n "${LSB_JOBINDEX}" ]] && jobindex=${LSB_JOBINDEX}
 279     [[ -n "${SGE_TASK_ID}" ]] && jobindex=${SGE_TASK_ID}
 280   fi
 281
 282   [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
 283   doneFile="${commonOutputPath}/meta/cpass1.job${jobindex}.run${runNumber}.done"
 284
 285   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
 286
 287   if [[ -n ${ALIROOT_FORCE_COREDUMP} ]]; then
 288     ulimit -c unlimited
 289     export ALIROOT_FORCE_COREDUMP
 290   fi
 291
 292   #the contents of this is stored in the tree and used later (e.g. AliAnalysisTaskPIDResponse)!
 293   #at the QA stage the pass number is guessed from the path stored here.
 294   #The Format is:
 295   #Packages= ;OutputDir= ;LPMPass= ;TriggerAlias= ;LPMRunNumber= ;LPMProductionType= ;LPMInteractionType= ;LPMProductionTag= ;LPMAnchorRun= ;LPMAnchorProduction= ;LPMAnchorYear=
 296   export PRODUCTION_METADATA="OutputDir=cpass1"
 297
 298   [[ ! -f ${inputList} && -z ${pretend} ]] && echo "input file ${inputList} not found, exiting..." && touch ${doneFile} && return 1
 299   if [[ "${inputList}" =~ \.root$ ]]; then
 300     infile=${inputList}
 301   else
 302     infile=$(sed -ne "${jobindex}p" ${inputList} | egrep '\s*\w*/\w*')
 303   fi
 304   chunkName=${infile##*/}
 305
 306   outputDir=${targetDirectory}/${jobindex}_${chunkName%.*}
 307   mkdir -p ${outputDir}
 308   [[ ! -d ${outputDir} ]] && echo "cannot make ${outputDir}" && touch ${doneFile} && return 1
 309
 310   #runpath=${PWD}/rundir_cpass1_${runNumber}_${jobindex}
 311   runpath=${outputDir}
 312   [[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
 313   [[ ${reconstructInTemporaryDir} -eq 1 && -z ${TMPDIR} ]] && runpath=$(mktemp -d)
 314
 315   #MC
 316   if [[ "${infile}" =~ galice\.root ]]; then
 317     ln -s ${inputList%/*}/* ${runpath}
 318     infile=""
 319   fi
 320
 321   #init the running path
 322   mkdir -p ${runpath}
 323   [[ ! -d ${runpath} ]] && echo "cannot make runpath ${runpath}" && touch ${doneFile} && return 1
 324   cd ${runpath}
 325   [[ ! ${PWD} =~ ${runpath} ]] && echo "PWD=$PWD is not the runpath=${runpath}" && touch ${doneFile} && return 1
 326
 327   #this is needed for runCPass1.sh
 328   ln -s ${infile} ${runpath}/${chunkName}
 329
 330   logOutputDir=${runpath}
 331   [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir}
 332   [[ -z ${dontRedirectStdOutToLog} ]] && exec 1> ${logOutputDir}/stdout
 333   [[ -z ${dontRedirectStdOutToLog} ]] && exec 2> ${logOutputDir}/stderr
 334   echo "${0} $*"
 335
 336   echo "#####################"
 337   echo CPass1:
 338   echo JOB setup
 339   echo nEvents            ${nEvents}
 340   echo runNumber          ${runNumber}
 341   echo ocdbPath           ${ocdbPath}
 342   echo infile             ${infile}
 343   echo chunkName          ${chunkName}
 344   echo jobindex           ${jobindex}
 345   echo recoTriggerOptions ${recoTriggerOptions}
 346   echo targetDirectory    ${targetDirectory}
 347   echo commonOutputPath         ${commonOutputPath}
 348   echo doneFile      ${doneFile}
 349   echo runpath            ${runpath}
 350   echo outputDir          ${outputDir}
 351   echo ALICE_ROOT         ${ALICE_ROOT}
 352   echo PWD                ${PWD}
 353   echo "########## ###########"
 354
 355   alirootInfo > ALICE_ROOT.log
 356
 357   filesCPass1=(
 358                "${batchWorkingDirectory}/runCPass1.sh"
 359                "${batchWorkingDirectory}/recCPass1.C"
 360                "${batchWorkingDirectory}/recCPass1_OuterDet.C"
 361                "${batchWorkingDirectory}/runCalibTrain.C"
 362                "${batchWorkingDirectory}/QAtrain_duo.C"
 363                "${batchWorkingDirectory}/localOCDBaccessConfig.C"
 364                "${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz"
 365                "${batchWorkingDirectory}/OCDB.root"
 366                "${trustedQAtrainMacro}"
 367                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/runCPass1.sh"
 368                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/recCPass1.C"
 369                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/recCPass1_OuterDet.C"
 370                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/runCalibTrain.C"
 371                "${ALICE_ROOT}/ANALYSIS/macros/QAtrain_duo.C"
 372   )
 373
 374   for file in ${filesCPass1[*]}; do
 375     [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
 376     [[ ${file##*/} =~ .*\.sh ]] && chmod +x ${file##*/}
 377   done
 378
 379   echo "this directory (${PWD}) contents:"
 380   /bin/ls
 381   echo
 382
 383   sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh
 384
 385   if [[ -n ${postSetUpActionCPass1} ]]; then
 386     echo "running ${postSetUpActionCPass1}"
 387     eval ${postSetUpActionCPass1}
 388     echo
 389   fi
 390
 391   #configure local OCDB storage from CPass0 (creates the localOCDBaccessConfig.C script)
 392   if [[ -f cpass0.localOCDB.${runNumber}.tgz ]]; then
 393     echo goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz"
 394     goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz"
 395   else
 396     echo "WARNING: file cpass0.localOCDB.${runNumber}.tgz not found!"
 397   fi
 398
 399   if [[ ! $(/bin/ls -1 OCDB/*/*/*/*.root 2>/dev/null) ]]; then
 400     echo "cpass0 produced no calibration! exiting..."
 401     touch ${doneFile}
 402     return 1
 403   fi
 404
 405   #create the Barrel and OuterDet directories for CPass1 and link the local OCDB directory
 406   #there to make the localOCDBaccessConfig.C file work, since it may point to the OCDB
 407   #entries using a relative path, e.g. local://./OCDB
 408   mkdir Barrel OuterDet
 409   ln -s ../OCDB Barrel/OCDB
 410   ln -s ../OCDB OuterDet/OCDB
 411
 412   #setup the filtering
 413   #the following option enables the filtering task inside the QAtrain_duo.C
 414   [[ -n $runESDfiltering ]] && export QA_TaskFilteredTree=1
 415   #set the downscaling factors during the filtering fro expert QA (overrides the previous values)
 416   if [[ -n ${filteringFactorHighPt} ]]; then
 417     export AliAnalysisTaskFilteredTree_fLowPtTrackDownscaligF=${filteringFactorHighPt}
 418   fi
 419   if [[ -n ${filteringFactorV0s} ]]; then
 420     export AliAnalysisTaskFilteredTree_fLowPtV0DownscaligF=${filteringFactorV0s}
 421   fi
 422
 423   #run CPass1
 424   chmod u+x runCPass1.sh
 425   echo "${runpath}/runCPass1.sh ${infile} ${nEvents} ${runNumber} ${ocdbPath} ${recoTriggerOptions}"
 426   if [[ -n ${pretend} ]]; then
 427     touch AliESDs_Barrel.root
 428     touch AliESDfriends_Barrel.root
 429     touch AliESDfriends_v1.root
 430     touch QAresults_barrel.root
 431     touch EventStat_temp_barrel.root
 432     touch AODtpITS.root
 433     touch AliESDs_Outer.root
 434     touch AliESDfriends_Outer.root
 435     touch QAresults_outer.root
 436     touch EventStat_temp_outer.root
 437     touch rec.log
 438     touch calib.log
 439     touch qa.log
 440     touch filtering.log FilterEvents_Trees.root
 441   else
 442     ./runCPass1.sh "${infile}" "${nEvents}" "${runNumber}" "${ocdbPath}" "${recoTriggerOptions}"
 443
 444     [[ ! -f AliESDs_Barrel.root && -f Barrel/AliESDs.root ]] && mv Barrel/AliESDs.root AliESDs_Barrel.root
 445     [[ ! -f AliESDfriends_Barrel.root && -f Barrel/AliESDfriends.root ]] && mv Barrel/AliESDfriends.root AliESDfriends_Barrel.root
 446     [[ ! -f AliESDfriends_v1.root && -f Barrel/AliESDfriends_v1.root ]] && mv Barrel/AliESDfriends_v1.root .
 447     [[ ! -f QAresults_barrel.root && -f Barrel/QAresults_barrel.root ]] && mv Barrel/QAresults_barrel.root .
 448     [[ ! -f AliESDs_Outer.root && -f OuterDet/AliESDs.root ]] && mv OuterDet/AliESDs.root AliESDs_Outer.root
 449     [[ ! -f AliESDfriends_Outer.root && -f OuterDet/AliESDfriends.root ]] && mv OuterDet/AliESDfriends.root AliESDfriends_Outer.root
 450     [[ ! -f QAresults_outer.root && -f OuterDet/QAresults_outer.root ]] && mv OuterDet/QAresults_outer.root .
 451     [[ ! -f FilterEvents_Trees.root && -f Barrel/FilterEvents_Trees.root ]] && mv Barrel/FilterEvents_Trees.root .
 452
 453     #make the filtered tree (if requested and not already produced by QA
 454     [[ -f AliESDs_Barrel.root ]] && echo "AliESDs_Barrel.root" > filtered.list
 455     if [[ -n ${runESDfiltering} && ! -f FilterEvents_Trees.root ]]; then
 456       goMakeFilteredTrees ${PWD} ${runNumber} "filtered.list" ${filteringFactorHighPt} ${filteringFactorV0s} ${ocdbPath} 1000000 0 10000000 0 ${configFile} AliESDs_Barrel.root "${extraOpts[@]}"
 457     fi
 458
 459   fi
 460
 461   ##handle possible crashes in QA (happens often in trunk)
 462   ##rerun QA with a trusted aliroot version
 463   #if [[ $(validateLog qa_barrel.log) ]]; then
 464   #  echo "qa_barrel.log not validated!"
 465   #fi
 466   #if [[ ! -f QAresults_barrel.root && -f ${setupTrustedAliROOTenvInCurrentShell} || $(validateLog qa_barrel.log) ]]; then
 467   #  echo "WARNING: using trusted QA aliroot ${ALICE_ROOT}"
 468   #  source ${setupTrustedAliROOTenvInCurrentShell}
 469   #  cd Barrel
 470   #  rm QAresults_barrel.root
 471   #  rm EventStat_temp_barrel.root
 472   #  rm AODtpITS.root
 473   #  [[ ! -f AliESDs.root ]] && ln -s ../AliESDs_Barrel.root AliESDs.root
 474   #  [[ ! -f AliESDfriends.root ]] && ln -s ../AliESDfriends_Barrel.root AliESDfriends.root
 475   #  if [[ -n ${trustedQAtrainMacro} ]]; then
 476   #    eval "cp ${trustedQAtrainMacro} QAtrain_duo_trusted.C"
 477   #  fi
 478   #  echo executing aliroot -b -q "QAtrain_duo_trusted.C(\"_barrel\",${runNumber},\"wn.xml\",0,\"${ocdbPath}\")"
 479   #  time aliroot -b -q "QAtrain_duo.C(\"_barrel\",${runNumber},\"wn.xml\",0,\"${ocdbPath}\")" &> ../qa_barrel_trusted.log
 480   #  cd ../
 481   #fi
 482
 483   #move stuff to final destination
 484   echo "this directory (${PWD}) contents:"
 485   /bin/ls
 486   echo rm -f ./${chunkName}
 487   rm -f ./${chunkName}
 488   echo "cp --recursive ${runpath}/* ${outputDir}"
 489   cp -pf --recursive ${runpath}/* ${outputDir}
 490   echo
 491
 492   #validate CPass1
 493   cd ${outputDir}
 494   touch ${doneFile}
 495   echo "dir ${outputDir}" >> ${doneFile}
 496   if summarizeLogs >> ${doneFile}; then
 497     [[ -f AliESDs_Barrel.root ]] && echo "esd ${outputDir}/AliESDs_Barrel.root" >> ${doneFile}
 498     [[ -f AliESDfriends_v1.root ]] && echo "calibfile ${outputDir}/AliESDfriends_v1.root" >> ${doneFile}
 499     [[ -f QAresults_Barrel.root ]] && echo "qafile ${outputDir}/QAresults_Barrel.root" >> ${doneFile}
 500     [[ -f QAresults_Outer.root ]] && echo "qafile ${outputDir}/QAresults_Outer.root" >> ${doneFile}
 501     [[ -f FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFile}
 502   fi
 503
 504   [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
 505   return 0
 506 }
 507
 508
 509 goMergeCPass0()
 510 {
 511   #
 512   # find the output files and merge them
 513   #
 514
 515   outputDir=${1}
 516   ocdbStorage=${2}
 517   configFile=${3}
 518   runNumber=${4}
 519   calibrationFilesToMerge=${5}  #can be a non-existent file, will then be produced on the fly
 520   shift 5
 521   if ! parseConfig ${configFile} "$@"; then return 1; fi
 522
 523   #record the working directory provided by the batch system
 524   batchWorkingDirectory=${PWD}
 525
 526   [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
 527   doneFile="${commonOutputPath}/meta/merge.cpass0.run${runNumber}.done"
 528
 529   umask 0002
 530   ulimit -c unlimited
 531
 532   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
 533
 534   #runpath=${PWD}/rundir_cpass0_Merge_${runNumber}
 535   runpath=${outputDir}
 536   [[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
 537   [[ ${reconstructInTemporaryDir} -eq 1 && -z ${TMPDIR} ]] && runpath=$(mktemp -d)
 538
 539   mkdir -p ${runpath}
 540   [[ ! -d ${runpath} ]] && echo "not able to make the runpath ${runpath}" && touch ${doneFile} && return 1
 541   cd ${runpath}
 542   [[ ! ${PWD} =~ ${runpath} ]] && echo "PWD=$PWD is not the runpath=${runpath}" && touch ${doneFile} && return 1
 543
 544   logOutputDir=${runpath}
 545   [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir}
 546   [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/mergeMakeOCDB.log
 547   echo "${0} $*"
 548
 549   mergingScript="mergeMakeOCDB.byComponent.sh"
 550
 551   echo goMergeCPass0 SETUP:
 552   echo runNumber=${runNumber}
 553   echo outputDir=${outputDir}
 554   echo ocdbStorage=${ocdbStorage}
 555   echo calibrationFilesToMerge=${calibrationFilesToMerge}
 556   echo mergingScript=${mergingScript}
 557   echo commonOutputPath=${commonOutputPath}
 558   echo runpath=${runpath}
 559
 560   # copy files in case they are not already there
 561   filesMergeCPass0=(
 562                     "${batchWorkingDirectory}/${calibrationFilesToMerge}"
 563                     "${batchWorkingDirectory}/OCDB.root"
 564                     "${batchWorkingDirectory}/localOCDBaccessConfig.C"
 565                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/mergeMakeOCDB.byComponent.sh"
 566                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/mergeByComponent.C"
 567                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/makeOCDB.C"
 568                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/merge.C"
 569                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/mergeMakeOCDB.sh"
 570   )
 571   for file in ${filesMergeCPass0[*]}; do
 572     [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
 573     [[ ${file##*/} =~ .*\.sh ]] && chmod +x ${file##*/}
 574   done
 575
 576   sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh
 577
 578   alirootInfo > ALICE_ROOT.log
 579
 580   #
 581   echo "PWD"
 582   /bin/ls
 583   echo "PWD/.."
 584   /bin/ls ../
 585
 586
 587   #merge calibration
 588   chmod u+x ${mergingScript}
 589   mkdir -p ./OCDB
 590   if [[ ! -f ${calibrationFilesToMerge} ]]; then
 591     echo "/bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root > ${calibrationFilesToMerge}"
 592     /bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root 2>/dev/null > ${calibrationFilesToMerge}
 593   fi
 594
 595   echo "${mergingScript} ${calibrationFilesToMerge} ${runNumber} local://./OCDB ${ocdbStorage}"
 596   if [[ -n ${pretend} ]]; then
 597     touch CalibObjects.root
 598     touch ocdb.log
 599     touch merge.log
 600     touch dcsTime.root
 601     mkdir -p ./OCDB/TPC/Calib/TimeGain/
 602     mkdir -p ./OCDB/TPC/Calib/TimeDrift/
 603     touch ./OCDB/TPC/Calib/TimeGain/someCalibObject_0-999999_cpass0.root
 604     touch ./OCDB/TPC/Calib/TimeDrift/otherCalibObject_0-999999_cpass0.root
 605   else
 606     ./${mergingScript} ${calibrationFilesToMerge} ${runNumber} "local://./OCDB" ${ocdbStorage}
 607
 608     #produce the calib trees for expert QA (dcsTime.root)
 609     goMakeLocalOCDBaccessConfig ./OCDB
 610     echo aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")"
 611     aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")"
 612   fi
 613
 614   ### produce the output
 615   #tar the produced OCDB for reuse
 616   tar czf ${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz ./OCDB
 617
 618   /bin/ls
 619
 620   #copy all to output dir
 621   cp -pf --recursive ${runpath}/* ${outputDir}
 622
 623   if [[ -n ${generateMC} ]]; then
 624     goPrintValues sim ${commonOutputPath}/meta/sim.run${runNumber}.list ${commonOutputPath}/meta/cpass0.job*.run${runNumber}.done
 625   fi
 626
 627   #validate merging cpass0
 628   cd ${outputDir}
 629   touch ${doneFile}
 630   echo "dir ${outputDir}" >> ${doneFile}
 631   if summarizeLogs >> ${doneFile}; then
 632     [[ -f CalibObjects.root ]] && echo "calibfile ${outputDir}/CalibObjects.root" >> ${doneFile}
 633     [[ -f dcsTime.root ]] && echo "dcsTree ${outputDir}/dcsTime.root" >> ${doneFile}
 634   fi
 635
 636   [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
 637   return 0
 638 }
 639
 640 goMergeCPass1()
 641 {
 642   #
 643   # find the output files and merge them
 644   #
 645
 646   outputDir=${1}
 647   ocdbStorage=${2}
 648   configFile=${3}
 649   runNumber=${4}
 650   calibrationFilesToMerge=${5}
 651   qaFilesToMerge=${6}
 652   filteredFilesToMerge=${7}
 653   shift 7
 654   if ! parseConfig ${configFile} "$@"; then return 1; fi
 655
 656   #record the working directory provided by the batch system
 657   batchWorkingDirectory=${PWD}
 658
 659   [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
 660   doneFile="${commonOutputPath}/meta/merge.cpass1.run${runNumber}.done"
 661
 662   umask 0002
 663   ulimit -c unlimited
 664
 665   #clean up first:
 666   rm -f ${outputDir}/*.log
 667   rm -f ${outputDir}/*.root
 668   rm -f ${outputDir}/*done
 669
 670   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
 671
 672   #runpath=${PWD}/rundir_cpass1_Merge_${runNumber}
 673   runpath=${outputDir}
 674   [[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
 675   [[ ${reconstructInTemporaryDir} -eq 1 && -z ${TMPDIR} ]] && runpath=$(mktemp -d)
 676
 677   mkdir -p ${runpath}
 678   [[ ! -d ${runpath} ]] && echo "not able to make the runpath ${runpath}" && touch ${doneFile} && return 1
 679   cd ${runpath}
 680   [[ ! ${PWD} =~ ${runpath} ]] && echo "PWD=$PWD is not the runpath=${runpath}" && touch ${doneFile} && return 1
 681
 682   logOutputDir=${runpath}
 683   [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir}
 684   [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/mergeMakeOCDB.log
 685   echo "${0} $*"
 686
 687   calibrationOutputFileName='AliESDfriends_v1.root'
 688   qaOutputFileName='QAresults*.root'
 689   mergingScript="mergeMakeOCDB.byComponent.sh"
 690   #important to have the string "Stage.txt" in the filename to trigger the merging
 691   #it has to be a list of directories containing the files
 692   qaMergedOutputFileName="QAresults_merged.root"
 693
 694   echo goMergeCPass1 SETUP:
 695   echo runNumber=${runNumber}
 696   echo outputDir=${outputDir}
 697   echo ocdbStorage=${ocdbStorage}
 698   echo calibrationFilesToMerge=$calibrationFilesToMerge
 699   echo qaFilesToMerge=$qaFilesToMerge
 700   echo calibrationOutputFileName=${calibrationOutputFileName}
 701   echo mergingScript=${mergingScript}
 702
 703   # copy files in case they are not already there
 704   filesMergeCPass1=(
 705                     "${batchWorkingDirectory}/${calibrationFilesToMerge}"
 706                     "${batchWorkingDirectory}/${qaFilesToMerge}"
 707                     "${batchWorkingDirectory}/OCDB.root"
 708                     "${batchWorkingDirectory}/localOCDBaccessConfig.C"
 709                     "${batchWorkingDirectory}/meta/cpass0.localOCDB.${runNumber}.tgz"
 710                     "${batchWorkingDirectory}/QAtrain_duo.C"
 711                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/mergeMakeOCDB.byComponent.sh"
 712                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/mergeByComponent.C"
 713                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/makeOCDB.C"
 714                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/merge.C"
 715                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/mergeMakeOCDB.sh"
 716                     "${trustedQAtrainMacro}"
 717                     "${ALICE_ROOT}/ANALYSIS/macros/QAtrain_duo.C"
 718   )
 719   for file in ${filesMergeCPass1[*]}; do
 720     [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
 721     [[ ${file##*/} =~ .*\.sh ]] && chmod +x ${file##*/}
 722   done
 723
 724   sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh
 725
 726   #configure local OCDB storage from CPass0 (creates the localOCDBaccessConfig.C script)
 727   if [[ -f cpass0.localOCDB.${runNumber}.tgz ]]; then
 728     echo goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz"
 729     goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz"
 730   else
 731     echo "WARNING: file cpass0.localOCDB.${runNumber}.tgz not found!"
 732   fi
 733
 734   alirootInfo > ALICE_ROOT.log
 735
 736   #
 737   /bin/ls
 738
 739   #merge calibration
 740   chmod u+x ${mergingScript}
 741   mkdir -p OCDB
 742
 743   #if not provided, create the lists of files to merge
 744   if [[ ! -f ${filteredFilesToMerge} ]]; then
 745     echo "/bin/ls -1 ${outputDir}/*/FilterEvents_Trees.root > ${filteredFilesToMerge}"
 746     /bin/ls -1 ${outputDir}/*/FilterEvents_Trees.root 2>/dev/null > ${filteredFilesToMerge}
 747   fi
 748   if [[ ! -f ${calibrationFilesToMerge} ]]; then
 749     echo "/bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root > ${calibrationFilesToMerge}"
 750     /bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root 2>/dev/null > ${calibrationFilesToMerge}
 751   fi
 752   if [[ ! -f ${qaFilesToMerge} ]]; then
 753     #find the files, but only store the directories (QAtrain_duo.C requires this)
 754     echo "/bin/ls -1 ${outputDir}/*/QAresults*.root | while read x; do echo ${x%/*}; done | sort | uniq > ${qaFilesToMerge}"
 755     /bin/ls -1 ${outputDir}/*/QAresults*.root | while read x; do echo ${x%/*}; done | sort | uniq > ${qaFilesToMerge}
 756   fi
 757
 758   echo "${mergingScript} ${calibrationFilesToMerge} ${runNumber} local://./OCDB ${ocdbStorage}"
 759   if [[ -n ${pretend} ]]; then
 760     touch ocdb.log
 761     touch cpass1.localOCDB.${runNumber}.tgz
 762     touch ${qaMergedOutputFileName}
 763     touch merge.log
 764     touch trending.root
 765     touch FilterEvents_Trees.root
 766     touch CalibObjects.root
 767     touch dcsTime.root
 768     touch ${qaMergedOutputFileName}
 769     mkdir -p OCDB
 770   else
 771     ./${mergingScript} ${calibrationFilesToMerge} ${runNumber} "local://./OCDB" ${ocdbStorage}
 772
 773     #merge QA (and filtered trees)
 774     [[ -n ${AliAnalysisTaskFilteredTree_fLowPtTrackDownscaligF} ]] && export AliAnalysisTaskFilteredTree_fLowPtTrackDownscaligF
 775     [[ -n ${AliAnalysisTaskFilteredTree_fLowPtV0DownscaligF} ]] && export AliAnalysisTaskFilteredTree_fLowPtV0DownscaligF
 776
 777     #echo aliroot -l -b -q "merge.C(\"${qaFilesToMerge}\",\"\",kFALSE,\"${qaMergedOutputFileName}\")"
 778     echo aliroot -b -q "QAtrain_duo.C(\"_barrel\",${runNumber},\"${qaFilesToMerge}\",1,\"${ocdbStorage}\")"
 779     #aliroot -l -b -q "merge.C(\"${qaFilesToMerge}\",\"\",kFALSE,\"${qaMergedOutputFileName}\")"
 780     aliroot -b -q "QAtrain_duo.C(\"_barrel\",${runNumber},\"${qaFilesToMerge}\",1,\"${ocdbStorage}\")" > mergeQA.log
 781     mv QAresults_barrel.root ${qaMergedOutputFileName}
 782     mv trending_barrel.root trending.root
 783
 784     #merge filtered trees
 785     echo aliroot -l -b -q "merge.C(\"${qaFilesToMerge}\",\"\",kFALSE,\"${qaMergedOutputFileName}\")"
 786     aliroot -l -b -q "merge.C(\"${filteredFilesToMerge}\",\"\",kFALSE,\"FilterEvents_Trees.root\")"
 787
 788     #produce the calib trees for expert QA
 789     echo aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")"
 790     aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")"
 791   fi
 792
 793   tar czf ${commonOutputPath}/meta/cpass1.localOCDB.${runNumber}.tgz ./OCDB
 794
 795   /bin/ls
 796
 797   #copy all to output dir
 798   cp -pf --recursive ${runpath}/* ${outputDir}
 799
 800   #validate merge cpass1
 801   cd ${outputDir}
 802   touch ${doneFile}
 803   echo "dir ${outputDir}" >> ${doneFile}
 804   if summarizeLogs >>  ${doneFile}; then
 805     [[ -f CalibObjects.root ]] && echo "calibfile ${outputDir}/CalibObjects.root" >> ${doneFile}
 806     [[ -f ${qaMergedOutputFileName} ]] && echo "qafile ${outputDir}/${qaMergedOutputFileName}" >> ${doneFile}
 807     [[ -f trending.root ]] && echo "trendingfile ${outputDir}/trending.root" >> ${doneFile}
 808     [[ -f dcsTime.root ]] && echo "dcsTree ${outputDir}/dcsTime.root" >> ${doneFile}
 809     [[ -f FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFile}
 810   fi
 811
 812   [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
 813   return 0
 814 }
 815
 816 goMerge()
 817 {
 818   #generic root merge using CPass1 merge.C script
 819   inputList=${1}
 820   outputFile=${2}
 821   configFile=${3-"becnhmark.config"}
 822   shift 3
 823   if ! parseConfig ${configFile} "$@"; then return 1; fi
 824
 825   #record the working directory provided by the batch system
 826   batchWorkingDirectory=${PWD}
 827
 828   [[ ! -f ${inputList} ]] && echo "inputList ${inputList} does not exist!" && return 1
 829   [[ ! -f ${configFile} ]] && echo "configFile ${configFile} does not exist!" && return 1
 830   umask 0002
 831   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
 832   rm -f ${outputFile}
 833   aliroot -b -q "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/merge.C(\"${inputList}\",\"\",kFALSE,\"${outputFile}\")" > merge_${inputList}.log
 834   return 0
 835 }
 836
 837 goSubmitMakeflow()
 838 {
 839   #run
 840   productionID=${1}
 841   inputFileList=${2}
 842   configFile=${3}
 843   shift 3
 844   extraOpts=("$@")
 845   if ! parseConfig ${configFile} "${extraOpts[@]}"; then return 1; fi
 846
 847   #record the working directory provided by the batch system
 848   batchWorkingDirectory=${PWD}
 849
 850   [[ -z ${configFile} ]] && configFile="benchmark.config"
 851   [[ ! -f ${configFile} ]] && echo "no config file found (${configFile})" && return 1
 852
 853   if [[ ! $(which makeflow &>/dev/null) && -n ${makeflowPath} ]]; then
 854     echo "setting the makflow path from the config: "
 855     echo "  export PATH=${makeflowPath}:${PATH}"
 856     export PATH=${makeflowPath}:${PATH}
 857   fi
 858
 859   #create the common output dir and the meta dir
 860   commonOutputPath=${baseOutputDirectory}/${productionID}
 861   if [[ -d ${commonOutputPath} ]]; then
 862     echo "output dir ${commonOutputPath} exists!"
 863     return 1
 864   else
 865     mkdir -p ${commonOutputPath}
 866   fi
 867   mkdir -p ${commonOutputPath}/meta
 868
 869   #submit - use makeflow if available, fall back to old stuff when makeflow not there
 870   if which makeflow; then
 871
 872     goGenerateMakeflow ${productionID} ${inputFileList} ${configFile} "${extraOpts[@]}" commonOutputPath=${commonOutputPath} > benchmark.makeflow
 873
 874     makeflow ${makeflowOptions} benchmark.makeflow
 875     cd ../
 876   else
 877     echo "no makeflow!"
 878   fi
 879   return 0
 880 }
 881
 882 goGenerateMakeflow()
 883 {
 884   #generate the makeflow file
 885   [[ $# -lt 3 ]] && echo "args: id inputFileList configFile" && return 1
 886   productionID=${1}
 887   inputFileList=${2}
 888   configFile=${3}
 889   shift 3
 890   extraOpts=("$@")
 891   if ! parseConfig ${configFile} "${extraOpts[@]}" &>/dev/null; then return 1; fi
 892
 893   #extra safety
 894   if [[ -z ${commonOutputPath} ]]; then
 895     commonOutputPath=${baseOutputDirectory}/${productionID}
 896     extraOpts=( "${extraOpts[@]}" "commonOutputPath=${commonOutputPath}" )
 897   fi
 898
 899   #record the working directory provided by the batch system
 900   batchWorkingDirectory=${PWD}
 901
 902   [[ -z ${configFile} ]] && configFile="benchmark.config"
 903   [[ ! -f ${configFile} ]] && echo "no config file found (${configFile})" && return 1
 904
 905   self=$(readlink -f "${0}")
 906
 907   #these files will be made a dependency - will be copied to the working dir of the jobs
 908   declare -a copyFiles
 909   inputFiles=(
 910               "OCDB.root"
 911               "localOCDBaccessConfig.C"
 912               "QAtrain_duo.C"
 913               "runCPass1.sh"
 914               "recCPass1.C"
 915               "recCPass1_OuterDet.C"
 916               "runCalibTrain.C"
 917               "runCPass0.sh"
 918               "recCPass0.C"
 919   )
 920   for file in ${inputFiles[*]}; do
 921     [[ -f ${file} ]] && copyFiles+=("${file}")
 922   done
 923
 924   #create the makeflow file
 925   [[ -n ${batchFlags} ]] && echo "BATCH_OPTIONS = ${batchFlags}"
 926   declare -A arr_cpass0_merged arr_cpass1_merged
 927   declare -A arr_cpass0_calib_list arr_cpass1_calib_list
 928   declare -A arr_cpass1_QA_list arr_cpass1_ESD_list arr_cpass1_filtered_list
 929   declare -A arr_cpass0_profiled_outputs
 930   declare -A listOfRuns
 931   [[ -n ${runNumber} ]] && listOfRuns[${runNumber}]=1
 932   while read x; do tmpRun=$(guessRunNumber ${x}); [[ -n ${tmpRun} ]] && listOfRuns[${tmpRun}]=1; done < ${inputFileList}
 933   for runNumber in "${!listOfRuns[@]}"; do
 934     [[ -z ${runNumber} ]] && continue
 935     [[ ! ${runNumber} =~ ^[0-9]*[0-9]$ ]] && continue
 936
 937     unset arr_cpass0_outputs
 938     unset arr_cpass1_outputs
 939     declare -a arr_cpass0_outputs
 940     declare -a arr_cpass1_outputs
 941
 942     jobindex=0
 943     inputFile=""
 944     while read inputFile; do
 945       currentDefaultOCDB=${defaultOCDB}
 946       [[ -z ${autoOCDB} ]] && autoOCDB=1
 947       if [[ ${autoOCDB} -ne 0 ]]; then
 948         currentDefaultOCDB=$(setYear ${inputFile} ${defaultOCDB})
 949       fi
 950
 951       #CPass0
 952       arr_cpass0_outputs[${jobindex}]="${commonOutputPath}/meta/cpass0.job${jobindex}.run${runNumber}.done"
 953       echo "${arr_cpass0_outputs[${jobindex}]} : benchmark.sh ${configFile} ${copyFiles[@]}"
 954       echo -n " ${alirootEnv} ./benchmark.sh CPass0 ${commonOutputPath}/000${runNumber}/cpass0 ${inputFile} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} ${jobindex}"" "
 955       for extraOption in "${extraOpts[@]}"; do echo -n \'${extraOption}\'" "; done; echo
 956       echo
 957
 958       #CPass1
 959       arr_cpass1_outputs[${jobindex}]="${commonOutputPath}/meta/cpass1.job${jobindex}.run${runNumber}.done"
 960       echo "${arr_cpass1_outputs[${jobindex}]} : benchmark.sh ${configFile} ${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz ${copyFiles[@]}"
 961       echo -n " ${alirootEnv} ./benchmark.sh CPass1 ${commonOutputPath}/000${runNumber}/cpass1 ${inputFile} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} ${jobindex}"" "
 962       for extraOption in "${extraOpts[@]}"; do echo -n \'${extraOption}\'" "; done; echo
 963       echo
 964       ((jobindex++))
 965
 966     done< <(grep "/000${runNumber}/" ${inputFileList})
 967
 968     #CPass0 list of Calib files to merge
 969     arr_cpass0_calib_list[${runNumber}]="${commonOutputPath}/meta/cpass0.calib.run${runNumber}.list"
 970     echo "${arr_cpass0_calib_list[${runNumber}]} : benchmark.sh ${arr_cpass0_outputs[*]}"
 971     echo "  ./benchmark.sh PrintValues calibfile ${arr_cpass0_calib_list[${runNumber}]} ${arr_cpass0_outputs[*]}"
 972     echo
 973
 974     #CPass0 merging
 975     arr_cpass0_merged[${runNumber}]="${commonOutputPath}/meta/merge.cpass0.run${runNumber}.done"
 976     echo "${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz ${arr_cpass0_merged[${runNumber}]} : benchmark.sh ${configFile} ${arr_cpass0_calib_list[${runNumber}]} ${copyFiles[@]}"
 977     echo -n " ${alirootEnv} ./benchmark.sh MergeCPass0 ${commonOutputPath}/000${runNumber}/cpass0 ${currentDefaultOCDB} ${configFile} ${runNumber} ${arr_cpass0_calib_list[${runNumber}]}"" "
 978     for extraOption in "${extraOpts[@]}"; do echo -n \'${extraOption}\'" "; done; echo
 979     echo
 980
 981     #CPass1 list of Calib/QA/ESD/filtered files
 982     # the trick with QA is to have the string "Stage.txt" in the file name of the list of directories with QA output to trigger
 983     # the production of the QA trending tree (only then the task->Finish() will be called in QAtrain_duo.C, on the grid
 984     # this corresponds to the last merging stage)
 985     arr_cpass1_QA_list[${runNumber}]="${commonOutputPath}/meta/cpass1.QA.run${runNumber}.lastMergingStage.txt.list"
 986     echo "${arr_cpass1_QA_list[${runNumber}]}: benchmark.sh ${arr_cpass1_outputs[*]}"
 987     echo "  ./benchmark.sh PrintValues dir ${arr_cpass1_QA_list[${runNumber}]} ${arr_cpass1_outputs[*]}"
 988     echo
 989     arr_cpass1_calib_list[${runNumber}]="${commonOutputPath}/meta/cpass1.calib.run${runNumber}.list"
 990     echo "${arr_cpass1_calib_list[${runNumber}]} : benchmark.sh ${arr_cpass1_outputs[*]}"
 991     echo "  ./benchmark.sh PrintValues calibfile ${arr_cpass1_calib_list[${runNumber}]} ${arr_cpass1_outputs[*]};"
 992     echo
 993     arr_cpass1_ESD_list[${runNumber}]="${commonOutputPath}/meta/cpass1.ESD.run${runNumber}.list"
 994     echo "${arr_cpass1_ESD_list[${runNumber}]} : benchmark.sh ${arr_cpass1_outputs[*]}"
 995     echo "  ./benchmark.sh PrintValues esd ${arr_cpass1_ESD_list[${runNumber}]} ${arr_cpass1_outputs[*]}"
 996     echo
 997     arr_cpass1_filtered_list[${runNumber}]="${commonOutputPath}/meta/cpass1.filtered.run${runNumber}.list"
 998     echo "${arr_cpass1_filtered_list[${runNumber}]} : benchmark.sh ${arr_cpass1_outputs[*]}"
 999     echo "  ./benchmark.sh PrintValues filteredTree ${arr_cpass1_filtered_list[${runNumber}]} ${arr_cpass1_outputs[*]}"
1000     echo
1001
1002     #CPass1 merging
1003     arr_cpass1_merged[${runNumber}]="${commonOutputPath}/meta/merge.cpass1.run${runNumber}.done"
1004     echo "${commonOutputPath}/meta/cpass1.localOCDB.${runNumber}.tgz ${arr_cpass1_merged[${runNumber}]} :  benchmark.sh ${configFile} ${arr_cpass1_calib_list[${runNumber}]} ${arr_cpass1_QA_list[${runNumber}]} ${copyFiles[@]}"
1005     echo -n " ${alirootEnv} ./benchmark.sh MergeCPass1 ${commonOutputPath}/000${runNumber}/cpass1 ${currentDefaultOCDB} ${configFile} ${runNumber} ${arr_cpass1_calib_list[${runNumber}]} ${arr_cpass1_QA_list[${runNumber}]} ${arr_cpass1_filtered_list[${runNumber}]}"" "
1006     for extraOption in "${extraOpts[@]}"; do echo -n \'${extraOption}\'" "; done; echo
1007     echo
1008
1009     #CPass0 wrapped in a profiling tool (valgrind,....)
1010     if [[ -n ${profilingCommand} ]]; then
1011       arr_cpass0_profiled_outputs[${runNumber}]="${commonOutputPath}/meta/cpass0.jobProfiled.run${runNumber}.done"
1012       echo "${arr_cpass0_profiled_outputs[${runNumber}]} : benchmark.sh ${configFile} ${copyFiles[@]}"
1013       echo -n " ${alirootEnv} ./benchmark.sh CPass0 ${commonOutputPath}/000${runNumber}/profiled ${inputFile} ${nEventsProfiling} ${currentDefaultOCDB} ${configFile} ${runNumber} profiling"" "
1014       for extraOption in "${extraOpts[@]}"; do echo -n \'${extraOption}\'" "; done;
1015       echo "\"useProfilingCommand=${profilingCommand}\""
1016       echo
1017     fi
1018
1019   done #runs
1020
1021   #Summary
1022   echo "${commonOutputPath}/summary.log : benchmark.sh ${configFile} ${arr_cpass1_merged[*]}"
1023   echo -n "  ${alirootEnv} ./benchmark.sh MakeSummary ${configFile}"" "
1024   for extraOption in "${extraOpts[@]}"; do echo -n \'${extraOption}\'" "; done; echo
1025   echo
1026
1027   return 0
1028 }
1029
1030 goPrintValues()
1031 {
1032   #print the values given the key from any number of files (store in output file on second argument)
1033   if [[ $# -lt 3 ]]; then
1034     echo "goPrintValues key outputFile inputFiles"
1035     echo "if outputFile is \"-\" dont redirect to a file"
1036     return
1037   fi
1038   key=${1}
1039   outputFile=${2}
1040   [[ ${outputFile} =~ "-" ]] && outputFile=""
1041   shift 2 #remove 2 first arguments from arg list to only pass the input files to awk
1042   awk -v key=${key} '$0 ~ key" " {print $2}' "$@" | tee ${outputFile}
1043   return 0
1044 }
1045
1046 goCreateQAplots()
1047 {
1048   umask 0002
1049   mergedQAfileList=${1}
1050   productionID=${2}
1051   outputDir=${3}
1052   configFile=${4}
1053   shift 4
1054   if ! parseConfig ${configFile} ${@}; then return 1; fi
1055
1056   #record the working directory provided by the batch system
1057   batchWorkingDirectory=${PWD}
1058
1059   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
1060
1061   [[ -z ${logOutputDir} ]] && logOutputDir=${PWD}
1062   [[ -z ${dontRedirectStdOutToLog} ]] && exec 2>&1 > ${logOutputDir}/makeQAplots.log
1063   echo "${0} $*"
1064
1065   olddir=${PWD}
1066   mkdir -p ${outputDir}
1067   cd ${outputDir}
1068   [[ ! "${PWD}" =~ "${outputDir}" ]] && echo "cannot make ${outputDir}... exiting" && return 1
1069
1070   echo ${ALICE_ROOT}/PWGPP/QA/scripts/runQA.sh inputList=${mergedQAfileList}
1071   ${ALICE_ROOT}/PWGPP/QA/scripts/runQA.sh inputList="${mergedQAfileList}" inputListHighPtTrees="${filteringList}"
1072
1073   cd ${olddir}
1074   return 0
1075 }
1076
1077 goTest()
1078 {
1079   umask 0002
1080   exec 2>&1
1081   exec > >(tee test.log)
1082   echo "$@"
1083   echo something
1084   return 0
1085 }
1086
1087 alirootInfo()
1088 {
1089   umask 0002
1090   # save aliroot repository info
1091   [[ -z "${ALICE_ROOT}" ]] && return 1
1092
1093   echo "\${ALICE_ROOT}=${ALICE_ROOT}"
1094   echo "\${ROOTSYS}=${ROOTSYS}"
1095   echo "\${PATH}=${PATH}"
1096   echo "\${LD_LIBRARY_PATH}=${LD_LIBRARY_PATH}"
1097
1098   pushd ${PWD}
1099   cd ${ALICE_ROOT}
1100   echo
1101
1102   currentBranch=$(git rev-parse --abbrev-ref HEAD)
1103   git status
1104   echo ""
1105   echo ""
1106   git diff ${currentBranch}
1107   popd
1108   return 0
1109 }
1110
1111 setYear()
1112 {
1113   #set the year
1114   #  ${1} - year to be set
1115   #  ${2} - where to set the year
1116   year1=$(guessYear ${1})
1117   year2=$(guessYear ${2})
1118   local path=${2}
1119   [[ ${year1} -ne ${year2} && -n ${year2} && -n ${year1} ]] && path=${2/\/${year2}\//\/${year1}\/}
1120   echo ${path}
1121   return 0
1122 }
1123
1124 guessPeriod()
1125 {
1126   #guess the period from the path, pick the rightmost one
1127   local IFS="/"
1128   declare -a path=( ${1} )
1129   local dirDepth=${#path[*]}
1130   for ((x=${dirDepth}-1;x>=0;x--)); do
1131     local field=${path[${x}]}
1132     [[ ${field} =~ ^LHC[0-9][0-9][a-z]$ ]] && period=${field} && break
1133   done
1134   echo ${period}
1135   return 0
1136 }
1137
1138 guessYear()
1139 {
1140   #guess the year from the path, pick the rightmost one
1141   local IFS="/"
1142   declare -a path=( ${1} )
1143   local dirDepth=${#path[*]}
1144   for ((x=${dirDepth}-1;x>=0;x--)); do
1145     local field=${path[${x}]}
1146     [[ ${field} =~ ^20[0-9][0-9]$ ]] && year=${field} && break
1147   done
1148   echo ${year}
1149   return 0
1150 }
1151
1152 guessRunNumber()
1153 {
1154   #guess the run number from the path, pick the rightmost one
1155   #works for /path/foo/000123456/bar/...
1156   #and       /path/foo.run123456.bar
1157   local IFS="/."
1158   declare -a path=( ${1} )
1159   local dirDepth=${#path[*]}
1160   for ((x=${dirDepth}-1;x>=0;x--)); do
1161     local field=${path[${x}]}
1162     field=${field/run/000}
1163     [[ ${field} =~ [0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && runNumber=${field#000} && break
1164   done
1165   echo ${runNumber}
1166   return 0
1167 }
1168
1169 validateLog()
1170 {
1171   log=${1}
1172   errorConditions=(
1173                     'There was a crash'
1174                     'floating'
1175                     'error while loading shared libraries'
1176                     'std::bad_alloc'
1177                     's_err_syswatch_'
1178                     'Thread [0-9]* (Thread'
1179                     'AliFatal'
1180                     'core dumped'
1181                     '\.C.*error:.*\.h: No such file'
1182   )
1183
1184   warningConditions=(
1185                      'This is serious !'
1186                      'rocVoltage out of range:'
1187   )
1188
1189   local logstatus=0
1190   local errorSummary=""
1191   local warningSummary=""
1192
1193   for ((i=0; i<${#errorConditions[@]};i++)); do
1194     local tmp=$(grep -m1 -e "${errorConditions[${i}]}" ${log})
1195     [[ -n ${tmp} ]] && tmp+=" : "
1196     errorSummary+=${tmp}
1197   done
1198
1199   for ((i=0; i<${#warningConditions[@]};i++)); do
1200     local tmp=$(grep -m1 -e "${warningConditions[${i}]}" ${log})
1201     [[ -n ${tmp} ]] && tmp+=" : "
1202     warningSummary+=${tmp}
1203   done
1204
1205   if [[ -n ${errorSummary} ]]; then
1206     echo "${errorSummary}"
1207     return 1
1208   fi
1209
1210   if [[ -n ${warningSummary} ]]; then
1211     echo "${warningSummary}"
1212     return 2
1213   fi
1214
1215   return 0
1216 }
1217
1218 summarizeLogs()
1219 {
1220   #print a summary of logs
1221   logFiles=(
1222             "*.log"
1223             "stdout"
1224             "stderr"
1225   )
1226
1227   #check logs
1228   local logstatus=0
1229   for log in ${logFiles[*]}; do
1230     finallog=${PWD%/}/${log}
1231     [[ ! -f ${log} ]] && continue
1232     errorSummary=$(validateLog ${log})
1233     validationStatus=$?
1234     if [[ ${validationStatus} -eq 0 ]]; then
1235       #in pretend mode randomly report an error in rec.log some cases
1236       if [[ -n ${pretend} && "${log}" == "rec.log" ]]; then
1237         #[[ $(( ${RANDOM}%2 )) -ge 1 ]] && echo "${finallog} BAD random error" || echo "${finallog} OK"
1238         echo "${finallog} OK"
1239       else
1240         echo "${finallog} OK"
1241       fi
1242     elif [[ ${validationStatus} -eq 1 ]]; then
1243       echo "${finallog} BAD ${errorSummary}"
1244       logstatus=1
1245     elif [[ ${validationStatus} -eq 2 ]]; then
1246       echo "${finallog} OK MWAH ${errorSummary}"
1247     fi
1248   done
1249
1250   #report core files
1251   while read x; do
1252     echo ${x}
1253     chmod 644 ${x}
1254     gdb --batch --quiet -ex "bt" -ex "quit" aliroot ${x} > stacktrace_${x//\//_}.log
1255   done < <(/bin/ls ${PWD}/*/core 2>/dev/null; /bin/ls ${PWD}/core 2>/dev/null)
1256
1257   return ${logstatus}
1258 }
1259
1260 spitOutLocalOCDBaccessConfig()
1261 {
1262   umask 0002
1263   #find ${1} -name "*root" | \
1264   /bin/ls -1 ${1}/*/*/*/*.root 2>/dev/null | \
1265   while read line
1266   do
1267     local tmp=${line#${1}}
1268     echo ${tmp%/*} | \
1269     awk -v ocdb=${1} '{print "  man->SetSpecificStorage(\""$1"\",\"local://"ocdb"\");"}'
1270   done
1271   return 0
1272 }
1273
1274 goMakeLocalOCDBaccessConfig()
1275 {
1276   umask 0002
1277   # make a script that sets the specific storages form all the root files produced by CPass0
1278   local localOCDBpathCPass0=${1}
1279   local OCDBpathPrefix=${2}
1280   [[ -z ${OCDBpathPrefix} ]] && OCDBpathPrefix="."
1281
1282   if [[ -f ${localOCDBpathCPass0} && ${localOCDBpathCPass0} =~ \.tgz$ ]]; then
1283     tar xzf ${localOCDBpathCPass0}
1284     local localOCDBpathCPass0="${OCDBpathPrefix}/OCDB"
1285   fi
1286
1287   echo
1288   echo creating the specific storage script
1289   echo   localOCDBaccessConfig.C
1290   echo   based on OCDB: ${localOCDBaccessConfig}
1291   echo
1292
1293   local tempLocalOCDB=""
1294   if [[ -f localOCDBaccessConfig.C ]]; then
1295     tempLocalOCDB=$(mktemp)
1296     echo "egrep "SetSpecificStorage" localOCDBaccessConfig.C > ${tempLocalOCDB}"
1297     egrep "SetSpecificStorage" localOCDBaccessConfig.C > ${tempLocalOCDB}
1298   fi
1299
1300   echo "localOCDBaccessConfig()"                               >  localOCDBaccessConfig.C
1301   echo "{"                                                     >> localOCDBaccessConfig.C
1302   echo "  AliCDBManager* man = AliCDBManager::Instance();"     >> localOCDBaccessConfig.C
1303   spitOutLocalOCDBaccessConfig ${localOCDBpathCPass0}|sort|uniq  >> localOCDBaccessConfig.C
1304   [[ -f "${tempLocalOCDB}" ]] && cat ${tempLocalOCDB}              >> localOCDBaccessConfig.C
1305   echo "}"                                                     >> localOCDBaccessConfig.C
1306
1307   [[ -f "${tempLocalOCDB}" ]] && rm -f ${tempLocalOCDB}
1308
1309   if ! grep SetSpecificStorage localOCDBaccessConfig.C; then
1310     echo
1311     echo "!!!!!!! CPass0 produced no OCDB entries"
1312     return 1
1313   fi
1314   return 0
1315 }
1316
1317 goMakeFilteredTrees()
1318 {
1319   outputDir=${1}
1320   runNumber=${2}
1321   #get path to input list
1322   inputListfiles=${3}
1323   #get scale number for tracks
1324   filterT=${4}
1325   #get scale number for V0s
1326   filterV=${5}
1327   #get OCDB path (optional)
1328   OCDBpath=${6}
1329   #get max number of files
1330   maxFiles=${7-"1000000"}
1331   #get offset of first file
1332   offsetFile=${8-"0"}
1333   #get max number of events
1334   maxEvents=${9-"30000000"}
1335   #get offset of first event
1336   offsetEvent=${10-"0"}
1337   configFile=${11-"benchmark.config"}
1338   esdFileName=${12-"AliESDs_Barrel.root"}
1339   shift 12
1340   if ! parseConfig ${configFile} "$@"; then return 1; fi
1341
1342   #record the working directory provided by the batch system
1343   batchWorkingDirectory=${PWD}
1344
1345
1346   commonOutputPath=${PWD}
1347   doneFile=${commonOutputPath}/meta/filtering.cpass1.run${runNumber}.done
1348
1349   runpath=${outputDir}
1350   [[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
1351   [[ ${reconstructInTemporaryDir} -eq 1 && -z ${TMPDIR} ]] && runpath=$(mktemp -d)
1352
1353
1354   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
1355
1356   mkdir -p ${outputDir}
1357   mkdir -p ${runpath}
1358
1359   cd ${runpath}
1360   [[ ! ${PWD} =~ ${runpath} ]] && echo "PWD=$PWD is not the runpath=${runpath}" && touch ${doneFile} && return 1
1361
1362   cat > filtering.log << EOF
1363   goMakeFilteredTrees config:
1364   runpath=${runpath}
1365   outputDir=${outputDir}
1366   commonOutputPath=${commonOutputPath}
1367   ALICE_ROOT=${ALICE_ROOT}
1368   PATH=${PATH}
1369   offsetEvent=$offsetEvent
1370   configFile=$configFile
1371   esdFileName=$esdFileName
1372 EOF
1373
1374   if [[ -z ${pretend} ]];then
1375     aliroot -l -b -q "${ALICE_ROOT}/PWGPP/macros/runFilteringTask.C(\"${inputListfiles}\",${filterT},${filterV},\"${OCDBpath}\",${maxFiles},${offsetFile},${maxEvents},${offsetEvent},\"${esdFileName}\")" &>> filtering.log
1376   else
1377     touch filtering.log FilterEvents_Trees.root
1378   fi
1379   pwd
1380   /bin/ls
1381   touch ${doneFile}
1382   summarizeLogs >>  ${doneFile}
1383   echo mv -f * ${outputDir}
1384   mv -f * ${outputDir}
1385   [[ -f ${outputDir}/FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFile}
1386   cd ${commonOutputPath}
1387   [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
1388   return 0
1389 }
1390
1391 submit()
1392 {
1393   umask 0002
1394   [[ $# -lt 5 ]] && echo "at least 5 args needed, you supplied $#" && return 1
1395   JobID=${1}
1396   startID=${2}
1397   endID=${3}
1398   waitForJOBID=${4}
1399   command=${5}
1400   shift 5
1401   commandArgs=("$@")
1402
1403   #add quote strings around the extra arguments
1404   for ((i=0; i<${#commandArgs[@]}; i++)); do
1405     commandArgs[i]=\"${commandArgs[i]}\"
1406   done
1407
1408   [[ -z ${waitForJOBID} ]] && waitForJOBID=0
1409
1410   newFarm=$(which qsub|grep "^/usr/bin/qsub")
1411
1412   batchSystem="SGE"
1413
1414   if [[ -z "${newFarm}" ]]
1415   then
1416     #old LSF
1417     # submit it (as job array)
1418     nFiles=$(( ${endID}-${startID}+1 ))
1419     while [ ${startID} -le ${nFiles}  ] ; do
1420       if [ $(expr ${nFiles} - ${startID}) -gt 999 ] ; then
1421         endID=$(expr ${startID} + 999)
1422       else
1423         endID=${nFiles}
1424       fi
1425     if [[ ${waitForJOBID} -eq 0 ]]; then
1426         echo ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}"
1427         ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}"
1428       else
1429         echo ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -w "ended(${waitForJOBID})" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}"
1430         ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -w "ended(${waitForJOBID})" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}"
1431       fi
1432       startID=$(expr ${endID} + 1)
1433     done
1434   else
1435     #new SGE farm
1436     if [[ ${waitForJOBID} =~ "000" ]]; then
1437       echo ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}"
1438       ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}"
1439     else
1440       echo ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -hold_jid "${waitForJOBID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}"
1441       ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -hold_jid "${waitForJOBID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}"
1442     fi
1443   fi
1444   return 0
1445 }
1446
1447 goSubmitBatch()
1448 {
1449   if [[ $# -lt 3 ]]; then
1450     echo "minimal use:"
1451     echo " ${0} submit fileList productionID configFile"
1452     return 0
1453   fi
1454
1455   productionID=${1}
1456   inputList=${2}
1457   configFile=${3:-"benchmark.config"}
1458   configFile=$(readlink -f ${configFile})
1459   shift 3
1460   extraOpts=("$@")
1461   if ! parseConfig ${configFile} "${extraOpts[@]}"; then return 1; fi
1462
1463   #record the working directory provided by the batch system
1464   batchWorkingDirectory=${PWD}
1465
1466   #redirect all output to submit.log
1467   echo "redirecting all output to ${PWD}/submit_${productionID//"/"/_}.log"
1468   exec 7>&1
1469   exec 1>submit_${productionID//"/"/_}.log 2>&1
1470
1471   umask 0002
1472   echo ${0}" submit $*"
1473   if [[ -z "${inputList}" || -z "${productionID}" ]]
1474   then
1475     echo
1476     echo " Usage: ${0} submit inputList productionID [configFile=benchmark.config]"
1477     echo
1478     return
1479   fi
1480
1481   # check if config file is there
1482   if [ ! -f ${configFile} ]; then
1483     echo "ERROR! Config File '${configFile}' not found" >&2
1484     return
1485   else
1486     echo "Using Config File: '${configFile}'"
1487   fi
1488
1489   [[ ! -f ${alirootEnv} ]] && echo "alirootEnv script ${alirootEnv} not found!..." && return 1
1490
1491   #move the script, config and some other stuff to ${commonOutputPath} first, then use them from there
1492   self=$(readlink -f "${0}")
1493   configPath=$(dirname ${configFile})
1494   export commonOutputPath=${baseOutputDirectory}/${productionID}
1495
1496   mkdir -p ${commonOutputPath}
1497   mkdir -p ${commonOutputPath}/logs
1498   mkdir -p ${commonOutputPath}/meta
1499
1500   cp ${self} ${commonOutputPath}
1501   cp ${configFile} ${commonOutputPath}
1502   cp ${inputList} ${commonOutputPath}
1503   self=${commonOutputPath}/${self##*/}
1504   chmod u+x ${self}
1505   configFile=${commonOutputPath}/${configFile##*/}
1506   inputList=${commonOutputPath}/${inputList##*/}
1507
1508   #convert to absolut pathnames
1509   inputList=$(readlink -f "${inputList}")
1510   #make list of runs
1511   if [[ -z ${runNumber} ]]; then
1512     listOfRuns=($(while read x; do guessRunNumber ${x}; done < ${inputList} | sort | uniq))
1513   else
1514     listOfRuns=${runNumber}
1515   fi
1516
1517   alirootSource=$(readlink -f "${alirootSource}")
1518
1519   echo ""
1520   echo "### BEGIN CONFIGURATION ###"
1521   echo ""
1522   echo "GENERAL:"
1523   echo ""
1524   echo "    productionID:    ${productionID}"
1525   echo "    batchCommand:    ${batchCommand}"
1526   echo "    batchFlags:      ${batchFlags}"
1527   echo "    alirootEnv:   ${alirootEnv}"
1528   ${alirootEnv} echo '    ALICE_ROOT:      ${ALICE_ROOT}'
1529   ${alirootEnv} echo '    ALIROOT_RELEASE: ${ALICE_RELEASE}'
1530   echo "    inputList:       ${inputList}"
1531   echo "    configPath:      ${configPath}"
1532   echo "    commonOutputPath:      ${commonOutputPath}"
1533   echo "    defaultOCDB:     ${defaultOCDB}"
1534   echo "      autoOCDB: ${autoOCDB}"
1535   echo "    recoTriggerOptions:   ${recoTriggerOptions}"
1536   echo "    runs:"
1537   echo "      ${listOfRuns[*]}"
1538   echo ""
1539   echo "THE TRAIN WILL RUN:"
1540
1541   if [ ${runCPass0reco} -eq 1 ]; then
1542     echo "    Pass0 - Recontruction"
1543   fi
1544
1545   if [ ${runCPass0MergeMakeOCDB} -eq 1 ]; then
1546     echo "    Pass0 - merging and OCDB export"
1547   fi
1548
1549   if [ ${runCPass1reco} -eq 1 ]; then
1550     echo "    Pass1 - Recontruction"
1551   fi
1552   if [ ${runCPass1MergeMakeOCDB} -eq 1 ]; then
1553     echo "    Pass1 - merging and OCDB export"
1554   fi
1555
1556   echo ""
1557   echo "LIMITS:"
1558   echo "    max. Events/Chunk:   ${nEvents}"
1559   echo "    max. Number of Chunks per Run:     ${nMaxChunks}"
1560   echo ""
1561   echo "### END CONFIGURATION ###"
1562   echo ""
1563
1564
1565   # check if input file is there
1566   if [ ! -f ${inputList} ]; then
1567     echo "ERROR! Input List '${inputList}' not found" >&2
1568     return
1569   fi
1570
1571   # define jobid (for dependent jobs)
1572   date=$(date +%Y_%m_%d_%H%M%S)
1573   #for each run we submit one jobarray:
1574   for runNumber in ${listOfRuns[*]}; do
1575
1576     [[ -z ${runNumber} ]] && continue
1577     [[ ! ${runNumber} =~ ^[0-9]*[0-9]$ ]] && continue
1578
1579     JOBpostfix="${productionID//"/"/_}_${runNumber}_${date}"
1580     JOBID1="p0_${JOBpostfix}"
1581     JOBID1wait="w0_${JOBpostfix}"
1582     JOBID2="m0_${JOBpostfix}"
1583     JOBID2wait="wm0_${JOBpostfix}"
1584     JOBID3="op0_${JOBpostfix}"
1585     JOBID3wait="wop0_${JOBpostfix}"
1586     JOBID4="p1_${JOBpostfix}"
1587     JOBID4wait="w1_${JOBpostfix}"
1588     JOBID5="m1_${JOBpostfix}"
1589     JOBID5wait="wm1_${JOBpostfix}"
1590     JOBID6="s1_${JOBpostfix}"
1591     JOBID6wait="ws1_${JOBpostfix}"
1592     JOBID7="QA_${JOBpostfix}"
1593     JOBmakeESDlistCPass1="lp1_${JOBpostfix}"
1594     JOBfilterESDcpass1="fp1_${JOBpostfix}"
1595     LASTJOB="000"
1596
1597     oneInputFile=$(egrep -m1 "${runNumber}\/" ${inputList})
1598
1599     currentDefaultOCDB=${defaultOCDB}
1600     [[ -z ${autoOCDB} ]] && autoOCDB=1
1601     if [[ ${autoOCDB} -ne 0 ]]; then
1602       currentDefaultOCDB=$(setYear ${oneInputFile} ${defaultOCDB})
1603     fi
1604
1605     echo "submitting run ${runNumber} with OCDB ${currentDefaultOCDB}"
1606
1607     ###############################################################################
1608     #run one chunk with valgrind:
1609     if [[ -n ${profilingCommand} ]]; then
1610       [[ -z ${nEventsProfiling} ]] && nEventsProfiling=2
1611       [[ -z ${profilingCommand} ]] && profilingCommand="/usr/bin/valgrind --tool=callgrind --num-callers=40 -v --trace-children=yes"
1612       submit "valgrind" 1 1 000 "${alirootEnv} ${self}" CPass0 ${commonOutputPath}/000${runNumber}/valgrind ${oneInputFile} ${nEventsProfiling} ${currentDefaultOCDB} ${configFile} ${runNumber} valgrind useProfilingCommand=${profilingCommand} "${extraOpts[@]}"
1613     fi
1614
1615     ################################################################################
1616     ################################################################################
1617     # run the CPass0 if requested
1618
1619     if [ ${runCPass0reco} -eq 1 ]; then
1620
1621       echo
1622       echo "starting CPass0... for run ${runNumber}"
1623       echo
1624
1625       # create directory and copy all files that are needed
1626       targetDirectory="${commonOutputPath}/000${runNumber}/cpass0"
1627       mkdir -p ${targetDirectory}
1628
1629       filesCPass0=(
1630                     "${configPath}/runCPass0.sh"
1631                     "${configPath}/recCPass0.C"
1632                     "${configPath}/runCalibTrain.C"
1633                     "${configPath}/localOCDBaccessConfig.C"
1634                     "${configPath}/OCDB*.root"
1635                     "${configPath}/sim.C"
1636                     "${configPath}/rec.C"
1637                     "${configPath}/Config.C"
1638       )
1639       for file in ${filesCPass0[*]}; do
1640         [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
1641       done
1642
1643       localInputList=${targetDirectory}/${inputList##*/}
1644       [[ ! -f ${localInputList} ]] && egrep "\/000${runNumber}\/" ${inputList} > ${localInputList}
1645       # limit nFiles to nMaxChunks
1646       nFiles=$(wc -l < ${localInputList})
1647       [[ ${nFiles} -eq 0 ]] && echo "list contains ZERO files! exiting..." && return 1
1648       echo "raw files in list:    ${nFiles}"
1649       if [[ ${nMaxChunks} -gt 0 && ${nMaxChunks} -le ${nFiles} ]]; then
1650         nFiles=${nMaxChunks}
1651       fi
1652       echo "raw files to process: ${nFiles}"
1653       [[ -z "${percentProcessedFilesToContinue}" ]] && percentProcessedFilesToContinue=100
1654       if [[ ${percentProcessedFilesToContinue} -eq 100 ]]; then
1655         nFilesToWaitFor=${nFiles}
1656       else
1657         nFilesToWaitFor=$(( ${nFiles}-${nFiles}/(100/(100-${percentProcessedFilesToContinue})) ))
1658       fi
1659       echo "requested success rate is ${percentProcessedFilesToContinue}%"
1660       echo "merging will start after ${nFilesToWaitFor} jobs are done"
1661
1662       submit ${JOBID1} 1 ${nFiles} 000 "${alirootEnv} ${self}" CPass0 ${targetDirectory} ${localInputList} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} -1 "${extraOpts[@]}"
1663
1664       ## submit a monitoring job that will run until a certain number of jobs are done with reconstruction
1665       submit "${JOBID1wait}" 1 1 000 "${alirootEnv} ${self}" WaitForOutput ${commonOutputPath} "meta/cpass0.job*.run${runNumber}.done" ${nFilesToWaitFor} ${maxSecondsToWait} '-maxdepth 1'
1666       LASTJOB=${JOBID1wait}
1667
1668     fi #end running CPass0
1669     ################################################################################
1670
1671
1672     ################################################################################
1673     # submit merging of CPass0, depends on the reconstruction
1674
1675     if [ ${runCPass0MergeMakeOCDB} -eq 1 ]; then
1676
1677       echo
1678       echo "submit CPass0 merging for run ${runNumber}"
1679       echo
1680
1681       targetDirectory="${commonOutputPath}/000${runNumber}/cpass0"
1682       mkdir -p ${targetDirectory}
1683
1684       #copy the scripts
1685       filesMergeCPass0=(
1686                         "${configPath}/OCDB.root"
1687                         "${configPath}/mergeMakeOCDB.byComponent.sh"
1688                         "${configPath}/mergeMakeOCDB.sh"
1689                         "${configPath}/localOCDBaccessConfig.C"
1690                         "${configPath}/mergeByComponent.C"
1691                         "${configPath}/makeOCDB.C"
1692                         "${configPath}/merge.C"
1693       )
1694       for file in ${filesMergeCPass0[*]}; do
1695         [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
1696       done
1697
1698       submit ${JOBID2} 1 1 "${LASTJOB}" "${alirootEnv} ${self}" MergeCPass0 ${targetDirectory} ${currentDefaultOCDB} ${configFile} ${runNumber} cpass0.calib.run${runNumber}.list "${extraOpts[@]}"
1699       LASTJOB=${JOBID2}
1700
1701       if [[ -n ${generateMC} ]]; then
1702         submit "mrl${JOBpostfix}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" PrintValues sim ${commonOutputPath}/meta/sim.run${runNumber}.list ${commonOutputPath}/meta/cpass0.job*.run${runNumber}.done
1703         LASTJOB="mrl${JOBpostfix}"
1704       fi
1705
1706       echo
1707     fi
1708     # end of merging CPass0
1709     ################################################################################
1710
1711     ################################################################################
1712     ################################################################################
1713     # run the CPass1 if requested
1714
1715     if [ ${runCPass1reco} -eq 1 ]; then
1716
1717       targetDirectory="${commonOutputPath}/000${runNumber}/cpass1"
1718       rm -f ${commonOutputPath}/meta/cpass1.job*.run${runNumber}.done
1719
1720       # safety feature: if we are re-running for any reason we want to delete the previous output first.
1721       [[ -d ${targetDirectory} ]] && rm -rf ${targetDirectory}/* && echo "removed old output at ${targetDirectory}/*"
1722
1723       echo
1724       echo "starting CPass1... for run ${runNumber}"
1725       echo
1726
1727       # create directory and copy all files that are needed
1728       mkdir -p ${targetDirectory}
1729
1730       filesCPass1=(
1731                     "${configPath}/runCPass1.sh"
1732                     "${configPath}/recCPass1.C"
1733                     "${configPath}/recCPass1_OuterDet.C"
1734                     "${configPath}/runCalibTrain.C"
1735                     "${configPath}/QAtrain_duo.C"
1736                     "${configPath}/localOCDBaccessConfig.C"
1737                     "${configPath}/OCDB.root"
1738       )
1739       for file in ${filesCPass1[*]}; do
1740         [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
1741       done
1742
1743       if [[ -n ${generateMC} ]]; then
1744         localInputList=${commonOutputPath}/meta/sim.run${runNumber}.list
1745       else
1746         localInputList=${targetDirectory}/${inputList##*/}
1747         [[ ! -f ${localInputList} ]] && egrep "\/000${runNumber}\/" ${inputList} > ${localInputList}
1748       fi
1749       # limit nFiles to nMaxChunks
1750       nFiles=$(wc -l < ${localInputList})
1751       [[ ${nFiles} -eq 0 ]] && echo "list contains ZERO files! continuing..." && continue
1752       echo "raw files in list:    ${nFiles}"
1753       if [[ ${nMaxChunks} -gt 0 && ${nMaxChunks} -le ${nFiles} ]]; then
1754         nFiles=${nMaxChunks}
1755       fi
1756       echo "raw files to process: ${nFiles}"
1757       [[ -z "${percentProcessedFilesToContinue}" ]] && percentProcessedFilesToContinue=100
1758       if [[ ${percentProcessedFilesToContinue} -eq 100 ]]; then
1759         nFilesToWaitFor=${nFiles}
1760       else
1761         nFilesToWaitFor=$(( ${nFiles}-${nFiles}/(100/(100-${percentProcessedFilesToContinue})) ))
1762       fi
1763       echo "requested success rate is ${percentProcessedFilesToContinue}%"
1764       echo "merging will start after ${nFilesToWaitFor} jobs are done"
1765
1766       submit ${JOBID4} 1 ${nFiles} "${LASTJOB}" "${alirootEnv} ${self}" CPass1 ${targetDirectory} ${localInputList} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} -1 "${extraOpts[@]}"
1767
1768       ################################################################################
1769       ## submit a monitoring job that will run until a certain number of jobs are done with reconstruction
1770       submit "${JOBID4wait}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" WaitForOutput ${commonOutputPath} "meta/cpass1.job*.run${runNumber}.done" ${nFilesToWaitFor} ${maxSecondsToWait} '-maxdepth 1'
1771       LASTJOB=${JOBID4wait}
1772       ################################################################################
1773
1774       echo
1775     fi #end running CPass1
1776
1777     ################################################################################
1778     # submit merging of CPass1, depends on the reconstruction
1779     if [ ${runCPass1MergeMakeOCDB} -eq 1 ]; then
1780
1781       echo
1782       echo "submit CPass1 merging for run ${runNumber}"
1783       echo
1784
1785       targetDirectory="${commonOutputPath}/000${runNumber}/cpass1"
1786       rm -f ${commonOutputPath}/meta/merge.cpass1.run${runNumber}.done
1787       mkdir -p ${targetDirectory}
1788
1789       # copy files
1790       filesMergeCPass1=(
1791                         "${configPath}/OCDB.root"
1792                         "${configPath}/localOCDBaccessConfig.C"
1793                         "${configPath}/mergeMakeOCDB.byComponent.sh"
1794                         "${configPath}/mergeByComponent.C"
1795                         "${configPath}/makeOCDB.C"
1796                         "${configPath}/merge.C"
1797                         "${configPath}/mergeMakeOCDB.sh"
1798                         "${configPath}/QAtrain_duo.C"
1799       )
1800       for file in ${filesMergeCPass1[*]}; do
1801         [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
1802       done
1803
1804       submit "${JOBID5}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" MergeCPass1 ${targetDirectory} ${currentDefaultOCDB} ${configFile} ${runNumber} cpass1.calib.run${runNumber}.list cpass1.QA.run${runNumber}.lastMergingStage.txt.list cpass1.filtered.run${runNumber}.list "${extraOpts[@]}"
1805       LASTJOB=${JOBID5}
1806       echo
1807     fi
1808
1809     ###############################
1810     #if [ ${runESDfiltering} -eq 1 ]; then
1811     #  rm -f ${commonOutputPath}/cpass1.ESD.run${runNumber}.list
1812     #  rm -f ${commonOutputPath}/meta/filtering.cpass1.run*.done
1813     #  echo
1814     #  echo submitting filtering for run ${runNumber}
1815     #  echo
1816     #  submit "${JOBmakeESDlistCPass1}" 1 1 "${LASTJOB}" "${self}" PrintValues esd ${commonOutputPath}/meta/cpass1.ESD.run${runNumber}.list ${commonOutputPath}/meta/cpass1.job*.run${runNumber}.done
1817     #  submit "${JOBfilterESDcpass1}" 1 1 "${JOBmakeESDlistCPass1}" "${alirootEnv} ${self}" MakeFilteredTrees ${commonOutputPath}/000${runNumber}/cpass1 ${runNumber} ${commonOutputPath}/meta/cpass1.ESD.run${runNumber}.list ${filteringFactorHighPt} ${filteringFactorV0s} ${currentDefaultOCDB} 1000000 0 10000000 0 ${configFile} AliESDs_Barrel.root "${extraOpts[@]}"
1818     #  LASTJOB=${JOBfilterESDcpass1}
1819     #fi
1820
1821   done
1822
1823   #################################################################################
1824   #################################################################################
1825   #if [ ${runESDfiltering} -eq 1 ]; then
1826   #  submit "${JOBID5wait}" 1 1 "${LASTJOB}" "${self}" WaitForOutput ${commonOutputPath} "meta/filtering.cpass1.run*.done" "${#listOfRuns[@]}" ${maxSecondsToWait}
1827   #else
1828     submit "${JOBID5wait}" 1 1 "${LASTJOB}" "${self}" WaitForOutput ${commonOutputPath} "meta/merge.cpass1.run*.done" ${#listOfRuns[@]} ${maxSecondsToWait}
1829   #fi
1830   LASTJOB=${JOBID5wait}
1831
1832   #################################################################################
1833   echo
1834   echo "submit make a summary"
1835   echo
1836
1837   submit "${JOBID6}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" MakeSummary ${configFile}
1838   LASTJOB=${JOBID6}
1839   #################################################################################
1840
1841   #restore stdout
1842   exec 1>&7 7>&-
1843   echo "jobs submitted."
1844   return 0
1845 }
1846
1847 goWaitForOutput()
1848 {
1849   umask 0002
1850   [[ $# -lt 3 ]] && echo "goWaitForOutput() wrong number of arguments, exiting.." && return 1
1851   echo searchPath=${1}
1852   echo fileName=${2}
1853   echo numberOfFiles=${3}
1854   echo maxSecondsToWait=${4}
1855   searchPath=${1}
1856   fileName=${2}
1857   numberOfFiles=${3}
1858   maxSecondsToWait=${4}
1859   extraFindOptions=${5}
1860   echo "command to be executed: /bin/ls -1 ${searchPath}/${fileName} ${extraFindOptions}"
1861   [[ -z "${maxSecondsToWait}" ]] && maxSecondsToWait=$(( 3600*12 ))
1862   while true; do
1863     #n=$(find ${searchPath} ${extraFindOptions} -name "${fileName}" | wc -l)
1864     n=$(/bin/ls -1 ${searchPath}/${fileName} 2>/dev/null | wc -l)
1865     [[ ${n} -gt 0 ]] && echo "found ${n} X ${fileName}"
1866     [[ ${n} -ge ${numberOfFiles} ]] && break
1867     [[ ${SECONDS} -gt ${maxSecondsToWait} ]] && echo "timeout of ${maxSecondsToWait}s!" && break
1868     sleep 60
1869   done
1870   echo "DONE! exiting..."
1871   return 0
1872 }
1873
1874 mergeSysLogs()
1875 {
1876   outputFile=${1}
1877   shift
1878   inputFiles="$@"
1879   i=0
1880   if ! ls -1 ${inputFiles} &>/dev/null; then echo "the files dont exist!: ${inputFiles}"; return 1; fi
1881   while read x; do
1882     runNumber=$(guessRunNumber ${x})
1883     [[ -z ${runNumber} ]] && echo "run number cannot be guessed for ${x}" && continue
1884     awk -v run=${runNumber} -v i=${i} 'NR > 1 {print run" "$0} NR==1 && i==0 {print "run/I:"$0}' ${x}
1885     (( i++ ))
1886   done < <(ls -1 ${inputFiles}) > ${outputFile}
1887   return 0
1888 }
1889
1890 goMakeMergedSummaryTree()
1891 {
1892   # create list of calibration entries
1893   # takes no arguments, just run it in the base output
1894   # directory with the following files in the working directory
1895   #
1896   #  Calibration file lists:
1897   #       cpass0.dcsTree.list, cpass1.dcsTree.list
1898   #  QA trending root file:
1899   #       trending.root
1900   #
1901   #  Production infoo ascii files:
1902   #       summary_pass0.tree
1903   #       summary_pass1.tree
1904   #
1905
1906   [[ ! -f cpass0.dcsTree.list ]] && echo "no cpass0.dcsTree.list" && return 1
1907   [[ ! -f cpass1.dcsTree.list ]] && echo "no cpass1.dcsTree.list" && return 1
1908   [[ ! -f trending.root ]] && echo "no trending.root" && return 1
1909   [[ ! -f summary_pass0.tree ]] && echo "no summary_pass0.tree" && return 1
1910   [[ ! -f summary_pass1.tree ]] && echo "no summary_pass1.tree" && return 1
1911
1912   #first, dump the C macro to file
1913   cat << EOF > mergeTree.C
1914   //
1915   // Merge summary information
1916   // Following files are expected to be in the working directory
1917   //
1918   // Calibration file lists:
1919   //      cpass0.dcsTree.list, cpass1.dcsTree.list
1920   // QA trending root file:
1921   //      trending.root
1922   //
1923   // Production infoo ascii files:
1924   //      summary_pass0.tree
1925   //      summary_pass1.tree
1926   //
1927   void mergeTree(){
1928     //
1929     //
1930     //
1931     // Calibration values dump
1932     //
1933     AliXRDPROOFtoolkit::MakeTreeFromList("Calib.TPC.CPass0.root", "dcs","dcs","cpass0.dcsTree.list",1);
1934     AliXRDPROOFtoolkit::MakeTreeFromList("Calib.TPC.CPass1.root", "dcs","dcs","cpass1.dcsTree.list",1);
1935     //
1936     // Calibration status dump
1937     //
1938     TFile *fprod = TFile::Open("fproduction.root","recreate");
1939     TTree  tree0, tree1;
1940     tree0.ReadFile("summary_pass0.tree");
1941     tree1.ReadFile("summary_pass1.tree");
1942     tree0.Write("CPass0");
1943     tree1.Write("CPass1");
1944     fprod->Close();
1945     //
1946     //
1947     //
1948     TString stringSetup="";
1949     stringSetup+="1#QA.TPC#run#SummaryTPCQA/tpcQA#trending.root+";  //
1950     stringSetup+="1#Calib.TPC.CPass0#run#dcs#Calib.TPC.CPass0.root+";  //
1951     stringSetup+="1#Calib.TPC.CPass1#run#dcs#Calib.TPC.CPass1.root+";  //
1952     //
1953     stringSetup+="1#CPass0#runnumber#CPass0#fproduction.root+";  //
1954     stringSetup+="1#CPass1#runnumber#CPass1#fproduction.root+";  //
1955     //
1956     AliXRDPROOFtoolkit::JoinTreesIndex("outAll.root","joinAll","run",stringSetup.Data(), 1);
1957   }
1958 EOF
1959
1960   aliroot -b -q "mergeTree.C" > mergeTrees.log
1961   return $?
1962 }
1963
1964 goMakeSummary()
1965 {
1966   #all the final stuff goes in here for ease of use:
1967   # summary logs
1968   # qa plot making
1969   # final file lists
1970   #some defaults:
1971   log="summary.log"
1972   productionID="qa"
1973
1974   configFile=${1}
1975   shift 1
1976   extraOpts=("$@")
1977   if ! parseConfig ${configFile} "${extraOpts[@]}"; then return 1; fi
1978
1979   #record the working directory provided by the batch system
1980   batchWorkingDirectory=${PWD}
1981
1982   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
1983
1984   [[ ! -f ${configFile} ]] && echo "no config file ${configFile}!" && return
1985
1986   [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
1987
1988   #copy some useful stuff
1989   #and go to the commonOutputPath
1990   cp ${configFile} ${commonOutputPath}
1991   cd ${commonOutputPath}
1992
1993   exec &> >(tee ${log})
1994
1995   #summarize the global stuff
1996   echo "env script: ${alirootSource} ${alirootEnv}"
1997   echo "\$ALICE_ROOT=${ALICE_ROOT}"
1998   echo "commonOutputPath=${commonOutputPath}"
1999
2000   #summarize the stacktraces
2001   awk '
2002        BEGIN {
2003                print "frame/I:method/C:line/C:cpass/I:aliroot/I";
2004                RS="#[0-9]*";
2005                aliroot=0;
2006              }
2007              {
2008                if ($3 ~ /Ali*/) aliroot=1; else aliroot=0;
2009                gsub("#","",RT);
2010                if ($NF!="" && RT!="" && $3!="") print RT" "$3" "$NF" "0" "aliroot
2011              }
2012       ' 000*/cpass0/*/stacktrace* 2>/dev/null > stacktrace.tree
2013   awk '
2014        BEGIN {
2015                RS="#[0-9]*";
2016                aliroot=0;
2017              }
2018              {
2019                if ($3 ~ /Ali*/) aliroot=1; else aliroot=0;
2020                gsub("#","",RT);
2021                if ($NF!="" && RT!="" && $3!="") print RT" "$3" "$NF" "1" "aliroot
2022              }
2023       ' 000*/cpass1/*/stacktrace* 2>/dev/null >> stacktrace.tree
2024
2025   echo total numbers for the production:
2026   echo
2027   awk 'BEGIN {nFiles=0;nCore=0;}
2028   /^calibfile/ {nFiles++;}
2029   /core dumped/ {nCore++i;}
2030   END {print     "cpass0 produced "nFiles" calib files, "nCore" core files";}' meta/cpass0.job*done 2>/dev/null
2031   awk 'BEGIN {nOK=0; nBAD=0; }
2032   /\/rec.log OK/ {nOK++;}
2033   /\/rec.log BAD/ {nBAD++;}
2034   /stderr BAD/ {if ($0 ~ /rec.log/){nBAD++;}}
2035   END {print     "cpass0 reco:  OK: "nOK"\tBAD: "nBAD;}' meta/cpass0.job*done 2>/dev/null
2036   awk 'BEGIN {nOK=0; nBAD=0; }
2037   /\/calib.log OK/ {nOK++;}
2038   /\/calib.log BAD/ {nBAD++;}
2039   END {print "cpass0 calib: OK: "nOK"\tBAD: "nBAD;}' meta/cpass0.job*done 2>/dev/null
2040
2041   awk 'BEGIN {nOK=0; nBAD=0; }
2042   /merge.log OK/ {nOK++;}
2043   /merge.log BAD/ {nBAD++;}
2044   END {print "cpass0 merge: OK: "nOK"\tBAD: "nBAD;}' meta/merge.cpass0*done 2>/dev/null
2045   awk 'BEGIN {nOK=0; nBAD=0; }
2046   /ocdb.log OK/ {nOK++;}
2047   /ocdb.log BAD/ {nBAD++;}
2048   END {print   "cpass0 OCDB:  OK: "nOK"\tBAD: "nBAD;}' meta/merge.cpass0*done 2>/dev/null
2049
2050   echo
2051   awk 'BEGIN {nFiles=0;nCore=0;}
2052   /^calibfile/ {nFiles++;}
2053   /core dumped/ {nCore++;}
2054   END {print     "cpass1 produced "nFiles" calib files, "nCore" core files";}' meta/cpass1.job*done 2>/dev/null
2055   awk 'BEGIN {nOK=0; nBAD=0; }
2056   /\/rec.log OK/ {nOK++;}
2057   /\/rec.log BAD/ {nBAD++;}
2058   /stderr BAD/ {if ($0 ~ /rec.log/){nBAD++;}}
2059   END {print     "cpass1 reco:  OK: "nOK"\tBAD: "nBAD;}' meta/cpass1.job*done 2>/dev/null
2060   awk 'BEGIN {nOK=0; nBAD=0; }
2061   /\/calib.log OK/ {nOK++;}
2062   /\/calib.log BAD/ {nBAD++;}
2063   END {print "cpass1 calib: OK: "nOK"\tBAD: "nBAD;}' meta/cpass1.job*done 2>/dev/null
2064
2065   awk 'BEGIN {nOK=0; nBAD=0; }
2066   /merge.log OK/ {nOK++;}
2067   /merge.log BAD/ {nBAD++;}
2068   END {print "cpass1 merge: OK: "nOK"\tBAD: "nBAD;}' meta/merge.cpass1*done 2>/dev/null
2069   awk 'BEGIN {nOK=0; nBAD=0; }
2070   /ocdb.log OK/ {nOK++;}
2071   /ocdb.log BAD/ {nBAD++;}
2072   END {print   "cpass1 OCDB:  OK: "nOK"\tBAD: "nBAD;}' meta/merge.cpass1*done 2>/dev/null
2073
2074   echo
2075   echo per run stats:
2076   /bin/ls -1 meta/merge.cpass0.run*.done | while read x
2077 do
2078   dir=$(goPrintValues calibfile - ${x})
2079   runNumber=$(guessRunNumber ${dir})
2080   [[ -z ${runNumber} ]] && continue
2081
2082   if $(/bin/ls meta/cpass0.job*.run${runNumber}.done &> /dev/null); then
2083     statusCPass0=( $(
2084     awk 'BEGIN {nOKrec=0;nBADrec=0;nOKcalib=0;nBADcalib=0;nOKstderr=0;nBADstderr=0;}
2085     /\/rec.log OK/ {nOKrec++;}
2086     /\/rec.log BAD/ {nBADrec++;}
2087     /stderr BAD/ {if ($0 ~ /rec.log/) {nBADrec++;} nBADstderr++;}
2088     /stderr OK/ {nOKstderr++;}
2089     /\/calib.log OK/ {nOKcalib++;}
2090     /\/calib.log BAD/ {nBADcalib++}
2091     END {print ""nOKrec" "nBADrec" "nOKstderr" "nBADstderr" "nOKcalib" "nBADcalib;}' meta/cpass0.job*.run${runNumber}.done 2>/dev/null
2092     ) )
2093   fi
2094
2095   if $(/bin/ls meta/cpass1.job*.run${runNumber}.done &>/dev/null); then
2096     statusCPass1=( $(
2097     awk 'BEGIN {nOKrec=0;nBADrec=0;nOKcalib=0;nBADcalib=0;nOKstderr=0;nBADstderr=0;nQAbarrelOK=0;nQAbarrelBAD=0;nQAouterOK=0;nQAouterBAD=0;}
2098     /\/rec.log OK/ {nOKrec++;}
2099     /\/rec.log BAD/ {nBADrec++;}
2100     /stderr BAD/ {if ($0 ~ /rec.log/) nBADrec++;nBADstderr++;}
2101     /stderr OK/ {nOKstderr++;}
2102     /\/calib.log OK/ {nOKcalib++;}
2103     /\/calib.log BAD/ {nBADcalib++}
2104     /\/qa_barrel.log OK/ {nQAbarrelOK++;}
2105     /\/qa_barrel.log BAD/ {nQAbarrelBAD++;}
2106     /\/qa_outer.log OK/ {nQAouterOK++;}
2107     /\/qa_outer.log BAD/ {nQAouterBAD++;}
2108     END {print ""nOKrec" "nBADrec" "nOKstderr" "nBADstderr" "nOKcalib" "nBADcalib" "nQAbarrelOK" "nQAbarrelBAD" "nQAouterOK" "nQAouterBAD;}' meta/cpass1.job*.run${runNumber}.done 2>/dev/null
2109     ) )
2110   fi
2111
2112   statusOCDBcpass0=$(awk '/ocdb.log/ {print $2} ' ${x} 2>/dev/null)
2113   statusOCDBcpass1=$(awk '/ocdb.log/ {print $2}' ${x/cpass0/cpass1} 2>/dev/null)
2114   statusQA=$(awk '/mergeMakeOCDB.log/ {print $2}' ${x/cpass0/cpass1} 2>/dev/null)
2115
2116   printf "%s\t ocdb.log cpass0: %s\t ocdb.log cpass1: %s\tqa.log:%s\t| cpass0: rec:%s/%s stderr:%s/%s calib:%s/%s cpass1: rec:%s/%s stderr:%s/%s calib:%s/%s QAbarrel:%s/%s QAouter:%s/%s\n" ${runNumber} ${statusOCDBcpass0} ${statusOCDBcpass1} ${statusQA} ${statusCPass0[0]} ${statusCPass0[1]} ${statusCPass0[2]} ${statusCPass0[3]} ${statusCPass0[4]} ${statusCPass0[5]} ${statusCPass1[0]} ${statusCPass1[1]} ${statusCPass1[2]} ${statusCPass1[3]} ${statusCPass1[4]} ${statusCPass1[5]} ${statusCPass1[6]} ${statusCPass1[7]} ${statusCPass1[8]} ${statusCPass1[9]}
2117 done
2118
2119   #make lists with output files - QA, trending, filtering and calibration
2120   ### wait for the merging of all runs to be over ###
2121   rm -f qa.list
2122   goPrintValues qafile qa.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2123   rm -f calib.list
2124   goPrintValues calibfile calib.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2125   rm -f trending.list
2126   goPrintValues trendingfile trending.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2127   rm -f filtering.list
2128   goPrintValues filteredTree filtering.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2129   rm -f cpass0.dcsTree.list
2130   goPrintValues dcsTree cpass0.dcsTree.list ${commonOutputPath}/meta/merge.cpass0.run*.done &>/dev/null
2131   rm -f cpass1.dcsTree.list
2132   goPrintValues dcsTree cpass1.dcsTree.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2133
2134   #merge trending
2135   rm -f ${commonOutputPath}/trending_merged.root
2136   goMerge trending.list ${commonOutputPath}/trending.root ${configFile} "${extraOpts[@]}" &> mergeTrending.log
2137
2138   goMakeSummaryTree ${commonOutputPath} 0
2139   goMakeSummaryTree ${commonOutputPath} 1
2140
2141   goCreateQAplots "qa.list" "${productionID}" "QAplots" "${configFile}" "${extraOpts[@]}" filteringList="filtering.list" &>createQAplots.log
2142
2143   #make a merged summary tree out of the QA trending, dcs trees and log summary trees
2144   goMakeMergedSummaryTree
2145
2146   #if set, email the summary
2147   [[ -n ${MAILTO} ]] && cat ${log} | mail -s "benchmark ${productionID} done" ${MAILTO}
2148
2149   return 0
2150 }
2151
2152 goMakeSummaryTree()
2153 {
2154   if [[ $# -lt 1 ]] ; then
2155     return
2156   fi
2157   #1. define vars/arrays
2158   DIR=${1} #use input or exec in current dir
2159   pass=${2-"0"} #pass from input
2160   outfile="summary_pass${pass}.tree"
2161   Ncolumns=0
2162   test -f ${outfile} && : >${outfile}
2163   errfile=${outfile/tree/err}
2164   test -f ${errfile} && : >${errfile}
2165
2166   declare -a counterString=(TOFevents TOFtracks TPCevents TPCtracks TRDevents TRDtracks T0events SDDevents SDDtracks MeanVertexevents)
2167   Ncounter=${#counterString[@]}
2168
2169   declare -a statusString=(TRDStatus TOFStatus TPCStatus T0Status MeanVertexStatus)
2170   Nstatus=${#statusString[@]}
2171
2172
2173   declare -a ratesString=(rec stderr calib qa_barrel qa_outer)
2174   Nrates=${#ratesString[@]}
2175
2176   runs=( $(ls -1 ${DIR}/meta/merge.cpass0* | while read x; do guessRunNumber $x; done) )
2177   Nruns=${#runs[@]}
2178
2179   echo -n runnumber/I >>${outfile}
2180   echo -n :cpass${pass}status/I >>${outfile}
2181   echo -n :cpass${pass}QAstatus/I >>${outfile}
2182   for i in ${ratesString[@]}; do
2183     echo -n :${i}OK/I >>${outfile}
2184     echo -n :${i}BAD/I >>${outfile}
2185
2186   done
2187   for i in ${counterString[@]} ${statusString[@]} ; do
2188     echo -n :${i}/I >>${outfile}
2189   done
2190   Ncolumns=$((2 + 2*Nrates + Ncounter + Nstatus))
2191   echo >> ${outfile}
2192
2193   #2. loop runs
2194
2195   for runnumber in ${runs[@]} ; do
2196
2197
2198
2199     filejob="${DIR}/meta/cpass${pass}.job*.run${runnumber}.done"
2200     filemerge="${DIR}/meta/merge.cpass${pass}.run${runnumber}.done"
2201     fileOCDB=$(grep /ocdb.log ${filemerge} | awk '{print $1}')
2202     if ! $(/bin/ls ${filemerge} &>/dev/null) ; then
2203       echo "${filemerge} does not exist!" >>${errfile}
2204       continue
2205     elif ! $(/bin/ls ${filejob} &>/dev/null) ; then
2206       echo "${filejob} does not exist!" >>${errfile}
2207       echo -n ${runnumber} >> ${outfile}
2208       for i in $(seq ${Ncolumns}) ; do
2209         echo -n "-1" >> ${outfile}
2210       done
2211       echo >> ${outfile}
2212       continue
2213     fi
2214     echo -n ${runnumber} >> ${outfile}
2215     #pass0status= grep '/ocdb.log' ${filemerge} |  cut -d' ' -f2 | tr OK x1 | tr BAD xx0 | tr -d 'x'
2216     passStatus=$(grep '/ocdb.log' ${filemerge} | grep OK | wc -l)
2217     echo -n " ${passStatus}" >> ${outfile}
2218     qaStatus=$(grep '/mergeMakeOCDB.log' ${filemerge} | grep OK | wc -l)
2219     echo -n " ${qaStatus}" >> ${outfile}
2220
2221
2222     #fill OK/BAD rates
2223     for i in $(seq 0 $((${Nrates}-1))) ; do
2224       var1=$(grep "/${ratesString[${i}]}.log" ${filejob} | grep OK | wc -l)
2225       var2=$(grep "/${ratesString[${i}]}.log" ${filejob} | grep BAD | wc -l)
2226
2227       if [[ ${ratesString[${i}]} == "stderr" ]] ; then
2228         var1=$(grep "stderr" ${filejob} | grep OK | wc -l)
2229         var2=$(grep "stderr" ${filejob} | grep "rec.log" | grep BAD | wc -l)
2230       fi
2231       echo -n " ${var1}" >> ${outfile}
2232       echo -n " ${var2}" >> ${outfile}
2233     done
2234
2235     if [[ -f ${fileOCDB} ]] ; then
2236       #fill counter
2237       for i in $(seq 0 $((${Ncounter}-1))) ; do
2238         var1=$(grep Monalisa ${fileOCDB} | grep ${counterString[${i}]} | cut -f2)
2239         echo -n " ${var1:-"-1"}" >> ${outfile}
2240       done
2241
2242       #fill status
2243       for i in $(seq 0 $((${Nstatus}-1))) ; do
2244         var1=$(grep "calibration status=" ${fileOCDB} | grep ${statusString[${i}]/Status/} | cut -d'=' -f2)
2245         echo -n " ${var1:-"-1"}" >> ${outfile}
2246       done
2247     fi
2248     echo >> ${outfile}
2249   done
2250
2251   return 0
2252 }
2253
2254 parseConfig()
2255 {
2256   configFile=${1}
2257   shift
2258   args=("$@")
2259
2260   #some defaults
2261   #autoOCDB=0
2262   defaultOCDB="raw://"
2263   #runNumber=167123
2264   #makeflowPath="/hera/alice/aux/cctools/bin"
2265   #makeflowOptions="-T wq -N alice -d all -C ali-copilot.cern.ch:9097"
2266   #makeflowOptions="-T wq -N alice -C ali-copilot.cern.ch:9097"
2267   makeflowOptions=""
2268   #batchCommand="/usr/bin/qsub"
2269   batchFlags="-b y -cwd -l h_rt=24:0:0,h_rss=4G "
2270   baseOutputDirectory="$PWD/output"
2271   #alirootEnv="/cvmfs/alice.cern.ch/bin/alienv setenv AliRoot/v5-04-34-AN -c"
2272   #alirootEnv="/home/mkrzewic/alisoft/balice_master.sh"
2273   #trustedQAtrainMacro='/hera/alice/mkrzewic/gsisvn/Calibration/QAtrain_duo.C'
2274   reconstructInTemporaryDir=0
2275   recoTriggerOptions="\"\""
2276   percentProcessedFilesToContinue=100
2277   maxSecondsToWait=$(( 3600*24 ))
2278   nEvents=-1
2279   nMaxChunks=0
2280   postSetUpActionCPass0=""
2281   postSetUpActionCPass1=""
2282   runCPass0reco=1
2283   runCPass0MergeMakeOCDB=1
2284   runCPass1reco=1
2285   runCPass1MergeMakeOCDB=1
2286   runESDfiltering=1
2287   filteringFactorHighPt=1e2
2288   filteringFactorV0s=1e1
2289   MAILTO=""
2290   #pretend=1
2291   #dontRedirectStdOutToLog=1
2292   logToFinalDestination=1
2293   ALIROOT_FORCE_COREDUMP=1
2294
2295   #first, source the config file
2296   if [ -f ${configFile} ]; then
2297     source ${configFile}
2298   else
2299     echo "config file ${configFile} not found!, skipping..."
2300   fi
2301
2302   #then, parse the options as they override the options from file
2303   for opt in "${args[@]}"; do
2304     if [[ ! "${opt}" =~ .*=.* ]]; then
2305       echo "badly formatted option ${var}, should be: option=value, stopping..."
2306       return 1
2307     fi
2308     local var="${opt%%=*}"
2309     local value="${opt#*=}"
2310     echo "${var} = ${value}"
2311     export ${var}="${value}"
2312   done
2313
2314   #do some checking
2315   [[ -z ${alirootEnv} ]] && echo "alirootEnv not defined!" && return 1
2316
2317   #export the aliroot function if defined to override normal behaviour
2318   [[ $(type -t aliroot) =~ "function" ]] && export -f aliroot
2319
2320   return 0
2321 }
2322
2323 aliroot()
2324 {
2325   args="$@"
2326   if [[ -n ${useProfilingCommand} ]]; then
2327     valgrindLogFile="cpu.txt"
2328     [[ "${args}" =~ rec ]] && valgrindLogFile="cpu_rec.txt"
2329     [[ "${args}}" =~ Calib ]] && valgrindLogFile="cpu_calib.txt"
2330     [[ -n ${useProfilingCommand} ]] && useProfilingCommand="${useProfilingCommand} --log-file=${valgrindLogFile}"
2331     echo running ${useProfilingCommand} aliroot ${args}
2332     ${useProfilingCommand} aliroot ${args}
2333   else
2334     #to prevent an infinite recursion use "command aliroot" to disable
2335     #aliases and functions
2336     echo running command aliroot ${args}
2337     command aliroot "$@"
2338   fi
2339   return 0
2340 }
2341
2342 guessRunData()
2343 {
2344   #guess the period from the path, pick the rightmost one
2345   period=""
2346   runNumber=""
2347   year=""
2348   pass=""
2349   legoTrainRunNumber=""
2350   dataType=""
2351
2352   local shortRunNumber=""
2353   local IFS="/"
2354   declare -a path=( $1 )
2355   local dirDepth=$(( ${#path[*]}-1 ))
2356   i=0
2357   #for ((x=${dirDepth};x>=0;x--)); do
2358   for ((x=0;x<=${dirDepth};x++)); do
2359
2360     [[ $((x-1)) -ge 0 ]] && local fieldPrev=${path[$((x-1))]}
2361     local field=${path[${x}]}
2362     local fieldNext=${path[$((x+1))]}
2363
2364     [[ ${field} =~ ^[0-9]*$ && ${fieldNext} =~ (.*\.zip$|.*\.root$) ]] && legoTrainRunNumber=${field}
2365     [[ -n ${legoTrainRunNumber} && -z ${pass} ]] && pass=${fieldPrev}
2366     [[ ${field} =~ ^LHC[0-9][0-9][a-z].*$ ]] && period=${field%_*}
2367     [[ ${field} =~ ^000[0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && runNumber=${field#000}
2368     [[ ${field} =~ ^[0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && shortRunNumber=${field}
2369     [[ ${field} =~ ^20[0-9][0-9]$ ]] && year=${field}
2370     [[ ${field} =~ ^(^sim$|^data$) ]] && dataType=${field}
2371     (( i++ ))
2372   done
2373   [[ -z ${legoTrainRunNumber} ]] && pass=${path[$((dirDepth-1))]}
2374   [[ "${dataType}" =~ ^sim$ ]] && pass="passMC" && runNumber=${shortRunNumber}
2375
2376   #if [[ -z ${dataType} || -z ${year} || -z ${period} || -z ${runNumber}} || -z ${pass} ]];
2377   if [[ -z ${runNumber}} ]];
2378   then
2379     #error condition
2380     return 1
2381   else
2382     #ALL OK
2383     return 0
2384   fi
2385   return 0
2386 }
2387
2388 main "$@"