PWGPP/benchmark/benchmark.sh

   1 #!/bin/bash
   2 #include benchmark.config
   3
   4 # this script runs the CPass0/CPass1 train
   5 # produced OCDB updates are local
   6
   7 main()
   8 {
   9   #run in proper mode depending on the selection
  10   if [[ $# -lt 1 ]]; then
  11     if [[ ! "${0}" =~ "bash" ]]; then
  12       echo "uses makeflow:"
  13       echo " ${0} \"run\" productionID inputList configFile [extraOpts]"
  14       echo "uses a batch system (SGE):"
  15       echo " ${0} \"submit\" productionID inputList configFile [extraOpts]"
  16       echo "extraOpts if non-empty override the config file, e.g.:"
  17       echo " ${0} submit test1 benchmark.list benchmark.config runNumber=169123 nEvents=10"
  18     fi
  19     return
  20   fi
  21
  22   #define some aliases - default is to call one of the functions directly
  23   runMode=${1}
  24   umask 0002
  25   shift
  26   case ${runMode} in
  27     "CPass0") goCPass0 "$@";;
  28     "CPass1") goCPass1 "$@";;
  29     "MakeLocalOCDBaccessConfig") goMakeLocalOCDBaccessConfig "$@";;
  30     "MergeCPass0") goMergeCPass0 "$@";;
  31     "MergeCPass1") goMergeCPass1 "$@";;
  32     "MakeFilteredTrees") goMakeFilteredTrees "$@";;
  33     "MakeSummary") goMakeSummary "$@";;
  34     "run") goSubmitMakeflow "$@";;
  35     "submit") goSubmitBatch "$@";;
  36     "test") goTest "$@";;
  37     "GenerateMakeflow") goGenerateMakeflow "$@";;
  38     "PrintValues") goPrintValues "$@";;
  39     "CreateQAplots") goCreateQAplots "$@";;
  40     "WaitForOutput") goWaitForOutput "$@";;
  41     "Merge") goMerge "$@";;
  42     *)
  43       ${runMode} "$@"
  44     ;;
  45   esac
  46 }
  47
  48 generateMC()
  49 {
  50   #generate one raw chunk in current directory
  51   SEED=${JOB_ID}${SGE_TASK_ID}
  52   export CONFIG_SEED=${SEED}
  53   runNumber=${1}
  54   OCDBpath=${2}
  55   nEventsim=${3}
  56   if [[ -n ${pretend} ]]; then
  57     touch galice.root
  58   else
  59     if [[ -f sim.C && -f Config.C ]] ; then
  60         time aliroot -b -q -x sim.C\(${runNumber},\"${OCDBpath}\",${nEventsim}\) >sim.log 2>&1
  61         mv syswatch.log simwatch.log
  62     fi
  63   fi
  64 }
  65
  66 goCPass0()
  67 {
  68   umask 0002
  69
  70   targetDirectory=${1}
  71   inputList=${2}
  72   nEvents=${3}
  73   ocdbPath=${4}
  74   configFile=${5}
  75   runNumber=${6}
  76   jobindex=${7}
  77   shift 7
  78   if ! parseConfig ${configFile} "$@"; then return 1; fi
  79
  80   #use the jobindex only if set and non-negative
  81   if [[ -z ${jobindex} || ${jobindex} -lt 0 ]]; then
  82     [[ -n "${LSB_JOBINDEX}" ]] && jobindex=${LSB_JOBINDEX}
  83     [[ -n "${SGE_TASK_ID}" ]] && jobindex=${SGE_TASK_ID}
  84   fi
  85
  86   [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
  87   doneFile="${commonOutputPath}/meta/cpass0.job${jobindex}.run${runNumber}.done"
  88
  89   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
  90
  91   if [[ -n ${ALIROOT_FORCE_COREDUMP} ]]; then
  92     ulimit -c unlimited
  93     export ALIROOT_FORCE_COREDUMP
  94   fi
  95
  96   #the contents of this is stored in the tree and used later (e.g. AliAnalysisTaskPIDResponse)!
  97   #at the QA stage the pass number is guessed from the path stored here.
  98   #The Format is:
  99   #Packages= ;OutputDir= ;LPMPass= ;TriggerAlias= ;LPMRunNumber= ;LPMProductionType= ;LPMInteractionType= ;LPMProductionTag= ;LPMAnchorRun= ;LPMAnchorProduction= ;LPMAnchorYear=
 100   export PRODUCTION_METADATA="OutputDir=cpass0"
 101
 102   if [[ "${inputList}" =~ \.root$ ]]; then
 103     infile=${inputList}
 104   else
 105     infile=$(sed -ne "${jobindex}p" ${inputList} | egrep '\s*\w*/\w*')
 106   fi
 107   chunkName=${infile##*/}
 108
 109   outputDir=${targetDirectory}/${jobindex}_${chunkName%.*}
 110   mkdir -p ${outputDir}
 111   [[ ! -d ${outputDir} ]] && echo "cannot make ${outputDir}" && touch ${doneFile} && return 1
 112
 113   #runpath=${PWD}/rundir_cpass0_${runNumber}_${jobindex}
 114   runpath=${outputDir}
 115   [[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
 116   [[ ${reconstructInTemporaryDir} -eq 1 && -z ${TMPDIR} ]] && runpath=$(mktemp -d)
 117   mkdir -p ${runpath}
 118   [[ ! -d ${runpath} ]] && echo "cannot make runpath ${runpath}" && touch ${doneFile} && return 1
 119   cd ${runpath}
 120
 121   #runCPassX/C expects the raw chunk to be linked in the run dir
 122   #despite it being accessed by the full path
 123   ln -s ${infile} ${runpath}/${chunkName}
 124
 125   #####MC
 126   if [[ -n ${generateMC} ]]; then
 127     olddir=${PWD}
 128     outputDirMC=${commonOutputPath}/000${runNumber}/sim/${jobindex}
 129     simrunpath=${outputDirMC}
 130     [[ ${simulateInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && simrunpath=${TMPDIR}
 131     [[ ${simulateInTemporaryDir} -eq 1 && -z ${TMPDIR} ]] && simrunpath=$(mktemp -d)
 132     mkdir -p ${outputDirMC}
 133     mkdir -p ${simrunpath}
 134     cd ${simrunpath}
 135
 136     filesMC=(
 137               "${commonOutputPath}/sim.C"
 138               "${commonOutputPath}/rec.C"
 139               "${commonOutputPath}/Config.C"
 140               "${commonOutputPath}/OCDB_*.root"
 141     )
 142     for file in ${filesMC[*]}; do
 143       [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
 144     done
 145
 146     generateMC ${runNumber} ${ocdbPath} ${nEvents}
 147
 148     [[ ! "${simrunpath}" =~ "${outputDirMC}" ]] && mv * ${outputDirMC} #TODO check if it works
 149     cd ${olddir}
 150
 151     ln -s ${outputDirMC}/* ${runpath}/
 152
 153     inputList=${outputDirMC}/galice.root #TODO not valid outside shell !!!
 154     infile=""
 155   fi
 156   ######
 157
 158   [[ ! -f ${inputList} && -z ${pretend} ]] && echo "input file ${inputList} not found, exiting..." && touch ${doneFile} && return 1
 159
 160   logOutputDir=${runpath}
 161   [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir}
 162   [[ -z ${dontRedirectStdOutToLog} ]] && exec 1> ${logOutputDir}/stdout
 163   [[ -z ${dontRedirectStdOutToLog} ]] && exec 2> ${logOutputDir}/stderr
 164   echo "${0} $*"
 165
 166   echo "#####################"
 167   echo CPass0:
 168   echo JOB setup
 169   echo nEvents            ${nEvents}
 170   echo runNumber          ${runNumber}
 171   echo ocdbPath           ${ocdbPath}
 172   echo infile             ${infile}
 173   echo chunkName          ${chunkName}
 174   echo jobindex           ${jobindex}
 175   echo recoTriggerOptions ${recoTriggerOptions}
 176   echo targetDirectory    ${targetDirectory}
 177   echo commonOutputPath         ${commonOutputPath}
 178   echo doneFile      ${doneFile}
 179   echo runpath            ${runpath}
 180   echo outputDir          ${outputDir}
 181   echo ALICE_ROOT         ${ALICE_ROOT}
 182   echo PWD                ${PWD}
 183   echo "########## ###########"
 184
 185   alirootInfo > ALICE_ROOT.log
 186
 187   filesCPass0=(
 188                "${commonOutputPath}/runCPass0.sh"
 189                "${commonOutputPath}/recCPass0.C"
 190                "${commonOutputPath}/runCalibTrain.C"
 191                "${commonOutputPath}/localOCDBaccessConfig.C"
 192                "${commonOutputPath}/OCDB.root"
 193                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/runCPass0.sh"
 194                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/recCPass0.C"
 195                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/runCalibTrain.C"
 196   )
 197
 198   for file in ${filesCPass0[*]}; do
 199     [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
 200   done
 201
 202   echo "this directory (${PWD}) contents:"
 203   /bin/ls
 204   echo
 205   chmod u+x runCPass0.sh
 206
 207   sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh
 208
 209   if [[ -n ${postSetUpActionCPass0} ]]; then
 210     echo "running ${postSetUpActionCPass0}"
 211     eval ${postSetUpActionCPass0}
 212   fi
 213
 214   #run CPass0
 215   echo "${runpath}/runCPass0.sh ${infile} ${nEvents} ${runNumber} ${ocdbPath} ${recoTriggerOptions}"
 216   if [[ -n ${pretend} ]]; then
 217     touch AliESDs.root
 218     touch AliESDfriends.root
 219     touch AliESDfriends_v1.root
 220     touch rec.log
 221     touch calib.log
 222   else
 223     echo ./runCPass0.sh "${infile}" "${nEvents}" "${runNumber}" "${ocdbPath}" "${recoTriggerOptions}"
 224     ./runCPass0.sh "${infile}" "${nEvents}" "${runNumber}" "${ocdbPath}" "${recoTriggerOptions}"
 225   fi
 226
 227   #move stuff to final destination
 228   echo "this directory (${PWD}) contents:"
 229   /bin/ls
 230   echo
 231
 232   echo rm -f ./${chunkName}
 233   rm -f ./${chunkName}
 234   echo "cp --recursive ${runpath}/* ${outputDir}"
 235   cp -p --recursive ${runpath}/* ${outputDir}
 236   echo
 237
 238   #validate CPass0
 239   cd ${outputDir}
 240   touch ${doneFile}
 241   echo "dir ${outputDir}" >> ${doneFile}
 242   if summarizeLogs >> ${doneFile}; then
 243     [[ -f ${outputDirMC}/galice.root ]] && echo "sim ${outputDirMC}/galice.root" >> ${doneFile}
 244     [[ -f AliESDfriends_v1.root ]] && echo "calibfile ${outputDir}/AliESDfriends_v1.root" >> ${doneFile}
 245     [[ -f AliESDs.root ]] && echo "esd ${outputDir}/AliESDs.root" >> ${doneFile}
 246   fi
 247
 248   [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
 249   return 0
 250 }
 251
 252 goCPass1()
 253 {
 254   umask 0002
 255
 256   targetDirectory=${1}
 257   inputList=${2}
 258   nEvents=${3}
 259   ocdbPath=${4}
 260   configFile=${5}
 261   runNumber=${6}
 262   jobindex=${7}
 263   shift 7
 264   extraOpts=("$@")
 265   if ! parseConfig ${configFile} "$@"; then return 1; fi
 266
 267   #use the jobindex only if set and non-negative
 268   if [[ -z ${jobindex} || ${jobindex} -lt 0 ]]; then
 269     [[ -n "${LSB_JOBINDEX}" ]] && jobindex=${LSB_JOBINDEX}
 270     [[ -n "${SGE_TASK_ID}" ]] && jobindex=${SGE_TASK_ID}
 271   fi
 272
 273   [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
 274   doneFile="${commonOutputPath}/meta/cpass1.job${jobindex}.run${runNumber}.done"
 275
 276   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
 277
 278   if [[ -n ${ALIROOT_FORCE_COREDUMP} ]]; then
 279     ulimit -c unlimited
 280     export ALIROOT_FORCE_COREDUMP
 281   fi
 282
 283   #the contents of this is stored in the tree and used later (e.g. AliAnalysisTaskPIDResponse)!
 284   #at the QA stage the pass number is guessed from the path stored here.
 285   #The Format is:
 286   #Packages= ;OutputDir= ;LPMPass= ;TriggerAlias= ;LPMRunNumber= ;LPMProductionType= ;LPMInteractionType= ;LPMProductionTag= ;LPMAnchorRun= ;LPMAnchorProduction= ;LPMAnchorYear=
 287   export PRODUCTION_METADATA="OutputDir=cpass1"
 288
 289   [[ ! -f ${inputList} && -z ${pretend} ]] && echo "input file ${inputList} not found, exiting..." && touch ${doneFile} && return 1
 290   if [[ "${inputList}" =~ \.root$ ]]; then
 291     infile=${inputList}
 292   else
 293     infile=$(sed -ne "${jobindex}p" ${inputList} | egrep '\s*\w*/\w*')
 294   fi
 295   chunkName=${infile##*/}
 296
 297   outputDir=${targetDirectory}/${jobindex}_${chunkName%.*}
 298   mkdir -p ${outputDir}
 299   [[ ! -d ${outputDir} ]] && echo "cannot make ${outputDir}" && touch ${doneFile} && return 1
 300
 301   #runpath=${PWD}/rundir_cpass1_${runNumber}_${jobindex}
 302   runpath=${outputDir}
 303   [[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
 304   [[ ${reconstructInTemporaryDir} -eq 1 && -z ${TMPDIR} ]] && runpath=$(mktemp -d)
 305
 306   #MC
 307   if [[ "${infile}" =~ galice\.root ]]; then
 308     ln -s ${inputList%/*}/* ${runpath}
 309     infile=""
 310   fi
 311
 312   #init the running path
 313   mkdir -p ${runpath}
 314   [[ ! -d ${runpath} ]] && echo "cannot make runpath ${runpath}" && touch ${doneFile} && return 1
 315   cd ${runpath}
 316
 317   #this is needed for runCPass1.sh
 318   ln -s ${infile} ${runpath}/${chunkName}
 319
 320   logOutputDir=${runpath}
 321   [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir}
 322   [[ -z ${dontRedirectStdOutToLog} ]] && exec 1> ${logOutputDir}/stdout
 323   [[ -z ${dontRedirectStdOutToLog} ]] && exec 2> ${logOutputDir}/stderr
 324   echo "${0} $*"
 325
 326   echo "#####################"
 327   echo CPass1:
 328   echo JOB setup
 329   echo nEvents            ${nEvents}
 330   echo runNumber          ${runNumber}
 331   echo ocdbPath           ${ocdbPath}
 332   echo infile             ${infile}
 333   echo chunkName          ${chunkName}
 334   echo jobindex           ${jobindex}
 335   echo recoTriggerOptions ${recoTriggerOptions}
 336   echo targetDirectory    ${targetDirectory}
 337   echo commonOutputPath         ${commonOutputPath}
 338   echo doneFile      ${doneFile}
 339   echo runpath            ${runpath}
 340   echo outputDir          ${outputDir}
 341   echo ALICE_ROOT         ${ALICE_ROOT}
 342   echo PWD                ${PWD}
 343   echo "########## ###########"
 344
 345   alirootInfo > ALICE_ROOT.log
 346
 347   filesCPass1=(
 348                "${commonOutputPath}/runCPass1.sh"
 349                "${commonOutputPath}/recCPass1.C"
 350                "${commonOutputPath}/recCPass1_OuterDet.C"
 351                "${commonOutputPath}/runCalibTrain.C"
 352                "${commonOutputPath}/QAtrain_duo.C"
 353                "${commonOutputPath}/localOCDBaccessConfig.C"
 354                "${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz"
 355                "${commonOutputPath}/OCDB.root"
 356                "${trustedQAtrainMacro}"
 357                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/runCPass1.sh"
 358                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/recCPass1.C"
 359                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/recCPass1_OuterDet.C"
 360                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/runCalibTrain.C"
 361                "${ALICE_ROOT}/ANALYSIS/macros/QAtrain_duo.C"
 362   )
 363
 364   for file in ${filesCPass1[*]}; do
 365     [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
 366   done
 367
 368   echo "this directory (${PWD}) contents:"
 369   /bin/ls
 370   echo
 371
 372   sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh
 373
 374   if [[ -n ${postSetUpActionCPass1} ]]; then
 375     echo "running ${postSetUpActionCPass1}"
 376     eval ${postSetUpActionCPass1}
 377     echo
 378   fi
 379
 380   #configure local OCDB storage from CPass0 (creates the localOCDBaccessConfig.C script)
 381   if [[ -f cpass0.localOCDB.${runNumber}.tgz ]]; then
 382     echo goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz"
 383     goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz"
 384   else
 385     echo "WARNING: file cpass0.localOCDB.${runNumber}.tgz not found!"
 386   fi
 387
 388   if [[ ! $(/bin/ls -1 OCDB/*/*/*/*.root 2>/dev/null) ]]; then
 389     echo "cpass0 produced no calibration! exiting..."
 390     touch ${doneFile}
 391     return 1
 392   fi
 393
 394   #create the Barrel and OuterDet directories for CPass1 and link the local OCDB directory
 395   #there to make the localOCDBaccessConfig.C file work, since it may point to the OCDB
 396   #entries using a relative path, e.g. local://./OCDB
 397   mkdir Barrel OuterDet
 398   ln -s ../OCDB Barrel/OCDB
 399   ln -s ../OCDB OuterDet/OCDB
 400
 401   #setup the filtering
 402   #the following option enables the filtering task inside the QAtrain_duo.C
 403   [[ -n $runESDfiltering ]] && export QA_TaskFilteredTree=1
 404   #set the downscaling factors during the filtering fro expert QA (overrides the previous values)
 405   if [[ -n ${filteringFactorHighPt} ]]; then
 406     export AliAnalysisTaskFilteredTree_fLowPtTrackDownscaligF=${filteringFactorHighPt}
 407   fi
 408   if [[ -n ${filteringFactorV0s} ]]; then
 409     export AliAnalysisTaskFilteredTree_fLowPtV0DownscaligF=${filteringFactorV0s}
 410   fi
 411
 412   #run CPass1
 413   chmod u+x runCPass1.sh
 414   echo "${runpath}/runCPass1.sh ${infile} ${nEvents} ${runNumber} ${ocdbPath} ${recoTriggerOptions}"
 415   if [[ -n ${pretend} ]]; then
 416     touch AliESDs_Barrel.root
 417     touch AliESDfriends_Barrel.root
 418     touch AliESDfriends_v1.root
 419     touch QAresults_barrel.root
 420     touch EventStat_temp_barrel.root
 421     touch AODtpITS.root
 422     touch AliESDs_Outer.root
 423     touch AliESDfriends_Outer.root
 424     touch QAresults_outer.root
 425     touch EventStat_temp_outer.root
 426     touch rec.log
 427     touch calib.log
 428     touch qa.log
 429     touch filtering.log FilterEvents_Trees.root
 430   else
 431     ./runCPass1.sh "${infile}" "${nEvents}" "${runNumber}" "${ocdbPath}" "${recoTriggerOptions}"
 432
 433     [[ ! -f AliESDs_Barrel.root && -f Barrel/AliESDs.root ]] && mv Barrel/AliESDs.root AliESDs_Barrel.root
 434     [[ ! -f AliESDfriends_Barrel.root && -f Barrel/AliESDfriends.root ]] && mv Barrel/AliESDfriends.root AliESDfriends_Barrel.root
 435     [[ ! -f AliESDfriends_v1.root && -f Barrel/AliESDfriends_v1.root ]] && mv Barrel/AliESDfriends_v1.root .
 436     [[ ! -f QAresults_barrel.root && -f Barrel/QAresults_barrel.root ]] && mv Barrel/QAresults_barrel.root .
 437     [[ ! -f AliESDs_Outer.root && -f OuterDet/AliESDs.root ]] && mv OuterDet/AliESDs.root AliESDs_Outer.root
 438     [[ ! -f AliESDfriends_Outer.root && -f OuterDet/AliESDfriends.root ]] && mv OuterDet/AliESDfriends.root AliESDfriends_Outer.root
 439     [[ ! -f QAresults_outer.root && -f OuterDet/QAresults_outer.root ]] && mv OuterDet/QAresults_outer.root .
 440     [[ ! -f FilterEvents_Trees.root && -f Barrel/FilterEvents_Trees.root ]] && mv Barrel/FilterEvents_Trees.root .
 441
 442     #make the filtered tree (if requested and not already produced by QA
 443     [[ -f AliESDs_Barrel.root ]] && echo "AliESDs_Barrel.root" > filtered.list
 444     if [[ -n ${runESDfiltering} && ! -f FilterEvents_Trees.root ]]; then
 445       goMakeFilteredTrees ${PWD} ${runNumber} "filtered.list" ${filteringFactorHighPt} ${filteringFactorV0s} ${ocdbPath} 1000000 0 10000000 0 ${configFile} AliESDs_Barrel.root "${extraOpts[@]}"
 446     fi
 447
 448   fi
 449
 450   ##handle possible crashes in QA (happens often in trunk)
 451   ##rerun QA with a trusted aliroot version
 452   #if [[ $(validateLog qa_barrel.log) ]]; then
 453   #  echo "qa_barrel.log not validated!"
 454   #fi
 455   #if [[ ! -f QAresults_barrel.root && -f ${setupTrustedAliROOTenvInCurrentShell} || $(validateLog qa_barrel.log) ]]; then
 456   #  echo "WARNING: using trusted QA aliroot ${ALICE_ROOT}"
 457   #  source ${setupTrustedAliROOTenvInCurrentShell}
 458   #  cd Barrel
 459   #  rm QAresults_barrel.root
 460   #  rm EventStat_temp_barrel.root
 461   #  rm AODtpITS.root
 462   #  [[ ! -f AliESDs.root ]] && ln -s ../AliESDs_Barrel.root AliESDs.root
 463   #  [[ ! -f AliESDfriends.root ]] && ln -s ../AliESDfriends_Barrel.root AliESDfriends.root
 464   #  if [[ -n ${trustedQAtrainMacro} ]]; then
 465   #    eval "cp ${trustedQAtrainMacro} QAtrain_duo_trusted.C"
 466   #  fi
 467   #  echo executing aliroot -b -q "QAtrain_duo_trusted.C(\"_barrel\",${runNumber},\"wn.xml\",0,\"${ocdbPath}\")"
 468   #  time aliroot -b -q "QAtrain_duo.C(\"_barrel\",${runNumber},\"wn.xml\",0,\"${ocdbPath}\")" &> ../qa_barrel_trusted.log
 469   #  cd ../
 470   #fi
 471
 472   #move stuff to final destination
 473   echo "this directory (${PWD}) contents:"
 474   /bin/ls
 475   echo rm -f ./${chunkName}
 476   rm -f ./${chunkName}
 477   echo "cp --recursive ${runpath}/* ${outputDir}"
 478   cp -pf --recursive ${runpath}/* ${outputDir}
 479   echo
 480
 481   #validate CPass1
 482   cd ${outputDir}
 483   touch ${doneFile}
 484   echo "dir ${outputDir}" >> ${doneFile}
 485   if summarizeLogs >> ${doneFile}; then
 486     [[ -f AliESDs_Barrel.root ]] && echo "esd ${outputDir}/AliESDs_Barrel.root" >> ${doneFile}
 487     [[ -f AliESDfriends_v1.root ]] && echo "calibfile ${outputDir}/AliESDfriends_v1.root" >> ${doneFile}
 488     [[ -f QAresults_Barrel.root ]] && echo "qafile ${outputDir}/QAresults_Barrel.root" >> ${doneFile}
 489     [[ -f QAresults_Outer.root ]] && echo "qafile ${outputDir}/QAresults_Outer.root" >> ${doneFile}
 490     [[ -f FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFile}
 491   fi
 492
 493   [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
 494   return 0
 495 }
 496
 497
 498 goMergeCPass0()
 499 {
 500   #
 501   # find the output files and merge them
 502   #
 503
 504   outputDir=${1}
 505   ocdbStorage=${2}
 506   configFile=${3}
 507   runNumber=${4}
 508   calibrationFilesToMerge=${5}  #can be a non-existent file, will then be produced on the fly
 509   shift 5
 510   if ! parseConfig ${configFile} "$@"; then return 1; fi
 511
 512   [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
 513   doneFile="${commonOutputPath}/meta/merge.cpass0.run${runNumber}.done"
 514
 515   umask 0002
 516   ulimit -c unlimited
 517
 518   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
 519
 520   #runpath=${PWD}/rundir_cpass0_Merge_${runNumber}
 521   runpath=${outputDir}
 522   [[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
 523   [[ ${reconstructInTemporaryDir} -eq 1 && -z ${TMPDIR} ]] && runpath=$(mktemp -d)
 524
 525   mkdir -p ${runpath}
 526   [[ ! -d ${runpath} ]] && echo "not able to make the runpath ${runpath}" && touch ${doneFile} && return 1
 527   cd ${runpath}
 528
 529   logOutputDir=${runpath}
 530   [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir}
 531   [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/mergeMakeOCDB.log
 532   echo "${0} $*"
 533
 534   mergingScript="mergeMakeOCDB.byComponent.sh"
 535
 536   echo goMergeCPass0 SETUP:
 537   echo runNumber=${runNumber}
 538   echo outputDir=${outputDir}
 539   echo ocdbStorage=${ocdbStorage}
 540   echo calibrationFilesToMerge=${calibrationFilesToMerge}
 541   echo mergingScript=${mergingScript}
 542   echo commonOutputPath=${commonOutputPath}
 543   echo runpath=${runpath}
 544
 545   # copy files in case they are not already there
 546   filesMergeCPass0=(
 547                     "${commonOutputPath}/${calibrationFilesToMerge}"
 548                     "${commonOutputPath}/OCDB.root"
 549                     "${commonOutputPath}/localOCDBaccessConfig.C"
 550                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/mergeMakeOCDB.byComponent.sh"
 551                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/mergeByComponent.C"
 552                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/makeOCDB.C"
 553                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/merge.C"
 554                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/mergeMakeOCDB.sh"
 555   )
 556   for file in ${filesMergeCPass0[*]}; do
 557     [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
 558   done
 559
 560   sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh
 561
 562   alirootInfo > ALICE_ROOT.log
 563
 564   #
 565   echo "PWD"
 566   /bin/ls
 567   echo "PWD/.."
 568   /bin/ls ../
 569
 570
 571   #merge calibration
 572   chmod u+x ${mergingScript}
 573   mkdir -p ./OCDB
 574   if [[ ! -f ${calibrationFilesToMerge} ]]; then
 575     echo "/bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root > ${calibrationFilesToMerge}"
 576     /bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root 2>/dev/null > ${calibrationFilesToMerge}
 577   fi
 578
 579   echo "${mergingScript} ${calibrationFilesToMerge} ${runNumber} local://./OCDB ${ocdbStorage}"
 580   if [[ -n ${pretend} ]]; then
 581     touch CalibObjects.root
 582     touch ocdb.log
 583     touch merge.log
 584     mkdir -p ./OCDB/TPC/Calib/TimeGain/
 585     mkdir -p ./OCDB/TPC/Calib/TimeDrift/
 586     touch ./OCDB/TPC/Calib/TimeGain/someCalibObject_0-999999_cpass0.root
 587     touch ./OCDB/TPC/Calib/TimeDrift/otherCalibObject_0-999999_cpass0.root
 588   else
 589     ./${mergingScript} ${calibrationFilesToMerge} ${runNumber} "local://./OCDB" ${ocdbStorage}
 590   fi
 591
 592   ### produce the output
 593   #tar the produced OCDB for reuse
 594   tar czf ${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz ./OCDB
 595
 596   #produce the calib trees for expert QA (dcsTime.root)
 597   goMakeLocalOCDBaccessConfig ./OCDB
 598   echo aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")"
 599   aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")"
 600
 601   /bin/ls
 602
 603   #copy all to output dir
 604   cp -pf --recursive ${runpath}/* ${outputDir}
 605
 606   if [[ -n ${generateMC} ]]; then
 607     goPrintValues sim ${commonOutputPath}/meta/sim.run${runNumber}.list ${commonOutputPath}/meta/cpass0.job*.run${runNumber}.done
 608   fi
 609
 610   #validate merging cpass0
 611   cd ${outputDir}
 612   touch ${doneFile}
 613   echo "dir ${outputDir}" >> ${doneFile}
 614   if summarizeLogs >> ${doneFile}; then
 615     [[ -f CalibObjects.root ]] && echo "calibfile ${outputDir}/CalibObjects.root" >> ${doneFile}
 616     [[ -f dcsTime.root ]] && echo "dcsTree ${outputDir}/dcsTime.root" >> ${doneFile}
 617   fi
 618
 619   [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
 620   return 0
 621 }
 622
 623 goMergeCPass1()
 624 {
 625   #
 626   # find the output files and merge them
 627   #
 628
 629   outputDir=${1}
 630   ocdbStorage=${2}
 631   configFile=${3}
 632   runNumber=${4}
 633   calibrationFilesToMerge=${5}
 634   qaFilesToMerge=${6}
 635   filteredFilesToMerge=${7}
 636   shift 7
 637   if ! parseConfig ${configFile} "$@"; then return 1; fi
 638
 639   [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
 640   doneFile="${commonOutputPath}/meta/merge.cpass1.run${runNumber}.done"
 641
 642   umask 0002
 643   ulimit -c unlimited
 644
 645   #clean up first:
 646   rm -f ${outputDir}/*.log
 647   rm -f ${outputDir}/*.root
 648   rm -f ${outputDir}/*done
 649
 650   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
 651
 652   #runpath=${PWD}/rundir_cpass1_Merge_${runNumber}
 653   runpath=${outputDir}
 654   [[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
 655   [[ ${reconstructInTemporaryDir} -eq 1 && -z ${TMPDIR} ]] && runpath=$(mktemp -d)
 656
 657   mkdir -p ${runpath}
 658   [[ ! -d ${runpath} ]] && echo "not able to make the runpath ${runpath}" && touch ${doneFile} && return 1
 659   cd ${runpath}
 660
 661   logOutputDir=${runpath}
 662   [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir}
 663   [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/mergeMakeOCDB.log
 664   echo "${0} $*"
 665
 666   calibrationOutputFileName='AliESDfriends_v1.root'
 667   qaOutputFileName='QAresults*.root'
 668   mergingScript="mergeMakeOCDB.byComponent.sh"
 669   #important to have the string "Stage.txt" in the filename to trigger the merging
 670   #it has to be a list of directories containing the files
 671   qaMergedOutputFileName="QAresults_merged.root"
 672
 673   echo goMergeCPass1 SETUP:
 674   echo runNumber=${runNumber}
 675   echo outputDir=${outputDir}
 676   echo ocdbStorage=${ocdbStorage}
 677   echo calibrationFilesToMerge=$calibrationFilesToMerge
 678   echo qaFilesToMerge=$qaFilesToMerge
 679   echo calibrationOutputFileName=${calibrationOutputFileName}
 680   echo mergingScript=${mergingScript}
 681
 682   # copy files in case they are not already there
 683   filesMergeCPass1=(
 684                     "${commonOutputPath}/${calibrationFilesToMerge}"
 685                     "${commonOutputPath}/${qaFilesToMerge}"
 686                     "${commonOutputPath}/OCDB.root"
 687                     "${commonOutputPath}/localOCDBaccessConfig.C"
 688                     "${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz"
 689                     "${commonOutputPath}/QAtrain_duo.C"
 690                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/mergeMakeOCDB.byComponent.sh"
 691                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/mergeByComponent.C"
 692                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/makeOCDB.C"
 693                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/merge.C"
 694                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/mergeMakeOCDB.sh"
 695                     "${trustedQAtrainMacro}"
 696                     "${ALICE_ROOT}/ANALYSIS/macros/QAtrain_duo.C"
 697   )
 698   for file in ${filesMergeCPass1[*]}; do
 699     [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
 700   done
 701
 702   sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh
 703
 704   #configure local OCDB storage from CPass0 (creates the localOCDBaccessConfig.C script)
 705   if [[ -f cpass0.localOCDB.${runNumber}.tgz ]]; then
 706     echo goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz"
 707     goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz"
 708   else
 709     echo "WARNING: file cpass0.localOCDB.${runNumber}.tgz not found!"
 710   fi
 711
 712   alirootInfo > ALICE_ROOT.log
 713
 714   #
 715   /bin/ls
 716
 717   #merge calibration
 718   chmod u+x ${mergingScript}
 719   mkdir -p OCDB
 720
 721   #if not provided, create the lists of files to merge
 722   if [[ ! -f ${filteredFilesToMerge} ]]; then
 723     echo "/bin/ls -1 ${outputDir}/*/FilterEvents_Trees.root > ${filteredFilesToMerge}"
 724     /bin/ls -1 ${outputDir}/*/FilterEvents_Trees.root 2>/dev/null > ${filteredFilesToMerge}
 725   fi
 726   if [[ ! -f ${calibrationFilesToMerge} ]]; then
 727     echo "/bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root > ${calibrationFilesToMerge}"
 728     /bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root 2>/dev/null > ${calibrationFilesToMerge}
 729   fi
 730   if [[ ! -f ${qaFilesToMerge} ]]; then
 731     #find the files, but only store the directories (QAtrain_duo.C requires this)
 732     echo "/bin/ls -1 ${outputDir}/*/QAresults*.root | while read x; do echo ${x%/*}; done | sort | uniq > ${qaFilesToMerge}"
 733     /bin/ls -1 ${outputDir}/*/QAresults*.root | while read x; do echo ${x%/*}; done | sort | uniq > ${qaFilesToMerge}
 734   fi
 735
 736   echo "${mergingScript} ${calibrationFilesToMerge} ${runNumber} local://./OCDB ${ocdbStorage}"
 737   if [[ -n ${pretend} ]]; then
 738     touch CalibObjects.root
 739     touch ocdb.log
 740     touch merge.log
 741     touch cpass1.localOCDB.${runNumber}.tgz
 742   else
 743     ./${mergingScript} ${calibrationFilesToMerge} ${runNumber} "local://./OCDB" ${ocdbStorage}
 744   fi
 745
 746   tar czf ${commonOutputPath}/meta/cpass1.localOCDB.${runNumber}.tgz ./OCDB
 747
 748   #merge QA (and filtered trees)
 749   [[ -n ${AliAnalysisTaskFilteredTree_fLowPtTrackDownscaligF} ]] && export AliAnalysisTaskFilteredTree_fLowPtTrackDownscaligF
 750   [[ -n ${AliAnalysisTaskFilteredTree_fLowPtV0DownscaligF} ]] && export AliAnalysisTaskFilteredTree_fLowPtV0DownscaligF
 751
 752   #echo aliroot -l -b -q "merge.C(\"${qaFilesToMerge}\",\"\",kFALSE,\"${qaMergedOutputFileName}\")"
 753   echo aliroot -b -q "QAtrain_duo.C(\"_barrel\",${runNumber},\"${qaFilesToMerge}\",1,\"${ocdbStorage}\")"
 754   if [[ -n ${pretend} ]]; then
 755     touch ${qaMergedOutputFileName}
 756     touch merge.log
 757     touch trending.root
 758     touch FilterEvents_Trees.root
 759     touch CalibObjects.root
 760     touch dcsTime.root
 761     touch ${qaMergedOutputFileName}
 762   else
 763     #aliroot -l -b -q "merge.C(\"${qaFilesToMerge}\",\"\",kFALSE,\"${qaMergedOutputFileName}\")"
 764     aliroot -b -q "QAtrain_duo.C(\"_barrel\",${runNumber},\"${qaFilesToMerge}\",1,\"${ocdbStorage}\")" > mergeQA.log
 765     mv QAresults_barrel.root ${qaMergedOutputFileName}
 766     mv trending_barrel.root trending.root
 767
 768     #merge filtered trees
 769     echo aliroot -l -b -q "merge.C(\"${qaFilesToMerge}\",\"\",kFALSE,\"${qaMergedOutputFileName}\")"
 770     aliroot -l -b -q "merge.C(\"${filteredFilesToMerge}\",\"\",kFALSE,\"FilterEvents_Trees.root\")"
 771   fi
 772
 773   #produce the calib trees for expert QA
 774   echo aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")"
 775   aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")"
 776
 777   /bin/ls
 778
 779   #copy all to output dir
 780   cp -pf --recursive ${runpath}/* ${outputDir}
 781
 782   #validate merge cpass1
 783   cd ${outputDir}
 784   touch ${doneFile}
 785   echo "dir ${outputDir}" >> ${doneFile}
 786   if summarizeLogs >>  ${doneFile}; then
 787     [[ -f CalibObjects.root ]] && echo "calibfile ${outputDir}/CalibObjects.root" >> ${doneFile}
 788     [[ -f ${qaMergedOutputFileName} ]] && echo "qafile ${outputDir}/${qaMergedOutputFileName}" >> ${doneFile}
 789     [[ -f trending.root ]] && echo "trendingfile ${outputDir}/trending.root" >> ${doneFile}
 790     [[ -f dcsTime.root ]] && echo "dcsTree ${outputDir}/dcsTime.root" >> ${doneFile}
 791     [[ -f FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFile}
 792   fi
 793
 794   [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
 795   return 0
 796 }
 797
 798 goMerge()
 799 {
 800   #generic root merge using CPass1 merge.C script
 801   inputList=${1}
 802   outputFile=${2}
 803   configFile=${3-"becnhmark.config"}
 804   shift 3
 805   if ! parseConfig ${configFile} "$@"; then return 1; fi
 806
 807   [[ ! -f ${inputList} ]] && echo "inputList ${inputList} does not exist!" && return 1
 808   [[ ! -f ${configFile} ]] && echo "configFile ${configFile} does not exist!" && return 1
 809   umask 0002
 810   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
 811   rm -f ${outputFile}
 812   aliroot -b -q "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/merge.C(\"${inputList}\",\"\",kFALSE,\"${outputFile}\")" > merge_${inputList}.log
 813   return 0
 814 }
 815
 816 goSubmitMakeflow()
 817 {
 818   #run
 819   productionID=${1}
 820   inputFileList=${2}
 821   configFile=${3}
 822   shift 3
 823   extraOpts=("$@")
 824   if ! parseConfig ${configFile} "${extraOpts[@]}"; then return 1; fi
 825
 826   [[ -z ${configFile} ]] && configFile="benchmark.config"
 827   [[ ! -f ${configFile} ]] && echo "no config file found (${configFile})" && return 1
 828
 829   #create the directopry for the metadata
 830   mkdir meta
 831
 832   if [[ ! $(which makeflow &>/dev/null) && -n ${makeflowPath} ]]; then
 833     echo "setting the makflow path from the config: "
 834     echo "  export PATH=${makeflowPath}:${PATH}"
 835     export PATH=${makeflowPath}:${PATH}
 836   fi
 837
 838   #submit - use makeflow if available, fall back to old stuff when makeflow not there
 839   if which makeflow; then
 840
 841     goGenerateMakeflow ${productionID} ${inputFileList} ${configFile} "${extraOpts[@]}" > benchmark.makeflow
 842
 843     makeflow ${makeflowOptions} benchmark.makeflow
 844     cd ../
 845   else
 846     echo "no makeflow!"
 847   fi
 848   return 0
 849 }
 850
 851 goGenerateMakeflow()
 852 {
 853   #generate the makeflow file
 854   [[ $# -lt 3 ]] && echo "args: id inputFileList configFile" && return 1
 855   productionID=${1}
 856   inputFileList=${2}
 857   configFile=${3}
 858   shift 3
 859   extraOpts=("$@")
 860   if ! parseConfig ${configFile} "${extraOpts[@]}"; then return 1; fi
 861
 862   [[ -z ${configFile} ]] && configFile="benchmark.config"
 863   [[ ! -f ${configFile} ]] && echo "no config file found (${configFile})" && return 1
 864
 865   commonOutputPath=${baseOutputDirectory}/${productionID}
 866
 867   self=$(readlink -f "${0}")
 868   echo "self: $self"
 869   #these files will be made a dependency - will be copied to the working dir of the jobs
 870   declare -a copyFiles
 871   inputFiles=(
 872               "OCDB.root"
 873               "localOCDBaccessConfig.C"
 874               "QAtrain_duo.C"
 875               "runCPass1.sh"
 876               "recCPass1.C"
 877               "recCPass1_OuterDet.C"
 878               "runCalibTrain.C"
 879               "runCPass0.sh"
 880               "recCPass0.C"
 881   )
 882   for file in ${inputFiles[*]}; do
 883     [[ -f ${file} ]] && copyFiles+=("${file}")
 884   done
 885
 886   #create the makeflow file
 887   [[ -n ${batchFlags} ]] && echo "BATCH_OPTIONS = ${batchFlags}"
 888   declare -A arr_cpass0_merged arr_cpass1_merged
 889   declare -A arr_cpass0_calib_list arr_cpass1_calib_list
 890   declare -A arr_cpass1_QA_list arr_cpass1_ESD_list arr_cpass1_filtered_list
 891   declare -A arr_cpass0_profiled_outputs
 892   declare -A listOfRuns
 893   [[ -n ${runNumber} ]] && listOfRuns[${runNumber}]=1
 894   while read x; do listOfRuns[$(guessRunNumber ${x})]=1; done < ${inputFileList}
 895   for runNumber in "${!listOfRuns[@]}"; do
 896     [[ -z ${runNumber} ]] && continue
 897     [[ ! ${runNumber} =~ ^[0-9]*[0-9]$ ]] && continue
 898
 899     unset arr_cpass0_outputs
 900     unset arr_cpass1_outputs
 901     declare -a arr_cpass0_outputs
 902     declare -a arr_cpass1_outputs
 903
 904     jobindex=0
 905     inputFile=""
 906     while read inputFile; do
 907       currentDefaultOCDB=${defaultOCDB}
 908       [[ -z ${autoOCDB} ]] && autoOCDB=1
 909       if [[ ${autoOCDB} -ne 0 ]]; then
 910         currentDefaultOCDB=$(setYear ${inputFile} ${defaultOCDB})
 911       fi
 912
 913       #CPass0
 914       arr_cpass0_outputs[${jobindex}]="meta/cpass0.job${jobindex}.run${runNumber}.done"
 915       echo "${arr_cpass0_outputs[${jobindex}]} : benchmark.sh ${configFile} ${copyFiles[@]}"
 916       echo -n " ${alirootEnv} ./benchmark.sh CPass0 ${commonOutputPath}/000${runNumber}/cpass0 ${inputFile} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} ${jobindex}"" "
 917       for extraOption in "${extraOpts[@]}"; do echo -n \"${extraOption}\"" "; done; echo
 918       echo
 919
 920       #CPass1
 921       arr_cpass1_outputs[${jobindex}]="meta/cpass1.job${jobindex}.run${runNumber}.done"
 922       echo "${arr_cpass1_outputs[${jobindex}]} : benchmark.sh ${configFile} meta/cpass0.localOCDB.${runNumber}.tgz ${copyFiles[@]}"
 923       echo -n " ${alirootEnv} ./benchmark.sh CPass1 ${commonOutputPath}/000${runNumber}/cpass1 ${inputFile} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} ${jobindex}"" "
 924       for extraOption in "${extraOpts[@]}"; do echo -n \"${extraOption}\"" "; done; echo
 925       echo
 926       ((jobindex++))
 927
 928     done< <(grep "/000${runNumber}/" ${inputFileList})
 929
 930     #CPass0 list of Calib files to merge
 931     arr_cpass0_calib_list[${runNumber}]="meta/cpass0.calib.run${runNumber}.list"
 932     echo "${arr_cpass0_calib_list[${runNumber}]} : benchmark.sh ${arr_cpass0_outputs[*]}"
 933     echo "  ./benchmark.sh PrintValues calibfile ${arr_cpass0_calib_list[${runNumber}]} ${arr_cpass0_outputs[*]}"
 934     echo
 935
 936     #CPass0 merging
 937     arr_cpass0_merged[${runNumber}]="meta/merge.cpass0.run${runNumber}.done"
 938     echo "meta/cpass0.localOCDB.${runNumber}.tgz ${arr_cpass0_merged[${runNumber}]} : benchmark.sh ${configFile} ${arr_cpass0_calib_list[${runNumber}]} ${copyFiles[@]}"
 939     echo -n " ${alirootEnv} ./benchmark.sh MergeCPass0 ${commonOutputPath}/000${runNumber}/cpass0 ${currentDefaultOCDB} ${configFile} ${runNumber} ${arr_cpass0_calib_list[${runNumber}]}"" "
 940     for extraOption in "${extraOpts[@]}"; do echo -n \"${extraOption}\"" "; done; echo
 941     echo
 942
 943     #CPass1 list of Calib/QA/ESD/filtered files
 944     # the trick with QA is to have the string "Stage.txt" in the file name of the list of directories with QA output to trigger
 945     # the production of the QA trending tree (only then the task->Finish() will be called in QAtrain_duo.C, on the grid
 946     # this corresponds to the last merging stage)
 947     arr_cpass1_QA_list[${runNumber}]="meta/cpass1.QA.run${runNumber}.lastMergingStage.txt.list"
 948     echo "${arr_cpass1_QA_list[${runNumber}]}: benchmark.sh ${arr_cpass1_outputs[*]}"
 949     echo "  ./benchmark.sh PrintValues dir ${arr_cpass1_QA_list[${runNumber}]} ${arr_cpass1_outputs[*]}"
 950     echo
 951     arr_cpass1_calib_list[${runNumber}]="meta/cpass1.calib.run${runNumber}.list"
 952     echo "${arr_cpass1_calib_list[${runNumber}]} : benchmark.sh ${arr_cpass1_outputs[*]}"
 953     echo "  ./benchmark.sh PrintValues calibfile ${arr_cpass1_calib_list[${runNumber}]} ${arr_cpass1_outputs[*]};"
 954     echo
 955     arr_cpass1_ESD_list[${runNumber}]="meta/cpass1.ESD.run${runNumber}.list"
 956     echo "${arr_cpass1_ESD_list[${runNumber}]} : benchmark.sh ${arr_cpass1_outputs[*]}"
 957     echo "  ./benchmark.sh PrintValues esd ${arr_cpass1_ESD_list[${runNumber}]} ${arr_cpass1_outputs[*]}"
 958     echo
 959     arr_cpass1_filtered_list[${runNumber}]="meta/cpass1.filtered.run${runNumber}.list"
 960     echo "${arr_cpass1_filtered_list[${runNumber}]} : benchmark.sh ${arr_cpass1_outputs[*]}"
 961     echo "  ./benchmark.sh PrintValues filteredTree ${arr_cpass1_filtered_list[${runNumber}]} ${arr_cpass1_outputs[*]}"
 962     echo
 963
 964     #CPass1 merging
 965     arr_cpass1_merged[${runNumber}]="meta/merge.cpass1.run${runNumber}.done"
 966     echo "meta/cpass1.localOCDB.${runNumber}.tgz ${arr_cpass1_merged[${runNumber}]} :  benchmark.sh ${configFile} ${arr_cpass1_calib_list[${runNumber}]} ${arr_cpass1_QA_list[${runNumber}]} ${copyFiles[@]}"
 967     echo -n " ${alirootEnv} ./benchmark.sh MergeCPass1 ${commonOutputPath}/000${runNumber}/cpass1 ${currentDefaultOCDB} ${configFile} ${runNumber} ${arr_cpass1_calib_list[${runNumber}]} ${arr_cpass1_QA_list[${runNumber}]} ${arr_cpass1_filtered_list[${runNumber}]}"" "
 968     for extraOption in "${extraOpts[@]}"; do echo -n \"${extraOption}\"" "; done; echo
 969     echo
 970
 971     #CPass0 wrapped in a profiling tool (valgrind,....)
 972     if [[ -n profilingCommand ]]; then
 973       arr_cpass0_profiled_outputs[${runNumber}]="meta/cpass0.jobProfiled.run${runNumber}.done"
 974       echo "${arr_cpass0_profiled_outputs[${runNumber}]} : benchmark.sh ${configFile} ${copyFiles[@]}"
 975       echo -n " ${alirootEnv} ./benchmark.sh CPass0 ${commonOutputPath}/000${runNumber}/profiled ${inputFile} ${nEventsProfiling} ${currentDefaultOCDB} ${configFile} ${runNumber} profiling"" "
 976       for extraOption in "${extraOpts[@]}"; do echo -n \"${extraOption}\"" "; done;
 977       echo "\"useProfilingCommand=${profilingCommand}\""
 978       echo
 979     fi
 980
 981   done #runs
 982
 983   #Summary
 984   echo "summary.log : benchmark.sh ${configFile} ${arr_cpass1_merged[*]}"
 985   echo " LOCAL ./benchmark.sh MakeSummary ${configFile}"
 986   echo
 987
 988   return 0
 989 }
 990
 991 goPrintValues()
 992 {
 993   #print the values given the key from any number of files (store in output file on second argument)
 994   if [[ $# -lt 3 ]]; then
 995     echo "goPrintValues key outputFile inputFiles"
 996     echo "if outputFile is \"-\" dont redirect to a file"
 997     return
 998   fi
 999   key=${1}
1000   outputFile=${2}
1001   [[ ${outputFile} =~ "-" ]] && outputFile=""
1002   shift 2 #remove 2 first arguments from arg list to only pass the input files to awk
1003   awk -v key=${key} '$0 ~ key" " {print $2}' "$@" | tee ${outputFile}
1004   return 0
1005 }
1006
1007 goCreateQAplots()
1008 {
1009   umask 0002
1010   mergedQAfileList=${1}
1011   productionID=${2}
1012   outputDir=${3}
1013   configFile=${4}
1014   shift 4
1015   if ! parseConfig ${configFile} ${@}; then return 1; fi
1016
1017   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
1018
1019   [[ -z ${logOutputDir} ]] && logOutputDir=${PWD}
1020   [[ -z ${dontRedirectStdOutToLog} ]] && exec 2>&1 > ${logOutputDir}/makeQAplots.log
1021   echo "${0} $*"
1022
1023   olddir=${PWD}
1024   mkdir -p ${outputDir}
1025   cd ${outputDir}
1026   [[ ! "${PWD}" = "${outputDir}" ]] && echo "cannot make ${outputDir}... exiting" && return 1
1027
1028   echo ${ALICE_ROOT}/PWGPP/QA/scripts/runQA.sh inputList=${mergedQAfileList}
1029   ${ALICE_ROOT}/PWGPP/QA/scripts/runQA.sh inputList="${mergedQAfileList}" inputListHighPtTrees="${filteringList}"
1030
1031   cd ${olddir}
1032   return 0
1033 }
1034
1035 goTest()
1036 {
1037   umask 0002
1038   exec 2>&1
1039   exec > >(tee test.log)
1040   echo "$@"
1041   echo something
1042   return 0
1043 }
1044
1045 alirootInfo()
1046 {
1047   umask 0002
1048   # save aliroot repository info
1049   [[ -z "${ALICE_ROOT}" ]] && return 1
1050
1051   echo "\${ALICE_ROOT}=${ALICE_ROOT}"
1052   echo "\${ROOTSYS}=${ROOTSYS}"
1053   echo "\${PATH}=${PATH}"
1054   echo "\${LD_LIBRARY_PATH}=${LD_LIBRARY_PATH}"
1055
1056   pushd ${PWD}
1057   cd ${ALICE_ROOT}
1058   echo
1059
1060   currentBranch=$(git rev-parse --abbrev-ref HEAD)
1061   git status
1062   echo ""
1063   echo ""
1064   git diff ${currentBranch}
1065   popd
1066   return 0
1067 }
1068
1069 setYear()
1070 {
1071   #set the year
1072   #  ${1} - year to be set
1073   #  ${2} - where to set the year
1074   year1=$(guessYear ${1})
1075   year2=$(guessYear ${2})
1076   local path=${2}
1077   [[ ${year1} -ne ${year2} && -n ${year2} && -n ${year1} ]] && path=${2/\/${year2}\//\/${year1}\/}
1078   echo ${path}
1079   return 0
1080 }
1081
1082 guessPeriod()
1083 {
1084   #guess the period from the path, pick the rightmost one
1085   local IFS="/"
1086   declare -a path=( ${1} )
1087   local dirDepth=${#path[*]}
1088   for ((x=${dirDepth}-1;x>=0;x--)); do
1089     local field=${path[${x}]}
1090     [[ ${field} =~ ^LHC[0-9][0-9][a-z]$ ]] && period=${field} && break
1091   done
1092   echo ${period}
1093   return 0
1094 }
1095
1096 guessYear()
1097 {
1098   #guess the year from the path, pick the rightmost one
1099   local IFS="/"
1100   declare -a path=( ${1} )
1101   local dirDepth=${#path[*]}
1102   for ((x=${dirDepth}-1;x>=0;x--)); do
1103     local field=${path[${x}]}
1104     [[ ${field} =~ ^20[0-9][0-9]$ ]] && year=${field} && break
1105   done
1106   echo ${year}
1107   return 0
1108 }
1109
1110 guessRunNumber()
1111 {
1112   #guess the run number from the path, pick the rightmost one
1113   #works for /path/foo/000123456/bar/...
1114   #and       /path/foo.run123456.bar
1115   local IFS="/."
1116   declare -a path=( ${1} )
1117   local dirDepth=${#path[*]}
1118   for ((x=${dirDepth}-1;x>=0;x--)); do
1119     local field=${path[${x}]}
1120     field=${field/run/000}
1121     [[ ${field} =~ ^000[0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && runNumber=${field#000} && break
1122   done
1123   echo ${runNumber}
1124   return 0
1125 }
1126
1127 validateLog()
1128 {
1129   log=${1}
1130   errorConditions=(
1131                     'There was a crash'
1132                     'floating'
1133                     'error while loading shared libraries'
1134                     'std::bad_alloc'
1135                     's_err_syswatch_'
1136                     'Thread [0-9]* (Thread'
1137                     'AliFatal'
1138                     'core dumped'
1139                     '\.C.*error:.*\.h: No such file'
1140   )
1141
1142   warningConditions=(
1143                      'This is serious !'
1144                      'rocVoltage out of range:'
1145   )
1146
1147   local logstatus=0
1148   local errorSummary=""
1149   local warningSummary=""
1150
1151   for ((i=0; i<${#errorConditions[@]};i++)); do
1152     local tmp=$(grep -m1 -e "${errorConditions[${i}]}" ${log})
1153     [[ -n ${tmp} ]] && tmp+=" : "
1154     errorSummary+=${tmp}
1155   done
1156
1157   for ((i=0; i<${#warningConditions[@]};i++)); do
1158     local tmp=$(grep -m1 -e "${warningConditions[${i}]}" ${log})
1159     [[ -n ${tmp} ]] && tmp+=" : "
1160     warningSummary+=${tmp}
1161   done
1162
1163   if [[ -n ${errorSummary} ]]; then
1164     echo "${errorSummary}"
1165     return 1
1166   fi
1167
1168   if [[ -n ${warningSummary} ]]; then
1169     echo "${warningSummary}"
1170     return 2
1171   fi
1172
1173   return 0
1174 }
1175
1176 summarizeLogs()
1177 {
1178   #print a summary of logs
1179   logFiles=(
1180             "*.log"
1181             "stdout"
1182             "stderr"
1183   )
1184
1185   #check logs
1186   local logstatus=0
1187   for log in ${logFiles[*]}; do
1188     finallog=${PWD%/}/${log}
1189     [[ ! -f ${log} ]] && continue
1190     errorSummary=$(validateLog ${log})
1191     validationStatus=$?
1192     if [[ ${validationStatus} -eq 0 ]]; then
1193       #in pretend mode randomly report an error in rec.log some cases
1194       if [[ -n ${pretend} && "${log}" == "rec.log" ]]; then
1195         [[ $(( ${RANDOM}%2 )) -ge 1 ]] && echo "${finallog} BAD random error" || echo "${finallog} OK"
1196       else
1197         echo "${finallog} OK"
1198       fi
1199     elif [[ ${validationStatus} -eq 1 ]]; then
1200       echo "${finallog} BAD ${errorSummary}"
1201       logstatus=1
1202     elif [[ ${validationStatus} -eq 2 ]]; then
1203       echo "${finallog} OK MWAH ${errorSummary}"
1204     fi
1205   done
1206
1207   #report core files
1208   while read x; do
1209     echo ${x}
1210     chmod 644 ${x}
1211     gdb --batch --quiet -ex "bt" -ex "quit" aliroot ${x} > stacktrace_${x//\//_}.log
1212   done < <(/bin/ls ${PWD}/*/core 2>/dev/null; /bin/ls ${PWD}/core 2>/dev/null)
1213
1214   return ${logstatus}
1215 }
1216
1217 spitOutLocalOCDBaccessConfig()
1218 {
1219   umask 0002
1220   #find ${1} -name "*root" | \
1221   /bin/ls -1 ${1}/*/*/*/*.root 2>/dev/null | \
1222   while read line
1223   do
1224     local tmp=${line#${1}}
1225     echo ${tmp%/*} | \
1226     awk -v ocdb=${1} '{print "  man->SetSpecificStorage(\""$1"\",\"local://"ocdb"\");"}'
1227   done
1228   return 0
1229 }
1230
1231 goMakeLocalOCDBaccessConfig()
1232 {
1233   umask 0002
1234   # make a script that sets the specific storages form all the root files produced by CPass0
1235   local localOCDBpathCPass0=${1}
1236   local OCDBpathPrefix=${2}
1237   [[ -z ${OCDBpathPrefix} ]] && OCDBpathPrefix="."
1238
1239   if [[ -f ${localOCDBpathCPass0} && ${localOCDBpathCPass0} =~ \.tgz$ ]]; then
1240     tar xzf ${localOCDBpathCPass0}
1241     local localOCDBpathCPass0="${OCDBpathPrefix}/OCDB"
1242   fi
1243
1244   echo
1245   echo creating the specific storage script
1246   echo   localOCDBaccessConfig.C
1247   echo   based on OCDB: ${localOCDBaccessConfig}
1248   echo
1249
1250   local tempLocalOCDB=""
1251   if [[ -f localOCDBaccessConfig.C ]]; then
1252     tempLocalOCDB=$(mktemp)
1253     echo "egrep "SetSpecificStorage" localOCDBaccessConfig.C > ${tempLocalOCDB}"
1254     egrep "SetSpecificStorage" localOCDBaccessConfig.C > ${tempLocalOCDB}
1255   fi
1256
1257   echo "localOCDBaccessConfig()"                               >  localOCDBaccessConfig.C
1258   echo "{"                                                     >> localOCDBaccessConfig.C
1259   echo "  AliCDBManager* man = AliCDBManager::Instance();"     >> localOCDBaccessConfig.C
1260   spitOutLocalOCDBaccessConfig ${localOCDBpathCPass0}|sort|uniq  >> localOCDBaccessConfig.C
1261   [[ -f "${tempLocalOCDB}" ]] && cat ${tempLocalOCDB}              >> localOCDBaccessConfig.C
1262   echo "}"                                                     >> localOCDBaccessConfig.C
1263
1264   [[ -f "${tempLocalOCDB}" ]] && rm -f ${tempLocalOCDB}
1265
1266   if ! grep SetSpecificStorage localOCDBaccessConfig.C; then
1267     echo
1268     echo "!!!!!!! CPass0 produced no OCDB entries"
1269     return 1
1270   fi
1271   return 0
1272 }
1273
1274 goMakeFilteredTrees()
1275 {
1276   outputDir=${1}
1277   runNumber=${2}
1278   #get path to input list
1279   inputListfiles=${3}
1280   #get scale number for tracks
1281   filterT=${4}
1282   #get scale number for V0s
1283   filterV=${5}
1284   #get OCDB path (optional)
1285   OCDBpath=${6}
1286   #get max number of files
1287   maxFiles=${7-"1000000"}
1288   #get offset of first file
1289   offsetFile=${8-"0"}
1290   #get max number of events
1291   maxEvents=${9-"30000000"}
1292   #get offset of first event
1293   offsetEvent=${10-"0"}
1294   configFile=${11-"benchmark.config"}
1295   esdFileName=${12-"AliESDs_Barrel.root"}
1296   shift 12
1297   if ! parseConfig ${configFile} "$@"; then return 1; fi
1298
1299   commonOutputPath=${PWD}
1300   doneFile=${commonOutputPath}/meta/filtering.cpass1.run${runNumber}.done
1301
1302   runpath=${outputDir}
1303   [[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
1304   [[ ${reconstructInTemporaryDir} -eq 1 && -z ${TMPDIR} ]] && runpath=$(mktemp -d)
1305
1306
1307   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
1308
1309   mkdir -p ${outputDir}
1310   mkdir -p ${runpath}
1311
1312   cd ${runpath}
1313
1314   cat > filtering.log << EOF
1315   goMakeFilteredTrees config:
1316   runpath=${runpath}
1317   outputDir=${outputDir}
1318   commonOutputPath=${commonOutputPath}
1319   ALICE_ROOT=${ALICE_ROOT}
1320   PATH=${PATH}
1321   offsetEvent=$offsetEvent
1322   configFile=$configFile
1323   esdFileName=$esdFileName
1324 EOF
1325
1326   if [[ -z ${pretend} ]];then
1327     aliroot -l -b -q "${ALICE_ROOT}/PWGPP/macros/runFilteringTask.C(\"${inputListfiles}\",${filterT},${filterV},\"${OCDBpath}\",${maxFiles},${offsetFile},${maxEvents},${offsetEvent},\"${esdFileName}\")" &>> filtering.log
1328   else
1329     touch filtering.log FilterEvents_Trees.root
1330   fi
1331   pwd
1332   /bin/ls
1333   touch ${doneFile}
1334   summarizeLogs >>  ${doneFile}
1335   echo mv -f * ${outputDir}
1336   mv -f * ${outputDir}
1337   [[ -f ${outputDir}/FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFile}
1338   cd ${commonOutputPath}
1339   [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
1340   return 0
1341 }
1342
1343 submit()
1344 {
1345   umask 0002
1346   [[ $# -lt 5 ]] && echo "at least 5 args needed, you supplied $#" && return 1
1347   JobID=${1}
1348   startID=${2}
1349   endID=${3}
1350   waitForJOBID=${4}
1351   command=${5}
1352   shift 5
1353   commandArgs=("$@")
1354
1355   #add quote strings around the extra arguments
1356   for ((i=0; i<${#commandArgs[@]}; i++)); do
1357     commandArgs[i]=\"${commandArgs[i]}\"
1358   done
1359
1360   [[ -z ${waitForJOBID} ]] && waitForJOBID=0
1361
1362   newFarm=$(which qsub|grep "^/usr/bin/qsub")
1363
1364   batchSystem="SGE"
1365
1366   if [[ -z "${newFarm}" ]]
1367   then
1368     #old LSF
1369     # submit it (as job array)
1370     nFiles=$(( ${endID}-${startID}+1 ))
1371     while [ ${startID} -le ${nFiles}  ] ; do
1372       if [ $(expr ${nFiles} - ${startID}) -gt 999 ] ; then
1373         endID=$(expr ${startID} + 999)
1374       else
1375         endID=${nFiles}
1376       fi
1377     if [[ ${waitForJOBID} -eq 0 ]]; then
1378         echo ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}"
1379         ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}"
1380       else
1381         echo ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -w "ended(${waitForJOBID})" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}"
1382         ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -w "ended(${waitForJOBID})" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}"
1383       fi
1384       startID=$(expr ${endID} + 1)
1385     done
1386   else
1387     #new SGE farm
1388     if [[ ${waitForJOBID} =~ "000" ]]; then
1389       echo ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}"
1390       ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}"
1391     else
1392       echo ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -hold_jid "${waitForJOBID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}"
1393       ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -hold_jid "${waitForJOBID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}"
1394     fi
1395   fi
1396   return 0
1397 }
1398
1399 goSubmitBatch()
1400 {
1401   if [[ $# -lt 3 ]]; then
1402     echo "minimal use:"
1403     echo " ${0} submit fileList productionID configFile"
1404     return 0
1405   fi
1406
1407   productionID=${1}
1408   inputList=${2}
1409   configFile=${3:-"benchmark.config"}
1410   configFile=$(readlink -f ${configFile})
1411   shift 3
1412   extraOpts=("$@")
1413   if ! parseConfig ${configFile} "${extraOpts[@]}"; then return 1; fi
1414
1415   #redirect all output to submit.log
1416   echo "redirecting all output to ${PWD}/submit_${productionID//"/"/_}.log"
1417   exec 7>&1
1418   exec 1>submit_${productionID//"/"/_}.log 2>&1
1419
1420   umask 0002
1421   echo ${0}" submit $*"
1422   if [[ -z "${inputList}" || -z "${productionID}" ]]
1423   then
1424     echo
1425     echo " Usage: ${0} submit inputList productionID [configFile=benchmark.config]"
1426     echo
1427     return
1428   fi
1429
1430   # check if config file is there
1431   if [ ! -f ${configFile} ]; then
1432     echo "ERROR! Config File '${configFile}' not found" >&2
1433     return
1434   else
1435     echo "Using Config File: '${configFile}'"
1436   fi
1437
1438   [[ ! -f ${alirootEnv} ]] && echo "alirootEnv script ${alirootEnv} not found!..." && return 1
1439
1440   #move the script, config and some other stuff to ${commonOutputPath} first, then use them from there
1441   self=$(readlink -f "${0}")
1442   configPath=$(dirname ${configFile})
1443   export commonOutputPath=${baseOutputDirectory}/${productionID}
1444
1445   mkdir -p ${commonOutputPath}
1446   mkdir -p ${commonOutputPath}/logs
1447   mkdir -p ${commonOutputPath}/meta
1448
1449   cp ${self} ${commonOutputPath}
1450   cp ${configFile} ${commonOutputPath}
1451   cp ${inputList} ${commonOutputPath}
1452   self=${commonOutputPath}/${self##*/}
1453   chmod u+x ${self}
1454   configFile=${commonOutputPath}/${configFile##*/}
1455   inputList=${commonOutputPath}/${inputList##*/}
1456
1457   #convert to absolut pathnames
1458   inputList=$(readlink -f "${inputList}")
1459   #make list of runs
1460   if [[ -z ${runNumber} ]]; then
1461     listOfRuns=($(while read x; do guessRunNumber ${x}; done < ${inputList} | sort | uniq))
1462   else
1463     listOfRuns=${runNumber}
1464   fi
1465
1466   alirootSource=$(readlink -f "${alirootSource}")
1467
1468   echo ""
1469   echo "### BEGIN CONFIGURATION ###"
1470   echo ""
1471   echo "GENERAL:"
1472   echo ""
1473   echo "    productionID:    ${productionID}"
1474   echo "    batchCommand:    ${batchCommand}"
1475   echo "    batchFlags:      ${batchFlags}"
1476   echo "    alirootEnv:   ${alirootEnv}"
1477   ${alirootEnv} echo '    ALICE_ROOT:      ${ALICE_ROOT}'
1478   ${alirootEnv} echo '    ALIROOT_RELEASE: ${ALICE_RELEASE}'
1479   echo "    inputList:       ${inputList}"
1480   echo "    configPath:      ${configPath}"
1481   echo "    commonOutputPath:      ${commonOutputPath}"
1482   echo "    defaultOCDB:     ${defaultOCDB}"
1483   echo "      autoOCDB: ${autoOCDB}"
1484   echo "    recoTriggerOptions:   ${recoTriggerOptions}"
1485   echo "    runs:"
1486   echo "      ${listOfRuns[*]}"
1487   echo ""
1488   echo "THE TRAIN WILL RUN:"
1489
1490   if [ ${runCPass0reco} -eq 1 ]; then
1491     echo "    Pass0 - Recontruction"
1492   fi
1493
1494   if [ ${runCPass0MergeMakeOCDB} -eq 1 ]; then
1495     echo "    Pass0 - merging and OCDB export"
1496   fi
1497
1498   if [ ${runCPass1reco} -eq 1 ]; then
1499     echo "    Pass1 - Recontruction"
1500   fi
1501   if [ ${runCPass1MergeMakeOCDB} -eq 1 ]; then
1502     echo "    Pass1 - merging and OCDB export"
1503   fi
1504
1505   echo ""
1506   echo "LIMITS:"
1507   echo "    max. Events/Chunk:   ${nEvents}"
1508   echo "    max. Number of Chunks per Run:     ${nMaxChunks}"
1509   echo ""
1510   echo "### END CONFIGURATION ###"
1511   echo ""
1512
1513
1514   # check if input file is there
1515   if [ ! -f ${inputList} ]; then
1516     echo "ERROR! Input List '${inputList}' not found" >&2
1517     return
1518   fi
1519
1520   # define jobid (for dependent jobs)
1521   date=$(date +%Y_%m_%d_%H%M%S)
1522   #for each run we submit one jobarray:
1523   for runNumber in ${listOfRuns[*]}; do
1524
1525     [[ -z ${runNumber} ]] && continue
1526     [[ ! ${runNumber} =~ ^[0-9]*[0-9]$ ]] && continue
1527
1528     JOBpostfix="${productionID//"/"/_}_${runNumber}_${date}"
1529     JOBID1="p0_${JOBpostfix}"
1530     JOBID1wait="w0_${JOBpostfix}"
1531     JOBID2="m0_${JOBpostfix}"
1532     JOBID2wait="wm0_${JOBpostfix}"
1533     JOBID3="op0_${JOBpostfix}"
1534     JOBID3wait="wop0_${JOBpostfix}"
1535     JOBID4="p1_${JOBpostfix}"
1536     JOBID4wait="w1_${JOBpostfix}"
1537     JOBID5="m1_${JOBpostfix}"
1538     JOBID5wait="wm1_${JOBpostfix}"
1539     JOBID6="s1_${JOBpostfix}"
1540     JOBID6wait="ws1_${JOBpostfix}"
1541     JOBID7="QA_${JOBpostfix}"
1542     JOBmakeESDlistCPass1="lp1_${JOBpostfix}"
1543     JOBfilterESDcpass1="fp1_${JOBpostfix}"
1544     LASTJOB="000"
1545
1546     oneInputFile=$(egrep -m1 "${runNumber}\/" ${inputList})
1547
1548     currentDefaultOCDB=${defaultOCDB}
1549     [[ -z ${autoOCDB} ]] && autoOCDB=1
1550     if [[ ${autoOCDB} -ne 0 ]]; then
1551       currentDefaultOCDB=$(setYear ${oneInputFile} ${defaultOCDB})
1552     fi
1553
1554     echo "submitting run ${runNumber} with OCDB ${currentDefaultOCDB}"
1555
1556     ###############################################################################
1557     #run one chunk with valgrind:
1558     if [[ -n ${profilingCommand} ]]; then
1559       [[ -z ${nEventsProfiling} ]] && nEventsProfiling=2
1560       [[ -z ${profilingCommand} ]] && profilingCommand="/usr/bin/valgrind --tool=callgrind --num-callers=40 -v --trace-children=yes"
1561       submit "valgrind" 1 1 000 "${alirootEnv} ${self}" CPass0 ${commonOutputPath}/000${runNumber}/valgrind ${oneInputFile} ${nEventsProfiling} ${currentDefaultOCDB} ${configFile} ${runNumber} valgrind useProfilingCommand=${profilingCommand} "${extraOpts[@]}"
1562     fi
1563
1564     ################################################################################
1565     ################################################################################
1566     # run the CPass0 if requested
1567
1568     if [ ${runCPass0reco} -eq 1 ]; then
1569
1570       echo
1571       echo "starting CPass0... for run ${runNumber}"
1572       echo
1573
1574       # create directory and copy all files that are needed
1575       targetDirectory="${commonOutputPath}/000${runNumber}/cpass0"
1576       mkdir -p ${targetDirectory}
1577
1578       filesCPass0=(
1579                     "${configPath}/runCPass0.sh"
1580                     "${configPath}/recCPass0.C"
1581                     "${configPath}/runCalibTrain.C"
1582                     "${configPath}/localOCDBaccessConfig.C"
1583                     "${configPath}/OCDB*.root"
1584                     "${configPath}/sim.C"
1585                     "${configPath}/rec.C"
1586                     "${configPath}/Config.C"
1587       )
1588       for file in ${filesCPass0[*]}; do
1589         [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
1590       done
1591
1592       localInputList=${targetDirectory}/${inputList##*/}
1593       [[ ! -f ${localInputList} ]] && egrep "\/000${runNumber}\/" ${inputList} > ${localInputList}
1594       # limit nFiles to nMaxChunks
1595       nFiles=$(wc -l < ${localInputList})
1596       [[ ${nFiles} -eq 0 ]] && echo "list contains ZERO files! exiting..." && return 1
1597       echo "raw files in list:    ${nFiles}"
1598       if [[ ${nMaxChunks} -gt 0 && ${nMaxChunks} -le ${nFiles} ]]; then
1599         nFiles=${nMaxChunks}
1600       fi
1601       echo "raw files to process: ${nFiles}"
1602       [[ -z "${percentProcessedFilesToContinue}" ]] && percentProcessedFilesToContinue=100
1603       if [[ ${percentProcessedFilesToContinue} -eq 100 ]]; then
1604         nFilesToWaitFor=${nFiles}
1605       else
1606         nFilesToWaitFor=$(( ${nFiles}-${nFiles}/(100/(100-${percentProcessedFilesToContinue})) ))
1607       fi
1608       echo "requested success rate is ${percentProcessedFilesToContinue}%"
1609       echo "merging will start after ${nFilesToWaitFor} jobs are done"
1610
1611       submit ${JOBID1} 1 ${nFiles} 000 "${alirootEnv} ${self}" CPass0 ${targetDirectory} ${localInputList} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} -1 "${extraOpts[@]}"
1612
1613       ## submit a monitoring job that will run until a certain number of jobs are done with reconstruction
1614       submit "${JOBID1wait}" 1 1 000 "${alirootEnv} ${self}" WaitForOutput ${commonOutputPath} "meta/cpass0.job*.run${runNumber}.done" ${nFilesToWaitFor} ${maxSecondsToWait} '-maxdepth 1'
1615       LASTJOB=${JOBID1wait}
1616
1617     fi #end running CPass0
1618     ################################################################################
1619
1620
1621     ################################################################################
1622     # submit merging of CPass0, depends on the reconstruction
1623
1624     if [ ${runCPass0MergeMakeOCDB} -eq 1 ]; then
1625
1626       echo
1627       echo "submit CPass0 merging for run ${runNumber}"
1628       echo
1629
1630       targetDirectory="${commonOutputPath}/000${runNumber}/cpass0"
1631       mkdir -p ${targetDirectory}
1632
1633       #copy the scripts
1634       filesMergeCPass0=(
1635                         "${configPath}/OCDB.root"
1636                         "${configPath}/mergeMakeOCDB.byComponent.sh"
1637                         "${configPath}/mergeMakeOCDB.sh"
1638                         "${configPath}/localOCDBaccessConfig.C"
1639                         "${configPath}/mergeByComponent.C"
1640                         "${configPath}/makeOCDB.C"
1641                         "${configPath}/merge.C"
1642       )
1643       for file in ${filesMergeCPass0[*]}; do
1644         [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
1645       done
1646
1647       submit ${JOBID2} 1 1 "${LASTJOB}" "${alirootEnv} ${self}" MergeCPass0 ${targetDirectory} ${currentDefaultOCDB} ${configFile} ${runNumber} cpass0.calib.run${runNumber}.list "${extraOpts[@]}"
1648       LASTJOB=${JOBID2}
1649
1650       if [[ -n ${generateMC} ]]; then
1651         submit "mrl${JOBpostfix}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" PrintValues sim ${commonOutputPath}/meta/sim.run${runNumber}.list ${commonOutputPath}/meta/cpass0.job*.run${runNumber}.done
1652         LASTJOB="mrl${JOBpostfix}"
1653       fi
1654
1655       echo
1656     fi
1657     # end of merging CPass0
1658     ################################################################################
1659
1660     ################################################################################
1661     ################################################################################
1662     # run the CPass1 if requested
1663
1664     if [ ${runCPass1reco} -eq 1 ]; then
1665
1666       targetDirectory="${commonOutputPath}/000${runNumber}/cpass1"
1667       rm -f ${commonOutputPath}/meta/cpass1.job*.run${runNumber}.done
1668
1669       # safety feature: if we are re-running for any reason we want to delete the previous output first.
1670       [[ -d ${targetDirectory} ]] && rm -rf ${targetDirectory}/* && echo "removed old output at ${targetDirectory}/*"
1671
1672       echo
1673       echo "starting CPass1... for run ${runNumber}"
1674       echo
1675
1676       # create directory and copy all files that are needed
1677       mkdir -p ${targetDirectory}
1678
1679       filesCPass1=(
1680                     "${configPath}/runCPass1.sh"
1681                     "${configPath}/recCPass1.C"
1682                     "${configPath}/recCPass1_OuterDet.C"
1683                     "${configPath}/runCalibTrain.C"
1684                     "${configPath}/QAtrain_duo.C"
1685                     "${configPath}/localOCDBaccessConfig.C"
1686                     "${configPath}/OCDB.root"
1687       )
1688       for file in ${filesCPass1[*]}; do
1689         [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
1690       done
1691
1692       if [[ -n ${generateMC} ]]; then
1693         localInputList=${commonOutputPath}/meta/sim.run${runNumber}.list
1694       else
1695         localInputList=${targetDirectory}/${inputList##*/}
1696         [[ ! -f ${localInputList} ]] && egrep "\/000${runNumber}\/" ${inputList} > ${localInputList}
1697       fi
1698       # limit nFiles to nMaxChunks
1699       nFiles=$(wc -l < ${localInputList})
1700       [[ ${nFiles} -eq 0 ]] && echo "list contains ZERO files! continuing..." && continue
1701       echo "raw files in list:    ${nFiles}"
1702       if [[ ${nMaxChunks} -gt 0 && ${nMaxChunks} -le ${nFiles} ]]; then
1703         nFiles=${nMaxChunks}
1704       fi
1705       echo "raw files to process: ${nFiles}"
1706       [[ -z "${percentProcessedFilesToContinue}" ]] && percentProcessedFilesToContinue=100
1707       if [[ ${percentProcessedFilesToContinue} -eq 100 ]]; then
1708         nFilesToWaitFor=${nFiles}
1709       else
1710         nFilesToWaitFor=$(( ${nFiles}-${nFiles}/(100/(100-${percentProcessedFilesToContinue})) ))
1711       fi
1712       echo "requested success rate is ${percentProcessedFilesToContinue}%"
1713       echo "merging will start after ${nFilesToWaitFor} jobs are done"
1714
1715       submit ${JOBID4} 1 ${nFiles} "${LASTJOB}" "${alirootEnv} ${self}" CPass1 ${targetDirectory} ${localInputList} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} -1 "${extraOpts[@]}"
1716
1717       ################################################################################
1718       ## submit a monitoring job that will run until a certain number of jobs are done with reconstruction
1719       submit "${JOBID4wait}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" WaitForOutput ${commonOutputPath} "meta/cpass1.job*.run${runNumber}.done" ${nFilesToWaitFor} ${maxSecondsToWait} '-maxdepth 1'
1720       LASTJOB=${JOBID4wait}
1721       ################################################################################
1722
1723       echo
1724     fi #end running CPass1
1725
1726     ################################################################################
1727     # submit merging of CPass1, depends on the reconstruction
1728     if [ ${runCPass1MergeMakeOCDB} -eq 1 ]; then
1729
1730       echo
1731       echo "submit CPass1 merging for run ${runNumber}"
1732       echo
1733
1734       targetDirectory="${commonOutputPath}/000${runNumber}/cpass1"
1735       rm -f ${commonOutputPath}/meta/merge.cpass1.run${runNumber}.done
1736       mkdir -p ${targetDirectory}
1737
1738       # copy files
1739       filesMergeCPass1=(
1740                         "${configPath}/OCDB.root"
1741                         "${configPath}/localOCDBaccessConfig.C"
1742                         "${configPath}/mergeMakeOCDB.byComponent.sh"
1743                         "${configPath}/mergeByComponent.C"
1744                         "${configPath}/makeOCDB.C"
1745                         "${configPath}/merge.C"
1746                         "${configPath}/mergeMakeOCDB.sh"
1747                         "${configPath}/QAtrain_duo.C"
1748       )
1749       for file in ${filesMergeCPass1[*]}; do
1750         [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
1751       done
1752
1753       submit "${JOBID5}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" MergeCPass1 ${targetDirectory} ${currentDefaultOCDB} ${configFile} ${runNumber} cpass1.calib.run${runNumber}.list cpass1.QA.run${runNumber}.lastMergingStage.txt.list cpass1.filtered.run${runNumber}.list "${extraOpts[@]}"
1754       LASTJOB=${JOBID5}
1755       echo
1756     fi
1757
1758     ###############################
1759     #if [ ${runESDfiltering} -eq 1 ]; then
1760     #  rm -f ${commonOutputPath}/cpass1.ESD.run${runNumber}.list
1761     #  rm -f ${commonOutputPath}/meta/filtering.cpass1.run*.done
1762     #  echo
1763     #  echo submitting filtering for run ${runNumber}
1764     #  echo
1765     #  submit "${JOBmakeESDlistCPass1}" 1 1 "${LASTJOB}" "${self}" PrintValues esd ${commonOutputPath}/meta/cpass1.ESD.run${runNumber}.list ${commonOutputPath}/meta/cpass1.job*.run${runNumber}.done
1766     #  submit "${JOBfilterESDcpass1}" 1 1 "${JOBmakeESDlistCPass1}" "${alirootEnv} ${self}" MakeFilteredTrees ${commonOutputPath}/000${runNumber}/cpass1 ${runNumber} ${commonOutputPath}/meta/cpass1.ESD.run${runNumber}.list ${filteringFactorHighPt} ${filteringFactorV0s} ${currentDefaultOCDB} 1000000 0 10000000 0 ${configFile} AliESDs_Barrel.root "${extraOpts[@]}"
1767     #  LASTJOB=${JOBfilterESDcpass1}
1768     #fi
1769
1770   done
1771
1772   #################################################################################
1773   #################################################################################
1774   #if [ ${runESDfiltering} -eq 1 ]; then
1775   #  submit "${JOBID5wait}" 1 1 "${LASTJOB}" "${self}" WaitForOutput ${commonOutputPath} "meta/filtering.cpass1.run*.done" "${#listOfRuns[@]}" ${maxSecondsToWait}
1776   #else
1777     submit "${JOBID5wait}" 1 1 "${LASTJOB}" "${self}" WaitForOutput ${commonOutputPath} "meta/merge.cpass1.run*.done" ${#listOfRuns[@]} ${maxSecondsToWait}
1778   #fi
1779   LASTJOB=${JOBID5wait}
1780
1781   #################################################################################
1782   echo
1783   echo "submit make a summary"
1784   echo
1785
1786   submit "${JOBID6}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" MakeSummary ${configFile}
1787   LASTJOB=${JOBID6}
1788   #################################################################################
1789
1790   #restore stdout
1791   exec 1>&7 7>&-
1792   echo "jobs submitted."
1793   return 0
1794 }
1795
1796 goWaitForOutput()
1797 {
1798   umask 0002
1799   [[ $# -lt 3 ]] && echo "goWaitForOutput() wrong number of arguments, exiting.." && return 1
1800   echo searchPath=${1}
1801   echo fileName=${2}
1802   echo numberOfFiles=${3}
1803   echo maxSecondsToWait=${4}
1804   searchPath=${1}
1805   fileName=${2}
1806   numberOfFiles=${3}
1807   maxSecondsToWait=${4}
1808   extraFindOptions=${5}
1809   echo "command to be executed: /bin/ls -1 ${searchPath}/${fileName} ${extraFindOptions}"
1810   [[ -z "${maxSecondsToWait}" ]] && maxSecondsToWait=$(( 3600*12 ))
1811   while true; do
1812     #n=$(find ${searchPath} ${extraFindOptions} -name "${fileName}" | wc -l)
1813     n=$(/bin/ls -1 ${searchPath}/${fileName} 2>/dev/null | wc -l)
1814     [[ ${n} -gt 0 ]] && echo "found ${n} X ${fileName}"
1815     [[ ${n} -ge ${numberOfFiles} ]] && break
1816     [[ ${SECONDS} -gt ${maxSecondsToWait} ]] && echo "timeout of ${maxSecondsToWait}s!" && break
1817     sleep 60
1818   done
1819   echo "DONE! exiting..."
1820   return 0
1821 }
1822
1823 mergeSysLogs()
1824 {
1825   outputFile=${1}
1826   shift
1827   inputFiles="$@"
1828   i=0
1829   if ! ls -1 ${inputFiles} &>/dev/null; then echo "the files dont exist!: ${inputFiles}"; return 1; fi
1830   while read x; do
1831     runNumber=$(guessRunNumber ${x})
1832     [[ -z ${runNumber} ]] && echo "run number cannot be guessed for ${x}" && continue
1833     awk -v run=${runNumber} -v i=${i} 'NR > 1 {print run" "$0} NR==1 && i==0 {print "run/I:"$0}' ${x}
1834     (( i++ ))
1835   done < <(ls -1 ${inputFiles}) > ${outputFile}
1836   return 0
1837 }
1838
1839 goMakeMergedSummaryTree()
1840 {
1841   # create list of calibration entries
1842   # takes no arguments, just run it in the base output
1843   # directory with the following files in the working directory
1844   #
1845   #  Calibration file lists:
1846   #       cpass0.dcsTree.list, cpass1.dcsTree.list
1847   #  QA trending root file:
1848   #       trending.root
1849   #
1850   #  Production infoo ascii files:
1851   #       summary_pass0.tree
1852   #       summary_pass1.tree
1853   #
1854
1855   [[ ! -f cpass0.dcsTree.list ]] && echo "no cpass0.dcsTree.list" && return 1
1856   [[ ! -f cpass1.dcsTree.list ]] && echo "no cpass1.dcsTree.list" && return 1
1857   [[ ! -f trending.root ]] && echo "no trending.root" && return 1
1858   [[ ! -f summary_pass0.tree ]] && echo "no summary_pass0.tree" && return 1
1859   [[ ! -f summary_pass1.tree ]] && echo "no summary_pass1.tree" && return 1
1860
1861   #first, dump the C macro to file
1862   cat << EOF > mergeTree.C
1863   //
1864   // Merge summary information
1865   // Following files are expected to be in the working directory
1866   //
1867   // Calibration file lists:
1868   //      cpass0.dcsTree.list, cpass1.dcsTree.list
1869   // QA trending root file:
1870   //      trending.root
1871   //
1872   // Production infoo ascii files:
1873   //      summary_pass0.tree
1874   //      summary_pass1.tree
1875   //
1876   void mergeTree(){
1877     //
1878     //
1879     //
1880     // Calibration values dump
1881     //
1882     AliXRDPROOFtoolkit::MakeTreeFromList("Calib.TPC.CPass0.root", "dcs","dcs","cpass0.dcsTree.list",1);
1883     AliXRDPROOFtoolkit::MakeTreeFromList("Calib.TPC.CPass1.root", "dcs","dcs","cpass1.dcsTree.list",1);
1884     //
1885     // Calibration status dump
1886     //
1887     TFile *fprod = TFile::Open("fproduction.root","recreate");
1888     TTree  tree0, tree1;
1889     tree0.ReadFile("summary_pass0.tree");
1890     tree1.ReadFile("summary_pass1.tree");
1891     tree0.Write("CPass0");
1892     tree1.Write("CPass1");
1893     fprod->Close();
1894     //
1895     //
1896     //
1897     TString stringSetup="";
1898     stringSetup+="1#QA.TPC#run#SummaryTPCQA/tpcQA#trending.root+";  //
1899     stringSetup+="1#Calib.TPC.CPass0#run#dcs#Calib.TPC.CPass0.root+";  //
1900     stringSetup+="1#Calib.TPC.CPass1#run#dcs#Calib.TPC.CPass1.root+";  //
1901     //
1902     stringSetup+="1#CPass0#runnumber#CPass0#fproduction.root+";  //
1903     stringSetup+="1#CPass1#runnumber#CPass1#fproduction.root+";  //
1904     //
1905     AliXRDPROOFtoolkit::JoinTreesIndex("outAll.root","joinAll","run",stringSetup.Data(), 1);
1906   }
1907 EOF
1908
1909   aliroot -b -q "mergeTree.C" > mergeTrees.log
1910   return $?
1911 }
1912
1913 goMakeSummary()
1914 {
1915   #all the final stuff goes in here for ease of use:
1916   # summary logs
1917   # qa plot making
1918   # final file lists
1919   #some defaults:
1920   log="summary.log"
1921   productionID="qa"
1922
1923   configFile=${1}
1924   shift 1
1925   extraOpts=("$@")
1926   if ! parseConfig ${configFile} "${extraOpts[@]}"; then return 1; fi
1927
1928   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
1929
1930   exec &> >(tee ${log})
1931
1932   [[ ! -f ${configFile} ]] && echo "no config file ${configFile}!" && return
1933
1934   [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
1935
1936   #summarize the global stuff
1937   echo "env script: ${alirootSource} ${alirootEnv}"
1938   echo "\$ALICE_ROOT=${ALICE_ROOT}"
1939   echo "commonOutputPath=${commonOutputPath}"
1940
1941   #summarize the stacktraces
1942   awk '
1943        BEGIN {
1944                print "frame/I:method/C:line/C:cpass/I:aliroot/I";
1945                RS="#[0-9]*";
1946                aliroot=0;
1947              }
1948              {
1949                if ($3 ~ /Ali*/) aliroot=1; else aliroot=0;
1950                gsub("#","",RT);
1951                if ($NF!="" && RT!="" && $3!="") print RT" "$3" "$NF" "0" "aliroot
1952              }
1953       ' 000*/cpass0/*/stacktrace* 2>/dev/null > stacktrace.tree
1954   awk '
1955        BEGIN {
1956                RS="#[0-9]*";
1957                aliroot=0;
1958              }
1959              {
1960                if ($3 ~ /Ali*/) aliroot=1; else aliroot=0;
1961                gsub("#","",RT);
1962                if ($NF!="" && RT!="" && $3!="") print RT" "$3" "$NF" "1" "aliroot
1963              }
1964       ' 000*/cpass1/*/stacktrace* 2>/dev/null >> stacktrace.tree
1965
1966   echo total numbers for the production:
1967   echo
1968   awk 'BEGIN {nFiles=0;nCore=0;}
1969   /^calibfile/ {nFiles++;}
1970   /core dumped/ {nCore++i;}
1971   END {print     "cpass0 produced "nFiles" calib files, "nCore" core files";}' meta/cpass0.job*done 2>/dev/null
1972   awk 'BEGIN {nOK=0; nBAD=0; }
1973   /\/rec.log OK/ {nOK++;}
1974   /\/rec.log BAD/ {nBAD++;}
1975   /stderr BAD/ {if ($0 ~ /rec.log/){nBAD++;}}
1976   END {print     "cpass0 reco:  OK: "nOK"\tBAD: "nBAD;}' meta/cpass0.job*done 2>/dev/null
1977   awk 'BEGIN {nOK=0; nBAD=0; }
1978   /\/calib.log OK/ {nOK++;}
1979   /\/calib.log BAD/ {nBAD++;}
1980   END {print "cpass0 calib: OK: "nOK"\tBAD: "nBAD;}' meta/cpass0.job*done 2>/dev/null
1981
1982   awk 'BEGIN {nOK=0; nBAD=0; }
1983   /merge.log OK/ {nOK++;}
1984   /merge.log BAD/ {nBAD++;}
1985   END {print "cpass0 merge: OK: "nOK"\tBAD: "nBAD;}' meta/merge.cpass0*done 2>/dev/null
1986   awk 'BEGIN {nOK=0; nBAD=0; }
1987   /ocdb.log OK/ {nOK++;}
1988   /ocdb.log BAD/ {nBAD++;}
1989   END {print   "cpass0 OCDB:  OK: "nOK"\tBAD: "nBAD;}' meta/merge.cpass0*done 2>/dev/null
1990
1991   echo
1992   awk 'BEGIN {nFiles=0;nCore=0;}
1993   /^calibfile/ {nFiles++;}
1994   /core dumped/ {nCore++;}
1995   END {print     "cpass1 produced "nFiles" calib files, "nCore" core files";}' meta/cpass1.job*done 2>/dev/null
1996   awk 'BEGIN {nOK=0; nBAD=0; }
1997   /\/rec.log OK/ {nOK++;}
1998   /\/rec.log BAD/ {nBAD++;}
1999   /stderr BAD/ {if ($0 ~ /rec.log/){nBAD++;}}
2000   END {print     "cpass1 reco:  OK: "nOK"\tBAD: "nBAD;}' meta/cpass1.job*done 2>/dev/null
2001   awk 'BEGIN {nOK=0; nBAD=0; }
2002   /\/calib.log OK/ {nOK++;}
2003   /\/calib.log BAD/ {nBAD++;}
2004   END {print "cpass1 calib: OK: "nOK"\tBAD: "nBAD;}' meta/cpass1.job*done 2>/dev/null
2005
2006   awk 'BEGIN {nOK=0; nBAD=0; }
2007   /merge.log OK/ {nOK++;}
2008   /merge.log BAD/ {nBAD++;}
2009   END {print "cpass1 merge: OK: "nOK"\tBAD: "nBAD;}' meta/merge.cpass1*done 2>/dev/null
2010   awk 'BEGIN {nOK=0; nBAD=0; }
2011   /ocdb.log OK/ {nOK++;}
2012   /ocdb.log BAD/ {nBAD++;}
2013   END {print   "cpass1 OCDB:  OK: "nOK"\tBAD: "nBAD;}' meta/merge.cpass1*done 2>/dev/null
2014
2015   echo
2016   echo per run stats:
2017   /bin/ls -1 meta/merge.cpass0.run*.done | while read x
2018 do
2019   dir=$(goPrintValues calibfile - ${x})
2020   runNumber=$(guessRunNumber ${dir})
2021   [[ -z ${runNumber} ]] && continue
2022
2023   if $(/bin/ls meta/cpass0.job*.run${runNumber}.done &> /dev/null); then
2024     statusCPass0=( $(
2025     awk 'BEGIN {nOKrec=0;nBADrec=0;nOKcalib=0;nBADcalib=0;nOKstderr=0;nBADstderr=0;}
2026     /\/rec.log OK/ {nOKrec++;}
2027     /\/rec.log BAD/ {nBADrec++;}
2028     /stderr BAD/ {if ($0 ~ /rec.log/) {nBADrec++;} nBADstderr++;}
2029     /stderr OK/ {nOKstderr++;}
2030     /\/calib.log OK/ {nOKcalib++;}
2031     /\/calib.log BAD/ {nBADcalib++}
2032     END {print ""nOKrec" "nBADrec" "nOKstderr" "nBADstderr" "nOKcalib" "nBADcalib;}' meta/cpass0.job*.run${runNumber}.done 2>/dev/null
2033     ) )
2034   fi
2035
2036   if $(/bin/ls meta/cpass1.job*.run${runNumber}.done &>/dev/null); then
2037     statusCPass1=( $(
2038     awk 'BEGIN {nOKrec=0;nBADrec=0;nOKcalib=0;nBADcalib=0;nOKstderr=0;nBADstderr=0;nQAbarrelOK=0;nQAbarrelBAD=0;nQAouterOK=0;nQAouterBAD=0;}
2039     /\/rec.log OK/ {nOKrec++;}
2040     /\/rec.log BAD/ {nBADrec++;}
2041     /stderr BAD/ {if ($0 ~ /rec.log/) nBADrec++;nBADstderr++;}
2042     /stderr OK/ {nOKstderr++;}
2043     /\/calib.log OK/ {nOKcalib++;}
2044     /\/calib.log BAD/ {nBADcalib++}
2045     /\/qa_barrel.log OK/ {nQAbarrelOK++;}
2046     /\/qa_barrel.log BAD/ {nQAbarrelBAD++;}
2047     /\/qa_outer.log OK/ {nQAouterOK++;}
2048     /\/qa_outer.log BAD/ {nQAouterBAD++;}
2049     END {print ""nOKrec" "nBADrec" "nOKstderr" "nBADstderr" "nOKcalib" "nBADcalib" "nQAbarrelOK" "nQAbarrelBAD" "nQAouterOK" "nQAouterBAD;}' meta/cpass1.job*.run${runNumber}.done 2>/dev/null
2050     ) )
2051   fi
2052
2053   statusOCDBcpass0=$(awk '/ocdb.log/ {print $2} ' ${x} 2>/dev/null)
2054   statusOCDBcpass1=$(awk '/ocdb.log/ {print $2}' ${x/cpass0/cpass1} 2>/dev/null)
2055   statusQA=$(awk '/mergeMakeOCDB.log/ {print $2}' ${x/cpass0/cpass1} 2>/dev/null)
2056
2057   printf "%s\t ocdb.log cpass0: %s\t ocdb.log cpass1: %s\tqa.log:%s\t| cpass0: rec:%s/%s stderr:%s/%s calib:%s/%s cpass1: rec:%s/%s stderr:%s/%s calib:%s/%s QAbarrel:%s/%s QAouter:%s/%s\n" ${runNumber} ${statusOCDBcpass0} ${statusOCDBcpass1} ${statusQA} ${statusCPass0[0]} ${statusCPass0[1]} ${statusCPass0[2]} ${statusCPass0[3]} ${statusCPass0[4]} ${statusCPass0[5]} ${statusCPass1[0]} ${statusCPass1[1]} ${statusCPass1[2]} ${statusCPass1[3]} ${statusCPass1[4]} ${statusCPass1[5]} ${statusCPass1[6]} ${statusCPass1[7]} ${statusCPass1[8]} ${statusCPass1[9]}
2058 done
2059
2060   #make lists with output files - QA, trending, filtering and calibration
2061   ### wait for the merging of all runs to be over ###
2062   rm -f qa.list
2063   goPrintValues qafile qa.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2064   rm -f calib.list
2065   goPrintValues calibfile calib.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2066   rm -f trending.list
2067   goPrintValues trendingfile trending.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2068   rm -f filtering.list
2069   goPrintValues filteredTree filtering.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2070   #/bin/ls ${commonOutputPath}/*/cpass0/dcs* > cpass0.dcsTree.list
2071   rm -f cpass0.dcsTree.list
2072   goPrintValues dcsTree cpass0.dcsTree.list ${commonOutputPath}/meta/merge.cpass0.run*.done &>/dev/null
2073   #/bin/ls ${commonOutputPath}/*/cpass1/dcs* > cpass1.dcsTree.list
2074   rm -f cpass1.dcsTree.list
2075   goPrintValues dcsTree cpass1.dcsTree.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2076
2077   #merge trending
2078   rm -f ${commonOutputPath}/trending_merged.root
2079   goMerge trending.list ${commonOutputPath}/trending.root ${configFile} "${extraOpts[@]}" &> mergeTrending.log
2080
2081   goMakeSummaryTree ${commonOutputPath} 0
2082   goMakeSummaryTree ${commonOutputPath} 1
2083
2084   goCreateQAplots "qa.list" "${productionID}" "QAplots" "${configFile}" "${extraOpts[@]}" filteringList="filtering.list" &>createQAplots.log
2085
2086   #make a merged summary tree out of the QA trending, dcs trees and log summary trees
2087   goMakeMergedSummaryTree
2088
2089   #if set, email the summary
2090   [[ -n ${MAILTO} ]] && cat ${log} | mail -s "benchmark ${productionID} done" ${MAILTO}
2091
2092   return 0
2093 }
2094
2095 goMakeSummaryTree()
2096 {
2097   if [[ $# -lt 1 ]] ; then
2098     return
2099   fi
2100   #1. define vars/arrays
2101   DIR=${1} #use input or exec in current dir
2102   pass=${2-"0"} #pass from input
2103   outfile="summary_pass${pass}.tree"
2104   Ncolumns=0
2105   test -f ${outfile} && : >${outfile}
2106   errfile=${outfile/tree/err}
2107   test -f ${errfile} && : >${errfile}
2108
2109   declare -a counterString=(TOFevents TOFtracks TPCevents TPCtracks TRDevents TRDtracks T0events SDDevents SDDtracks MeanVertexevents)
2110   Ncounter=${#counterString[@]}
2111
2112   declare -a statusString=(TRDStatus TOFStatus TPCStatus T0Status MeanVertexStatus)
2113   Nstatus=${#statusString[@]}
2114
2115
2116   declare -a ratesString=(rec stderr calib qa_barrel qa_outer)
2117   Nrates=${#ratesString[@]}
2118
2119   runs=( $(ls -1 ${DIR}/meta/merge.cpass0* | while read x; do guessRunNumber $x; done) )
2120   Nruns=${#runs[@]}
2121
2122   echo -n runnumber/I >>${outfile}
2123   echo -n :cpass${pass}status/I >>${outfile}
2124   echo -n :cpass${pass}QAstatus/I >>${outfile}
2125   for i in ${ratesString[@]}; do
2126     echo -n :${i}OK/I >>${outfile}
2127     echo -n :${i}BAD/I >>${outfile}
2128
2129   done
2130   for i in ${counterString[@]} ${statusString[@]} ; do
2131     echo -n :${i}/I >>${outfile}
2132   done
2133   Ncolumns=$((2 + 2*Nrates + Ncounter + Nstatus))
2134   echo >> ${outfile}
2135
2136   #2. loop runs
2137
2138   for runnumber in ${runs[@]} ; do
2139
2140
2141
2142     filejob="${DIR}/meta/cpass${pass}.job*.run${runnumber}.done"
2143     filemerge="${DIR}/meta/merge.cpass${pass}.run${runnumber}.done"
2144     fileOCDB=$(grep /ocdb.log ${filemerge} | awk '{print $1}')
2145     if ! $(/bin/ls ${filemerge} &>/dev/null) ; then
2146       echo "${filemerge} does not exist!" >>${errfile}
2147       continue
2148     elif ! $(/bin/ls ${filejob} &>/dev/null) ; then
2149       echo "${filejob} does not exist!" >>${errfile}
2150       echo -n ${runnumber} >> ${outfile}
2151       for i in $(seq ${Ncolumns}) ; do
2152         echo -n "-1" >> ${outfile}
2153       done
2154       echo >> ${outfile}
2155       continue
2156     fi
2157     echo -n ${runnumber} >> ${outfile}
2158     #pass0status= grep '/ocdb.log' ${filemerge} |  cut -d' ' -f2 | tr OK x1 | tr BAD xx0 | tr -d 'x'
2159     passStatus=$(grep '/ocdb.log' ${filemerge} | grep OK | wc -l)
2160     echo -n " ${passStatus}" >> ${outfile}
2161     qaStatus=$(grep '/mergeMakeOCDB.log' ${filemerge} | grep OK | wc -l)
2162     echo -n " ${qaStatus}" >> ${outfile}
2163
2164
2165     #fill OK/BAD rates
2166     for i in $(seq 0 $((${Nrates}-1))) ; do
2167       var1=$(grep "/${ratesString[${i}]}.log" ${filejob} | grep OK | wc -l)
2168       var2=$(grep "/${ratesString[${i}]}.log" ${filejob} | grep BAD | wc -l)
2169
2170       if [[ ${ratesString[${i}]} == "stderr" ]] ; then
2171         var1=$(grep "stderr" ${filejob} | grep OK | wc -l)
2172         var2=$(grep "stderr" ${filejob} | grep "rec.log" | grep BAD | wc -l)
2173       fi
2174       echo -n " ${var1}" >> ${outfile}
2175       echo -n " ${var2}" >> ${outfile}
2176     done
2177
2178     if [[ -f ${fileOCDB} ]] ; then
2179       #fill counter
2180       for i in $(seq 0 $((${Ncounter}-1))) ; do
2181         var1=$(grep Monalisa ${fileOCDB} | grep ${counterString[${i}]} | cut -f2)
2182         echo -n " ${var1:-"-1"}" >> ${outfile}
2183       done
2184
2185       #fill status
2186       for i in $(seq 0 $((${Nstatus}-1))) ; do
2187         var1=$(grep "calibration status=" ${fileOCDB} | grep ${statusString[${i}]/Status/} | cut -d'=' -f2)
2188         echo -n " ${var1:-"-1"}" >> ${outfile}
2189       done
2190     fi
2191     echo >> ${outfile}
2192   done
2193
2194   return 0
2195 }
2196
2197 parseConfig()
2198 {
2199   configFile=${1}
2200   shift
2201   args=("$@")
2202
2203   #some defaults
2204   #autoOCDB=0
2205   defaultOCDB="raw://"
2206   #runNumber=167123
2207   #makeflowPath="/hera/alice/aux/cctools/bin"
2208   #makeflowOptions="-T wq -N alice -d all -C ali-copilot.cern.ch:9097"
2209   #makeflowOptions="-T wq -N alice -C ali-copilot.cern.ch:9097"
2210   makeflowOptions=""
2211   batchCommand="/usr/bin/qsub"
2212   batchFlags="-b y -cwd -l h_rt=24:0:0,h_rss=4G "
2213   baseOutputDirectory="$PWD/output"
2214   #alirootEnv="/cvmfs/alice.cern.ch/bin/alienv setenv AliRoot/v5-04-34-AN -c"
2215   #alirootEnv="/home/mkrzewic/alisoft/balice_master.sh"
2216   #trustedQAtrainMacro='/hera/alice/mkrzewic/gsisvn/Calibration/QAtrain_duo.C'
2217   reconstructInTemporaryDir=0
2218   recoTriggerOptions="\"\""
2219   percentProcessedFilesToContinue=100
2220   maxSecondsToWait=$(( 3600*24 ))
2221   nEvents=-1
2222   nMaxChunks=0
2223   postSetUpActionCPass0=""
2224   postSetUpActionCPass1=""
2225   runCPass0reco=1
2226   runCPass0MergeMakeOCDB=1
2227   runCPass1reco=1
2228   runCPass1MergeMakeOCDB=1
2229   runESDfiltering=1
2230   filteringFactorHighPt=1e2
2231   filteringFactorV0s=1e1
2232   MAILTO=""
2233   #pretend=1
2234   #dontRedirectStdOutToLog=1
2235   logToFinalDestination=1
2236   ALIROOT_FORCE_COREDUMP=1
2237
2238   #first, source the config file
2239   if [ -f ${configFile} ]; then
2240     source ${configFile}
2241   else
2242     echo "config file ${configFile} not found!, skipping..."
2243   fi
2244
2245   #then, parse the options as they override the options from file
2246   for opt in "${args[@]}"; do
2247     if [[ ! "${opt}" =~ .*=.* ]]; then
2248       echo "badly formatted option ${var}, should be: option=value, stopping..."
2249       return 1
2250     fi
2251     local var="${opt%%=*}"
2252     local value="${opt#*=}"
2253     echo "${var} = ${value}"
2254     export ${var}="${value}"
2255   done
2256
2257   #do some checking
2258   [[ -z ${alirootEnv} ]] && echo "alirootEnv not defined!" && return 1
2259
2260   #export the aliroot function if defined to override normal behaviour
2261   [[ $(type -t aliroot) =~ "function" ]] && export -f aliroot
2262
2263   return 0
2264 }
2265
2266 aliroot()
2267 {
2268   args="$@"
2269   if [[ -n ${useProfilingCommand} ]]; then
2270     valgrindLogFile="cpu.txt"
2271     [[ "${args}" =~ rec ]] && valgrindLogFile="cpu_rec.txt"
2272     [[ "${args}}" =~ Calib ]] && valgrindLogFile="cpu_calib.txt"
2273     [[ -n ${useProfilingCommand} ]] && useProfilingCommand="${useProfilingCommand} --log-file=${valgrindLogFile}"
2274     echo running ${useProfilingCommand} aliroot ${args}
2275     ${useProfilingCommand} aliroot ${args}
2276   else
2277     #to prevent an infinite recursion use "command aliroot" to disable
2278     #aliases and functions
2279     echo running command aliroot ${args}
2280     command aliroot "$@"
2281   fi
2282   return 0
2283 }
2284
2285 guessRunData()
2286 {
2287   #guess the period from the path, pick the rightmost one
2288   period=""
2289   runNumber=""
2290   year=""
2291   pass=""
2292   legoTrainRunNumber=""
2293   dataType=""
2294
2295   local shortRunNumber=""
2296   local IFS="/"
2297   declare -a path=( $1 )
2298   local dirDepth=$(( ${#path[*]}-1 ))
2299   i=0
2300   #for ((x=${dirDepth};x>=0;x--)); do
2301   for ((x=0;x<=${dirDepth};x++)); do
2302
2303     [[ $((x-1)) -ge 0 ]] && local fieldPrev=${path[$((x-1))]}
2304     local field=${path[${x}]}
2305     local fieldNext=${path[$((x+1))]}
2306
2307     [[ ${field} =~ ^[0-9]*$ && ${fieldNext} =~ (.*\.zip$|.*\.root$) ]] && legoTrainRunNumber=${field}
2308     [[ -n ${legoTrainRunNumber} && -z ${pass} ]] && pass=${fieldPrev}
2309     [[ ${field} =~ ^LHC[0-9][0-9][a-z].*$ ]] && period=${field%_*}
2310     [[ ${field} =~ ^000[0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && runNumber=${field#000}
2311     [[ ${field} =~ ^[0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && shortRunNumber=${field}
2312     [[ ${field} =~ ^20[0-9][0-9]$ ]] && year=${field}
2313     [[ ${field} =~ ^(^sim$|^data$) ]] && dataType=${field}
2314     (( i++ ))
2315   done
2316   [[ -z ${legoTrainRunNumber} ]] && pass=${path[$((dirDepth-1))]}
2317   [[ "${dataType}" =~ ^sim$ ]] && pass="passMC" && runNumber=${shortRunNumber}
2318
2319   #if [[ -z ${dataType} || -z ${year} || -z ${period} || -z ${runNumber}} || -z ${pass} ]];
2320   if [[ -z ${runNumber}} ]];
2321   then
2322     #error condition
2323     return 1
2324   else
2325     #ALL OK
2326     return 0
2327   fi
2328   return 0
2329 }
2330
2331 main "$@"