PWGPP/benchmark/benchmark.sh

   1 #!/bin/bash
   2 #include benchmark.config
   3
   4 # this script runs the CPass0/CPass1 train
   5 # produced OCDB updates are local
   6
   7 main()
   8 {
   9   #run in proper mode depending on the selection
  10   if [[ $# -lt 1 ]]; then
  11     if [[ ! "${0}" =~ "bash" ]]; then
  12       echo "uses makeflow:"
  13       echo " ${0} \"run\" productionID inputList configFile [extraOpts]"
  14       echo "uses a batch system (SGE):"
  15       echo " ${0} \"submit\" productionID inputList configFile [extraOpts]"
  16       echo "extraOpts if non-empty override the config file, e.g.:"
  17       echo " ${0} submit test1 benchmark.list benchmark.config runNumber=169123 nEvents=10"
  18     fi
  19     return
  20   fi
  21
  22   #define some aliases - default is to call one of the functions directly
  23   runMode=${1}
  24   umask 0002
  25   shift
  26   case ${runMode} in
  27     "CPass0") goCPass0 "$@";;
  28     "CPass1") goCPass1 "$@";;
  29     "MakeLocalOCDBaccessConfig") goMakeLocalOCDBaccessConfig "$@";;
  30     "MergeCPass0") goMergeCPass0 "$@";;
  31     "MergeCPass1") goMergeCPass1 "$@";;
  32     "MakeFilteredTrees") goMakeFilteredTrees "$@";;
  33     "MakeSummary") goMakeSummary "$@";;
  34     "run") goSubmitMakeflow "$@";;
  35     "submit") goSubmitBatch "$@";;
  36     "test") goTest "$@";;
  37     "GenerateMakeflow") goGenerateMakeflow "$@";;
  38     "PrintValues") goPrintValues "$@";;
  39     "CreateQAplots") goCreateQAplots "$@";;
  40     "WaitForOutput") goWaitForOutput "$@";;
  41     "Merge") goMerge "$@";;
  42     *)
  43       ${runMode} "$@"
  44     ;;
  45   esac
  46   return 0
  47 }
  48
  49 generateMC()
  50 {
  51   #generate one raw chunk in current directory
  52   SEED=${JOB_ID}${SGE_TASK_ID}
  53   export CONFIG_SEED=${SEED}
  54   runNumber=${1}
  55   OCDBpath=${2}
  56   nEventsim=${3}
  57   if [[ -n ${pretend} ]]; then
  58     touch galice.root
  59   else
  60     if [[ -f sim.C && -f Config.C ]] ; then
  61         time aliroot -b -q -x sim.C\(${runNumber},\"${OCDBpath}\",${nEventsim}\) >sim.log 2>&1
  62         mv syswatch.log simwatch.log
  63     fi
  64   fi
  65 }
  66
  67 goCPass0()
  68 {
  69   umask 0002
  70
  71   targetDirectory=${1}
  72   inputList=${2}
  73   nEvents=${3}
  74   ocdbPath=${4}
  75   configFile=${5}
  76   runNumber=${6}
  77   jobindex=${7}
  78   shift 7
  79   if ! parseConfig ${configFile} "$@"; then return 1; fi
  80
  81   #record the working directory provided by the batch system
  82   batchWorkingDirectory=${PWD}
  83
  84   #use the jobindex only if set and non-negative
  85   if [[ -z ${jobindex} || ${jobindex} -lt 0 ]]; then
  86     [[ -n "${LSB_JOBINDEX}" ]] && jobindex=${LSB_JOBINDEX}
  87     [[ -n "${SGE_TASK_ID}" ]] && jobindex=${SGE_TASK_ID}
  88   fi
  89
  90   [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
  91   doneFile="${commonOutputPath}/meta/cpass0.job${jobindex}.run${runNumber}.done"
  92
  93   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
  94
  95   if [[ -n ${ALIROOT_FORCE_COREDUMP} ]]; then
  96     ulimit -c unlimited
  97     export ALIROOT_FORCE_COREDUMP
  98   fi
  99
 100   #the contents of this is stored in the tree and used later (e.g. AliAnalysisTaskPIDResponse)!
 101   #at the QA stage the pass number is guessed from the path stored here.
 102   #The Format is:
 103   #Packages= ;OutputDir= ;LPMPass= ;TriggerAlias= ;LPMRunNumber= ;LPMProductionType= ;LPMInteractionType= ;LPMProductionTag= ;LPMAnchorRun= ;LPMAnchorProduction= ;LPMAnchorYear=
 104   export PRODUCTION_METADATA="OutputDir=cpass0"
 105
 106   if [[ "${inputList}" =~ \.root$ ]]; then
 107     infile=${inputList}
 108   else
 109     infile=$(sed -ne "${jobindex}p" ${inputList} | egrep '\s*\w*/\w*')
 110   fi
 111   chunkName=${infile##*/}
 112
 113   outputDir=${targetDirectory}/${jobindex}_${chunkName%.*}
 114   mkdir -p ${outputDir}
 115   [[ ! -d ${outputDir} ]] && echo "cannot make ${outputDir}" && touch ${doneFile} && return 1
 116
 117   #runpath=${PWD}/rundir_cpass0_${runNumber}_${jobindex}
 118   runpath=${outputDir}
 119   [[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
 120   [[ ${reconstructInTemporaryDir} -eq 1 && -z ${TMPDIR} ]] && runpath=$(mktemp -d)
 121   mkdir -p ${runpath}
 122   [[ ! -d ${runpath} ]] && echo "cannot make runpath ${runpath}" && touch ${doneFile} && return 1
 123   cd ${runpath}
 124
 125   #runCPassX/C expects the raw chunk to be linked in the run dir
 126   #despite it being accessed by the full path
 127   ln -s ${infile} ${runpath}/${chunkName}
 128
 129   #####MC
 130   if [[ -n ${generateMC} ]]; then
 131     olddir=${PWD}
 132     outputDirMC=${commonOutputPath}/000${runNumber}/sim/${jobindex}
 133     simrunpath=${outputDirMC}
 134     [[ ${simulateInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && simrunpath=${TMPDIR}
 135     [[ ${simulateInTemporaryDir} -eq 1 && -z ${TMPDIR} ]] && simrunpath=$(mktemp -d)
 136     mkdir -p ${outputDirMC}
 137     mkdir -p ${simrunpath}
 138     cd ${simrunpath}
 139
 140     filesMC=(
 141               "${batchWorkingDirectory}/sim.C"
 142               "${batchWorkingDirectory}/rec.C"
 143               "${batchWorkingDirectory}/Config.C"
 144               "${batchWorkingDirectory}/OCDB_*.root"
 145     )
 146     for file in ${filesMC[*]}; do
 147       [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
 148     done
 149
 150     generateMC ${runNumber} ${ocdbPath} ${nEvents}
 151
 152     [[ ! "${simrunpath}" =~ "${outputDirMC}" ]] && mv * ${outputDirMC} #TODO check if it works
 153     cd ${olddir}
 154
 155     ln -s ${outputDirMC}/* ${runpath}/
 156
 157     inputList=${outputDirMC}/galice.root #TODO not valid outside shell !!!
 158     infile=""
 159   fi
 160   ######
 161
 162   [[ ! -f ${inputList} && -z ${pretend} ]] && echo "input file ${inputList} not found, exiting..." && touch ${doneFile} && return 1
 163
 164   logOutputDir=${runpath}
 165   [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir}
 166   [[ -z ${dontRedirectStdOutToLog} ]] && exec 1> ${logOutputDir}/stdout
 167   [[ -z ${dontRedirectStdOutToLog} ]] && exec 2> ${logOutputDir}/stderr
 168   echo "${0} $*"
 169
 170   echo "#####################"
 171   echo CPass0:
 172   echo JOB setup
 173   echo nEvents            ${nEvents}
 174   echo runNumber          ${runNumber}
 175   echo ocdbPath           ${ocdbPath}
 176   echo infile             ${infile}
 177   echo chunkName          ${chunkName}
 178   echo jobindex           ${jobindex}
 179   echo recoTriggerOptions ${recoTriggerOptions}
 180   echo targetDirectory    ${targetDirectory}
 181   echo commonOutputPath         ${commonOutputPath}
 182   echo doneFile      ${doneFile}
 183   echo runpath            ${runpath}
 184   echo outputDir          ${outputDir}
 185   echo ALICE_ROOT         ${ALICE_ROOT}
 186   echo PWD                ${PWD}
 187   echo "########## ###########"
 188
 189   alirootInfo > ALICE_ROOT.log
 190
 191   filesCPass0=(
 192                "${batchWorkingDirectory}/runCPass0.sh"
 193                "${batchWorkingDirectory}/recCPass0.C"
 194                "${batchWorkingDirectory}/runCalibTrain.C"
 195                "${batchWorkingDirectory}/localOCDBaccessConfig.C"
 196                "${batchWorkingDirectory}/OCDB.root"
 197                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/runCPass0.sh"
 198                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/recCPass0.C"
 199                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/runCalibTrain.C"
 200   )
 201
 202   for file in ${filesCPass0[*]}; do
 203     [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
 204   done
 205
 206   echo "this directory (${PWD}) contents:"
 207   /bin/ls
 208   echo
 209   chmod u+x runCPass0.sh
 210
 211   sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh
 212
 213   if [[ -n ${postSetUpActionCPass0} ]]; then
 214     echo "running ${postSetUpActionCPass0}"
 215     eval ${postSetUpActionCPass0}
 216   fi
 217
 218   #run CPass0
 219   echo "${runpath}/runCPass0.sh ${infile} ${nEvents} ${runNumber} ${ocdbPath} ${recoTriggerOptions}"
 220   if [[ -n ${pretend} ]]; then
 221     touch AliESDs.root
 222     touch AliESDfriends.root
 223     touch AliESDfriends_v1.root
 224     touch rec.log
 225     touch calib.log
 226   else
 227     echo ./runCPass0.sh "${infile}" "${nEvents}" "${runNumber}" "${ocdbPath}" "${recoTriggerOptions}"
 228     ./runCPass0.sh "${infile}" "${nEvents}" "${runNumber}" "${ocdbPath}" "${recoTriggerOptions}"
 229   fi
 230
 231   #move stuff to final destination
 232   echo "this directory (${PWD}) contents:"
 233   /bin/ls
 234   echo
 235
 236   echo rm -f ./${chunkName}
 237   rm -f ./${chunkName}
 238   echo "cp --recursive ${runpath}/* ${outputDir}"
 239   cp -p --recursive ${runpath}/* ${outputDir}
 240   echo
 241
 242   #validate CPass0
 243   cd ${outputDir}
 244   touch ${doneFile}
 245   echo "dir ${outputDir}" >> ${doneFile}
 246   if summarizeLogs >> ${doneFile}; then
 247     [[ -f ${outputDirMC}/galice.root ]] && echo "sim ${outputDirMC}/galice.root" >> ${doneFile}
 248     [[ -f AliESDfriends_v1.root ]] && echo "calibfile ${outputDir}/AliESDfriends_v1.root" >> ${doneFile}
 249     [[ -f AliESDs.root ]] && echo "esd ${outputDir}/AliESDs.root" >> ${doneFile}
 250   fi
 251
 252   [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
 253   return 0
 254 }
 255
 256 goCPass1()
 257 {
 258   umask 0002
 259
 260   targetDirectory=${1}
 261   inputList=${2}
 262   nEvents=${3}
 263   ocdbPath=${4}
 264   configFile=${5}
 265   runNumber=${6}
 266   jobindex=${7}
 267   shift 7
 268   extraOpts=("$@")
 269   if ! parseConfig ${configFile} "$@"; then return 1; fi
 270
 271   #record the working directory provided by the batch system
 272   batchWorkingDirectory=${PWD}
 273
 274   #use the jobindex only if set and non-negative
 275   if [[ -z ${jobindex} || ${jobindex} -lt 0 ]]; then
 276     [[ -n "${LSB_JOBINDEX}" ]] && jobindex=${LSB_JOBINDEX}
 277     [[ -n "${SGE_TASK_ID}" ]] && jobindex=${SGE_TASK_ID}
 278   fi
 279
 280   [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
 281   doneFile="${commonOutputPath}/meta/cpass1.job${jobindex}.run${runNumber}.done"
 282
 283   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
 284
 285   if [[ -n ${ALIROOT_FORCE_COREDUMP} ]]; then
 286     ulimit -c unlimited
 287     export ALIROOT_FORCE_COREDUMP
 288   fi
 289
 290   #the contents of this is stored in the tree and used later (e.g. AliAnalysisTaskPIDResponse)!
 291   #at the QA stage the pass number is guessed from the path stored here.
 292   #The Format is:
 293   #Packages= ;OutputDir= ;LPMPass= ;TriggerAlias= ;LPMRunNumber= ;LPMProductionType= ;LPMInteractionType= ;LPMProductionTag= ;LPMAnchorRun= ;LPMAnchorProduction= ;LPMAnchorYear=
 294   export PRODUCTION_METADATA="OutputDir=cpass1"
 295
 296   [[ ! -f ${inputList} && -z ${pretend} ]] && echo "input file ${inputList} not found, exiting..." && touch ${doneFile} && return 1
 297   if [[ "${inputList}" =~ \.root$ ]]; then
 298     infile=${inputList}
 299   else
 300     infile=$(sed -ne "${jobindex}p" ${inputList} | egrep '\s*\w*/\w*')
 301   fi
 302   chunkName=${infile##*/}
 303
 304   outputDir=${targetDirectory}/${jobindex}_${chunkName%.*}
 305   mkdir -p ${outputDir}
 306   [[ ! -d ${outputDir} ]] && echo "cannot make ${outputDir}" && touch ${doneFile} && return 1
 307
 308   #runpath=${PWD}/rundir_cpass1_${runNumber}_${jobindex}
 309   runpath=${outputDir}
 310   [[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
 311   [[ ${reconstructInTemporaryDir} -eq 1 && -z ${TMPDIR} ]] && runpath=$(mktemp -d)
 312
 313   #MC
 314   if [[ "${infile}" =~ galice\.root ]]; then
 315     ln -s ${inputList%/*}/* ${runpath}
 316     infile=""
 317   fi
 318
 319   #init the running path
 320   mkdir -p ${runpath}
 321   [[ ! -d ${runpath} ]] && echo "cannot make runpath ${runpath}" && touch ${doneFile} && return 1
 322   cd ${runpath}
 323
 324   #this is needed for runCPass1.sh
 325   ln -s ${infile} ${runpath}/${chunkName}
 326
 327   logOutputDir=${runpath}
 328   [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir}
 329   [[ -z ${dontRedirectStdOutToLog} ]] && exec 1> ${logOutputDir}/stdout
 330   [[ -z ${dontRedirectStdOutToLog} ]] && exec 2> ${logOutputDir}/stderr
 331   echo "${0} $*"
 332
 333   echo "#####################"
 334   echo CPass1:
 335   echo JOB setup
 336   echo nEvents            ${nEvents}
 337   echo runNumber          ${runNumber}
 338   echo ocdbPath           ${ocdbPath}
 339   echo infile             ${infile}
 340   echo chunkName          ${chunkName}
 341   echo jobindex           ${jobindex}
 342   echo recoTriggerOptions ${recoTriggerOptions}
 343   echo targetDirectory    ${targetDirectory}
 344   echo commonOutputPath         ${commonOutputPath}
 345   echo doneFile      ${doneFile}
 346   echo runpath            ${runpath}
 347   echo outputDir          ${outputDir}
 348   echo ALICE_ROOT         ${ALICE_ROOT}
 349   echo PWD                ${PWD}
 350   echo "########## ###########"
 351
 352   alirootInfo > ALICE_ROOT.log
 353
 354   filesCPass1=(
 355                "${batchWorkingDirectory}/runCPass1.sh"
 356                "${batchWorkingDirectory}/recCPass1.C"
 357                "${batchWorkingDirectory}/recCPass1_OuterDet.C"
 358                "${batchWorkingDirectory}/runCalibTrain.C"
 359                "${batchWorkingDirectory}/QAtrain_duo.C"
 360                "${batchWorkingDirectory}/localOCDBaccessConfig.C"
 361                "${batchWorkingDirectory}/meta/cpass0.localOCDB.${runNumber}.tgz"
 362                "${batchWorkingDirectory}/OCDB.root"
 363                "${trustedQAtrainMacro}"
 364                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/runCPass1.sh"
 365                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/recCPass1.C"
 366                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/recCPass1_OuterDet.C"
 367                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/runCalibTrain.C"
 368                "${ALICE_ROOT}/ANALYSIS/macros/QAtrain_duo.C"
 369   )
 370
 371   for file in ${filesCPass1[*]}; do
 372     [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
 373   done
 374
 375   echo "this directory (${PWD}) contents:"
 376   /bin/ls
 377   echo
 378
 379   sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh
 380
 381   if [[ -n ${postSetUpActionCPass1} ]]; then
 382     echo "running ${postSetUpActionCPass1}"
 383     eval ${postSetUpActionCPass1}
 384     echo
 385   fi
 386
 387   #configure local OCDB storage from CPass0 (creates the localOCDBaccessConfig.C script)
 388   if [[ -f cpass0.localOCDB.${runNumber}.tgz ]]; then
 389     echo goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz"
 390     goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz"
 391   else
 392     echo "WARNING: file cpass0.localOCDB.${runNumber}.tgz not found!"
 393   fi
 394
 395   if [[ ! $(/bin/ls -1 OCDB/*/*/*/*.root 2>/dev/null) ]]; then
 396     echo "cpass0 produced no calibration! exiting..."
 397     touch ${doneFile}
 398     return 1
 399   fi
 400
 401   #create the Barrel and OuterDet directories for CPass1 and link the local OCDB directory
 402   #there to make the localOCDBaccessConfig.C file work, since it may point to the OCDB
 403   #entries using a relative path, e.g. local://./OCDB
 404   mkdir Barrel OuterDet
 405   ln -s ../OCDB Barrel/OCDB
 406   ln -s ../OCDB OuterDet/OCDB
 407
 408   #setup the filtering
 409   #the following option enables the filtering task inside the QAtrain_duo.C
 410   [[ -n $runESDfiltering ]] && export QA_TaskFilteredTree=1
 411   #set the downscaling factors during the filtering fro expert QA (overrides the previous values)
 412   if [[ -n ${filteringFactorHighPt} ]]; then
 413     export AliAnalysisTaskFilteredTree_fLowPtTrackDownscaligF=${filteringFactorHighPt}
 414   fi
 415   if [[ -n ${filteringFactorV0s} ]]; then
 416     export AliAnalysisTaskFilteredTree_fLowPtV0DownscaligF=${filteringFactorV0s}
 417   fi
 418
 419   #run CPass1
 420   chmod u+x runCPass1.sh
 421   echo "${runpath}/runCPass1.sh ${infile} ${nEvents} ${runNumber} ${ocdbPath} ${recoTriggerOptions}"
 422   if [[ -n ${pretend} ]]; then
 423     touch AliESDs_Barrel.root
 424     touch AliESDfriends_Barrel.root
 425     touch AliESDfriends_v1.root
 426     touch QAresults_barrel.root
 427     touch EventStat_temp_barrel.root
 428     touch AODtpITS.root
 429     touch AliESDs_Outer.root
 430     touch AliESDfriends_Outer.root
 431     touch QAresults_outer.root
 432     touch EventStat_temp_outer.root
 433     touch rec.log
 434     touch calib.log
 435     touch qa.log
 436     touch filtering.log FilterEvents_Trees.root
 437   else
 438     ./runCPass1.sh "${infile}" "${nEvents}" "${runNumber}" "${ocdbPath}" "${recoTriggerOptions}"
 439
 440     [[ ! -f AliESDs_Barrel.root && -f Barrel/AliESDs.root ]] && mv Barrel/AliESDs.root AliESDs_Barrel.root
 441     [[ ! -f AliESDfriends_Barrel.root && -f Barrel/AliESDfriends.root ]] && mv Barrel/AliESDfriends.root AliESDfriends_Barrel.root
 442     [[ ! -f AliESDfriends_v1.root && -f Barrel/AliESDfriends_v1.root ]] && mv Barrel/AliESDfriends_v1.root .
 443     [[ ! -f QAresults_barrel.root && -f Barrel/QAresults_barrel.root ]] && mv Barrel/QAresults_barrel.root .
 444     [[ ! -f AliESDs_Outer.root && -f OuterDet/AliESDs.root ]] && mv OuterDet/AliESDs.root AliESDs_Outer.root
 445     [[ ! -f AliESDfriends_Outer.root && -f OuterDet/AliESDfriends.root ]] && mv OuterDet/AliESDfriends.root AliESDfriends_Outer.root
 446     [[ ! -f QAresults_outer.root && -f OuterDet/QAresults_outer.root ]] && mv OuterDet/QAresults_outer.root .
 447     [[ ! -f FilterEvents_Trees.root && -f Barrel/FilterEvents_Trees.root ]] && mv Barrel/FilterEvents_Trees.root .
 448
 449     #make the filtered tree (if requested and not already produced by QA
 450     [[ -f AliESDs_Barrel.root ]] && echo "AliESDs_Barrel.root" > filtered.list
 451     if [[ -n ${runESDfiltering} && ! -f FilterEvents_Trees.root ]]; then
 452       goMakeFilteredTrees ${PWD} ${runNumber} "filtered.list" ${filteringFactorHighPt} ${filteringFactorV0s} ${ocdbPath} 1000000 0 10000000 0 ${configFile} AliESDs_Barrel.root "${extraOpts[@]}"
 453     fi
 454
 455   fi
 456
 457   ##handle possible crashes in QA (happens often in trunk)
 458   ##rerun QA with a trusted aliroot version
 459   #if [[ $(validateLog qa_barrel.log) ]]; then
 460   #  echo "qa_barrel.log not validated!"
 461   #fi
 462   #if [[ ! -f QAresults_barrel.root && -f ${setupTrustedAliROOTenvInCurrentShell} || $(validateLog qa_barrel.log) ]]; then
 463   #  echo "WARNING: using trusted QA aliroot ${ALICE_ROOT}"
 464   #  source ${setupTrustedAliROOTenvInCurrentShell}
 465   #  cd Barrel
 466   #  rm QAresults_barrel.root
 467   #  rm EventStat_temp_barrel.root
 468   #  rm AODtpITS.root
 469   #  [[ ! -f AliESDs.root ]] && ln -s ../AliESDs_Barrel.root AliESDs.root
 470   #  [[ ! -f AliESDfriends.root ]] && ln -s ../AliESDfriends_Barrel.root AliESDfriends.root
 471   #  if [[ -n ${trustedQAtrainMacro} ]]; then
 472   #    eval "cp ${trustedQAtrainMacro} QAtrain_duo_trusted.C"
 473   #  fi
 474   #  echo executing aliroot -b -q "QAtrain_duo_trusted.C(\"_barrel\",${runNumber},\"wn.xml\",0,\"${ocdbPath}\")"
 475   #  time aliroot -b -q "QAtrain_duo.C(\"_barrel\",${runNumber},\"wn.xml\",0,\"${ocdbPath}\")" &> ../qa_barrel_trusted.log
 476   #  cd ../
 477   #fi
 478
 479   #move stuff to final destination
 480   echo "this directory (${PWD}) contents:"
 481   /bin/ls
 482   echo rm -f ./${chunkName}
 483   rm -f ./${chunkName}
 484   echo "cp --recursive ${runpath}/* ${outputDir}"
 485   cp -pf --recursive ${runpath}/* ${outputDir}
 486   echo
 487
 488   #validate CPass1
 489   cd ${outputDir}
 490   touch ${doneFile}
 491   echo "dir ${outputDir}" >> ${doneFile}
 492   if summarizeLogs >> ${doneFile}; then
 493     [[ -f AliESDs_Barrel.root ]] && echo "esd ${outputDir}/AliESDs_Barrel.root" >> ${doneFile}
 494     [[ -f AliESDfriends_v1.root ]] && echo "calibfile ${outputDir}/AliESDfriends_v1.root" >> ${doneFile}
 495     [[ -f QAresults_Barrel.root ]] && echo "qafile ${outputDir}/QAresults_Barrel.root" >> ${doneFile}
 496     [[ -f QAresults_Outer.root ]] && echo "qafile ${outputDir}/QAresults_Outer.root" >> ${doneFile}
 497     [[ -f FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFile}
 498   fi
 499
 500   [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
 501   return 0
 502 }
 503
 504
 505 goMergeCPass0()
 506 {
 507   #
 508   # find the output files and merge them
 509   #
 510
 511   outputDir=${1}
 512   ocdbStorage=${2}
 513   configFile=${3}
 514   runNumber=${4}
 515   calibrationFilesToMerge=${5}  #can be a non-existent file, will then be produced on the fly
 516   shift 5
 517   if ! parseConfig ${configFile} "$@"; then return 1; fi
 518
 519   #record the working directory provided by the batch system
 520   batchWorkingDirectory=${PWD}
 521
 522   [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
 523   doneFile="${commonOutputPath}/meta/merge.cpass0.run${runNumber}.done"
 524
 525   umask 0002
 526   ulimit -c unlimited
 527
 528   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
 529
 530   #runpath=${PWD}/rundir_cpass0_Merge_${runNumber}
 531   runpath=${outputDir}
 532   [[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
 533   [[ ${reconstructInTemporaryDir} -eq 1 && -z ${TMPDIR} ]] && runpath=$(mktemp -d)
 534
 535   mkdir -p ${runpath}
 536   [[ ! -d ${runpath} ]] && echo "not able to make the runpath ${runpath}" && touch ${doneFile} && return 1
 537   cd ${runpath}
 538
 539   logOutputDir=${runpath}
 540   [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir}
 541   [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/mergeMakeOCDB.log
 542   echo "${0} $*"
 543
 544   mergingScript="mergeMakeOCDB.byComponent.sh"
 545
 546   echo goMergeCPass0 SETUP:
 547   echo runNumber=${runNumber}
 548   echo outputDir=${outputDir}
 549   echo ocdbStorage=${ocdbStorage}
 550   echo calibrationFilesToMerge=${calibrationFilesToMerge}
 551   echo mergingScript=${mergingScript}
 552   echo commonOutputPath=${commonOutputPath}
 553   echo runpath=${runpath}
 554
 555   # copy files in case they are not already there
 556   filesMergeCPass0=(
 557                     "${batchWorkingDirectory}/${calibrationFilesToMerge}"
 558                     "${batchWorkingDirectory}/OCDB.root"
 559                     "${batchWorkingDirectory}/localOCDBaccessConfig.C"
 560                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/mergeMakeOCDB.byComponent.sh"
 561                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/mergeByComponent.C"
 562                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/makeOCDB.C"
 563                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/merge.C"
 564                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/mergeMakeOCDB.sh"
 565   )
 566   for file in ${filesMergeCPass0[*]}; do
 567     [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
 568   done
 569
 570   sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh
 571
 572   alirootInfo > ALICE_ROOT.log
 573
 574   #
 575   echo "PWD"
 576   /bin/ls
 577   echo "PWD/.."
 578   /bin/ls ../
 579
 580
 581   #merge calibration
 582   chmod u+x ${mergingScript}
 583   mkdir -p ./OCDB
 584   if [[ ! -f ${calibrationFilesToMerge} ]]; then
 585     echo "/bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root > ${calibrationFilesToMerge}"
 586     /bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root 2>/dev/null > ${calibrationFilesToMerge}
 587   fi
 588
 589   echo "${mergingScript} ${calibrationFilesToMerge} ${runNumber} local://./OCDB ${ocdbStorage}"
 590   if [[ -n ${pretend} ]]; then
 591     touch CalibObjects.root
 592     touch ocdb.log
 593     touch merge.log
 594     mkdir -p ./OCDB/TPC/Calib/TimeGain/
 595     mkdir -p ./OCDB/TPC/Calib/TimeDrift/
 596     touch ./OCDB/TPC/Calib/TimeGain/someCalibObject_0-999999_cpass0.root
 597     touch ./OCDB/TPC/Calib/TimeDrift/otherCalibObject_0-999999_cpass0.root
 598   else
 599     ./${mergingScript} ${calibrationFilesToMerge} ${runNumber} "local://./OCDB" ${ocdbStorage}
 600   fi
 601
 602   ### produce the output
 603   #tar the produced OCDB for reuse
 604   tar czf ${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz ./OCDB
 605
 606   #produce the calib trees for expert QA (dcsTime.root)
 607   goMakeLocalOCDBaccessConfig ./OCDB
 608   echo aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")"
 609   aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")"
 610
 611   /bin/ls
 612
 613   #copy all to output dir
 614   cp -pf --recursive ${runpath}/* ${outputDir}
 615
 616   if [[ -n ${generateMC} ]]; then
 617     goPrintValues sim ${commonOutputPath}/meta/sim.run${runNumber}.list ${commonOutputPath}/meta/cpass0.job*.run${runNumber}.done
 618   fi
 619
 620   #validate merging cpass0
 621   cd ${outputDir}
 622   touch ${doneFile}
 623   echo "dir ${outputDir}" >> ${doneFile}
 624   if summarizeLogs >> ${doneFile}; then
 625     [[ -f CalibObjects.root ]] && echo "calibfile ${outputDir}/CalibObjects.root" >> ${doneFile}
 626     [[ -f dcsTime.root ]] && echo "dcsTree ${outputDir}/dcsTime.root" >> ${doneFile}
 627   fi
 628
 629   [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
 630   return 0
 631 }
 632
 633 goMergeCPass1()
 634 {
 635   #
 636   # find the output files and merge them
 637   #
 638
 639   outputDir=${1}
 640   ocdbStorage=${2}
 641   configFile=${3}
 642   runNumber=${4}
 643   calibrationFilesToMerge=${5}
 644   qaFilesToMerge=${6}
 645   filteredFilesToMerge=${7}
 646   shift 7
 647   if ! parseConfig ${configFile} "$@"; then return 1; fi
 648
 649   #record the working directory provided by the batch system
 650   batchWorkingDirectory=${PWD}
 651
 652   [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
 653   doneFile="${commonOutputPath}/meta/merge.cpass1.run${runNumber}.done"
 654
 655   umask 0002
 656   ulimit -c unlimited
 657
 658   #clean up first:
 659   rm -f ${outputDir}/*.log
 660   rm -f ${outputDir}/*.root
 661   rm -f ${outputDir}/*done
 662
 663   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
 664
 665   #runpath=${PWD}/rundir_cpass1_Merge_${runNumber}
 666   runpath=${outputDir}
 667   [[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
 668   [[ ${reconstructInTemporaryDir} -eq 1 && -z ${TMPDIR} ]] && runpath=$(mktemp -d)
 669
 670   mkdir -p ${runpath}
 671   [[ ! -d ${runpath} ]] && echo "not able to make the runpath ${runpath}" && touch ${doneFile} && return 1
 672   cd ${runpath}
 673
 674   logOutputDir=${runpath}
 675   [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir}
 676   [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/mergeMakeOCDB.log
 677   echo "${0} $*"
 678
 679   calibrationOutputFileName='AliESDfriends_v1.root'
 680   qaOutputFileName='QAresults*.root'
 681   mergingScript="mergeMakeOCDB.byComponent.sh"
 682   #important to have the string "Stage.txt" in the filename to trigger the merging
 683   #it has to be a list of directories containing the files
 684   qaMergedOutputFileName="QAresults_merged.root"
 685
 686   echo goMergeCPass1 SETUP:
 687   echo runNumber=${runNumber}
 688   echo outputDir=${outputDir}
 689   echo ocdbStorage=${ocdbStorage}
 690   echo calibrationFilesToMerge=$calibrationFilesToMerge
 691   echo qaFilesToMerge=$qaFilesToMerge
 692   echo calibrationOutputFileName=${calibrationOutputFileName}
 693   echo mergingScript=${mergingScript}
 694
 695   # copy files in case they are not already there
 696   filesMergeCPass1=(
 697                     "${batchWorkingDirectory}/${calibrationFilesToMerge}"
 698                     "${batchWorkingDirectory}/${qaFilesToMerge}"
 699                     "${batchWorkingDirectory}/OCDB.root"
 700                     "${batchWorkingDirectory}/localOCDBaccessConfig.C"
 701                     "${batchWorkingDirectory}/meta/cpass0.localOCDB.${runNumber}.tgz"
 702                     "${batchWorkingDirectory}/QAtrain_duo.C"
 703                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/mergeMakeOCDB.byComponent.sh"
 704                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/mergeByComponent.C"
 705                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/makeOCDB.C"
 706                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/merge.C"
 707                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/mergeMakeOCDB.sh"
 708                     "${trustedQAtrainMacro}"
 709                     "${ALICE_ROOT}/ANALYSIS/macros/QAtrain_duo.C"
 710   )
 711   for file in ${filesMergeCPass1[*]}; do
 712     [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
 713   done
 714
 715   sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh
 716
 717   #configure local OCDB storage from CPass0 (creates the localOCDBaccessConfig.C script)
 718   if [[ -f cpass0.localOCDB.${runNumber}.tgz ]]; then
 719     echo goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz"
 720     goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz"
 721   else
 722     echo "WARNING: file cpass0.localOCDB.${runNumber}.tgz not found!"
 723   fi
 724
 725   alirootInfo > ALICE_ROOT.log
 726
 727   #
 728   /bin/ls
 729
 730   #merge calibration
 731   chmod u+x ${mergingScript}
 732   mkdir -p OCDB
 733
 734   #if not provided, create the lists of files to merge
 735   if [[ ! -f ${filteredFilesToMerge} ]]; then
 736     echo "/bin/ls -1 ${outputDir}/*/FilterEvents_Trees.root > ${filteredFilesToMerge}"
 737     /bin/ls -1 ${outputDir}/*/FilterEvents_Trees.root 2>/dev/null > ${filteredFilesToMerge}
 738   fi
 739   if [[ ! -f ${calibrationFilesToMerge} ]]; then
 740     echo "/bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root > ${calibrationFilesToMerge}"
 741     /bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root 2>/dev/null > ${calibrationFilesToMerge}
 742   fi
 743   if [[ ! -f ${qaFilesToMerge} ]]; then
 744     #find the files, but only store the directories (QAtrain_duo.C requires this)
 745     echo "/bin/ls -1 ${outputDir}/*/QAresults*.root | while read x; do echo ${x%/*}; done | sort | uniq > ${qaFilesToMerge}"
 746     /bin/ls -1 ${outputDir}/*/QAresults*.root | while read x; do echo ${x%/*}; done | sort | uniq > ${qaFilesToMerge}
 747   fi
 748
 749   echo "${mergingScript} ${calibrationFilesToMerge} ${runNumber} local://./OCDB ${ocdbStorage}"
 750   if [[ -n ${pretend} ]]; then
 751     touch CalibObjects.root
 752     touch ocdb.log
 753     touch merge.log
 754     touch cpass1.localOCDB.${runNumber}.tgz
 755   else
 756     ./${mergingScript} ${calibrationFilesToMerge} ${runNumber} "local://./OCDB" ${ocdbStorage}
 757   fi
 758
 759   tar czf ${commonOutputPath}/meta/cpass1.localOCDB.${runNumber}.tgz ./OCDB
 760
 761   #merge QA (and filtered trees)
 762   [[ -n ${AliAnalysisTaskFilteredTree_fLowPtTrackDownscaligF} ]] && export AliAnalysisTaskFilteredTree_fLowPtTrackDownscaligF
 763   [[ -n ${AliAnalysisTaskFilteredTree_fLowPtV0DownscaligF} ]] && export AliAnalysisTaskFilteredTree_fLowPtV0DownscaligF
 764
 765   #echo aliroot -l -b -q "merge.C(\"${qaFilesToMerge}\",\"\",kFALSE,\"${qaMergedOutputFileName}\")"
 766   echo aliroot -b -q "QAtrain_duo.C(\"_barrel\",${runNumber},\"${qaFilesToMerge}\",1,\"${ocdbStorage}\")"
 767   if [[ -n ${pretend} ]]; then
 768     touch ${qaMergedOutputFileName}
 769     touch merge.log
 770     touch trending.root
 771     touch FilterEvents_Trees.root
 772     touch CalibObjects.root
 773     touch dcsTime.root
 774     touch ${qaMergedOutputFileName}
 775   else
 776     #aliroot -l -b -q "merge.C(\"${qaFilesToMerge}\",\"\",kFALSE,\"${qaMergedOutputFileName}\")"
 777     aliroot -b -q "QAtrain_duo.C(\"_barrel\",${runNumber},\"${qaFilesToMerge}\",1,\"${ocdbStorage}\")" > mergeQA.log
 778     mv QAresults_barrel.root ${qaMergedOutputFileName}
 779     mv trending_barrel.root trending.root
 780
 781     #merge filtered trees
 782     echo aliroot -l -b -q "merge.C(\"${qaFilesToMerge}\",\"\",kFALSE,\"${qaMergedOutputFileName}\")"
 783     aliroot -l -b -q "merge.C(\"${filteredFilesToMerge}\",\"\",kFALSE,\"FilterEvents_Trees.root\")"
 784   fi
 785
 786   #produce the calib trees for expert QA
 787   echo aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")"
 788   aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")"
 789
 790   /bin/ls
 791
 792   #copy all to output dir
 793   cp -pf --recursive ${runpath}/* ${outputDir}
 794
 795   #validate merge cpass1
 796   cd ${outputDir}
 797   touch ${doneFile}
 798   echo "dir ${outputDir}" >> ${doneFile}
 799   if summarizeLogs >>  ${doneFile}; then
 800     [[ -f CalibObjects.root ]] && echo "calibfile ${outputDir}/CalibObjects.root" >> ${doneFile}
 801     [[ -f ${qaMergedOutputFileName} ]] && echo "qafile ${outputDir}/${qaMergedOutputFileName}" >> ${doneFile}
 802     [[ -f trending.root ]] && echo "trendingfile ${outputDir}/trending.root" >> ${doneFile}
 803     [[ -f dcsTime.root ]] && echo "dcsTree ${outputDir}/dcsTime.root" >> ${doneFile}
 804     [[ -f FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFile}
 805   fi
 806
 807   [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
 808   return 0
 809 }
 810
 811 goMerge()
 812 {
 813   #generic root merge using CPass1 merge.C script
 814   inputList=${1}
 815   outputFile=${2}
 816   configFile=${3-"becnhmark.config"}
 817   shift 3
 818   if ! parseConfig ${configFile} "$@"; then return 1; fi
 819
 820   #record the working directory provided by the batch system
 821   batchWorkingDirectory=${PWD}
 822
 823   [[ ! -f ${inputList} ]] && echo "inputList ${inputList} does not exist!" && return 1
 824   [[ ! -f ${configFile} ]] && echo "configFile ${configFile} does not exist!" && return 1
 825   umask 0002
 826   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
 827   rm -f ${outputFile}
 828   aliroot -b -q "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/merge.C(\"${inputList}\",\"\",kFALSE,\"${outputFile}\")" > merge_${inputList}.log
 829   return 0
 830 }
 831
 832 goSubmitMakeflow()
 833 {
 834   #run
 835   productionID=${1}
 836   inputFileList=${2}
 837   configFile=${3}
 838   shift 3
 839   extraOpts=("$@")
 840   if ! parseConfig ${configFile} "${extraOpts[@]}"; then return 1; fi
 841
 842   #record the working directory provided by the batch system
 843   batchWorkingDirectory=${PWD}
 844
 845   [[ -z ${configFile} ]] && configFile="benchmark.config"
 846   [[ ! -f ${configFile} ]] && echo "no config file found (${configFile})" && return 1
 847
 848   if [[ ! $(which makeflow &>/dev/null) && -n ${makeflowPath} ]]; then
 849     echo "setting the makflow path from the config: "
 850     echo "  export PATH=${makeflowPath}:${PATH}"
 851     export PATH=${makeflowPath}:${PATH}
 852   fi
 853
 854   #create the common output dir and the meta dir
 855   commonOutputPath=${baseOutputDirectory}/${productionID}
 856   if [[ -d ${commonOutputPath} ]]; then
 857     echo "output dir ${commonOutputPath} exists!"
 858     return 1
 859   else
 860     mkdir -p ${commonOutputPath}
 861   fi
 862   mkdir -p ${commonOutputPath}/meta
 863
 864   #submit - use makeflow if available, fall back to old stuff when makeflow not there
 865   if which makeflow; then
 866
 867     goGenerateMakeflow ${productionID} ${inputFileList} ${configFile} "${extraOpts[@]}" commonOutputPath=${commonOutputPath} > benchmark.makeflow
 868
 869     makeflow ${makeflowOptions} benchmark.makeflow
 870     cd ../
 871   else
 872     echo "no makeflow!"
 873   fi
 874   return 0
 875 }
 876
 877 goGenerateMakeflow()
 878 {
 879   #generate the makeflow file
 880   [[ $# -lt 3 ]] && echo "args: id inputFileList configFile" && return 1
 881   productionID=${1}
 882   inputFileList=${2}
 883   configFile=${3}
 884   shift 3
 885   extraOpts=("$@")
 886   if ! parseConfig ${configFile} "${extraOpts[@]}" &>/dev/null; then return 1; fi
 887
 888   #make sure this is sanely defined in any case
 889   [[ -z ${commonOutputPath} ]] && commonOutputPath=${baseOutputDirectory}/${productionID}
 890
 891   #record the working directory provided by the batch system
 892   batchWorkingDirectory=${PWD}
 893
 894   [[ -z ${configFile} ]] && configFile="benchmark.config"
 895   [[ ! -f ${configFile} ]] && echo "no config file found (${configFile})" && return 1
 896
 897   self=$(readlink -f "${0}")
 898
 899   #these files will be made a dependency - will be copied to the working dir of the jobs
 900   declare -a copyFiles
 901   inputFiles=(
 902               "OCDB.root"
 903               "localOCDBaccessConfig.C"
 904               "QAtrain_duo.C"
 905               "runCPass1.sh"
 906               "recCPass1.C"
 907               "recCPass1_OuterDet.C"
 908               "runCalibTrain.C"
 909               "runCPass0.sh"
 910               "recCPass0.C"
 911   )
 912   for file in ${inputFiles[*]}; do
 913     [[ -f ${file} ]] && copyFiles+=("${file}")
 914   done
 915
 916   #create the makeflow file
 917   [[ -n ${batchFlags} ]] && echo "BATCH_OPTIONS = ${batchFlags}"
 918   declare -A arr_cpass0_merged arr_cpass1_merged
 919   declare -A arr_cpass0_calib_list arr_cpass1_calib_list
 920   declare -A arr_cpass1_QA_list arr_cpass1_ESD_list arr_cpass1_filtered_list
 921   declare -A arr_cpass0_profiled_outputs
 922   declare -A listOfRuns
 923   [[ -n ${runNumber} ]] && listOfRuns[${runNumber}]=1
 924   while read x; do listOfRuns[$(guessRunNumber ${x})]=1; done < ${inputFileList}
 925   for runNumber in "${!listOfRuns[@]}"; do
 926     [[ -z ${runNumber} ]] && continue
 927     [[ ! ${runNumber} =~ ^[0-9]*[0-9]$ ]] && continue
 928
 929     unset arr_cpass0_outputs
 930     unset arr_cpass1_outputs
 931     declare -a arr_cpass0_outputs
 932     declare -a arr_cpass1_outputs
 933
 934     jobindex=0
 935     inputFile=""
 936     while read inputFile; do
 937       currentDefaultOCDB=${defaultOCDB}
 938       [[ -z ${autoOCDB} ]] && autoOCDB=1
 939       if [[ ${autoOCDB} -ne 0 ]]; then
 940         currentDefaultOCDB=$(setYear ${inputFile} ${defaultOCDB})
 941       fi
 942
 943       #CPass0
 944       arr_cpass0_outputs[${jobindex}]="${commonOutputPath}/meta/cpass0.job${jobindex}.run${runNumber}.done"
 945       echo "${arr_cpass0_outputs[${jobindex}]} : benchmark.sh ${configFile} ${copyFiles[@]}"
 946       echo -n " ${alirootEnv} ./benchmark.sh CPass0 ${commonOutputPath}/000${runNumber}/cpass0 ${inputFile} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} ${jobindex}"" "
 947       for extraOption in "${extraOpts[@]}"; do echo -n \'${extraOption}\'" "; done; echo
 948       echo
 949
 950       #CPass1
 951       arr_cpass1_outputs[${jobindex}]="${commonOutputPath}/meta/cpass1.job${jobindex}.run${runNumber}.done"
 952       echo "${arr_cpass1_outputs[${jobindex}]} : benchmark.sh ${configFile} ${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz ${copyFiles[@]}"
 953       echo -n " ${alirootEnv} ./benchmark.sh CPass1 ${commonOutputPath}/000${runNumber}/cpass1 ${inputFile} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} ${jobindex}"" "
 954       for extraOption in "${extraOpts[@]}"; do echo -n \'${extraOption}\'" "; done; echo
 955       echo
 956       ((jobindex++))
 957
 958     done< <(grep "/000${runNumber}/" ${inputFileList})
 959
 960     #CPass0 list of Calib files to merge
 961     arr_cpass0_calib_list[${runNumber}]="${commonOutputPath}/meta/cpass0.calib.run${runNumber}.list"
 962     echo "${arr_cpass0_calib_list[${runNumber}]} : benchmark.sh ${arr_cpass0_outputs[*]}"
 963     echo "  ./benchmark.sh PrintValues calibfile ${arr_cpass0_calib_list[${runNumber}]} ${arr_cpass0_outputs[*]}"
 964     echo
 965
 966     #CPass0 merging
 967     arr_cpass0_merged[${runNumber}]="${commonOutputPath}/meta/merge.cpass0.run${runNumber}.done"
 968     echo "${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz ${arr_cpass0_merged[${runNumber}]} : benchmark.sh ${configFile} ${arr_cpass0_calib_list[${runNumber}]} ${copyFiles[@]}"
 969     echo -n " ${alirootEnv} ./benchmark.sh MergeCPass0 ${commonOutputPath}/000${runNumber}/cpass0 ${currentDefaultOCDB} ${configFile} ${runNumber} ${arr_cpass0_calib_list[${runNumber}]}"" "
 970     for extraOption in "${extraOpts[@]}"; do echo -n \'${extraOption}\'" "; done; echo
 971     echo
 972
 973     #CPass1 list of Calib/QA/ESD/filtered files
 974     # the trick with QA is to have the string "Stage.txt" in the file name of the list of directories with QA output to trigger
 975     # the production of the QA trending tree (only then the task->Finish() will be called in QAtrain_duo.C, on the grid
 976     # this corresponds to the last merging stage)
 977     arr_cpass1_QA_list[${runNumber}]="${commonOutputPath}/meta/cpass1.QA.run${runNumber}.lastMergingStage.txt.list"
 978     echo "${arr_cpass1_QA_list[${runNumber}]}: benchmark.sh ${arr_cpass1_outputs[*]}"
 979     echo "  ./benchmark.sh PrintValues dir ${arr_cpass1_QA_list[${runNumber}]} ${arr_cpass1_outputs[*]}"
 980     echo
 981     arr_cpass1_calib_list[${runNumber}]="${commonOutputPath}/meta/cpass1.calib.run${runNumber}.list"
 982     echo "${arr_cpass1_calib_list[${runNumber}]} : benchmark.sh ${arr_cpass1_outputs[*]}"
 983     echo "  ./benchmark.sh PrintValues calibfile ${arr_cpass1_calib_list[${runNumber}]} ${arr_cpass1_outputs[*]};"
 984     echo
 985     arr_cpass1_ESD_list[${runNumber}]="${commonOutputPath}/meta/cpass1.ESD.run${runNumber}.list"
 986     echo "${arr_cpass1_ESD_list[${runNumber}]} : benchmark.sh ${arr_cpass1_outputs[*]}"
 987     echo "  ./benchmark.sh PrintValues esd ${arr_cpass1_ESD_list[${runNumber}]} ${arr_cpass1_outputs[*]}"
 988     echo
 989     arr_cpass1_filtered_list[${runNumber}]="${commonOutputPath}/meta/cpass1.filtered.run${runNumber}.list"
 990     echo "${arr_cpass1_filtered_list[${runNumber}]} : benchmark.sh ${arr_cpass1_outputs[*]}"
 991     echo "  ./benchmark.sh PrintValues filteredTree ${arr_cpass1_filtered_list[${runNumber}]} ${arr_cpass1_outputs[*]}"
 992     echo
 993
 994     #CPass1 merging
 995     arr_cpass1_merged[${runNumber}]="${commonOutputPath}/meta/merge.cpass1.run${runNumber}.done"
 996     echo "${commonOutputPath}/meta/cpass1.localOCDB.${runNumber}.tgz ${arr_cpass1_merged[${runNumber}]} :  benchmark.sh ${configFile} ${arr_cpass1_calib_list[${runNumber}]} ${arr_cpass1_QA_list[${runNumber}]} ${copyFiles[@]}"
 997     echo -n " ${alirootEnv} ./benchmark.sh MergeCPass1 ${commonOutputPath}/000${runNumber}/cpass1 ${currentDefaultOCDB} ${configFile} ${runNumber} ${arr_cpass1_calib_list[${runNumber}]} ${arr_cpass1_QA_list[${runNumber}]} ${arr_cpass1_filtered_list[${runNumber}]}"" "
 998     for extraOption in "${extraOpts[@]}"; do echo -n \'${extraOption}\'" "; done; echo
 999     echo
1000
1001     #CPass0 wrapped in a profiling tool (valgrind,....)
1002     if [[ -n ${profilingCommand} ]]; then
1003       arr_cpass0_profiled_outputs[${runNumber}]="${commonOutputPath}/meta/cpass0.jobProfiled.run${runNumber}.done"
1004       echo "${arr_cpass0_profiled_outputs[${runNumber}]} : benchmark.sh ${configFile} ${copyFiles[@]}"
1005       echo -n " ${alirootEnv} ./benchmark.sh CPass0 ${commonOutputPath}/000${runNumber}/profiled ${inputFile} ${nEventsProfiling} ${currentDefaultOCDB} ${configFile} ${runNumber} profiling"" "
1006       for extraOption in "${extraOpts[@]}"; do echo -n \'${extraOption}\'" "; done;
1007       echo "\"useProfilingCommand=${profilingCommand}\""
1008       echo
1009     fi
1010
1011   done #runs
1012
1013   #Summary
1014   echo "summary.log : benchmark.sh ${configFile} ${arr_cpass1_merged[*]}"
1015   echo -n "  ${alirootEnv} ./benchmark.sh MakeSummary ${configFile}"" "
1016   for extraOption in "${extraOpts[@]}"; do echo -n \'${extraOption}\'" "; done; echo
1017   echo
1018
1019   return 0
1020 }
1021
1022 goPrintValues()
1023 {
1024   #print the values given the key from any number of files (store in output file on second argument)
1025   if [[ $# -lt 3 ]]; then
1026     echo "goPrintValues key outputFile inputFiles"
1027     echo "if outputFile is \"-\" dont redirect to a file"
1028     return
1029   fi
1030   key=${1}
1031   outputFile=${2}
1032   [[ ${outputFile} =~ "-" ]] && outputFile=""
1033   shift 2 #remove 2 first arguments from arg list to only pass the input files to awk
1034   awk -v key=${key} '$0 ~ key" " {print $2}' "$@" | tee ${outputFile}
1035   return 0
1036 }
1037
1038 goCreateQAplots()
1039 {
1040   umask 0002
1041   mergedQAfileList=${1}
1042   productionID=${2}
1043   outputDir=${3}
1044   configFile=${4}
1045   shift 4
1046   if ! parseConfig ${configFile} ${@}; then return 1; fi
1047
1048   #record the working directory provided by the batch system
1049   batchWorkingDirectory=${PWD}
1050
1051   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
1052
1053   [[ -z ${logOutputDir} ]] && logOutputDir=${PWD}
1054   [[ -z ${dontRedirectStdOutToLog} ]] && exec 2>&1 > ${logOutputDir}/makeQAplots.log
1055   echo "${0} $*"
1056
1057   olddir=${PWD}
1058   mkdir -p ${outputDir}
1059   cd ${outputDir}
1060   [[ ! "${PWD}" = "${outputDir}" ]] && echo "cannot make ${outputDir}... exiting" && return 1
1061
1062   echo ${ALICE_ROOT}/PWGPP/QA/scripts/runQA.sh inputList=${mergedQAfileList}
1063   ${ALICE_ROOT}/PWGPP/QA/scripts/runQA.sh inputList="${mergedQAfileList}" inputListHighPtTrees="${filteringList}"
1064
1065   cd ${olddir}
1066   return 0
1067 }
1068
1069 goTest()
1070 {
1071   umask 0002
1072   exec 2>&1
1073   exec > >(tee test.log)
1074   echo "$@"
1075   echo something
1076   return 0
1077 }
1078
1079 alirootInfo()
1080 {
1081   umask 0002
1082   # save aliroot repository info
1083   [[ -z "${ALICE_ROOT}" ]] && return 1
1084
1085   echo "\${ALICE_ROOT}=${ALICE_ROOT}"
1086   echo "\${ROOTSYS}=${ROOTSYS}"
1087   echo "\${PATH}=${PATH}"
1088   echo "\${LD_LIBRARY_PATH}=${LD_LIBRARY_PATH}"
1089
1090   pushd ${PWD}
1091   cd ${ALICE_ROOT}
1092   echo
1093
1094   currentBranch=$(git rev-parse --abbrev-ref HEAD)
1095   git status
1096   echo ""
1097   echo ""
1098   git diff ${currentBranch}
1099   popd
1100   return 0
1101 }
1102
1103 setYear()
1104 {
1105   #set the year
1106   #  ${1} - year to be set
1107   #  ${2} - where to set the year
1108   year1=$(guessYear ${1})
1109   year2=$(guessYear ${2})
1110   local path=${2}
1111   [[ ${year1} -ne ${year2} && -n ${year2} && -n ${year1} ]] && path=${2/\/${year2}\//\/${year1}\/}
1112   echo ${path}
1113   return 0
1114 }
1115
1116 guessPeriod()
1117 {
1118   #guess the period from the path, pick the rightmost one
1119   local IFS="/"
1120   declare -a path=( ${1} )
1121   local dirDepth=${#path[*]}
1122   for ((x=${dirDepth}-1;x>=0;x--)); do
1123     local field=${path[${x}]}
1124     [[ ${field} =~ ^LHC[0-9][0-9][a-z]$ ]] && period=${field} && break
1125   done
1126   echo ${period}
1127   return 0
1128 }
1129
1130 guessYear()
1131 {
1132   #guess the year from the path, pick the rightmost one
1133   local IFS="/"
1134   declare -a path=( ${1} )
1135   local dirDepth=${#path[*]}
1136   for ((x=${dirDepth}-1;x>=0;x--)); do
1137     local field=${path[${x}]}
1138     [[ ${field} =~ ^20[0-9][0-9]$ ]] && year=${field} && break
1139   done
1140   echo ${year}
1141   return 0
1142 }
1143
1144 guessRunNumber()
1145 {
1146   #guess the run number from the path, pick the rightmost one
1147   #works for /path/foo/000123456/bar/...
1148   #and       /path/foo.run123456.bar
1149   local IFS="/."
1150   declare -a path=( ${1} )
1151   local dirDepth=${#path[*]}
1152   for ((x=${dirDepth}-1;x>=0;x--)); do
1153     local field=${path[${x}]}
1154     field=${field/run/000}
1155     [[ ${field} =~ ^000[0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && runNumber=${field#000} && break
1156   done
1157   echo ${runNumber}
1158   return 0
1159 }
1160
1161 validateLog()
1162 {
1163   log=${1}
1164   errorConditions=(
1165                     'There was a crash'
1166                     'floating'
1167                     'error while loading shared libraries'
1168                     'std::bad_alloc'
1169                     's_err_syswatch_'
1170                     'Thread [0-9]* (Thread'
1171                     'AliFatal'
1172                     'core dumped'
1173                     '\.C.*error:.*\.h: No such file'
1174   )
1175
1176   warningConditions=(
1177                      'This is serious !'
1178                      'rocVoltage out of range:'
1179   )
1180
1181   local logstatus=0
1182   local errorSummary=""
1183   local warningSummary=""
1184
1185   for ((i=0; i<${#errorConditions[@]};i++)); do
1186     local tmp=$(grep -m1 -e "${errorConditions[${i}]}" ${log})
1187     [[ -n ${tmp} ]] && tmp+=" : "
1188     errorSummary+=${tmp}
1189   done
1190
1191   for ((i=0; i<${#warningConditions[@]};i++)); do
1192     local tmp=$(grep -m1 -e "${warningConditions[${i}]}" ${log})
1193     [[ -n ${tmp} ]] && tmp+=" : "
1194     warningSummary+=${tmp}
1195   done
1196
1197   if [[ -n ${errorSummary} ]]; then
1198     echo "${errorSummary}"
1199     return 1
1200   fi
1201
1202   if [[ -n ${warningSummary} ]]; then
1203     echo "${warningSummary}"
1204     return 2
1205   fi
1206
1207   return 0
1208 }
1209
1210 summarizeLogs()
1211 {
1212   #print a summary of logs
1213   logFiles=(
1214             "*.log"
1215             "stdout"
1216             "stderr"
1217   )
1218
1219   #check logs
1220   local logstatus=0
1221   for log in ${logFiles[*]}; do
1222     finallog=${PWD%/}/${log}
1223     [[ ! -f ${log} ]] && continue
1224     errorSummary=$(validateLog ${log})
1225     validationStatus=$?
1226     if [[ ${validationStatus} -eq 0 ]]; then
1227       #in pretend mode randomly report an error in rec.log some cases
1228       if [[ -n ${pretend} && "${log}" == "rec.log" ]]; then
1229         [[ $(( ${RANDOM}%2 )) -ge 1 ]] && echo "${finallog} BAD random error" || echo "${finallog} OK"
1230       else
1231         echo "${finallog} OK"
1232       fi
1233     elif [[ ${validationStatus} -eq 1 ]]; then
1234       echo "${finallog} BAD ${errorSummary}"
1235       logstatus=1
1236     elif [[ ${validationStatus} -eq 2 ]]; then
1237       echo "${finallog} OK MWAH ${errorSummary}"
1238     fi
1239   done
1240
1241   #report core files
1242   while read x; do
1243     echo ${x}
1244     chmod 644 ${x}
1245     gdb --batch --quiet -ex "bt" -ex "quit" aliroot ${x} > stacktrace_${x//\//_}.log
1246   done < <(/bin/ls ${PWD}/*/core 2>/dev/null; /bin/ls ${PWD}/core 2>/dev/null)
1247
1248   return ${logstatus}
1249 }
1250
1251 spitOutLocalOCDBaccessConfig()
1252 {
1253   umask 0002
1254   #find ${1} -name "*root" | \
1255   /bin/ls -1 ${1}/*/*/*/*.root 2>/dev/null | \
1256   while read line
1257   do
1258     local tmp=${line#${1}}
1259     echo ${tmp%/*} | \
1260     awk -v ocdb=${1} '{print "  man->SetSpecificStorage(\""$1"\",\"local://"ocdb"\");"}'
1261   done
1262   return 0
1263 }
1264
1265 goMakeLocalOCDBaccessConfig()
1266 {
1267   umask 0002
1268   # make a script that sets the specific storages form all the root files produced by CPass0
1269   local localOCDBpathCPass0=${1}
1270   local OCDBpathPrefix=${2}
1271   [[ -z ${OCDBpathPrefix} ]] && OCDBpathPrefix="."
1272
1273   if [[ -f ${localOCDBpathCPass0} && ${localOCDBpathCPass0} =~ \.tgz$ ]]; then
1274     tar xzf ${localOCDBpathCPass0}
1275     local localOCDBpathCPass0="${OCDBpathPrefix}/OCDB"
1276   fi
1277
1278   echo
1279   echo creating the specific storage script
1280   echo   localOCDBaccessConfig.C
1281   echo   based on OCDB: ${localOCDBaccessConfig}
1282   echo
1283
1284   local tempLocalOCDB=""
1285   if [[ -f localOCDBaccessConfig.C ]]; then
1286     tempLocalOCDB=$(mktemp)
1287     echo "egrep "SetSpecificStorage" localOCDBaccessConfig.C > ${tempLocalOCDB}"
1288     egrep "SetSpecificStorage" localOCDBaccessConfig.C > ${tempLocalOCDB}
1289   fi
1290
1291   echo "localOCDBaccessConfig()"                               >  localOCDBaccessConfig.C
1292   echo "{"                                                     >> localOCDBaccessConfig.C
1293   echo "  AliCDBManager* man = AliCDBManager::Instance();"     >> localOCDBaccessConfig.C
1294   spitOutLocalOCDBaccessConfig ${localOCDBpathCPass0}|sort|uniq  >> localOCDBaccessConfig.C
1295   [[ -f "${tempLocalOCDB}" ]] && cat ${tempLocalOCDB}              >> localOCDBaccessConfig.C
1296   echo "}"                                                     >> localOCDBaccessConfig.C
1297
1298   [[ -f "${tempLocalOCDB}" ]] && rm -f ${tempLocalOCDB}
1299
1300   if ! grep SetSpecificStorage localOCDBaccessConfig.C; then
1301     echo
1302     echo "!!!!!!! CPass0 produced no OCDB entries"
1303     return 1
1304   fi
1305   return 0
1306 }
1307
1308 goMakeFilteredTrees()
1309 {
1310   outputDir=${1}
1311   runNumber=${2}
1312   #get path to input list
1313   inputListfiles=${3}
1314   #get scale number for tracks
1315   filterT=${4}
1316   #get scale number for V0s
1317   filterV=${5}
1318   #get OCDB path (optional)
1319   OCDBpath=${6}
1320   #get max number of files
1321   maxFiles=${7-"1000000"}
1322   #get offset of first file
1323   offsetFile=${8-"0"}
1324   #get max number of events
1325   maxEvents=${9-"30000000"}
1326   #get offset of first event
1327   offsetEvent=${10-"0"}
1328   configFile=${11-"benchmark.config"}
1329   esdFileName=${12-"AliESDs_Barrel.root"}
1330   shift 12
1331   if ! parseConfig ${configFile} "$@"; then return 1; fi
1332
1333   #record the working directory provided by the batch system
1334   batchWorkingDirectory=${PWD}
1335
1336
1337   commonOutputPath=${PWD}
1338   doneFile=${commonOutputPath}/meta/filtering.cpass1.run${runNumber}.done
1339
1340   runpath=${outputDir}
1341   [[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
1342   [[ ${reconstructInTemporaryDir} -eq 1 && -z ${TMPDIR} ]] && runpath=$(mktemp -d)
1343
1344
1345   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
1346
1347   mkdir -p ${outputDir}
1348   mkdir -p ${runpath}
1349
1350   cd ${runpath}
1351
1352   cat > filtering.log << EOF
1353   goMakeFilteredTrees config:
1354   runpath=${runpath}
1355   outputDir=${outputDir}
1356   commonOutputPath=${commonOutputPath}
1357   ALICE_ROOT=${ALICE_ROOT}
1358   PATH=${PATH}
1359   offsetEvent=$offsetEvent
1360   configFile=$configFile
1361   esdFileName=$esdFileName
1362 EOF
1363
1364   if [[ -z ${pretend} ]];then
1365     aliroot -l -b -q "${ALICE_ROOT}/PWGPP/macros/runFilteringTask.C(\"${inputListfiles}\",${filterT},${filterV},\"${OCDBpath}\",${maxFiles},${offsetFile},${maxEvents},${offsetEvent},\"${esdFileName}\")" &>> filtering.log
1366   else
1367     touch filtering.log FilterEvents_Trees.root
1368   fi
1369   pwd
1370   /bin/ls
1371   touch ${doneFile}
1372   summarizeLogs >>  ${doneFile}
1373   echo mv -f * ${outputDir}
1374   mv -f * ${outputDir}
1375   [[ -f ${outputDir}/FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFile}
1376   cd ${commonOutputPath}
1377   [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
1378   return 0
1379 }
1380
1381 submit()
1382 {
1383   umask 0002
1384   [[ $# -lt 5 ]] && echo "at least 5 args needed, you supplied $#" && return 1
1385   JobID=${1}
1386   startID=${2}
1387   endID=${3}
1388   waitForJOBID=${4}
1389   command=${5}
1390   shift 5
1391   commandArgs=("$@")
1392
1393   #add quote strings around the extra arguments
1394   for ((i=0; i<${#commandArgs[@]}; i++)); do
1395     commandArgs[i]=\"${commandArgs[i]}\"
1396   done
1397
1398   [[ -z ${waitForJOBID} ]] && waitForJOBID=0
1399
1400   newFarm=$(which qsub|grep "^/usr/bin/qsub")
1401
1402   batchSystem="SGE"
1403
1404   if [[ -z "${newFarm}" ]]
1405   then
1406     #old LSF
1407     # submit it (as job array)
1408     nFiles=$(( ${endID}-${startID}+1 ))
1409     while [ ${startID} -le ${nFiles}  ] ; do
1410       if [ $(expr ${nFiles} - ${startID}) -gt 999 ] ; then
1411         endID=$(expr ${startID} + 999)
1412       else
1413         endID=${nFiles}
1414       fi
1415     if [[ ${waitForJOBID} -eq 0 ]]; then
1416         echo ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}"
1417         ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}"
1418       else
1419         echo ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -w "ended(${waitForJOBID})" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}"
1420         ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -w "ended(${waitForJOBID})" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}"
1421       fi
1422       startID=$(expr ${endID} + 1)
1423     done
1424   else
1425     #new SGE farm
1426     if [[ ${waitForJOBID} =~ "000" ]]; then
1427       echo ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}"
1428       ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}"
1429     else
1430       echo ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -hold_jid "${waitForJOBID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}"
1431       ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -hold_jid "${waitForJOBID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}"
1432     fi
1433   fi
1434   return 0
1435 }
1436
1437 goSubmitBatch()
1438 {
1439   if [[ $# -lt 3 ]]; then
1440     echo "minimal use:"
1441     echo " ${0} submit fileList productionID configFile"
1442     return 0
1443   fi
1444
1445   productionID=${1}
1446   inputList=${2}
1447   configFile=${3:-"benchmark.config"}
1448   configFile=$(readlink -f ${configFile})
1449   shift 3
1450   extraOpts=("$@")
1451   if ! parseConfig ${configFile} "${extraOpts[@]}"; then return 1; fi
1452
1453   #record the working directory provided by the batch system
1454   batchWorkingDirectory=${PWD}
1455
1456   #redirect all output to submit.log
1457   echo "redirecting all output to ${PWD}/submit_${productionID//"/"/_}.log"
1458   exec 7>&1
1459   exec 1>submit_${productionID//"/"/_}.log 2>&1
1460
1461   umask 0002
1462   echo ${0}" submit $*"
1463   if [[ -z "${inputList}" || -z "${productionID}" ]]
1464   then
1465     echo
1466     echo " Usage: ${0} submit inputList productionID [configFile=benchmark.config]"
1467     echo
1468     return
1469   fi
1470
1471   # check if config file is there
1472   if [ ! -f ${configFile} ]; then
1473     echo "ERROR! Config File '${configFile}' not found" >&2
1474     return
1475   else
1476     echo "Using Config File: '${configFile}'"
1477   fi
1478
1479   [[ ! -f ${alirootEnv} ]] && echo "alirootEnv script ${alirootEnv} not found!..." && return 1
1480
1481   #move the script, config and some other stuff to ${commonOutputPath} first, then use them from there
1482   self=$(readlink -f "${0}")
1483   configPath=$(dirname ${configFile})
1484   export commonOutputPath=${baseOutputDirectory}/${productionID}
1485
1486   mkdir -p ${commonOutputPath}
1487   mkdir -p ${commonOutputPath}/logs
1488   mkdir -p ${commonOutputPath}/meta
1489
1490   cp ${self} ${commonOutputPath}
1491   cp ${configFile} ${commonOutputPath}
1492   cp ${inputList} ${commonOutputPath}
1493   self=${commonOutputPath}/${self##*/}
1494   chmod u+x ${self}
1495   configFile=${commonOutputPath}/${configFile##*/}
1496   inputList=${commonOutputPath}/${inputList##*/}
1497
1498   #convert to absolut pathnames
1499   inputList=$(readlink -f "${inputList}")
1500   #make list of runs
1501   if [[ -z ${runNumber} ]]; then
1502     listOfRuns=($(while read x; do guessRunNumber ${x}; done < ${inputList} | sort | uniq))
1503   else
1504     listOfRuns=${runNumber}
1505   fi
1506
1507   alirootSource=$(readlink -f "${alirootSource}")
1508
1509   echo ""
1510   echo "### BEGIN CONFIGURATION ###"
1511   echo ""
1512   echo "GENERAL:"
1513   echo ""
1514   echo "    productionID:    ${productionID}"
1515   echo "    batchCommand:    ${batchCommand}"
1516   echo "    batchFlags:      ${batchFlags}"
1517   echo "    alirootEnv:   ${alirootEnv}"
1518   ${alirootEnv} echo '    ALICE_ROOT:      ${ALICE_ROOT}'
1519   ${alirootEnv} echo '    ALIROOT_RELEASE: ${ALICE_RELEASE}'
1520   echo "    inputList:       ${inputList}"
1521   echo "    configPath:      ${configPath}"
1522   echo "    commonOutputPath:      ${commonOutputPath}"
1523   echo "    defaultOCDB:     ${defaultOCDB}"
1524   echo "      autoOCDB: ${autoOCDB}"
1525   echo "    recoTriggerOptions:   ${recoTriggerOptions}"
1526   echo "    runs:"
1527   echo "      ${listOfRuns[*]}"
1528   echo ""
1529   echo "THE TRAIN WILL RUN:"
1530
1531   if [ ${runCPass0reco} -eq 1 ]; then
1532     echo "    Pass0 - Recontruction"
1533   fi
1534
1535   if [ ${runCPass0MergeMakeOCDB} -eq 1 ]; then
1536     echo "    Pass0 - merging and OCDB export"
1537   fi
1538
1539   if [ ${runCPass1reco} -eq 1 ]; then
1540     echo "    Pass1 - Recontruction"
1541   fi
1542   if [ ${runCPass1MergeMakeOCDB} -eq 1 ]; then
1543     echo "    Pass1 - merging and OCDB export"
1544   fi
1545
1546   echo ""
1547   echo "LIMITS:"
1548   echo "    max. Events/Chunk:   ${nEvents}"
1549   echo "    max. Number of Chunks per Run:     ${nMaxChunks}"
1550   echo ""
1551   echo "### END CONFIGURATION ###"
1552   echo ""
1553
1554
1555   # check if input file is there
1556   if [ ! -f ${inputList} ]; then
1557     echo "ERROR! Input List '${inputList}' not found" >&2
1558     return
1559   fi
1560
1561   # define jobid (for dependent jobs)
1562   date=$(date +%Y_%m_%d_%H%M%S)
1563   #for each run we submit one jobarray:
1564   for runNumber in ${listOfRuns[*]}; do
1565
1566     [[ -z ${runNumber} ]] && continue
1567     [[ ! ${runNumber} =~ ^[0-9]*[0-9]$ ]] && continue
1568
1569     JOBpostfix="${productionID//"/"/_}_${runNumber}_${date}"
1570     JOBID1="p0_${JOBpostfix}"
1571     JOBID1wait="w0_${JOBpostfix}"
1572     JOBID2="m0_${JOBpostfix}"
1573     JOBID2wait="wm0_${JOBpostfix}"
1574     JOBID3="op0_${JOBpostfix}"
1575     JOBID3wait="wop0_${JOBpostfix}"
1576     JOBID4="p1_${JOBpostfix}"
1577     JOBID4wait="w1_${JOBpostfix}"
1578     JOBID5="m1_${JOBpostfix}"
1579     JOBID5wait="wm1_${JOBpostfix}"
1580     JOBID6="s1_${JOBpostfix}"
1581     JOBID6wait="ws1_${JOBpostfix}"
1582     JOBID7="QA_${JOBpostfix}"
1583     JOBmakeESDlistCPass1="lp1_${JOBpostfix}"
1584     JOBfilterESDcpass1="fp1_${JOBpostfix}"
1585     LASTJOB="000"
1586
1587     oneInputFile=$(egrep -m1 "${runNumber}\/" ${inputList})
1588
1589     currentDefaultOCDB=${defaultOCDB}
1590     [[ -z ${autoOCDB} ]] && autoOCDB=1
1591     if [[ ${autoOCDB} -ne 0 ]]; then
1592       currentDefaultOCDB=$(setYear ${oneInputFile} ${defaultOCDB})
1593     fi
1594
1595     echo "submitting run ${runNumber} with OCDB ${currentDefaultOCDB}"
1596
1597     ###############################################################################
1598     #run one chunk with valgrind:
1599     if [[ -n ${profilingCommand} ]]; then
1600       [[ -z ${nEventsProfiling} ]] && nEventsProfiling=2
1601       [[ -z ${profilingCommand} ]] && profilingCommand="/usr/bin/valgrind --tool=callgrind --num-callers=40 -v --trace-children=yes"
1602       submit "valgrind" 1 1 000 "${alirootEnv} ${self}" CPass0 ${commonOutputPath}/000${runNumber}/valgrind ${oneInputFile} ${nEventsProfiling} ${currentDefaultOCDB} ${configFile} ${runNumber} valgrind useProfilingCommand=${profilingCommand} "${extraOpts[@]}"
1603     fi
1604
1605     ################################################################################
1606     ################################################################################
1607     # run the CPass0 if requested
1608
1609     if [ ${runCPass0reco} -eq 1 ]; then
1610
1611       echo
1612       echo "starting CPass0... for run ${runNumber}"
1613       echo
1614
1615       # create directory and copy all files that are needed
1616       targetDirectory="${commonOutputPath}/000${runNumber}/cpass0"
1617       mkdir -p ${targetDirectory}
1618
1619       filesCPass0=(
1620                     "${configPath}/runCPass0.sh"
1621                     "${configPath}/recCPass0.C"
1622                     "${configPath}/runCalibTrain.C"
1623                     "${configPath}/localOCDBaccessConfig.C"
1624                     "${configPath}/OCDB*.root"
1625                     "${configPath}/sim.C"
1626                     "${configPath}/rec.C"
1627                     "${configPath}/Config.C"
1628       )
1629       for file in ${filesCPass0[*]}; do
1630         [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
1631       done
1632
1633       localInputList=${targetDirectory}/${inputList##*/}
1634       [[ ! -f ${localInputList} ]] && egrep "\/000${runNumber}\/" ${inputList} > ${localInputList}
1635       # limit nFiles to nMaxChunks
1636       nFiles=$(wc -l < ${localInputList})
1637       [[ ${nFiles} -eq 0 ]] && echo "list contains ZERO files! exiting..." && return 1
1638       echo "raw files in list:    ${nFiles}"
1639       if [[ ${nMaxChunks} -gt 0 && ${nMaxChunks} -le ${nFiles} ]]; then
1640         nFiles=${nMaxChunks}
1641       fi
1642       echo "raw files to process: ${nFiles}"
1643       [[ -z "${percentProcessedFilesToContinue}" ]] && percentProcessedFilesToContinue=100
1644       if [[ ${percentProcessedFilesToContinue} -eq 100 ]]; then
1645         nFilesToWaitFor=${nFiles}
1646       else
1647         nFilesToWaitFor=$(( ${nFiles}-${nFiles}/(100/(100-${percentProcessedFilesToContinue})) ))
1648       fi
1649       echo "requested success rate is ${percentProcessedFilesToContinue}%"
1650       echo "merging will start after ${nFilesToWaitFor} jobs are done"
1651
1652       submit ${JOBID1} 1 ${nFiles} 000 "${alirootEnv} ${self}" CPass0 ${targetDirectory} ${localInputList} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} -1 "${extraOpts[@]}"
1653
1654       ## submit a monitoring job that will run until a certain number of jobs are done with reconstruction
1655       submit "${JOBID1wait}" 1 1 000 "${alirootEnv} ${self}" WaitForOutput ${commonOutputPath} "meta/cpass0.job*.run${runNumber}.done" ${nFilesToWaitFor} ${maxSecondsToWait} '-maxdepth 1'
1656       LASTJOB=${JOBID1wait}
1657
1658     fi #end running CPass0
1659     ################################################################################
1660
1661
1662     ################################################################################
1663     # submit merging of CPass0, depends on the reconstruction
1664
1665     if [ ${runCPass0MergeMakeOCDB} -eq 1 ]; then
1666
1667       echo
1668       echo "submit CPass0 merging for run ${runNumber}"
1669       echo
1670
1671       targetDirectory="${commonOutputPath}/000${runNumber}/cpass0"
1672       mkdir -p ${targetDirectory}
1673
1674       #copy the scripts
1675       filesMergeCPass0=(
1676                         "${configPath}/OCDB.root"
1677                         "${configPath}/mergeMakeOCDB.byComponent.sh"
1678                         "${configPath}/mergeMakeOCDB.sh"
1679                         "${configPath}/localOCDBaccessConfig.C"
1680                         "${configPath}/mergeByComponent.C"
1681                         "${configPath}/makeOCDB.C"
1682                         "${configPath}/merge.C"
1683       )
1684       for file in ${filesMergeCPass0[*]}; do
1685         [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
1686       done
1687
1688       submit ${JOBID2} 1 1 "${LASTJOB}" "${alirootEnv} ${self}" MergeCPass0 ${targetDirectory} ${currentDefaultOCDB} ${configFile} ${runNumber} cpass0.calib.run${runNumber}.list "${extraOpts[@]}"
1689       LASTJOB=${JOBID2}
1690
1691       if [[ -n ${generateMC} ]]; then
1692         submit "mrl${JOBpostfix}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" PrintValues sim ${commonOutputPath}/meta/sim.run${runNumber}.list ${commonOutputPath}/meta/cpass0.job*.run${runNumber}.done
1693         LASTJOB="mrl${JOBpostfix}"
1694       fi
1695
1696       echo
1697     fi
1698     # end of merging CPass0
1699     ################################################################################
1700
1701     ################################################################################
1702     ################################################################################
1703     # run the CPass1 if requested
1704
1705     if [ ${runCPass1reco} -eq 1 ]; then
1706
1707       targetDirectory="${commonOutputPath}/000${runNumber}/cpass1"
1708       rm -f ${commonOutputPath}/meta/cpass1.job*.run${runNumber}.done
1709
1710       # safety feature: if we are re-running for any reason we want to delete the previous output first.
1711       [[ -d ${targetDirectory} ]] && rm -rf ${targetDirectory}/* && echo "removed old output at ${targetDirectory}/*"
1712
1713       echo
1714       echo "starting CPass1... for run ${runNumber}"
1715       echo
1716
1717       # create directory and copy all files that are needed
1718       mkdir -p ${targetDirectory}
1719
1720       filesCPass1=(
1721                     "${configPath}/runCPass1.sh"
1722                     "${configPath}/recCPass1.C"
1723                     "${configPath}/recCPass1_OuterDet.C"
1724                     "${configPath}/runCalibTrain.C"
1725                     "${configPath}/QAtrain_duo.C"
1726                     "${configPath}/localOCDBaccessConfig.C"
1727                     "${configPath}/OCDB.root"
1728       )
1729       for file in ${filesCPass1[*]}; do
1730         [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
1731       done
1732
1733       if [[ -n ${generateMC} ]]; then
1734         localInputList=${commonOutputPath}/meta/sim.run${runNumber}.list
1735       else
1736         localInputList=${targetDirectory}/${inputList##*/}
1737         [[ ! -f ${localInputList} ]] && egrep "\/000${runNumber}\/" ${inputList} > ${localInputList}
1738       fi
1739       # limit nFiles to nMaxChunks
1740       nFiles=$(wc -l < ${localInputList})
1741       [[ ${nFiles} -eq 0 ]] && echo "list contains ZERO files! continuing..." && continue
1742       echo "raw files in list:    ${nFiles}"
1743       if [[ ${nMaxChunks} -gt 0 && ${nMaxChunks} -le ${nFiles} ]]; then
1744         nFiles=${nMaxChunks}
1745       fi
1746       echo "raw files to process: ${nFiles}"
1747       [[ -z "${percentProcessedFilesToContinue}" ]] && percentProcessedFilesToContinue=100
1748       if [[ ${percentProcessedFilesToContinue} -eq 100 ]]; then
1749         nFilesToWaitFor=${nFiles}
1750       else
1751         nFilesToWaitFor=$(( ${nFiles}-${nFiles}/(100/(100-${percentProcessedFilesToContinue})) ))
1752       fi
1753       echo "requested success rate is ${percentProcessedFilesToContinue}%"
1754       echo "merging will start after ${nFilesToWaitFor} jobs are done"
1755
1756       submit ${JOBID4} 1 ${nFiles} "${LASTJOB}" "${alirootEnv} ${self}" CPass1 ${targetDirectory} ${localInputList} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} -1 "${extraOpts[@]}"
1757
1758       ################################################################################
1759       ## submit a monitoring job that will run until a certain number of jobs are done with reconstruction
1760       submit "${JOBID4wait}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" WaitForOutput ${commonOutputPath} "meta/cpass1.job*.run${runNumber}.done" ${nFilesToWaitFor} ${maxSecondsToWait} '-maxdepth 1'
1761       LASTJOB=${JOBID4wait}
1762       ################################################################################
1763
1764       echo
1765     fi #end running CPass1
1766
1767     ################################################################################
1768     # submit merging of CPass1, depends on the reconstruction
1769     if [ ${runCPass1MergeMakeOCDB} -eq 1 ]; then
1770
1771       echo
1772       echo "submit CPass1 merging for run ${runNumber}"
1773       echo
1774
1775       targetDirectory="${commonOutputPath}/000${runNumber}/cpass1"
1776       rm -f ${commonOutputPath}/meta/merge.cpass1.run${runNumber}.done
1777       mkdir -p ${targetDirectory}
1778
1779       # copy files
1780       filesMergeCPass1=(
1781                         "${configPath}/OCDB.root"
1782                         "${configPath}/localOCDBaccessConfig.C"
1783                         "${configPath}/mergeMakeOCDB.byComponent.sh"
1784                         "${configPath}/mergeByComponent.C"
1785                         "${configPath}/makeOCDB.C"
1786                         "${configPath}/merge.C"
1787                         "${configPath}/mergeMakeOCDB.sh"
1788                         "${configPath}/QAtrain_duo.C"
1789       )
1790       for file in ${filesMergeCPass1[*]}; do
1791         [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
1792       done
1793
1794       submit "${JOBID5}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" MergeCPass1 ${targetDirectory} ${currentDefaultOCDB} ${configFile} ${runNumber} cpass1.calib.run${runNumber}.list cpass1.QA.run${runNumber}.lastMergingStage.txt.list cpass1.filtered.run${runNumber}.list "${extraOpts[@]}"
1795       LASTJOB=${JOBID5}
1796       echo
1797     fi
1798
1799     ###############################
1800     #if [ ${runESDfiltering} -eq 1 ]; then
1801     #  rm -f ${commonOutputPath}/cpass1.ESD.run${runNumber}.list
1802     #  rm -f ${commonOutputPath}/meta/filtering.cpass1.run*.done
1803     #  echo
1804     #  echo submitting filtering for run ${runNumber}
1805     #  echo
1806     #  submit "${JOBmakeESDlistCPass1}" 1 1 "${LASTJOB}" "${self}" PrintValues esd ${commonOutputPath}/meta/cpass1.ESD.run${runNumber}.list ${commonOutputPath}/meta/cpass1.job*.run${runNumber}.done
1807     #  submit "${JOBfilterESDcpass1}" 1 1 "${JOBmakeESDlistCPass1}" "${alirootEnv} ${self}" MakeFilteredTrees ${commonOutputPath}/000${runNumber}/cpass1 ${runNumber} ${commonOutputPath}/meta/cpass1.ESD.run${runNumber}.list ${filteringFactorHighPt} ${filteringFactorV0s} ${currentDefaultOCDB} 1000000 0 10000000 0 ${configFile} AliESDs_Barrel.root "${extraOpts[@]}"
1808     #  LASTJOB=${JOBfilterESDcpass1}
1809     #fi
1810
1811   done
1812
1813   #################################################################################
1814   #################################################################################
1815   #if [ ${runESDfiltering} -eq 1 ]; then
1816   #  submit "${JOBID5wait}" 1 1 "${LASTJOB}" "${self}" WaitForOutput ${commonOutputPath} "meta/filtering.cpass1.run*.done" "${#listOfRuns[@]}" ${maxSecondsToWait}
1817   #else
1818     submit "${JOBID5wait}" 1 1 "${LASTJOB}" "${self}" WaitForOutput ${commonOutputPath} "meta/merge.cpass1.run*.done" ${#listOfRuns[@]} ${maxSecondsToWait}
1819   #fi
1820   LASTJOB=${JOBID5wait}
1821
1822   #################################################################################
1823   echo
1824   echo "submit make a summary"
1825   echo
1826
1827   submit "${JOBID6}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" MakeSummary ${configFile}
1828   LASTJOB=${JOBID6}
1829   #################################################################################
1830
1831   #restore stdout
1832   exec 1>&7 7>&-
1833   echo "jobs submitted."
1834   return 0
1835 }
1836
1837 goWaitForOutput()
1838 {
1839   umask 0002
1840   [[ $# -lt 3 ]] && echo "goWaitForOutput() wrong number of arguments, exiting.." && return 1
1841   echo searchPath=${1}
1842   echo fileName=${2}
1843   echo numberOfFiles=${3}
1844   echo maxSecondsToWait=${4}
1845   searchPath=${1}
1846   fileName=${2}
1847   numberOfFiles=${3}
1848   maxSecondsToWait=${4}
1849   extraFindOptions=${5}
1850   echo "command to be executed: /bin/ls -1 ${searchPath}/${fileName} ${extraFindOptions}"
1851   [[ -z "${maxSecondsToWait}" ]] && maxSecondsToWait=$(( 3600*12 ))
1852   while true; do
1853     #n=$(find ${searchPath} ${extraFindOptions} -name "${fileName}" | wc -l)
1854     n=$(/bin/ls -1 ${searchPath}/${fileName} 2>/dev/null | wc -l)
1855     [[ ${n} -gt 0 ]] && echo "found ${n} X ${fileName}"
1856     [[ ${n} -ge ${numberOfFiles} ]] && break
1857     [[ ${SECONDS} -gt ${maxSecondsToWait} ]] && echo "timeout of ${maxSecondsToWait}s!" && break
1858     sleep 60
1859   done
1860   echo "DONE! exiting..."
1861   return 0
1862 }
1863
1864 mergeSysLogs()
1865 {
1866   outputFile=${1}
1867   shift
1868   inputFiles="$@"
1869   i=0
1870   if ! ls -1 ${inputFiles} &>/dev/null; then echo "the files dont exist!: ${inputFiles}"; return 1; fi
1871   while read x; do
1872     runNumber=$(guessRunNumber ${x})
1873     [[ -z ${runNumber} ]] && echo "run number cannot be guessed for ${x}" && continue
1874     awk -v run=${runNumber} -v i=${i} 'NR > 1 {print run" "$0} NR==1 && i==0 {print "run/I:"$0}' ${x}
1875     (( i++ ))
1876   done < <(ls -1 ${inputFiles}) > ${outputFile}
1877   return 0
1878 }
1879
1880 goMakeMergedSummaryTree()
1881 {
1882   # create list of calibration entries
1883   # takes no arguments, just run it in the base output
1884   # directory with the following files in the working directory
1885   #
1886   #  Calibration file lists:
1887   #       cpass0.dcsTree.list, cpass1.dcsTree.list
1888   #  QA trending root file:
1889   #       trending.root
1890   #
1891   #  Production infoo ascii files:
1892   #       summary_pass0.tree
1893   #       summary_pass1.tree
1894   #
1895
1896   [[ ! -f cpass0.dcsTree.list ]] && echo "no cpass0.dcsTree.list" && return 1
1897   [[ ! -f cpass1.dcsTree.list ]] && echo "no cpass1.dcsTree.list" && return 1
1898   [[ ! -f trending.root ]] && echo "no trending.root" && return 1
1899   [[ ! -f summary_pass0.tree ]] && echo "no summary_pass0.tree" && return 1
1900   [[ ! -f summary_pass1.tree ]] && echo "no summary_pass1.tree" && return 1
1901
1902   #first, dump the C macro to file
1903   cat << EOF > mergeTree.C
1904   //
1905   // Merge summary information
1906   // Following files are expected to be in the working directory
1907   //
1908   // Calibration file lists:
1909   //      cpass0.dcsTree.list, cpass1.dcsTree.list
1910   // QA trending root file:
1911   //      trending.root
1912   //
1913   // Production infoo ascii files:
1914   //      summary_pass0.tree
1915   //      summary_pass1.tree
1916   //
1917   void mergeTree(){
1918     //
1919     //
1920     //
1921     // Calibration values dump
1922     //
1923     AliXRDPROOFtoolkit::MakeTreeFromList("Calib.TPC.CPass0.root", "dcs","dcs","cpass0.dcsTree.list",1);
1924     AliXRDPROOFtoolkit::MakeTreeFromList("Calib.TPC.CPass1.root", "dcs","dcs","cpass1.dcsTree.list",1);
1925     //
1926     // Calibration status dump
1927     //
1928     TFile *fprod = TFile::Open("fproduction.root","recreate");
1929     TTree  tree0, tree1;
1930     tree0.ReadFile("summary_pass0.tree");
1931     tree1.ReadFile("summary_pass1.tree");
1932     tree0.Write("CPass0");
1933     tree1.Write("CPass1");
1934     fprod->Close();
1935     //
1936     //
1937     //
1938     TString stringSetup="";
1939     stringSetup+="1#QA.TPC#run#SummaryTPCQA/tpcQA#trending.root+";  //
1940     stringSetup+="1#Calib.TPC.CPass0#run#dcs#Calib.TPC.CPass0.root+";  //
1941     stringSetup+="1#Calib.TPC.CPass1#run#dcs#Calib.TPC.CPass1.root+";  //
1942     //
1943     stringSetup+="1#CPass0#runnumber#CPass0#fproduction.root+";  //
1944     stringSetup+="1#CPass1#runnumber#CPass1#fproduction.root+";  //
1945     //
1946     AliXRDPROOFtoolkit::JoinTreesIndex("outAll.root","joinAll","run",stringSetup.Data(), 1);
1947   }
1948 EOF
1949
1950   aliroot -b -q "mergeTree.C" > mergeTrees.log
1951   return $?
1952 }
1953
1954 goMakeSummary()
1955 {
1956   #all the final stuff goes in here for ease of use:
1957   # summary logs
1958   # qa plot making
1959   # final file lists
1960   #some defaults:
1961   log="summary.log"
1962   productionID="qa"
1963
1964   configFile=${1}
1965   shift 1
1966   extraOpts=("$@")
1967   if ! parseConfig ${configFile} "${extraOpts[@]}"; then return 1; fi
1968
1969   #record the working directory provided by the batch system
1970   batchWorkingDirectory=${PWD}
1971
1972   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
1973
1974   [[ ! -f ${configFile} ]] && echo "no config file ${configFile}!" && return
1975
1976   [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
1977
1978   #copy some useful stuff
1979   #and go to the commonOutputPath
1980   cp ${configFile} ${commonOutputPath}
1981   cd ${commonOutputPath}
1982
1983   exec &> >(tee ${log})
1984
1985   #summarize the global stuff
1986   echo "env script: ${alirootSource} ${alirootEnv}"
1987   echo "\$ALICE_ROOT=${ALICE_ROOT}"
1988   echo "commonOutputPath=${commonOutputPath}"
1989
1990   #summarize the stacktraces
1991   awk '
1992        BEGIN {
1993                print "frame/I:method/C:line/C:cpass/I:aliroot/I";
1994                RS="#[0-9]*";
1995                aliroot=0;
1996              }
1997              {
1998                if ($3 ~ /Ali*/) aliroot=1; else aliroot=0;
1999                gsub("#","",RT);
2000                if ($NF!="" && RT!="" && $3!="") print RT" "$3" "$NF" "0" "aliroot
2001              }
2002       ' 000*/cpass0/*/stacktrace* 2>/dev/null > stacktrace.tree
2003   awk '
2004        BEGIN {
2005                RS="#[0-9]*";
2006                aliroot=0;
2007              }
2008              {
2009                if ($3 ~ /Ali*/) aliroot=1; else aliroot=0;
2010                gsub("#","",RT);
2011                if ($NF!="" && RT!="" && $3!="") print RT" "$3" "$NF" "1" "aliroot
2012              }
2013       ' 000*/cpass1/*/stacktrace* 2>/dev/null >> stacktrace.tree
2014
2015   echo total numbers for the production:
2016   echo
2017   awk 'BEGIN {nFiles=0;nCore=0;}
2018   /^calibfile/ {nFiles++;}
2019   /core dumped/ {nCore++i;}
2020   END {print     "cpass0 produced "nFiles" calib files, "nCore" core files";}' meta/cpass0.job*done 2>/dev/null
2021   awk 'BEGIN {nOK=0; nBAD=0; }
2022   /\/rec.log OK/ {nOK++;}
2023   /\/rec.log BAD/ {nBAD++;}
2024   /stderr BAD/ {if ($0 ~ /rec.log/){nBAD++;}}
2025   END {print     "cpass0 reco:  OK: "nOK"\tBAD: "nBAD;}' meta/cpass0.job*done 2>/dev/null
2026   awk 'BEGIN {nOK=0; nBAD=0; }
2027   /\/calib.log OK/ {nOK++;}
2028   /\/calib.log BAD/ {nBAD++;}
2029   END {print "cpass0 calib: OK: "nOK"\tBAD: "nBAD;}' meta/cpass0.job*done 2>/dev/null
2030
2031   awk 'BEGIN {nOK=0; nBAD=0; }
2032   /merge.log OK/ {nOK++;}
2033   /merge.log BAD/ {nBAD++;}
2034   END {print "cpass0 merge: OK: "nOK"\tBAD: "nBAD;}' meta/merge.cpass0*done 2>/dev/null
2035   awk 'BEGIN {nOK=0; nBAD=0; }
2036   /ocdb.log OK/ {nOK++;}
2037   /ocdb.log BAD/ {nBAD++;}
2038   END {print   "cpass0 OCDB:  OK: "nOK"\tBAD: "nBAD;}' meta/merge.cpass0*done 2>/dev/null
2039
2040   echo
2041   awk 'BEGIN {nFiles=0;nCore=0;}
2042   /^calibfile/ {nFiles++;}
2043   /core dumped/ {nCore++;}
2044   END {print     "cpass1 produced "nFiles" calib files, "nCore" core files";}' meta/cpass1.job*done 2>/dev/null
2045   awk 'BEGIN {nOK=0; nBAD=0; }
2046   /\/rec.log OK/ {nOK++;}
2047   /\/rec.log BAD/ {nBAD++;}
2048   /stderr BAD/ {if ($0 ~ /rec.log/){nBAD++;}}
2049   END {print     "cpass1 reco:  OK: "nOK"\tBAD: "nBAD;}' meta/cpass1.job*done 2>/dev/null
2050   awk 'BEGIN {nOK=0; nBAD=0; }
2051   /\/calib.log OK/ {nOK++;}
2052   /\/calib.log BAD/ {nBAD++;}
2053   END {print "cpass1 calib: OK: "nOK"\tBAD: "nBAD;}' meta/cpass1.job*done 2>/dev/null
2054
2055   awk 'BEGIN {nOK=0; nBAD=0; }
2056   /merge.log OK/ {nOK++;}
2057   /merge.log BAD/ {nBAD++;}
2058   END {print "cpass1 merge: OK: "nOK"\tBAD: "nBAD;}' meta/merge.cpass1*done 2>/dev/null
2059   awk 'BEGIN {nOK=0; nBAD=0; }
2060   /ocdb.log OK/ {nOK++;}
2061   /ocdb.log BAD/ {nBAD++;}
2062   END {print   "cpass1 OCDB:  OK: "nOK"\tBAD: "nBAD;}' meta/merge.cpass1*done 2>/dev/null
2063
2064   echo
2065   echo per run stats:
2066   /bin/ls -1 meta/merge.cpass0.run*.done | while read x
2067 do
2068   dir=$(goPrintValues calibfile - ${x})
2069   runNumber=$(guessRunNumber ${dir})
2070   [[ -z ${runNumber} ]] && continue
2071
2072   if $(/bin/ls meta/cpass0.job*.run${runNumber}.done &> /dev/null); then
2073     statusCPass0=( $(
2074     awk 'BEGIN {nOKrec=0;nBADrec=0;nOKcalib=0;nBADcalib=0;nOKstderr=0;nBADstderr=0;}
2075     /\/rec.log OK/ {nOKrec++;}
2076     /\/rec.log BAD/ {nBADrec++;}
2077     /stderr BAD/ {if ($0 ~ /rec.log/) {nBADrec++;} nBADstderr++;}
2078     /stderr OK/ {nOKstderr++;}
2079     /\/calib.log OK/ {nOKcalib++;}
2080     /\/calib.log BAD/ {nBADcalib++}
2081     END {print ""nOKrec" "nBADrec" "nOKstderr" "nBADstderr" "nOKcalib" "nBADcalib;}' meta/cpass0.job*.run${runNumber}.done 2>/dev/null
2082     ) )
2083   fi
2084
2085   if $(/bin/ls meta/cpass1.job*.run${runNumber}.done &>/dev/null); then
2086     statusCPass1=( $(
2087     awk 'BEGIN {nOKrec=0;nBADrec=0;nOKcalib=0;nBADcalib=0;nOKstderr=0;nBADstderr=0;nQAbarrelOK=0;nQAbarrelBAD=0;nQAouterOK=0;nQAouterBAD=0;}
2088     /\/rec.log OK/ {nOKrec++;}
2089     /\/rec.log BAD/ {nBADrec++;}
2090     /stderr BAD/ {if ($0 ~ /rec.log/) nBADrec++;nBADstderr++;}
2091     /stderr OK/ {nOKstderr++;}
2092     /\/calib.log OK/ {nOKcalib++;}
2093     /\/calib.log BAD/ {nBADcalib++}
2094     /\/qa_barrel.log OK/ {nQAbarrelOK++;}
2095     /\/qa_barrel.log BAD/ {nQAbarrelBAD++;}
2096     /\/qa_outer.log OK/ {nQAouterOK++;}
2097     /\/qa_outer.log BAD/ {nQAouterBAD++;}
2098     END {print ""nOKrec" "nBADrec" "nOKstderr" "nBADstderr" "nOKcalib" "nBADcalib" "nQAbarrelOK" "nQAbarrelBAD" "nQAouterOK" "nQAouterBAD;}' meta/cpass1.job*.run${runNumber}.done 2>/dev/null
2099     ) )
2100   fi
2101
2102   statusOCDBcpass0=$(awk '/ocdb.log/ {print $2} ' ${x} 2>/dev/null)
2103   statusOCDBcpass1=$(awk '/ocdb.log/ {print $2}' ${x/cpass0/cpass1} 2>/dev/null)
2104   statusQA=$(awk '/mergeMakeOCDB.log/ {print $2}' ${x/cpass0/cpass1} 2>/dev/null)
2105
2106   printf "%s\t ocdb.log cpass0: %s\t ocdb.log cpass1: %s\tqa.log:%s\t| cpass0: rec:%s/%s stderr:%s/%s calib:%s/%s cpass1: rec:%s/%s stderr:%s/%s calib:%s/%s QAbarrel:%s/%s QAouter:%s/%s\n" ${runNumber} ${statusOCDBcpass0} ${statusOCDBcpass1} ${statusQA} ${statusCPass0[0]} ${statusCPass0[1]} ${statusCPass0[2]} ${statusCPass0[3]} ${statusCPass0[4]} ${statusCPass0[5]} ${statusCPass1[0]} ${statusCPass1[1]} ${statusCPass1[2]} ${statusCPass1[3]} ${statusCPass1[4]} ${statusCPass1[5]} ${statusCPass1[6]} ${statusCPass1[7]} ${statusCPass1[8]} ${statusCPass1[9]}
2107 done
2108
2109   #make lists with output files - QA, trending, filtering and calibration
2110   ### wait for the merging of all runs to be over ###
2111   rm -f qa.list
2112   goPrintValues qafile qa.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2113   rm -f calib.list
2114   goPrintValues calibfile calib.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2115   rm -f trending.list
2116   goPrintValues trendingfile trending.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2117   rm -f filtering.list
2118   goPrintValues filteredTree filtering.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2119   #/bin/ls ${commonOutputPath}/*/cpass0/dcs* > cpass0.dcsTree.list
2120   rm -f cpass0.dcsTree.list
2121   goPrintValues dcsTree cpass0.dcsTree.list ${commonOutputPath}/meta/merge.cpass0.run*.done &>/dev/null
2122   #/bin/ls ${commonOutputPath}/*/cpass1/dcs* > cpass1.dcsTree.list
2123   rm -f cpass1.dcsTree.list
2124   goPrintValues dcsTree cpass1.dcsTree.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2125
2126   #merge trending
2127   rm -f ${commonOutputPath}/trending_merged.root
2128   goMerge trending.list ${commonOutputPath}/trending.root ${configFile} "${extraOpts[@]}" &> mergeTrending.log
2129
2130   goMakeSummaryTree ${commonOutputPath} 0
2131   goMakeSummaryTree ${commonOutputPath} 1
2132
2133   goCreateQAplots "qa.list" "${productionID}" "QAplots" "${configFile}" "${extraOpts[@]}" filteringList="filtering.list" &>createQAplots.log
2134
2135   #make a merged summary tree out of the QA trending, dcs trees and log summary trees
2136   goMakeMergedSummaryTree
2137
2138   #if set, email the summary
2139   [[ -n ${MAILTO} ]] && cat ${log} | mail -s "benchmark ${productionID} done" ${MAILTO}
2140
2141   return 0
2142 }
2143
2144 goMakeSummaryTree()
2145 {
2146   if [[ $# -lt 1 ]] ; then
2147     return
2148   fi
2149   #1. define vars/arrays
2150   DIR=${1} #use input or exec in current dir
2151   pass=${2-"0"} #pass from input
2152   outfile="summary_pass${pass}.tree"
2153   Ncolumns=0
2154   test -f ${outfile} && : >${outfile}
2155   errfile=${outfile/tree/err}
2156   test -f ${errfile} && : >${errfile}
2157
2158   declare -a counterString=(TOFevents TOFtracks TPCevents TPCtracks TRDevents TRDtracks T0events SDDevents SDDtracks MeanVertexevents)
2159   Ncounter=${#counterString[@]}
2160
2161   declare -a statusString=(TRDStatus TOFStatus TPCStatus T0Status MeanVertexStatus)
2162   Nstatus=${#statusString[@]}
2163
2164
2165   declare -a ratesString=(rec stderr calib qa_barrel qa_outer)
2166   Nrates=${#ratesString[@]}
2167
2168   runs=( $(ls -1 ${DIR}/meta/merge.cpass0* | while read x; do guessRunNumber $x; done) )
2169   Nruns=${#runs[@]}
2170
2171   echo -n runnumber/I >>${outfile}
2172   echo -n :cpass${pass}status/I >>${outfile}
2173   echo -n :cpass${pass}QAstatus/I >>${outfile}
2174   for i in ${ratesString[@]}; do
2175     echo -n :${i}OK/I >>${outfile}
2176     echo -n :${i}BAD/I >>${outfile}
2177
2178   done
2179   for i in ${counterString[@]} ${statusString[@]} ; do
2180     echo -n :${i}/I >>${outfile}
2181   done
2182   Ncolumns=$((2 + 2*Nrates + Ncounter + Nstatus))
2183   echo >> ${outfile}
2184
2185   #2. loop runs
2186
2187   for runnumber in ${runs[@]} ; do
2188
2189
2190
2191     filejob="${DIR}/meta/cpass${pass}.job*.run${runnumber}.done"
2192     filemerge="${DIR}/meta/merge.cpass${pass}.run${runnumber}.done"
2193     fileOCDB=$(grep /ocdb.log ${filemerge} | awk '{print $1}')
2194     if ! $(/bin/ls ${filemerge} &>/dev/null) ; then
2195       echo "${filemerge} does not exist!" >>${errfile}
2196       continue
2197     elif ! $(/bin/ls ${filejob} &>/dev/null) ; then
2198       echo "${filejob} does not exist!" >>${errfile}
2199       echo -n ${runnumber} >> ${outfile}
2200       for i in $(seq ${Ncolumns}) ; do
2201         echo -n "-1" >> ${outfile}
2202       done
2203       echo >> ${outfile}
2204       continue
2205     fi
2206     echo -n ${runnumber} >> ${outfile}
2207     #pass0status= grep '/ocdb.log' ${filemerge} |  cut -d' ' -f2 | tr OK x1 | tr BAD xx0 | tr -d 'x'
2208     passStatus=$(grep '/ocdb.log' ${filemerge} | grep OK | wc -l)
2209     echo -n " ${passStatus}" >> ${outfile}
2210     qaStatus=$(grep '/mergeMakeOCDB.log' ${filemerge} | grep OK | wc -l)
2211     echo -n " ${qaStatus}" >> ${outfile}
2212
2213
2214     #fill OK/BAD rates
2215     for i in $(seq 0 $((${Nrates}-1))) ; do
2216       var1=$(grep "/${ratesString[${i}]}.log" ${filejob} | grep OK | wc -l)
2217       var2=$(grep "/${ratesString[${i}]}.log" ${filejob} | grep BAD | wc -l)
2218
2219       if [[ ${ratesString[${i}]} == "stderr" ]] ; then
2220         var1=$(grep "stderr" ${filejob} | grep OK | wc -l)
2221         var2=$(grep "stderr" ${filejob} | grep "rec.log" | grep BAD | wc -l)
2222       fi
2223       echo -n " ${var1}" >> ${outfile}
2224       echo -n " ${var2}" >> ${outfile}
2225     done
2226
2227     if [[ -f ${fileOCDB} ]] ; then
2228       #fill counter
2229       for i in $(seq 0 $((${Ncounter}-1))) ; do
2230         var1=$(grep Monalisa ${fileOCDB} | grep ${counterString[${i}]} | cut -f2)
2231         echo -n " ${var1:-"-1"}" >> ${outfile}
2232       done
2233
2234       #fill status
2235       for i in $(seq 0 $((${Nstatus}-1))) ; do
2236         var1=$(grep "calibration status=" ${fileOCDB} | grep ${statusString[${i}]/Status/} | cut -d'=' -f2)
2237         echo -n " ${var1:-"-1"}" >> ${outfile}
2238       done
2239     fi
2240     echo >> ${outfile}
2241   done
2242
2243   return 0
2244 }
2245
2246 parseConfig()
2247 {
2248   configFile=${1}
2249   shift
2250   args=("$@")
2251
2252   #some defaults
2253   #autoOCDB=0
2254   defaultOCDB="raw://"
2255   #runNumber=167123
2256   #makeflowPath="/hera/alice/aux/cctools/bin"
2257   #makeflowOptions="-T wq -N alice -d all -C ali-copilot.cern.ch:9097"
2258   #makeflowOptions="-T wq -N alice -C ali-copilot.cern.ch:9097"
2259   makeflowOptions=""
2260   #batchCommand="/usr/bin/qsub"
2261   batchFlags="-b y -cwd -l h_rt=24:0:0,h_rss=4G "
2262   baseOutputDirectory="$PWD/output"
2263   #alirootEnv="/cvmfs/alice.cern.ch/bin/alienv setenv AliRoot/v5-04-34-AN -c"
2264   #alirootEnv="/home/mkrzewic/alisoft/balice_master.sh"
2265   #trustedQAtrainMacro='/hera/alice/mkrzewic/gsisvn/Calibration/QAtrain_duo.C'
2266   reconstructInTemporaryDir=0
2267   recoTriggerOptions="\"\""
2268   percentProcessedFilesToContinue=100
2269   maxSecondsToWait=$(( 3600*24 ))
2270   nEvents=-1
2271   nMaxChunks=0
2272   postSetUpActionCPass0=""
2273   postSetUpActionCPass1=""
2274   runCPass0reco=1
2275   runCPass0MergeMakeOCDB=1
2276   runCPass1reco=1
2277   runCPass1MergeMakeOCDB=1
2278   runESDfiltering=1
2279   filteringFactorHighPt=1e2
2280   filteringFactorV0s=1e1
2281   MAILTO=""
2282   #pretend=1
2283   #dontRedirectStdOutToLog=1
2284   logToFinalDestination=1
2285   ALIROOT_FORCE_COREDUMP=1
2286
2287   #first, source the config file
2288   if [ -f ${configFile} ]; then
2289     source ${configFile}
2290   else
2291     echo "config file ${configFile} not found!, skipping..."
2292   fi
2293
2294   #then, parse the options as they override the options from file
2295   for opt in "${args[@]}"; do
2296     if [[ ! "${opt}" =~ .*=.* ]]; then
2297       echo "badly formatted option ${var}, should be: option=value, stopping..."
2298       return 1
2299     fi
2300     local var="${opt%%=*}"
2301     local value="${opt#*=}"
2302     echo "${var} = ${value}"
2303     export ${var}="${value}"
2304   done
2305
2306   #do some checking
2307   [[ -z ${alirootEnv} ]] && echo "alirootEnv not defined!" && return 1
2308
2309   #export the aliroot function if defined to override normal behaviour
2310   [[ $(type -t aliroot) =~ "function" ]] && export -f aliroot
2311
2312   return 0
2313 }
2314
2315 aliroot()
2316 {
2317   args="$@"
2318   if [[ -n ${useProfilingCommand} ]]; then
2319     valgrindLogFile="cpu.txt"
2320     [[ "${args}" =~ rec ]] && valgrindLogFile="cpu_rec.txt"
2321     [[ "${args}}" =~ Calib ]] && valgrindLogFile="cpu_calib.txt"
2322     [[ -n ${useProfilingCommand} ]] && useProfilingCommand="${useProfilingCommand} --log-file=${valgrindLogFile}"
2323     echo running ${useProfilingCommand} aliroot ${args}
2324     ${useProfilingCommand} aliroot ${args}
2325   else
2326     #to prevent an infinite recursion use "command aliroot" to disable
2327     #aliases and functions
2328     echo running command aliroot ${args}
2329     command aliroot "$@"
2330   fi
2331   return 0
2332 }
2333
2334 guessRunData()
2335 {
2336   #guess the period from the path, pick the rightmost one
2337   period=""
2338   runNumber=""
2339   year=""
2340   pass=""
2341   legoTrainRunNumber=""
2342   dataType=""
2343
2344   local shortRunNumber=""
2345   local IFS="/"
2346   declare -a path=( $1 )
2347   local dirDepth=$(( ${#path[*]}-1 ))
2348   i=0
2349   #for ((x=${dirDepth};x>=0;x--)); do
2350   for ((x=0;x<=${dirDepth};x++)); do
2351
2352     [[ $((x-1)) -ge 0 ]] && local fieldPrev=${path[$((x-1))]}
2353     local field=${path[${x}]}
2354     local fieldNext=${path[$((x+1))]}
2355
2356     [[ ${field} =~ ^[0-9]*$ && ${fieldNext} =~ (.*\.zip$|.*\.root$) ]] && legoTrainRunNumber=${field}
2357     [[ -n ${legoTrainRunNumber} && -z ${pass} ]] && pass=${fieldPrev}
2358     [[ ${field} =~ ^LHC[0-9][0-9][a-z].*$ ]] && period=${field%_*}
2359     [[ ${field} =~ ^000[0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && runNumber=${field#000}
2360     [[ ${field} =~ ^[0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && shortRunNumber=${field}
2361     [[ ${field} =~ ^20[0-9][0-9]$ ]] && year=${field}
2362     [[ ${field} =~ ^(^sim$|^data$) ]] && dataType=${field}
2363     (( i++ ))
2364   done
2365   [[ -z ${legoTrainRunNumber} ]] && pass=${path[$((dirDepth-1))]}
2366   [[ "${dataType}" =~ ^sim$ ]] && pass="passMC" && runNumber=${shortRunNumber}
2367
2368   #if [[ -z ${dataType} || -z ${year} || -z ${period} || -z ${runNumber}} || -z ${pass} ]];
2369   if [[ -z ${runNumber}} ]];
2370   then
2371     #error condition
2372     return 1
2373   else
2374     #ALL OK
2375     return 0
2376   fi
2377   return 0
2378 }
2379
2380 main "$@"