PWGPP/benchmark/benchmark.sh

   1 #!/bin/bash
   2 #include benchmark.config
   3
   4 # this script runs the CPass0/CPass1 train
   5 # produced OCDB updates are local
   6
   7 main()
   8 {
   9   #run in proper mode depending on the selection
  10   if [[ $# -lt 1 ]]; then
  11     if [[ ! "${0}" =~ "bash" ]]; then
  12       echo "uses makeflow:"
  13       echo " ${0} \"run\" productionID inputList configFile [extraOpts]"
  14       echo "uses a batch system (SGE):"
  15       echo " ${0} \"submit\" productionID inputList configFile [extraOpts]"
  16       echo "extraOpts if non-empty override the config file, e.g.:"
  17       echo " ${0} submit test1 benchmark.list benchmark.config runNumber=169123 nEvents=10"
  18     fi
  19     return
  20   fi
  21
  22   #define some aliases - default is to call one of the functions directly
  23   runMode=${1}
  24   umask 0002
  25   shift
  26   case ${runMode} in
  27     "CPass0") goCPass0 "$@";;
  28     "CPass1") goCPass1 "$@";;
  29     "MakeLocalOCDBaccessConfig") goMakeLocalOCDBaccessConfig "$@";;
  30     "MergeCPass0") goMergeCPass0 "$@";;
  31     "MergeCPass1") goMergeCPass1 "$@";;
  32     "MakeFilteredTrees") goMakeFilteredTrees "$@";;
  33     "MakeSummary") goMakeSummary "$@";;
  34     "run") goSubmitMakeflow "$@";;
  35     "submit") goSubmitBatch "$@";;
  36     "test") goTest "$@";;
  37     "GenerateMakeflow") goGenerateMakeflow "$@";;
  38     "PrintValues") goPrintValues "$@";;
  39     "CreateQAplots") goCreateQAplots "$@";;
  40     "WaitForOutput") goWaitForOutput "$@";;
  41     "Merge") goMerge "$@";;
  42     *)
  43       ${runMode} "$@"
  44     ;;
  45   esac
  46   return 0
  47 }
  48
  49 generateMC()
  50 {
  51   #generate one raw chunk in current directory
  52   SEED=${JOB_ID}${SGE_TASK_ID}
  53   export CONFIG_SEED=${SEED}
  54   runNumber=${1}
  55   OCDBpath=${2}
  56   nEventsim=${3}
  57   if [[ -n ${pretend} ]]; then
  58     sleep ${pretendDelay}
  59     touch galice.root
  60   else
  61     if [[ -f sim.C && -f Config.C ]] ; then
  62         time aliroot -b -q -x sim.C\(${runNumber},\"${OCDBpath}\",${nEventsim}\) >sim.log 2>&1
  63         mv syswatch.log simwatch.log
  64     fi
  65   fi
  66 }
  67
  68 goCPass0()
  69 (
  70   umask 0002
  71
  72   targetDirectory=${1}
  73   inputList=${2}
  74   nEvents=${3}
  75   ocdbPath=${4}
  76   configFile=${5}
  77   runNumber=${6}
  78   jobindex=${7}
  79   shift 7
  80   if ! parseConfig ${configFile} "$@"; then return 1; fi
  81
  82   #record the working directory provided by the batch system
  83   batchWorkingDirectory=${PWD}
  84
  85   #use the jobindex only if set and non-negative
  86   if [[ -z ${jobindex} || ${jobindex} -lt 0 ]]; then
  87     [[ -n "${LSB_JOBINDEX}" ]] && jobindex=${LSB_JOBINDEX}
  88     [[ -n "${SGE_TASK_ID}" ]] && jobindex=${SGE_TASK_ID}
  89     if [[ -z ${jobindex} ]]; then
  90       echo "no jobindex!"
  91       return 1
  92     fi
  93   fi
  94
  95   [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
  96   doneFile="${commonOutputPath}/meta/cpass0.job${jobindex}.run${runNumber}.done"
  97   [[ -n ${useProfilingCommand} ]] && doneFile="${commonOutputPath}/meta/profiling.cpass0.job${jobindex}.run${runNumber}.done"
  98
  99   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
 100
 101   if [[ -n ${ALIROOT_FORCE_COREDUMP} ]]; then
 102     ulimit -c unlimited
 103     export ALIROOT_FORCE_COREDUMP
 104   fi
 105
 106   #the contents of this is stored in the tree and used later (e.g. AliAnalysisTaskPIDResponse)!
 107   #at the QA stage the pass number is guessed from the path stored here.
 108   #The Format is:
 109   #Packages= ;OutputDir= ;LPMPass= ;TriggerAlias= ;LPMRunNumber= ;LPMProductionType= ;LPMInteractionType= ;LPMProductionTag= ;LPMAnchorRun= ;LPMAnchorProduction= ;LPMAnchorYear=
 110   export PRODUCTION_METADATA="OutputDir=cpass0"
 111
 112   if [[ "${inputList}" =~ \.root$ ]]; then
 113     infile=${inputList}
 114   else
 115     infile=$(sed -ne "${jobindex}p" ${inputList} | egrep '\s*\w*/\w*')
 116   fi
 117   chunkName=${infile##*/}
 118
 119   outputDir=${targetDirectory}/${jobindex}_${chunkName%.*}
 120   mkdir -p ${outputDir}
 121   if [[ ! -d ${outputDir} ]]; then
 122     touch ${doneFile}
 123     echo "cannot make ${outputDir}" >> ${doneFile}
 124     return 1
 125   fi
 126
 127   #runpath=${PWD}/rundir_cpass0_${runNumber}_${jobindex}
 128   runpath=${outputDir}
 129   #[[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
 130   [[ ${reconstructInTemporaryDir} -eq 1 ]] && runpath=$(mktemp -d -t cpass0.XXXXXX)
 131   mkdir -p ${runpath}
 132   if [[ ! -d ${runpath} ]]; then
 133     touch ${doneFile}
 134     echo "cannot make runpath ${runpath}" >> ${doneFile}
 135     return 1
 136   fi
 137   if ! cd ${runpath}; then
 138     touch ${doneFile}
 139     echo "PWD=$PWD is not the runpath=${runpath}" >> ${doneFile}
 140     return 1
 141   fi
 142
 143   #runCPassX/C expects the raw chunk to be linked in the run dir
 144   #despite it being accessed by the full path
 145   ln -s ${infile} ${runpath}/${chunkName}
 146
 147   #####MC
 148   if [[ -n ${generateMC} ]]; then
 149     olddir=${PWD}
 150     outputDirMC=${commonOutputPath}/000${runNumber}/sim/${jobindex}
 151     simrunpath=${outputDirMC}
 152     #[[ ${simulateInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && simrunpath=${TMPDIR}
 153     [[ ${simulateInTemporaryDir} -eq 1 ]] && simrunpath=$(mktemp -d -t cpass0MC.XXXXXX)
 154     mkdir -p ${outputDirMC}
 155     mkdir -p ${simrunpath}
 156     if cd ${simrunpath}; then
 157
 158       filesMC=(
 159       "${batchWorkingDirectory}/sim.C"
 160       "${batchWorkingDirectory}/rec.C"
 161       "${batchWorkingDirectory}/Config.C"
 162       "${batchWorkingDirectory}/OCDB_*.root"
 163       )
 164       for file in ${filesMC[*]}; do
 165         [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
 166       done
 167
 168       generateMC ${runNumber} ${ocdbPath} ${nEvents}
 169
 170       [[ ! "${simrunpath}" =~ "${outputDirMC}" ]] && mv * ${outputDirMC} #TODO check if it works
 171       cd ${olddir}
 172
 173       ln -s ${outputDirMC}/* ${runpath}/
 174
 175       inputList=${outputDirMC}/galice.root #TODO not valid outside shell !!!
 176       infile=""
 177     fi
 178   fi
 179   ######
 180
 181   if [[ ! -f ${inputList} && -z ${pretend} ]]; then
 182     touch ${doneFile}
 183     echo "input file ${inputList} not found, exiting..." >> ${doneFile}
 184     return 1
 185   fi
 186
 187   logOutputDir=${runpath}
 188   [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir}
 189   [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/stdout
 190   #[[ -z ${dontRedirectStdOutToLog} ]] && exec 2> ${logOutputDir}/stderr
 191   echo "${0} $*"
 192
 193   echo "#####################"
 194   echo CPass0:
 195   echo JOB setup
 196   echo nEvents            ${nEvents}
 197   echo runNumber          ${runNumber}
 198   echo ocdbPath           ${ocdbPath}
 199   echo infile             ${infile}
 200   echo chunkName          ${chunkName}
 201   echo jobindex           ${jobindex}
 202   echo recoTriggerOptions ${recoTriggerOptions}
 203   echo targetDirectory    ${targetDirectory}
 204   echo commonOutputPath         ${commonOutputPath}
 205   echo doneFile      ${doneFile}
 206   echo batchWorkingDirectory=$batchWorkingDirectory
 207   echo runpath            ${runpath}
 208   echo outputDir          ${outputDir}
 209   echo PWD                ${PWD}
 210   echo ALICE_ROOT         ${ALICE_ROOT}
 211   echo "########## ###########"
 212
 213   alirootInfo > ALICE_ROOT.log
 214
 215   filesCPass0=(
 216                "${batchWorkingDirectory}/runCPass0.sh"
 217                "${batchWorkingDirectory}/recCPass0.C"
 218                "${batchWorkingDirectory}/runCalibTrain.C"
 219                "${batchWorkingDirectory}/localOCDBaccessConfig.C"
 220                "${batchWorkingDirectory}/OCDB.root"
 221                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/runCPass0.sh"
 222                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/recCPass0.C"
 223                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/runCalibTrain.C"
 224   )
 225
 226   for file in ${filesCPass0[*]}; do
 227     [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
 228     [[ ${file##*/} =~ .*\.sh ]] && chmod +x ${file##*/}
 229   done
 230
 231   echo "this directory (${PWD}) contents:"
 232   /bin/ls
 233   echo
 234   chmod u+x runCPass0.sh
 235
 236   sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh
 237
 238   if [[ -n ${postSetUpActionCPass0} ]]; then
 239     echo "running ${postSetUpActionCPass0}"
 240     eval ${postSetUpActionCPass0}
 241   fi
 242
 243   #run CPass0
 244   echo "${runpath}/runCPass0.sh ${infile} ${nEvents} ${runNumber} ${ocdbPath} ${recoTriggerOptions}"
 245   if [[ -n ${pretend} ]]; then
 246     sleep ${pretendDelay}
 247     touch AliESDs.root
 248     touch AliESDfriends.root
 249     touch AliESDfriends_v1.root
 250     touch rec.log
 251     touch calib.log
 252   else
 253     echo ./runCPass0.sh "${infile}" "${nEvents}" "${runNumber}" "${ocdbPath}" "${recoTriggerOptions}"
 254     ./runCPass0.sh "${infile}" "${nEvents}" "${runNumber}" "${ocdbPath}" "${recoTriggerOptions}"
 255   fi
 256
 257   #move stuff to final destination
 258   echo "this directory (${PWD}) contents:"
 259   /bin/ls
 260   echo
 261
 262   echo rm -f ./${chunkName}
 263   rm -f ./${chunkName}
 264   echo "cp -R ${runpath}/* ${outputDir}"
 265   cp -p -R ${runpath}/* ${outputDir}
 266   echo
 267
 268   #validate CPass0
 269   cd ${outputDir}
 270   touch ${doneFile}
 271   echo "dir ${outputDir}" >> ${doneFile}
 272   if summarizeLogs >> ${doneFile}; then
 273     [[ -f ${outputDirMC}/galice.root ]] && echo "sim ${outputDirMC}/galice.root" >> ${doneFile}
 274     [[ -f AliESDfriends_v1.root ]] && echo "calibfile ${outputDir}/AliESDfriends_v1.root" >> ${doneFile}
 275     [[ -f AliESDs.root ]] && echo "esd ${outputDir}/AliESDs.root" >> ${doneFile}
 276   fi
 277
 278   [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath} && echo "removing ${runpath}"
 279   return 0
 280 )
 281
 282 goCPass1()
 283 (
 284   umask 0002
 285
 286   targetDirectory=${1}
 287   inputList=${2}
 288   nEvents=${3}
 289   ocdbPath=${4}
 290   configFile=${5}
 291   runNumber=${6}
 292   jobindex=${7}
 293   shift 7
 294   extraOpts=("$@")
 295   if ! parseConfig ${configFile} "$@"; then return 1; fi
 296
 297   #record the working directory provided by the batch system
 298   batchWorkingDirectory=${PWD}
 299
 300   #use the jobindex only if set and non-negative
 301   if [[ -z ${jobindex} || ${jobindex} -lt 0 ]]; then
 302     [[ -n "${LSB_JOBINDEX}" ]] && jobindex=${LSB_JOBINDEX}
 303     [[ -n "${SGE_TASK_ID}" ]] && jobindex=${SGE_TASK_ID}
 304     if [[ -z ${jobindex} ]]; then
 305       echo "no jobindex!"
 306       return 1
 307     fi
 308   fi
 309
 310   [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
 311   doneFile="${commonOutputPath}/meta/cpass1.job${jobindex}.run${runNumber}.done"
 312   [[ -n ${useProfilingCommand} ]] && doneFile="${commonOutputPath}/meta/profiling.cpass1.job${jobindex}.run${runNumber}.done"
 313
 314   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
 315
 316   if [[ -n ${ALIROOT_FORCE_COREDUMP} ]]; then
 317     ulimit -c unlimited
 318     export ALIROOT_FORCE_COREDUMP
 319   fi
 320
 321   #the contents of this is stored in the tree and used later (e.g. AliAnalysisTaskPIDResponse)!
 322   #at the QA stage the pass number is guessed from the path stored here.
 323   #The Format is:
 324   #Packages= ;OutputDir= ;LPMPass= ;TriggerAlias= ;LPMRunNumber= ;LPMProductionType= ;LPMInteractionType= ;LPMProductionTag= ;LPMAnchorRun= ;LPMAnchorProduction= ;LPMAnchorYear=
 325   export PRODUCTION_METADATA="OutputDir=cpass1"
 326
 327   if [[ ! -f ${inputList} && -z ${pretend} ]]; then
 328     touch ${doneFile}
 329     echo "input file ${inputList} not found, exiting..." >> ${doneFile}
 330     return 1
 331   fi
 332   if [[ "${inputList}" =~ \.root$ ]]; then
 333     infile=${inputList}
 334   else
 335     infile=$(sed -ne "${jobindex}p" ${inputList} | egrep '\s*\w*/\w*')
 336   fi
 337   chunkName=${infile##*/}
 338
 339   outputDir=${targetDirectory}/${jobindex}_${chunkName%.*}
 340   mkdir -p ${outputDir}
 341   if [[ ! -d ${outputDir} ]];then
 342     touch ${doneFile}
 343     echo "cannot make ${outputDir}" >> ${doneFile}
 344     return 1
 345   fi
 346
 347   #runpath=${PWD}/rundir_cpass1_${runNumber}_${jobindex}
 348   runpath=${outputDir}
 349   #[[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
 350   [[ ${reconstructInTemporaryDir} -eq 1 ]] && runpath=$(mktemp -d -t cpass1.XXXXXX)
 351
 352   #MC
 353   if [[ "${infile}" =~ galice\.root ]]; then
 354     ln -s ${inputList%/*}/* ${runpath}
 355     infile=""
 356   fi
 357
 358   #init the running path
 359   mkdir -p ${runpath}
 360   if [[ ! -d ${runpath} ]]; then
 361    touch ${doneFile}
 362    echo "cannot make runpath ${runpath}" >> ${doneFile}
 363    return 1
 364  fi
 365   if ! cd ${runpath}; then
 366     touch ${doneFile}
 367     echo "PWD=$PWD is not the runpath=${runpath}" >> ${doneFile}
 368     return 1
 369   fi
 370
 371   #this is needed for runCPass1.sh
 372   ln -s ${infile} ${runpath}/${chunkName}
 373
 374   logOutputDir=${runpath}
 375   [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir}
 376   [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/stdout
 377   #[[ -z ${dontRedirectStdOutToLog} ]] && exec 2> ${logOutputDir}/stderr
 378   echo "${0} $*"
 379
 380   echo "#####################"
 381   echo CPass1:
 382   echo JOB setup
 383   echo nEvents            ${nEvents}
 384   echo runNumber          ${runNumber}
 385   echo ocdbPath           ${ocdbPath}
 386   echo infile             ${infile}
 387   echo chunkName          ${chunkName}
 388   echo jobindex           ${jobindex}
 389   echo recoTriggerOptions ${recoTriggerOptions}
 390   echo targetDirectory    ${targetDirectory}
 391   echo commonOutputPath         ${commonOutputPath}
 392   echo doneFile      ${doneFile}
 393   echo runpath            ${runpath}
 394   echo outputDir          ${outputDir}
 395   echo batchWorkingDirectory ${batchWorkingDirectory}
 396   echo ALICE_ROOT         ${ALICE_ROOT}
 397   echo PWD                ${PWD}
 398   echo "########## ###########"
 399
 400   alirootInfo > ALICE_ROOT.log
 401
 402   filesCPass1=(
 403                "${batchWorkingDirectory}/runCPass1.sh"
 404                "${batchWorkingDirectory}/recCPass1.C"
 405                "${batchWorkingDirectory}/recCPass1_OuterDet.C"
 406                "${batchWorkingDirectory}/runCalibTrain.C"
 407                "${batchWorkingDirectory}/QAtrain_duo.C"
 408                "${batchWorkingDirectory}/localOCDBaccessConfig.C"
 409                "${batchWorkingDirectory}/${configFile}"
 410                "${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz"
 411                "${batchWorkingDirectory}/OCDB.root"
 412                "${trustedQAtrainMacro}"
 413                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/runCPass1.sh"
 414                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/recCPass1.C"
 415                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/recCPass1_OuterDet.C"
 416                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/runCalibTrain.C"
 417                "${ALICE_ROOT}/ANALYSIS/macros/QAtrain_duo.C"
 418   )
 419
 420   for file in "${filesCPass1[@]}"; do
 421     [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
 422     [[ ${file##*/} =~ .*\.sh ]] && echo "making ${file##*/} executable" && chmod +x ${file##*/}
 423   done
 424
 425   echo "this directory (${PWD}) contents:"
 426   /bin/ls
 427   echo
 428
 429   #remove spaces around commas from calls to root
 430   sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh
 431
 432   if [[ -n ${postSetUpActionCPass1} ]]; then
 433     echo "running ${postSetUpActionCPass1}"
 434     eval ${postSetUpActionCPass1}
 435     echo
 436   fi
 437
 438   #configure local OCDB storage from CPass0 (creates the localOCDBaccessConfig.C script)
 439   if [[ -f cpass0.localOCDB.${runNumber}.tgz ]]; then
 440     echo goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz"
 441     goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz"
 442   else
 443     echo "WARNING: file cpass0.localOCDB.${runNumber}.tgz not found!"
 444   fi
 445
 446   if [[ ! $(/bin/ls -1 OCDB/*/*/*/*.root 2>/dev/null) ]]; then
 447     touch ${doneFile}
 448     echo "cpass0 produced no calibration! exiting..." >> ${doneFile}
 449     return 1
 450   fi
 451
 452   #create the Barrel and OuterDet directories for CPass1 and link the local OCDB directory
 453   #there to make the localOCDBaccessConfig.C file work, since it may point to the OCDB
 454   #entries using a relative path, e.g. local://./OCDB
 455   echo "linking the OCDB/ for Barrel and OuterDet directories"
 456   mkdir Barrel OuterDet
 457   ls -l
 458   ln -s ../OCDB Barrel/OCDB
 459   ln -s ../OCDB OuterDet/OCDB
 460
 461   #setup the filtering
 462   #the following option enables the filtering task inside the QAtrain_duo.C
 463   [[ -n $runESDfiltering ]] && export QA_TaskFilteredTree=1
 464   #set the downscaling factors during the filtering fro expert QA (overrides the previous values)
 465   if [[ -n ${filteringFactorHighPt} ]]; then
 466     export AliAnalysisTaskFilteredTree_fLowPtTrackDownscaligF=${filteringFactorHighPt}
 467   fi
 468   if [[ -n ${filteringFactorV0s} ]]; then
 469     export AliAnalysisTaskFilteredTree_fLowPtV0DownscaligF=${filteringFactorV0s}
 470   fi
 471
 472   #run CPass1
 473   chmod u+x runCPass1.sh
 474   echo "${runpath}/runCPass1.sh ${infile} ${nEvents} ${runNumber} ${ocdbPath} ${recoTriggerOptions}"
 475   if [[ -n ${pretend} ]]; then
 476     sleep ${pretendDelay}
 477     touch AliESDs_Barrel.root
 478     touch AliESDfriends_Barrel.root
 479     touch AliESDfriends_v1.root
 480     touch QAresults_barrel.root
 481     touch EventStat_temp_barrel.root
 482     touch AODtpITS.root
 483     touch AliESDs_Outer.root
 484     touch AliESDfriends_Outer.root
 485     touch QAresults_outer.root
 486     touch EventStat_temp_outer.root
 487     touch rec.log
 488     touch calib.log
 489     touch qa.log
 490     touch filtering.log FilterEvents_Trees.root
 491   else
 492     ./runCPass1.sh "${infile}" "${nEvents}" "${runNumber}" "${ocdbPath}" "${recoTriggerOptions}"
 493
 494     [[ ! -f AliESDs_Barrel.root && -f Barrel/AliESDs.root ]] && mv Barrel/AliESDs.root AliESDs_Barrel.root
 495     [[ ! -f AliESDfriends_Barrel.root && -f Barrel/AliESDfriends.root ]] && mv Barrel/AliESDfriends.root AliESDfriends_Barrel.root
 496     [[ ! -f AliESDfriends_v1.root && -f Barrel/AliESDfriends_v1.root ]] && mv Barrel/AliESDfriends_v1.root .
 497     [[ ! -f QAresults_barrel.root && -f Barrel/QAresults_barrel.root ]] && mv Barrel/QAresults_barrel.root .
 498     [[ ! -f AliESDs_Outer.root && -f OuterDet/AliESDs.root ]] && mv OuterDet/AliESDs.root AliESDs_Outer.root
 499     [[ ! -f AliESDfriends_Outer.root && -f OuterDet/AliESDfriends.root ]] && mv OuterDet/AliESDfriends.root AliESDfriends_Outer.root
 500     [[ ! -f QAresults_outer.root && -f OuterDet/QAresults_outer.root ]] && mv OuterDet/QAresults_outer.root .
 501     [[ ! -f FilterEvents_Trees.root && -f Barrel/FilterEvents_Trees.root ]] && mv Barrel/FilterEvents_Trees.root .
 502
 503     #make the filtered tree (if requested and not already produced by QA
 504     [[ -f AliESDs_Barrel.root ]] && echo "AliESDs_Barrel.root" > filtered.list
 505     if [[ -n ${runESDfiltering} && ! -f FilterEvents_Trees.root && -f filtered.list ]]; then
 506       goMakeFilteredTrees ${PWD} ${runNumber} "${PWD}/filtered.list" ${filteringFactorHighPt} ${filteringFactorV0s} ${ocdbPath} 1000000 0 10000000 0 ${configFile} AliESDs_Barrel.root "${extraOpts[@]}" >filtering.log
 507     else
 508       echo ""
 509     fi
 510
 511   fi
 512
 513   ##handle possible crashes in QA (happens often in trunk)
 514   ##rerun QA with a trusted aliroot version
 515   #if [[ $(validateLog qa_barrel.log) ]]; then
 516   #  echo "qa_barrel.log not validated!"
 517   #fi
 518   #if [[ ! -f QAresults_barrel.root && -f ${setupTrustedAliROOTenvInCurrentShell} || $(validateLog qa_barrel.log) ]]; then
 519   #  echo "WARNING: using trusted QA aliroot ${ALICE_ROOT}"
 520   #  source ${setupTrustedAliROOTenvInCurrentShell}
 521   #  cd Barrel
 522   #  rm QAresults_barrel.root
 523   #  rm EventStat_temp_barrel.root
 524   #  rm AODtpITS.root
 525   #  [[ ! -f AliESDs.root ]] && ln -s ../AliESDs_Barrel.root AliESDs.root
 526   #  [[ ! -f AliESDfriends.root ]] && ln -s ../AliESDfriends_Barrel.root AliESDfriends.root
 527   #  if [[ -n ${trustedQAtrainMacro} ]]; then
 528   #    eval "cp ${trustedQAtrainMacro} QAtrain_duo_trusted.C"
 529   #  fi
 530   #  echo executing aliroot -b -q "QAtrain_duo_trusted.C(\"_barrel\",${runNumber},\"wn.xml\",0,\"${ocdbPath}\")"
 531   #  time aliroot -b -q "QAtrain_duo.C(\"_barrel\",${runNumber},\"wn.xml\",0,\"${ocdbPath}\")" &> ../qa_barrel_trusted.log
 532   #  cd ../
 533   #fi
 534
 535   #move stuff to final destination
 536   echo "this directory (${PWD}) contents:"
 537   /bin/ls
 538   echo rm -f ./${chunkName}
 539   rm -f ./${chunkName}
 540   echo "cp -R ${runpath}/* ${outputDir}"
 541   cp -pf -R ${runpath}/* ${outputDir}
 542   echo
 543
 544   #validate CPass1
 545   cd ${outputDir}
 546   touch ${doneFile}
 547   echo "dir ${outputDir}" >> ${doneFile}
 548   if summarizeLogs >> ${doneFile}; then
 549     [[ -f AliESDs_Barrel.root ]] && echo "esd ${outputDir}/AliESDs_Barrel.root" >> ${doneFile}
 550     [[ -f AliESDfriends_v1.root ]] && echo "calibfile ${outputDir}/AliESDfriends_v1.root" >> ${doneFile}
 551     [[ -f QAresults_Barrel.root ]] && echo "qafile ${outputDir}/QAresults_Barrel.root" >> ${doneFile}
 552     [[ -f QAresults_Outer.root ]] && echo "qafile ${outputDir}/QAresults_Outer.root" >> ${doneFile}
 553     [[ -f QAresults_barrel.root ]] && echo "qafile ${outputDir}/QAresults_barrel.root" >> ${doneFile}
 554     [[ -f QAresults_outer.root ]] && echo "qafile ${outputDir}/QAresults_outer.root" >> ${doneFile}
 555     [[ -f FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFile}
 556   else
 557     if grep "qa_outer.log.*OK" ${doneFile} > /dev/null; then
 558       [[ -f QAresults_Outer.root ]] && echo "qafile ${outputDir}/QAresults_Outer.root" >> ${doneFile}
 559       [[ -f QAresults_outer.root ]] && echo "qafile ${outputDir}/QAresults_outer.root" >> ${doneFile}
 560     fi
 561     if grep "qa_barrel.log.*OK" ${doneFile} > /dev/null; then
 562       [[ -f QAresults_Barrel.root ]] && echo "qafile ${outputDir}/QAresults_Barrel.root" >> ${doneFile}
 563       [[ -f QAresults_barrel.root ]] && echo "qafile ${outputDir}/QAresults_barrel.root" >> ${doneFile}
 564     fi
 565     if grep "filtering.log.*OK" ${doneFile} > /dev/null; then
 566       [[ -f FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFile}
 567     fi
 568   fi
 569
 570   [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
 571   return 0
 572 )
 573
 574
 575 goMergeCPass0()
 576 (
 577   #
 578   # find the output files and merge them
 579   #
 580
 581   outputDir=${1}
 582   ocdbStorage=${2}
 583   configFile=${3}
 584   runNumber=${4}
 585   calibrationFilesToMerge=${5}  #can be a non-existent file, will then be produced on the fly
 586   shift 5
 587   if ! parseConfig ${configFile} "$@"; then return 1; fi
 588
 589   #record the working directory provided by the batch system
 590   batchWorkingDirectory=${PWD}
 591
 592   [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
 593   doneFile="${commonOutputPath}/meta/merge.cpass0.run${runNumber}.done"
 594
 595   umask 0002
 596   ulimit -c unlimited
 597
 598   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
 599
 600   #runpath=${PWD}/rundir_cpass0_Merge_${runNumber}
 601   runpath=${outputDir}
 602   #[[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
 603   [[ ${reconstructInTemporaryDir} -eq 1 ]] && runpath=$(mktemp -d -t mergeCPass0.XXXXXX)
 604
 605   mkdir -p ${runpath}
 606   if [[ ! -d ${runpath} ]]; then
 607     touch ${doneFile}
 608     echo "not able to make the runpath ${runpath}" >> ${doneFile}
 609     return 1
 610   fi
 611   if ! cd ${runpath}; then
 612     touch ${doneFile}
 613     echo "PWD=$PWD is not the runpath=${runpath}" >> ${doneFile}
 614     return 1
 615   fi
 616
 617   logOutputDir=${runpath}
 618   [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir}
 619   [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/stdout
 620   echo "${0} $*"
 621
 622   mergingScript="mergeMakeOCDB.byComponent.sh"
 623
 624   echo goMergeCPass0 SETUP:
 625   echo runNumber=${runNumber}
 626   echo outputDir=${outputDir}
 627   echo ocdbStorage=${ocdbStorage}
 628   echo calibrationFilesToMerge=${calibrationFilesToMerge}
 629   echo mergingScript=${mergingScript}
 630   echo commonOutputPath=${commonOutputPath}
 631   echo runpath=${runpath}
 632
 633   # copy files in case they are not already there
 634   filesMergeCPass0=(
 635                     "${batchWorkingDirectory}/${calibrationFilesToMerge}"
 636                     "${batchWorkingDirectory}/OCDB.root"
 637                     "${batchWorkingDirectory}/localOCDBaccessConfig.C"
 638                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/mergeMakeOCDB.byComponent.sh"
 639                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/mergeByComponent.C"
 640                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/makeOCDB.C"
 641                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/merge.C"
 642                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/mergeMakeOCDB.sh"
 643   )
 644   for file in ${filesMergeCPass0[*]}; do
 645     [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
 646     [[ ${file##*/} =~ .*\.sh ]] && chmod +x ${file##*/}
 647   done
 648
 649   sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh
 650
 651   alirootInfo > ALICE_ROOT.log
 652
 653   #
 654   echo "PWD"
 655   /bin/ls
 656   echo "PWD/.."
 657   /bin/ls ../
 658
 659
 660   #merge calibration
 661   chmod u+x ${mergingScript}
 662   mkdir -p ./OCDB
 663   if [[ ! -f ${calibrationFilesToMerge} ]]; then
 664     echo "/bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root > ${calibrationFilesToMerge}"
 665     /bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root 2>/dev/null > ${calibrationFilesToMerge}
 666   fi
 667
 668   echo "${mergingScript} ${calibrationFilesToMerge} ${runNumber} local://./OCDB ${ocdbStorage}"
 669   if [[ -n ${pretend} ]]; then
 670     sleep ${pretendDelay}
 671     touch CalibObjects.root
 672     touch ocdb.log
 673     touch merge.log
 674     touch dcsTime.root
 675     mkdir -p ./OCDB/TPC/Calib/TimeGain/
 676     mkdir -p ./OCDB/TPC/Calib/TimeDrift/
 677     echo "some calibration" >> ./OCDB/TPC/Calib/TimeGain/someCalibObject_0-999999_cpass0.root
 678     echo "some calibration" >> ./OCDB/TPC/Calib/TimeDrift/otherCalibObject_0-999999_cpass0.root
 679   else
 680     ./${mergingScript} ${calibrationFilesToMerge} ${runNumber} "local://./OCDB" ${ocdbStorage} >> "mergeMakeOCDB.log"
 681
 682     #produce the calib trees for expert QA (dcsTime.root)
 683     goMakeLocalOCDBaccessConfig ./OCDB
 684     echo aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")"
 685     aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")"
 686   fi
 687
 688   ### produce the output
 689   #tar the produced OCDB for reuse
 690   tar czf ${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz ./OCDB
 691
 692   /bin/ls
 693
 694   #copy all to output dir
 695   cp -pf -R ${runpath}/* ${outputDir}
 696
 697   if [[ -n ${generateMC} ]]; then
 698     goPrintValues sim ${commonOutputPath}/meta/sim.run${runNumber}.list ${commonOutputPath}/meta/cpass0.job*.run${runNumber}.done
 699   fi
 700
 701   #validate merging cpass0
 702   cd ${outputDir}
 703   touch ${doneFile}
 704   echo "dir ${outputDir}" >> ${doneFile}
 705   if summarizeLogs >> ${doneFile}; then
 706     [[ -f CalibObjects.root ]] && echo "calibfile ${outputDir}/CalibObjects.root" >> ${doneFile}
 707     [[ -f dcsTime.root ]] && echo "dcsTree ${outputDir}/dcsTime.root" >> ${doneFile}
 708   fi
 709
 710   [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
 711   return 0
 712 )
 713
 714 goMergeCPass1()
 715 (
 716   #
 717   # find the output files and merge them
 718   #
 719
 720   outputDir=${1}
 721   ocdbStorage=${2}
 722   configFile=${3}
 723   runNumber=${4}
 724   calibrationFilesToMerge=${5}
 725   qaFilesToMerge=${6}
 726   filteredFilesToMerge=${7}
 727   shift 7
 728   if ! parseConfig ${configFile} "$@"; then return 1; fi
 729
 730   #record the working directory provided by the batch system
 731   batchWorkingDirectory=${PWD}
 732
 733   [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
 734   doneFile="${commonOutputPath}/meta/merge.cpass1.run${runNumber}.done"
 735
 736   umask 0002
 737   ulimit -c unlimited
 738
 739   #clean up first:
 740   rm -f ${outputDir}/*.log
 741   rm -f ${outputDir}/*.root
 742   rm -f ${outputDir}/*done
 743
 744   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
 745
 746   #runpath=${PWD}/rundir_cpass1_Merge_${runNumber}
 747   runpath=${outputDir}
 748   #[[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
 749   [[ ${reconstructInTemporaryDir} -eq 1 ]] && runpath=$(mktemp -d -t mergeCPass1.XXXXXX)
 750
 751   mkdir -p ${runpath}
 752   if [[ ! -d ${runpath} ]]; then
 753     touch ${doneFile}
 754     echo "not able to make the runpath ${runpath}" >> ${doneFile}
 755     return 1
 756   fi
 757   if ! cd ${runpath}; then
 758     touch ${doneFile}
 759     echo "PWD=$PWD is not the runpath=${runpath}" >> ${doneFile}
 760     return 1
 761   fi
 762
 763   logOutputDir=${runpath}
 764   [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir}
 765   [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/mergeMakeOCDB.log
 766   echo "${0} $*"
 767
 768   calibrationOutputFileName='AliESDfriends_v1.root'
 769   qaOutputFileName='QAresults*.root'
 770   mergingScript="mergeMakeOCDB.byComponent.sh"
 771   #important to have the string "Stage.txt" in the filename to trigger the merging
 772   #it has to be a list of directories containing the files
 773   qaMergedOutputFileName="QAresults_merged.root"
 774
 775   echo goMergeCPass1 SETUP:
 776   echo runNumber=${runNumber}
 777   echo outputDir=${outputDir}
 778   echo ocdbStorage=${ocdbStorage}
 779   echo calibrationFilesToMerge=$calibrationFilesToMerge
 780   echo qaFilesToMerge=$qaFilesToMerge
 781   echo calibrationOutputFileName=${calibrationOutputFileName}
 782   echo mergingScript=${mergingScript}
 783
 784   # copy files in case they are not already there
 785   filesMergeCPass1=(
 786                     "${batchWorkingDirectory}/${calibrationFilesToMerge}"
 787                     "${batchWorkingDirectory}/${qaFilesToMerge}"
 788                     "${batchWorkingDirectory}/OCDB.root"
 789                     "${batchWorkingDirectory}/localOCDBaccessConfig.C"
 790                     "${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz"
 791                     "${batchWorkingDirectory}/QAtrain_duo.C"
 792                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/mergeMakeOCDB.byComponent.sh"
 793                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/mergeByComponent.C"
 794                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/makeOCDB.C"
 795                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/merge.C"
 796                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/mergeMakeOCDB.sh"
 797                     "${trustedQAtrainMacro}"
 798                     "${ALICE_ROOT}/ANALYSIS/macros/QAtrain_duo.C"
 799   )
 800   for file in ${filesMergeCPass1[*]}; do
 801     [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
 802     [[ ${file##*/} =~ .*\.sh ]] && chmod +x ${file##*/}
 803   done
 804
 805   sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh
 806
 807   #configure local OCDB storage from CPass0 (creates the localOCDBaccessConfig.C script)
 808   if [[ -f cpass0.localOCDB.${runNumber}.tgz ]]; then
 809     echo goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz"
 810     goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz"
 811   else
 812     echo "WARNING: file cpass0.localOCDB.${runNumber}.tgz not found!"
 813   fi
 814
 815   alirootInfo > ALICE_ROOT.log
 816
 817   #
 818   /bin/ls
 819
 820   #merge calibration
 821   chmod u+x ${mergingScript}
 822   mkdir -p OCDB
 823
 824   #if not provided, create the lists of files to merge
 825   if [[ ! -f ${filteredFilesToMerge} ]]; then
 826     echo "/bin/ls -1 ${outputDir}/*/FilterEvents_Trees.root > ${filteredFilesToMerge}"
 827     /bin/ls -1 ${outputDir}/*/FilterEvents_Trees.root 2>/dev/null > ${filteredFilesToMerge}
 828   fi
 829   if [[ ! -f ${calibrationFilesToMerge} ]]; then
 830     echo "/bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root > ${calibrationFilesToMerge}"
 831     /bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root 2>/dev/null > ${calibrationFilesToMerge}
 832   fi
 833   if [[ ! -f ${qaFilesToMerge} ]]; then
 834     #find the files, but only store the directories (QAtrain_duo.C requires this)
 835     echo "/bin/ls -1 ${outputDir}/*/QAresults*.root | while read x; do echo ${x%/*}; done | sort | uniq > ${qaFilesToMerge}"
 836     /bin/ls -1 ${outputDir}/*/QAresults*.root | while read x; do echo ${x%/*}; done | sort | uniq > ${qaFilesToMerge}
 837   fi
 838
 839   echo "${mergingScript} ${calibrationFilesToMerge} ${runNumber} local://./OCDB ${ocdbStorage}"
 840   if [[ -n ${pretend} ]]; then
 841     sleep ${pretendDelay}
 842     touch ocdb.log
 843     touch cpass1.localOCDB.${runNumber}.tgz
 844     touch ${qaMergedOutputFileName}
 845     touch merge.log
 846     touch trending.root
 847     touch FilterEvents_Trees.root
 848     touch CalibObjects.root
 849     touch dcsTime.root
 850     touch ${qaMergedOutputFileName}
 851     mkdir -p OCDB
 852   else
 853     ./${mergingScript} ${calibrationFilesToMerge} ${runNumber} "local://./OCDB" ${ocdbStorage}
 854
 855     #merge QA (and filtered trees)
 856     [[ -n ${AliAnalysisTaskFilteredTree_fLowPtTrackDownscaligF} ]] && export AliAnalysisTaskFilteredTree_fLowPtTrackDownscaligF
 857     [[ -n ${AliAnalysisTaskFilteredTree_fLowPtV0DownscaligF} ]] && export AliAnalysisTaskFilteredTree_fLowPtV0DownscaligF
 858
 859     #echo aliroot -l -b -q "merge.C(\"${qaFilesToMerge}\",\"\",kFALSE,\"${qaMergedOutputFileName}\")"
 860     echo aliroot -b -q "QAtrain_duo.C(\"_barrel\",${runNumber},\"${qaFilesToMerge}\",1,\"${ocdbStorage}\")"
 861     #aliroot -l -b -q "merge.C(\"${qaFilesToMerge}\",\"\",kFALSE,\"${qaMergedOutputFileName}\")"
 862     aliroot -b -q "QAtrain_duo.C(\"_barrel\",${runNumber},\"${qaFilesToMerge}\",1,\"${ocdbStorage}\")" > mergeQA.log
 863     mv QAresults_barrel.root ${qaMergedOutputFileName}
 864     mv trending_barrel.root trending.root
 865
 866     #merge filtered trees
 867     echo aliroot -l -b -q "merge.C(\"${qaFilesToMerge}\",\"\",kFALSE,\"${qaMergedOutputFileName}\")"
 868     aliroot -l -b -q "merge.C(\"${filteredFilesToMerge}\",\"\",kFALSE,\"FilterEvents_Trees.root\")" > mergeFilteredTrees.log
 869
 870     #produce the calib trees for expert QA
 871     echo aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")"
 872     aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")" > calibTree.log
 873   fi
 874
 875   tar czf ${commonOutputPath}/meta/cpass1.localOCDB.${runNumber}.tgz ./OCDB
 876
 877   /bin/ls
 878
 879   #copy all to output dir
 880   cp -pf -R ${runpath}/* ${outputDir}
 881
 882   #validate merge cpass1
 883   cd ${outputDir}
 884   touch ${doneFile}
 885   echo "dir ${outputDir}" >> ${doneFile}
 886   if summarizeLogs >>  ${doneFile}; then
 887     [[ -f CalibObjects.root ]] && echo "calibfile ${outputDir}/CalibObjects.root" >> ${doneFile}
 888     [[ -f ${qaMergedOutputFileName} ]] && echo "qafile ${outputDir}/${qaMergedOutputFileName}" >> ${doneFile}
 889     [[ -f trending.root ]] && echo "trendingfile ${outputDir}/trending.root" >> ${doneFile}
 890     [[ -f dcsTime.root ]] && echo "dcsTree ${outputDir}/dcsTime.root" >> ${doneFile}
 891     [[ -f FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFile}
 892   else
 893     if grep "mergeQA.log.*OK" ${doneFile} > /dev/null; then
 894       [[ -f ${qaMergedOutputFileName} ]] && echo "qafile ${outputDir}/${qaMergedOutputFileName}" >> ${doneFile}
 895     fi
 896     if grep "mergeFilteredTrees.log.*OK" ${doneFile} > /dev/null; then
 897       [[ -f FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFile}
 898     fi
 899   fi
 900
 901   [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
 902   return 0
 903 )
 904
 905 goMerge()
 906 (
 907   #generic root merge using CPass1 merge.C script
 908   inputList=${1}
 909   outputFile=${2}
 910   configFile=${3-"benchmark.config"}
 911   shift 3
 912   if ! parseConfig ${configFile} "$@"; then return 1; fi
 913
 914   #record the working directory provided by the batch system
 915   batchWorkingDirectory=${PWD}
 916
 917   [[ ! -f ${inputList} ]] && echo "inputList ${inputList} does not exist!" && return 1
 918   [[ ! -f ${configFile} ]] && echo "configFile ${configFile} does not exist!" && return 1
 919   umask 0002
 920   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
 921   rm -f ${outputFile}
 922   aliroot -b -q "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/merge.C(\"${inputList}\",\"\",kFALSE,\"${outputFile}\")" > merge_${inputList}.log
 923   return 0
 924 )
 925
 926 goSubmitMakeflow()
 927 {
 928   #run
 929   productionID=${1}
 930   inputFileList=${2}
 931   configFile=${3}
 932   shift 3
 933   extraOpts=("$@")
 934   if ! parseConfig ${configFile} "${extraOpts[@]}"; then return 1; fi
 935
 936   #record the working directory provided by the batch system
 937   batchWorkingDirectory=${PWD}
 938
 939   [[ -z ${configFile} ]] && configFile="benchmark.config"
 940   [[ ! -f ${configFile} ]] && echo "no config file found (${configFile})" && return 1
 941
 942   if [[ ! $(which makeflow &>/dev/null) && -n ${makeflowPath} ]]; then
 943     echo "setting the makflow path from the config: "
 944     echo "  export PATH=${makeflowPath}:${PATH}"
 945     export PATH=${makeflowPath}:${PATH}
 946   fi
 947
 948   #create the common output dir and the meta dir
 949   commonOutputPath=${baseOutputDirectory}/${productionID}
 950   if [[ -d ${commonOutputPath} ]]; then
 951     echo "output dir ${commonOutputPath} exists!"
 952     #return 1
 953   else
 954     mkdir -p ${commonOutputPath}
 955   fi
 956   mkdir -p ${commonOutputPath}/meta
 957
 958   self=${0}
 959   #if which greadlink; then self=$(greadlink -f "${0}"); fi
 960
 961   #for reference copy the setup to the output dir
 962   cp ${self} ${commonOutputPath}
 963   cp ${configFile} ${commonOutputPath}
 964   cp ${inputFileList} ${commonOutputPath}
 965
 966   #submit - use makeflow if available, fall back to old stuff when makeflow not there
 967   if which makeflow; then
 968     goGenerateMakeflow ${productionID} ${inputFileList} ${configFile} "${extraOpts[@]}" commonOutputPath=${commonOutputPath} > benchmark.makeflow
 969     cp benchmark.makeflow ${commonOutputPath}
 970     makeflow ${makeflowOptions} benchmark.makeflow
 971   else
 972     echo "no makeflow!"
 973   fi
 974   return 0
 975 }
 976
 977 goGenerateMakeflow()
 978 (
 979   #generate the makeflow file
 980   [[ $# -lt 3 ]] && echo "args: id inputFileList configFile" && return 1
 981   productionID=${1}
 982   inputFileList=${2}
 983   configFile=${3}
 984   shift 3
 985   extraOpts=("$@")
 986
 987   #batch systems/makeflow sometimes handle spaces in arguments poorly, so encode them
 988   for (( i=0;i<${#extraOpts[@]};i++ )); do
 989     extraOpts[i]=$(encSpaces "${extraOpts[i]}")
 990   done
 991   extraOpts+=("encodedSpaces=1")
 992
 993   if ! parseConfig ${configFile} "${extraOpts[@]}" &>/dev/null; then return 1; fi
 994
 995   #extra safety
 996   if [[ -z ${commonOutputPath} ]]; then
 997     commonOutputPath=${baseOutputDirectory}/${productionID}
 998     extraOpts=( "${extraOpts[@]}" "commonOutputPath=${commonOutputPath}" )
 999   fi
1000
1001   #record the working directory provided by the batch system
1002   batchWorkingDirectory=${PWD}
1003
1004   [[ -z ${configFile} ]] && configFile="benchmark.config"
1005   [[ ! -f ${configFile} ]] && echo "no config file found (${configFile})" && return 1
1006
1007   #these files will be made a dependency - will be copied to the working dir of the jobs
1008   declare -a copyFiles
1009   inputFiles=(
1010               "OCDB.root"
1011               "localOCDBaccessConfig.C"
1012               "QAtrain_duo.C"
1013               "runCPass1.sh"
1014               "recCPass1.C"
1015               "recCPass1_OuterDet.C"
1016               "runCalibTrain.C"
1017               "runCPass0.sh"
1018               "recCPass0.C"
1019   )
1020   for file in ${inputFiles[*]}; do
1021     [[ -f ${file} ]] && copyFiles+=("${file}")
1022   done
1023
1024   #create the makeflow file
1025   [[ -n ${batchFlags} ]] && echo "BATCH_OPTIONS = ${batchFlags}"
1026   declare -A arr_cpass0_merged arr_cpass1_merged
1027   declare -A arr_cpass0_calib_list arr_cpass1_calib_list
1028   declare -A arr_cpass1_QA_list arr_cpass1_ESD_list arr_cpass1_filtered_list
1029   declare -A arr_cpass0_profiled_outputs
1030   declare -A listOfRuns
1031   [[ -n ${runNumber} ]] && listOfRuns[${runNumber}]=1
1032   while read x; do tmpRun=$(guessRunNumber ${x}); [[ -n ${tmpRun} ]] && listOfRuns[${tmpRun}]=1; done < ${inputFileList}
1033   for runNumber in "${!listOfRuns[@]}"; do
1034     [[ -z ${runNumber} ]] && continue
1035     [[ ! ${runNumber} =~ ^[0-9]*[0-9]$ ]] && continue
1036
1037     unset arr_cpass0_outputs
1038     unset arr_cpass1_outputs
1039     declare -a arr_cpass0_outputs
1040     declare -a arr_cpass1_outputs
1041
1042     jobindex=0
1043     inputFile=""
1044     while read inputFile; do
1045       currentDefaultOCDB=${defaultOCDB}
1046       [[ -z ${autoOCDB} ]] && autoOCDB=1
1047       if [[ ${autoOCDB} -ne 0 ]]; then
1048         currentDefaultOCDB=$(setYear ${inputFile} ${defaultOCDB})
1049       fi
1050
1051       #CPass0
1052       arr_cpass0_outputs[${jobindex}]="${commonOutputPath}/meta/cpass0.job${jobindex}.run${runNumber}.done"
1053       echo "${arr_cpass0_outputs[${jobindex}]} : benchmark.sh ${configFile} ${copyFiles[@]}"
1054       echo " ${alirootEnv} ./benchmark.sh CPass0 ${commonOutputPath}/000${runNumber}/cpass0 ${inputFile} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} ${jobindex} ${extraOpts[@]}"" "
1055
1056       #CPass1
1057       arr_cpass1_outputs[${jobindex}]="${commonOutputPath}/meta/cpass1.job${jobindex}.run${runNumber}.done"
1058       echo "${arr_cpass1_outputs[${jobindex}]} : benchmark.sh ${configFile} ${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz ${copyFiles[@]}"
1059       echo " ${alirootEnv} ./benchmark.sh CPass1 ${commonOutputPath}/000${runNumber}/cpass1 ${inputFile} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} ${jobindex} ${extraOpts[@]}"" "
1060       ((jobindex++))
1061
1062     done< <(grep "/000${runNumber}/" ${inputFileList})
1063
1064     #CPass0 list of Calib files to merge
1065     arr_cpass0_calib_list[${runNumber}]="${commonOutputPath}/meta/cpass0.calib.run${runNumber}.list"
1066     echo "${arr_cpass0_calib_list[${runNumber}]} : benchmark.sh ${arr_cpass0_outputs[*]}"
1067     echo "  ./benchmark.sh PrintValues calibfile ${arr_cpass0_calib_list[${runNumber}]} ${arr_cpass0_outputs[*]}"
1068     echo
1069
1070     #CPass0 merging
1071     arr_cpass0_merged[${runNumber}]="${commonOutputPath}/meta/merge.cpass0.run${runNumber}.done"
1072     echo "${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz ${arr_cpass0_merged[${runNumber}]} : benchmark.sh ${configFile} ${arr_cpass0_calib_list[${runNumber}]} ${copyFiles[@]}"
1073     echo " ${alirootEnv} ./benchmark.sh MergeCPass0 ${commonOutputPath}/000${runNumber}/cpass0 ${currentDefaultOCDB} ${configFile} ${runNumber} ${arr_cpass0_calib_list[${runNumber}]} ${extraOpts[@]}"" "
1074
1075     #CPass1 list of Calib/QA/ESD/filtered files
1076     # the trick with QA is to have the string "Stage.txt" in the file name of the list of directories with QA output to trigger
1077     # the production of the QA trending tree (only then the task->Finish() will be called in QAtrain_duo.C, on the grid
1078     # this corresponds to the last merging stage)
1079     arr_cpass1_QA_list[${runNumber}]="${commonOutputPath}/meta/cpass1.QA.run${runNumber}.lastMergingStage.txt.list"
1080     echo "${arr_cpass1_QA_list[${runNumber}]}: benchmark.sh ${arr_cpass1_outputs[*]}"
1081     echo "  ./benchmark.sh PrintValues dir ${arr_cpass1_QA_list[${runNumber}]} ${arr_cpass1_outputs[*]}"
1082     echo
1083     arr_cpass1_calib_list[${runNumber}]="${commonOutputPath}/meta/cpass1.calib.run${runNumber}.list"
1084     echo "${arr_cpass1_calib_list[${runNumber}]} : benchmark.sh ${arr_cpass1_outputs[*]}"
1085     echo "  ./benchmark.sh PrintValues calibfile ${arr_cpass1_calib_list[${runNumber}]} ${arr_cpass1_outputs[*]};"
1086     echo
1087     arr_cpass1_ESD_list[${runNumber}]="${commonOutputPath}/meta/cpass1.ESD.run${runNumber}.list"
1088     echo "${arr_cpass1_ESD_list[${runNumber}]} : benchmark.sh ${arr_cpass1_outputs[*]}"
1089     echo "  ./benchmark.sh PrintValues esd ${arr_cpass1_ESD_list[${runNumber}]} ${arr_cpass1_outputs[*]}"
1090     echo
1091     arr_cpass1_filtered_list[${runNumber}]="${commonOutputPath}/meta/cpass1.filtered.run${runNumber}.list"
1092     echo "${arr_cpass1_filtered_list[${runNumber}]} : benchmark.sh ${arr_cpass1_outputs[*]}"
1093     echo "  ./benchmark.sh PrintValues filteredTree ${arr_cpass1_filtered_list[${runNumber}]} ${arr_cpass1_outputs[*]}"
1094     echo
1095
1096     #CPass1 merging
1097     arr_cpass1_merged[${runNumber}]="${commonOutputPath}/meta/merge.cpass1.run${runNumber}.done"
1098     echo "${commonOutputPath}/meta/cpass1.localOCDB.${runNumber}.tgz ${arr_cpass1_merged[${runNumber}]} :  benchmark.sh ${configFile} ${arr_cpass1_calib_list[${runNumber}]} ${arr_cpass1_QA_list[${runNumber}]} ${copyFiles[@]}"
1099     echo " ${alirootEnv} ./benchmark.sh MergeCPass1 ${commonOutputPath}/000${runNumber}/cpass1 ${currentDefaultOCDB} ${configFile} ${runNumber} ${arr_cpass1_calib_list[${runNumber}]} ${arr_cpass1_QA_list[${runNumber}]} ${arr_cpass1_filtered_list[${runNumber}]} ${extraOpts[@]}"" "
1100
1101     #CPass0 wrapped in a profiling tool (valgrind,....)
1102     if [[ -n ${profilingCommand} ]]; then
1103       inputFile=$(grep -m1 "${runNumber}/" ${inputFileList})
1104       [[ -z ${nEventsProfiling} ]] && nEventsProfiling=2
1105       currentDefaultOCDB=$(setYear ${inputFile} ${defaultOCDB})
1106       jobindex="profiling"
1107
1108       arr_cpass0_profiled_outputs[${runNumber}]="${commonOutputPath}/meta/profiling.cpass0.job${jobindex}.run${runNumber}.done"
1109       echo "${arr_cpass0_profiled_outputs[${runNumber}]} : benchmark.sh ${configFile} ${copyFiles[@]}"
1110       profilingCommand=$(encSpaces "${profilingCommand}")
1111       echo " ${alirootEnv} ./benchmark.sh CPass0 ${commonOutputPath}/000${runNumber}/${jobindex} ${inputFile} ${nEventsProfiling} ${currentDefaultOCDB} ${configFile} ${runNumber} ${jobindex} ${extraOpts[@]} useProfilingCommand=${profilingCommand}"
1112     fi
1113
1114   done #runs
1115
1116   #Summary
1117   echo "${commonOutputPath}/summary.log : benchmark.sh ${configFile} ${arr_cpass1_merged[*]}"
1118   echo "  ${alirootEnv} ./benchmark.sh MakeSummary ${configFile} ${extraOpts[@]}"
1119   echo
1120
1121   return 0
1122 )
1123
1124 goPrintValues()
1125 (
1126   #print the values given the key from any number of files (store in output file on second argument)
1127   if [[ $# -lt 3 ]]; then
1128     echo "goPrintValues key outputFile inputFiles"
1129     echo "if outputFile is \"-\" dont redirect to a file"
1130     return
1131   fi
1132   key=${1}
1133   outputFile=${2}
1134   [[ ${outputFile} =~ "-" ]] && outputFile=""
1135   shift 2 #remove 2 first arguments from arg list to only pass the input files to awk
1136   awk -v key=${key} '$0 ~ key" " {print $2}' "$@" | tee ${outputFile}
1137   return 0
1138 )
1139
1140 goCreateQAplots()
1141 (
1142   umask 0002
1143   mergedQAfileList=${1}
1144   productionID=${2}
1145   outputDir=${3}
1146   configFile=${4}
1147   shift 4
1148   if ! parseConfig ${configFile} "$@"; then return 1; fi
1149
1150   #record the working directory provided by the batch system
1151   batchWorkingDirectory=${PWD}
1152
1153   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
1154
1155   [[ -z ${logOutputDir} ]] && logOutputDir=${PWD}
1156   [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/makeQAplots.log
1157   echo "${0} $*"
1158
1159   olddir=${PWD}
1160   mkdir -p ${outputDir}
1161   cd ${outputDir}
1162   [[ ! "${PWD}" =~ "${outputDir}" ]] && echo "PWD is not equal to outputDir=${outputDir}" && cd ${olddir} && return 1
1163
1164   if [[ -z ${pretend} ]]; then
1165     ${ALICE_ROOT}/PWGPP/QA/scripts/runQA.sh inputList="${mergedQAfileList}" inputListHighPtTrees="${filteringList}" ocdbStorage=${defaultOCDB}
1166   else
1167     touch pretendCreateQAplots.done
1168   fi
1169   cd ${olddir}
1170   return 0
1171 )
1172
1173 goTest()
1174 (
1175   echo AA
1176 )
1177
1178 alirootInfo()
1179 (
1180   umask 0002
1181   # save aliroot repository info
1182   [[ -z "${ALICE_ROOT}" ]] && return 1
1183
1184   echo "\${ALICE_ROOT}=${ALICE_ROOT}"
1185   echo "\${ROOTSYS}=${ROOTSYS}"
1186   echo "\${PATH}=${PATH}"
1187   echo "\${LD_LIBRARY_PATH}=${LD_LIBRARY_PATH}"
1188   echo
1189
1190   pushd ${PWD}
1191   cd ${ALICE_ROOT}
1192
1193   currentBranch=$(git rev-parse --abbrev-ref HEAD)
1194   git status
1195   echo ""
1196   echo ""
1197   git diff ${currentBranch}
1198   popd
1199   return 0
1200 )
1201
1202 setYear()
1203 (
1204   #set the year
1205   #  ${1} - year to be set
1206   #  ${2} - where to set the year
1207   year1=$(guessYear ${1})
1208   year2=$(guessYear ${2})
1209   local path=${2}
1210   [[ ${year1} -ne ${year2} && -n ${year2} && -n ${year1} ]] && path=${2/\/${year2}\//\/${year1}\/}
1211   echo ${path}
1212   return 0
1213 )
1214
1215 guessPeriod()
1216 (
1217   #guess the period from the path, pick the rightmost one
1218   local IFS="/"
1219   declare -a path=( ${1} )
1220   local dirDepth=${#path[*]}
1221   for ((x=${dirDepth}-1;x>=0;x--)); do
1222     local field=${path[${x}]}
1223     [[ ${field} =~ ^LHC[0-9][0-9][a-z]$ ]] && period=${field} && break
1224   done
1225   echo ${period}
1226   return 0
1227 )
1228
1229 guessYear()
1230 (
1231   #guess the year from the path, pick the rightmost one
1232   local IFS="/"
1233   declare -a path=( ${1} )
1234   local dirDepth=${#path[*]}
1235   for ((x=${dirDepth}-1;x>=0;x--)); do
1236     local field=${path[${x}]}
1237     [[ ${field} =~ ^20[0-9][0-9]$ ]] && year=${field} && break
1238   done
1239   echo ${year}
1240   return 0
1241 )
1242
1243 guessRunNumber()
1244 (
1245   #guess the run number from the path, pick the rightmost one
1246   #works for /path/foo/000123456/bar/...
1247   #and       /path/foo.run123456.bar
1248   local IFS="/."
1249   declare -a path=( ${1} )
1250   local dirDepth=${#path[*]}
1251   for ((x=${dirDepth}-1;x>=0;x--)); do
1252     local field=${path[${x}]}
1253     field=${field/run/000}
1254     [[ ${field} =~ [0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && runNumber=${field#000} && break
1255   done
1256   echo ${runNumber}
1257   return 0
1258 )
1259
1260 validateLog()
1261 (
1262   log=${1}
1263   errorConditions=(
1264                     'There was a crash'
1265                     'floating'
1266                     'error while loading shared libraries'
1267                     'std::bad_alloc'
1268                     's_err_syswatch_'
1269                     'Thread [0-9]* (Thread'
1270                     'AliFatal'
1271                     'core dumped'
1272                     '\.C.*error:.*\.h: No such file'
1273                     'line.*Aborted'
1274   )
1275
1276   warningConditions=(
1277                      'This is serious !'
1278                      'rocVoltage out of range:'
1279   )
1280
1281   local logstatus=0
1282   local errorSummary=""
1283   local warningSummary=""
1284
1285   for ((i=0; i<${#errorConditions[@]};i++)); do
1286     local tmp=$(grep -m1 -e "${errorConditions[${i}]}" ${log})
1287     [[ -n ${tmp} ]] && tmp+=" : "
1288     errorSummary+=${tmp}
1289   done
1290
1291   for ((i=0; i<${#warningConditions[@]};i++)); do
1292     local tmp=$(grep -m1 -e "${warningConditions[${i}]}" ${log})
1293     [[ -n ${tmp} ]] && tmp+=" : "
1294     warningSummary+=${tmp}
1295   done
1296
1297   if [[ -n ${errorSummary} ]]; then
1298     echo "${errorSummary}"
1299     return 1
1300   fi
1301
1302   if [[ -n ${warningSummary} ]]; then
1303     echo "${warningSummary}"
1304     return 2
1305   fi
1306
1307   return 0
1308 )
1309
1310 summarizeLogs()
1311 (
1312   #print a summary of logs
1313   logFiles=(
1314             "*.log"
1315             "stdout"
1316             "stderr"
1317   )
1318
1319   #check logs
1320   local logstatus=0
1321   for log in ${logFiles[*]}; do
1322     finallog=${PWD%/}/${log}
1323     [[ ! -f ${log} ]] && continue
1324     errorSummary=$(validateLog ${log})
1325     validationStatus=$?
1326     if [[ ${validationStatus} -eq 0 ]]; then
1327       #in pretend mode randomly report an error in rec.log some cases
1328       if [[ -n ${pretend} && "${log}" == "rec.log" ]]; then
1329         #[[ $(( ${RANDOM}%2 )) -ge 1 ]] && echo "${finallog} BAD random error" || echo "${finallog} OK"
1330         echo "${finallog} OK"
1331       else
1332         echo "${finallog} OK"
1333       fi
1334     elif [[ ${validationStatus} -eq 1 ]]; then
1335       echo "${finallog} BAD ${errorSummary}"
1336       logstatus=1
1337     elif [[ ${validationStatus} -eq 2 ]]; then
1338       echo "${finallog} OK MWAH ${errorSummary}"
1339     fi
1340   done
1341
1342   #report core files
1343   while read x; do
1344     echo ${x}
1345     chmod 644 ${x}
1346     gdb --batch --quiet -ex "bt" -ex "quit" aliroot ${x} > stacktrace_${x//\//_}.log
1347   done < <(/bin/ls ${PWD}/*/core 2>/dev/null; /bin/ls ${PWD}/core 2>/dev/null)
1348
1349   return ${logstatus}
1350 )
1351
1352 spitOutLocalOCDBaccessConfig()
1353 {
1354   umask 0002
1355   #find ${1} -name "*root" | \
1356   /bin/ls -1 ${1}/*/*/*/*.root 2>/dev/null | \
1357   while read line
1358   do
1359     local tmp=${line#${1}}
1360     echo ${tmp%/*} | \
1361     awk -v ocdb=${1} '{print "  man->SetSpecificStorage(\""$1"\",\"local://"ocdb"\");"}'
1362   done
1363   return 0
1364 }
1365
1366 goMakeLocalOCDBaccessConfig()
1367 {
1368   umask 0002
1369   # make a script that sets the specific storages form all the root files produced by CPass0
1370   local localOCDBpathCPass0=${1}
1371   local OCDBpathPrefix=${2}
1372   [[ -z ${OCDBpathPrefix} ]] && OCDBpathPrefix="."
1373
1374   if [[ -f ${localOCDBpathCPass0} && ${localOCDBpathCPass0} =~ \.tgz$ ]]; then
1375     tar xzf ${localOCDBpathCPass0}
1376     local localOCDBpathCPass0="${OCDBpathPrefix}/OCDB"
1377   fi
1378
1379   echo
1380   echo creating the specific storage script
1381   echo   localOCDBaccessConfig.C
1382   echo   based on OCDB: ${localOCDBaccessConfig}
1383   echo
1384
1385   local tempLocalOCDB=""
1386   if [[ -f localOCDBaccessConfig.C ]]; then
1387     tempLocalOCDB=$(mktemp -t tempLocalOCDB.XXXXXX)
1388     echo "egrep "SetSpecificStorage" localOCDBaccessConfig.C > ${tempLocalOCDB}"
1389     egrep "SetSpecificStorage" localOCDBaccessConfig.C > ${tempLocalOCDB}
1390   fi
1391
1392   echo "localOCDBaccessConfig()"                               >  localOCDBaccessConfig.C
1393   echo "{"                                                     >> localOCDBaccessConfig.C
1394   echo "  AliCDBManager* man = AliCDBManager::Instance();"     >> localOCDBaccessConfig.C
1395   spitOutLocalOCDBaccessConfig ${localOCDBpathCPass0}|sort|uniq  >> localOCDBaccessConfig.C
1396   [[ -f "${tempLocalOCDB}" ]] && cat ${tempLocalOCDB}              >> localOCDBaccessConfig.C
1397   echo "}"                                                     >> localOCDBaccessConfig.C
1398
1399   [[ -f "${tempLocalOCDB}" ]] && rm -f ${tempLocalOCDB}
1400
1401   if ! grep SetSpecificStorage localOCDBaccessConfig.C; then
1402     echo
1403     echo "!!!!!!! CPass0 produced no OCDB entries"
1404     return 1
1405   fi
1406   return 0
1407 }
1408
1409 goMakeFilteredTrees()
1410 (
1411   outputDir=${1}
1412   runNumber=${2}
1413   #get path to input list
1414   inputListfiles=${3}
1415   #get scale number for tracks
1416   filterT=${4}
1417   #get scale number for V0s
1418   filterV=${5}
1419   #get OCDB path (optional)
1420   OCDBpath=${6}
1421   #get max number of files
1422   maxFiles=${7-"1000000"}
1423   #get offset of first file
1424   offsetFile=${8-"0"}
1425   #get max number of events
1426   maxEvents=${9-"30000000"}
1427   #get offset of first event
1428   offsetEvent=${10-"0"}
1429   configFile=${11-"benchmark.config"}
1430   esdFileName=${12-"AliESDs_Barrel.root"}
1431   shift 12
1432   if ! parseConfig ${configFile} "$@"; then return 1; fi
1433
1434   #record the working directory provided by the batch system
1435   batchWorkingDirectory=${PWD}
1436
1437   [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
1438   doneFile=${commonOutputPath}/meta/filtering.cpass1.run${runNumber}.done
1439
1440   cat > filtering.log << EOF
1441   goMakeFilteredTrees config:
1442   runpath=${runpath}
1443   outputDir=${outputDir}
1444   commonOutputPath=${commonOutputPath}
1445   ALICE_ROOT=${ALICE_ROOT}
1446   PATH=${PATH}
1447   offsetEvent=$offsetEvent
1448   configFile=$configFile
1449   esdFileName=$esdFileName
1450   inputListfiles=$inputListfiles
1451   doneFile=$doneFile
1452 EOF
1453
1454   #runpath=${outputDir}
1455   #[[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
1456   #[[ ${reconstructInTemporaryDir} -eq 1 ]] && runpath=$(mktemp -d -t goMakeFilteredTrees.XXXXXX)
1457   #mkdir -p ${outputDir}
1458   #mkdir -p ${runpath}
1459   #if ! cd ${runpath}; then
1460   #  echo "PWD=$PWD is not the runpath=${runpath}"
1461   #  touch ${doneFile}
1462   #  return 1
1463   #fi
1464
1465   if [[ -z ${pretend} ]];then
1466     aliroot -l -b -q "${ALICE_ROOT}/PWGPP/macros/runFilteringTask.C(\"${inputListfiles}\",${filterT},${filterV},\"${OCDBpath}\",${maxFiles},${offsetFile},${maxEvents},${offsetEvent},\"${esdFileName}\")" &>> filtering.log
1467   else
1468     sleep ${pretendDelay}
1469     touch filtering.log FilterEvents_Trees.root
1470   fi
1471   pwd
1472   /bin/ls
1473   touch ${doneFile}
1474   summarizeLogs >>  ${doneFile}
1475
1476   #echo mv -f * ${outputDir}
1477   #mv -f * ${outputDir}
1478   #[[ -f ${outputDir}/FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFile}
1479   #cd ${commonOutputPath}
1480   #[[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
1481
1482   return 0
1483 )
1484
1485 submit()
1486 {
1487   umask 0002
1488   [[ $# -lt 5 ]] && echo "at least 5 args needed, you supplied $#" && return 1
1489   JobID=${1}
1490   startID=${2}
1491   endID=${3}
1492   waitForJOBID=${4}
1493   command=${5}
1494   shift 5
1495   local commandArgs=("$@")
1496
1497   #add quote strings around the extra arguments
1498   for ((i=0; i<${#commandArgs[@]}; i++)); do
1499     commandArgs[i]=\"${commandArgs[i]}\"
1500   done
1501
1502   [[ -z ${waitForJOBID} ]] && waitForJOBID=0
1503
1504   newFarm=$(which qsub|grep "^/usr/bin/qsub")
1505
1506   batchSystem="SGE"
1507
1508   if [[ -z "${newFarm}" ]]
1509   then
1510     #old LSF
1511     # submit it (as job array)
1512     nFiles=$(( ${endID}-${startID}+1 ))
1513     while [ ${startID} -le ${nFiles}  ] ; do
1514       if [ $(expr ${nFiles} - ${startID}) -gt 999 ] ; then
1515         endID=$(expr ${startID} + 999)
1516       else
1517         endID=${nFiles}
1518       fi
1519     if [[ ${waitForJOBID} -eq 0 ]]; then
1520         echo ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}"
1521         ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}"
1522       else
1523         echo ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -w "ended(${waitForJOBID})" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}"
1524         ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -w "ended(${waitForJOBID})" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}"
1525       fi
1526       startID=$(expr ${endID} + 1)
1527     done
1528   else
1529     #new SGE farm
1530     if [[ ${waitForJOBID} =~ "000" ]]; then
1531       echo ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}"
1532       ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}"
1533     else
1534       echo ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -hold_jid "${waitForJOBID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}"
1535       ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -hold_jid "${waitForJOBID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}"
1536     fi
1537   fi
1538   return 0
1539 }
1540
1541 goSubmitBatch()
1542 {
1543   if [[ $# -lt 3 ]]; then
1544     echo "minimal use:"
1545     echo " ${0} submit fileList productionID configFile"
1546     return 0
1547   fi
1548
1549   productionID=${1}
1550   inputList=${2}
1551   configFile=${3:-"benchmark.config"}
1552   #if which greadlink; then configFile=$(greadlink -f ${configFile}); fi
1553   shift 3
1554   extraOpts=("$@")
1555   if ! parseConfig ${configFile} "${extraOpts[@]}"; then return 1; fi
1556
1557   #batch systems/makeflow sometimes handle spaces in arguments poorly, so encode them
1558   for (( i=0;i<${#extraOpts[@]};i++ )); do
1559     extraOpts[i]=$(encSpaces "${extraOpts[i]}")
1560   done
1561   extraOpts+=("encodedSpaces=1")
1562
1563   #record the working directory provided by the batch system
1564   batchWorkingDirectory=${PWD}
1565
1566   #redirect all output to submit.log
1567   echo "redirecting all output to ${PWD}/submit_${productionID//"/"/_}.log"
1568   exec 7>&1
1569   exec 1>submit_${productionID//"/"/_}.log 2>&1
1570
1571   umask 0002
1572   echo ${0}" submit $*"
1573   if [[ -z "${inputList}" || -z "${productionID}" ]]
1574   then
1575     echo
1576     echo " Usage: ${0} submit inputList productionID [configFile=benchmark.config]"
1577     echo
1578     return
1579   fi
1580
1581   # check if config file is there
1582   if [ ! -f ${configFile} ]; then
1583     echo "ERROR! Config File '${configFile}' not found" >&2
1584     return
1585   else
1586     echo "Using Config File: '${configFile}'"
1587   fi
1588
1589   [[ ! -f ${alirootEnv} ]] && echo "alirootEnv script ${alirootEnv} not found!..." && return 1
1590
1591   #move the script, config and some other stuff to ${commonOutputPath} first, then use them from there
1592   self=${0}
1593   #if which greadlink; then self=$(greadlink -f "${0}"); fi
1594   configPath=$(dirname ${configFile})
1595   export commonOutputPath=${baseOutputDirectory}/${productionID}
1596
1597   mkdir -p ${commonOutputPath}
1598   mkdir -p ${commonOutputPath}/logs
1599   mkdir -p ${commonOutputPath}/meta
1600
1601   cp ${self} ${commonOutputPath}
1602   cp ${configFile} ${commonOutputPath}
1603   cp ${inputList} ${commonOutputPath}
1604   self=${commonOutputPath}/${self##*/}
1605   chmod u+x ${self}
1606   configFile=${commonOutputPath}/${configFile##*/}
1607   inputList=${commonOutputPath}/${inputList##*/}
1608
1609   #convert to absolut pathnames
1610   #if which greadlink; then inputList=$(greadlink -f "${inputList}"); fi
1611   #make list of runs
1612   if [[ -z ${runNumber} ]]; then
1613     listOfRuns=($(while read x; do guessRunNumber ${x}; done < ${inputList} | sort | uniq))
1614   else
1615     listOfRuns=${runNumber}
1616   fi
1617
1618   #if which greadlink; then alirootSource=$(greadlink -f "${alirootSource}"); fi
1619
1620   echo ""
1621   echo "### BEGIN CONFIGURATION ###"
1622   echo ""
1623   echo "GENERAL:"
1624   echo ""
1625   echo "    productionID:    ${productionID}"
1626   echo "    batchCommand:    ${batchCommand}"
1627   echo "    batchFlags:      ${batchFlags}"
1628   echo "    alirootEnv:   ${alirootEnv}"
1629   ${alirootEnv} echo '    ALICE_ROOT:      ${ALICE_ROOT}'
1630   ${alirootEnv} echo '    ALIROOT_RELEASE: ${ALICE_RELEASE}'
1631   echo "    inputList:       ${inputList}"
1632   echo "    configPath:      ${configPath}"
1633   echo "    commonOutputPath:      ${commonOutputPath}"
1634   echo "    defaultOCDB:     ${defaultOCDB}"
1635   echo "      autoOCDB: ${autoOCDB}"
1636   echo "    recoTriggerOptions:   ${recoTriggerOptions}"
1637   echo "    runs:"
1638   echo "      ${listOfRuns[*]}"
1639   echo ""
1640   echo "THE TRAIN WILL RUN:"
1641
1642   if [ ${runCPass0reco} -eq 1 ]; then
1643     echo "    Pass0 - Recontruction"
1644   fi
1645
1646   if [ ${runCPass0MergeMakeOCDB} -eq 1 ]; then
1647     echo "    Pass0 - merging and OCDB export"
1648   fi
1649
1650   if [ ${runCPass1reco} -eq 1 ]; then
1651     echo "    Pass1 - Recontruction"
1652   fi
1653   if [ ${runCPass1MergeMakeOCDB} -eq 1 ]; then
1654     echo "    Pass1 - merging and OCDB export"
1655   fi
1656
1657   echo ""
1658   echo "LIMITS:"
1659   echo "    max. Events/Chunk:   ${nEvents}"
1660   echo "    max. Number of Chunks per Run:     ${nMaxChunks}"
1661   echo ""
1662   echo "### END CONFIGURATION ###"
1663   echo ""
1664
1665
1666   # check if input file is there
1667   if [ ! -f ${inputList} ]; then
1668     echo "ERROR! Input List '${inputList}' not found" >&2
1669     return
1670   fi
1671
1672   # define jobid (for dependent jobs)
1673   date=$(date +%Y_%m_%d_%H%M%S)
1674   #for each run we submit one jobarray:
1675   for runNumber in ${listOfRuns[*]}; do
1676
1677     [[ -z ${runNumber} ]] && continue
1678     [[ ! ${runNumber} =~ ^[0-9]*[0-9]$ ]] && continue
1679
1680     JOBpostfix="${productionID//"/"/_}_${runNumber}_${date}"
1681     JOBID1="p0_${JOBpostfix}"
1682     JOBID1wait="w0_${JOBpostfix}"
1683     JOBID2="m0_${JOBpostfix}"
1684     JOBID2wait="wm0_${JOBpostfix}"
1685     JOBID3="op0_${JOBpostfix}"
1686     JOBID3wait="wop0_${JOBpostfix}"
1687     JOBID4="p1_${JOBpostfix}"
1688     JOBID4wait="w1_${JOBpostfix}"
1689     JOBID5="m1_${JOBpostfix}"
1690     JOBID5wait="wm1_${JOBpostfix}"
1691     JOBID6="s1_${JOBpostfix}"
1692     JOBID6wait="ws1_${JOBpostfix}"
1693     JOBID7="QA_${JOBpostfix}"
1694     JOBmakeESDlistCPass1="lp1_${JOBpostfix}"
1695     JOBfilterESDcpass1="fp1_${JOBpostfix}"
1696     LASTJOB="000"
1697
1698     oneInputFile=$(egrep -m1 "${runNumber}/" ${inputList})
1699
1700     currentDefaultOCDB=${defaultOCDB}
1701     [[ -z ${autoOCDB} ]] && autoOCDB=1
1702     if [[ ${autoOCDB} -ne 0 ]]; then
1703       currentDefaultOCDB=$(setYear ${oneInputFile} ${defaultOCDB})
1704     fi
1705
1706     echo "submitting run ${runNumber} with OCDB ${currentDefaultOCDB}"
1707
1708     ###############################################################################
1709     #run one chunk with valgrind:
1710     if [[ -n ${profilingCommand} ]]; then
1711       [[ -z ${nEventsProfiling} ]] && nEventsProfiling=2
1712       [[ -z ${profilingCommand} ]] && profilingCommand="/usr/bin/valgrind --tool=callgrind --num-callers=40 -v --trace-children=yes"
1713       submit "profile-${JOBpostfix}" 1 1 000 "${alirootEnv} ${self}" CPass0 ${commonOutputPath}/000${runNumber}/${jobindex} ${oneInputFile} ${nEventsProfiling} ${currentDefaultOCDB} ${configFile} ${runNumber} ${jobindex} useProfilingCommand=$(encSpaces "${profilingCommand}") "${extraOpts[@]}"
1714     fi
1715
1716     ################################################################################
1717     ################################################################################
1718     # run the CPass0 if requested
1719
1720     if [ ${runCPass0reco} -eq 1 ]; then
1721
1722       echo
1723       echo "starting CPass0... for run ${runNumber}"
1724       echo
1725
1726       # create directory and copy all files that are needed
1727       targetDirectory="${commonOutputPath}/000${runNumber}/cpass0"
1728       mkdir -p ${targetDirectory}
1729
1730       filesCPass0=(
1731                     "${configPath}/runCPass0.sh"
1732                     "${configPath}/recCPass0.C"
1733                     "${configPath}/runCalibTrain.C"
1734                     "${configPath}/localOCDBaccessConfig.C"
1735                     "${configPath}/OCDB*.root"
1736                     "${configPath}/sim.C"
1737                     "${configPath}/rec.C"
1738                     "${configPath}/Config.C"
1739       )
1740       for file in ${filesCPass0[*]}; do
1741         [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
1742       done
1743
1744       localInputList=${targetDirectory}/${inputList##*/}
1745       [[ ! -f ${localInputList} ]] && egrep "\/000${runNumber}\/" ${inputList} > ${localInputList}
1746       # limit nFiles to nMaxChunks
1747       nFiles=$(wc -l < ${localInputList})
1748       [[ ${nFiles} -eq 0 ]] && echo "list contains ZERO files! exiting..." && return 1
1749       echo "raw files in list:    ${nFiles}"
1750       if [[ ${nMaxChunks} -gt 0 && ${nMaxChunks} -le ${nFiles} ]]; then
1751         nFiles=${nMaxChunks}
1752       fi
1753       echo "raw files to process: ${nFiles}"
1754       [[ -z "${percentProcessedFilesToContinue}" ]] && percentProcessedFilesToContinue=100
1755       if [[ ${percentProcessedFilesToContinue} -eq 100 ]]; then
1756         nFilesToWaitFor=${nFiles}
1757       else
1758         nFilesToWaitFor=$(( ${nFiles}-${nFiles}/(100/(100-${percentProcessedFilesToContinue})) ))
1759       fi
1760       echo "requested success rate is ${percentProcessedFilesToContinue}%"
1761       echo "merging will start after ${nFilesToWaitFor} jobs are done"
1762
1763       submit ${JOBID1} 1 ${nFiles} 000 "${alirootEnv} ${self}" CPass0 ${targetDirectory} ${localInputList} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} -1 "${extraOpts[@]}"
1764
1765       ## submit a monitoring job that will run until a certain number of jobs are done with reconstruction
1766       submit "${JOBID1wait}" 1 1 000 "${alirootEnv} ${self}" WaitForOutput ${commonOutputPath} "meta/cpass0.job*.run${runNumber}.done" ${nFilesToWaitFor} ${maxSecondsToWait}
1767       LASTJOB=${JOBID1wait}
1768
1769     fi #end running CPass0
1770     ################################################################################
1771
1772
1773     ################################################################################
1774     # submit merging of CPass0, depends on the reconstruction
1775
1776     if [ ${runCPass0MergeMakeOCDB} -eq 1 ]; then
1777
1778       echo
1779       echo "submit CPass0 merging for run ${runNumber}"
1780       echo
1781
1782       targetDirectory="${commonOutputPath}/000${runNumber}/cpass0"
1783       mkdir -p ${targetDirectory}
1784
1785       #copy the scripts
1786       filesMergeCPass0=(
1787                         "${configPath}/OCDB.root"
1788                         "${configPath}/mergeMakeOCDB.byComponent.sh"
1789                         "${configPath}/mergeMakeOCDB.sh"
1790                         "${configPath}/localOCDBaccessConfig.C"
1791                         "${configPath}/mergeByComponent.C"
1792                         "${configPath}/makeOCDB.C"
1793                         "${configPath}/merge.C"
1794       )
1795       for file in ${filesMergeCPass0[*]}; do
1796         [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
1797       done
1798
1799       submit ${JOBID2} 1 1 "${LASTJOB}" "${alirootEnv} ${self}" MergeCPass0 ${targetDirectory} ${currentDefaultOCDB} ${configFile} ${runNumber} cpass0.calib.run${runNumber}.list "${extraOpts[@]}"
1800       LASTJOB=${JOBID2}
1801
1802       if [[ -n ${generateMC} ]]; then
1803         submit "mrl${JOBpostfix}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" PrintValues sim ${commonOutputPath}/meta/sim.run${runNumber}.list ${commonOutputPath}/meta/cpass0.job*.run${runNumber}.done
1804         LASTJOB="mrl${JOBpostfix}"
1805       fi
1806
1807       echo
1808     fi
1809     # end of merging CPass0
1810     ################################################################################
1811
1812     ################################################################################
1813     ################################################################################
1814     # run the CPass1 if requested
1815
1816     if [ ${runCPass1reco} -eq 1 ]; then
1817
1818       targetDirectory="${commonOutputPath}/000${runNumber}/cpass1"
1819       rm -f ${commonOutputPath}/meta/cpass1.job*.run${runNumber}.done
1820
1821       # safety feature: if we are re-running for any reason we want to delete the previous output first.
1822       [[ -d ${targetDirectory} ]] && rm -rf ${targetDirectory}/* && echo "removed old output at ${targetDirectory}/*"
1823
1824       echo
1825       echo "starting CPass1... for run ${runNumber}"
1826       echo
1827
1828       # create directory and copy all files that are needed
1829       mkdir -p ${targetDirectory}
1830
1831       filesCPass1=(
1832                     "${configPath}/runCPass1.sh"
1833                     "${configPath}/recCPass1.C"
1834                     "${configPath}/recCPass1_OuterDet.C"
1835                     "${configPath}/runCalibTrain.C"
1836                     "${configPath}/QAtrain_duo.C"
1837                     "${configPath}/localOCDBaccessConfig.C"
1838                     "${configPath}/OCDB.root"
1839       )
1840       for file in ${filesCPass1[*]}; do
1841         [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
1842       done
1843
1844       if [[ -n ${generateMC} ]]; then
1845         localInputList=${commonOutputPath}/meta/sim.run${runNumber}.list
1846       else
1847         localInputList=${targetDirectory}/${inputList##*/}
1848         [[ ! -f ${localInputList} ]] && egrep "\/000${runNumber}\/" ${inputList} > ${localInputList}
1849       fi
1850       # limit nFiles to nMaxChunks
1851       nFiles=$(wc -l < ${localInputList})
1852       [[ ${nFiles} -eq 0 ]] && echo "list contains ZERO files! continuing..." && continue
1853       echo "raw files in list:    ${nFiles}"
1854       if [[ ${nMaxChunks} -gt 0 && ${nMaxChunks} -le ${nFiles} ]]; then
1855         nFiles=${nMaxChunks}
1856       fi
1857       echo "raw files to process: ${nFiles}"
1858       [[ -z "${percentProcessedFilesToContinue}" ]] && percentProcessedFilesToContinue=100
1859       if [[ ${percentProcessedFilesToContinue} -eq 100 ]]; then
1860         nFilesToWaitFor=${nFiles}
1861       else
1862         nFilesToWaitFor=$(( ${nFiles}-${nFiles}/(100/(100-${percentProcessedFilesToContinue})) ))
1863       fi
1864       echo "requested success rate is ${percentProcessedFilesToContinue}%"
1865       echo "merging will start after ${nFilesToWaitFor} jobs are done"
1866
1867       submit ${JOBID4} 1 ${nFiles} "${LASTJOB}" "${alirootEnv} ${self}" CPass1 ${targetDirectory} ${localInputList} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} -1 "${extraOpts[@]}"
1868
1869       ################################################################################
1870       ## submit a monitoring job that will run until a certain number of jobs are done with reconstruction
1871       submit "${JOBID4wait}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" WaitForOutput ${commonOutputPath} "meta/cpass1.job*.run${runNumber}.done" ${nFilesToWaitFor} ${maxSecondsToWait}
1872       LASTJOB=${JOBID4wait}
1873       ################################################################################
1874
1875       echo
1876     fi #end running CPass1
1877
1878     ################################################################################
1879     # submit merging of CPass1, depends on the reconstruction
1880     if [ ${runCPass1MergeMakeOCDB} -eq 1 ]; then
1881
1882       echo
1883       echo "submit CPass1 merging for run ${runNumber}"
1884       echo
1885
1886       targetDirectory="${commonOutputPath}/000${runNumber}/cpass1"
1887       rm -f ${commonOutputPath}/meta/merge.cpass1.run${runNumber}.done
1888       mkdir -p ${targetDirectory}
1889
1890       # copy files
1891       filesMergeCPass1=(
1892                         "${configPath}/OCDB.root"
1893                         "${configPath}/localOCDBaccessConfig.C"
1894                         "${configPath}/mergeMakeOCDB.byComponent.sh"
1895                         "${configPath}/mergeByComponent.C"
1896                         "${configPath}/makeOCDB.C"
1897                         "${configPath}/merge.C"
1898                         "${configPath}/mergeMakeOCDB.sh"
1899                         "${configPath}/QAtrain_duo.C"
1900       )
1901       for file in ${filesMergeCPass1[*]}; do
1902         [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
1903       done
1904
1905       submit "${JOBID5}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" MergeCPass1 ${targetDirectory} ${currentDefaultOCDB} ${configFile} ${runNumber} cpass1.calib.run${runNumber}.list cpass1.QA.run${runNumber}.lastMergingStage.txt.list cpass1.filtered.run${runNumber}.list "${extraOpts[@]}"
1906       LASTJOB=${JOBID5}
1907       echo
1908     fi
1909
1910     ###############################
1911     #if [ ${runESDfiltering} -eq 1 ]; then
1912     #  rm -f ${commonOutputPath}/cpass1.ESD.run${runNumber}.list
1913     #  rm -f ${commonOutputPath}/meta/filtering.cpass1.run*.done
1914     #  echo
1915     #  echo submitting filtering for run ${runNumber}
1916     #  echo
1917     #  submit "${JOBmakeESDlistCPass1}" 1 1 "${LASTJOB}" "${self}" PrintValues esd ${commonOutputPath}/meta/cpass1.ESD.run${runNumber}.list ${commonOutputPath}/meta/cpass1.job*.run${runNumber}.done
1918     #  submit "${JOBfilterESDcpass1}" 1 1 "${JOBmakeESDlistCPass1}" "${alirootEnv} ${self}" MakeFilteredTrees ${commonOutputPath}/000${runNumber}/cpass1 ${runNumber} ${commonOutputPath}/meta/cpass1.ESD.run${runNumber}.list ${filteringFactorHighPt} ${filteringFactorV0s} ${currentDefaultOCDB} 1000000 0 10000000 0 ${configFile} AliESDs_Barrel.root "${extraOpts[@]}"
1919     #  LASTJOB=${JOBfilterESDcpass1}
1920     #fi
1921
1922   done
1923
1924   #################################################################################
1925   #################################################################################
1926   #if [ ${runESDfiltering} -eq 1 ]; then
1927   #  submit "${JOBID5wait}" 1 1 "${LASTJOB}" "${self}" WaitForOutput ${commonOutputPath} "meta/filtering.cpass1.run*.done" "${#listOfRuns[@]}" ${maxSecondsToWait}
1928   #else
1929     submit "${JOBID5wait}" 1 1 "${LASTJOB}" "${self}" WaitForOutput ${commonOutputPath} "meta/merge.cpass1.run*.done" ${#listOfRuns[@]} ${maxSecondsToWait}
1930   #fi
1931   LASTJOB=${JOBID5wait}
1932
1933   #################################################################################
1934   echo
1935   echo "submit make a summary"
1936   echo
1937
1938   submit "${JOBID6}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" MakeSummary ${configFile}
1939   LASTJOB=${JOBID6}
1940   #################################################################################
1941
1942   #restore stdout
1943   exec 1>&7 7>&-
1944   echo "jobs submitted."
1945   return 0
1946 }
1947
1948 goWaitForOutput()
1949 (
1950   umask 0002
1951   [[ $# -lt 3 ]] && echo "goWaitForOutput() wrong number of arguments, exiting.." && return 1
1952   echo searchPath=${1}
1953   echo fileName=${2}
1954   echo numberOfFiles=${3}
1955   echo maxSecondsToWait=${4}
1956   searchPath=${1}
1957   fileName=${2}
1958   numberOfFiles=${3}
1959   maxSecondsToWait=${4}
1960   echo "command to be executed: /bin/ls -1 ${searchPath}/${fileName}"
1961   [[ -z "${maxSecondsToWait}" ]] && maxSecondsToWait=$(( 3600*12 ))
1962   while true; do
1963     n=$(/bin/ls -1 ${searchPath}/${fileName} 2>/dev/null | wc -l)
1964     [[ ${n} -gt 0 ]] && echo "found ${n} X ${fileName}"
1965     [[ ${n} -ge ${numberOfFiles} ]] && break
1966     [[ ${SECONDS} -gt ${maxSecondsToWait} ]] && echo "timeout of ${maxSecondsToWait}s!" && break
1967     sleep 60
1968   done
1969   echo "DONE! exiting..."
1970   return 0
1971 )
1972
1973 mergeSysLogs()
1974 (
1975   outputFile=${1}
1976   shift
1977   inputFiles="$@"
1978   i=0
1979   if ! ls -1 ${inputFiles} &>/dev/null; then echo "the files dont exist!: ${inputFiles}"; return 1; fi
1980   while read x; do
1981     runNumber=$(guessRunNumber ${x})
1982     [[ -z ${runNumber} ]] && echo "run number cannot be guessed for ${x}" && continue
1983     awk -v run=${runNumber} -v i=${i} 'NR > 1 {print run" "$0} NR==1 && i==0 {print "run/I:"$0}' ${x}
1984     (( i++ ))
1985   done < <(ls -1 ${inputFiles}) > ${outputFile}
1986   return 0
1987 )
1988
1989 goMakeMergedSummaryTree()
1990 (
1991   # create list of calibration entries
1992   # takes no arguments, just run it in the base output
1993   # directory with the following files in the working directory
1994   #
1995   #  Calibration file lists:
1996   #       cpass0.dcsTree.list, cpass1.dcsTree.list
1997   #  QA trending root file:
1998   #       trending.root
1999   #
2000   #  Production infoo ascii files:
2001   #       summary_pass0.tree
2002   #       summary_pass1.tree
2003   #
2004
2005   [[ ! -f cpass0.dcsTree.list ]] && echo "no cpass0.dcsTree.list" && return 1
2006   [[ ! -f cpass1.dcsTree.list ]] && echo "no cpass1.dcsTree.list" && return 1
2007   [[ ! -f trending.root ]] && echo "no trending.root" && return 1
2008   [[ ! -f summary_pass0.tree ]] && echo "no summary_pass0.tree" && return 1
2009   [[ ! -f summary_pass1.tree ]] && echo "no summary_pass1.tree" && return 1
2010
2011   #first, dump the C macro to file
2012   cat << EOF > mergeTree.C
2013   //
2014   // Merge summary information
2015   // Following files are expected to be in the working directory
2016   //
2017   // Calibration file lists:
2018   //      cpass0.dcsTree.list, cpass1.dcsTree.list
2019   // QA trending root file:
2020   //      trending.root
2021   //
2022   // Production infoo ascii files:
2023   //      summary_pass0.tree
2024   //      summary_pass1.tree
2025   //
2026   void mergeTree(){
2027     //
2028     //
2029     //
2030     // Calibration values dump
2031     //
2032     //Printf("MakeTreeFromList cpass0.dcsTree.list");
2033     AliXRDPROOFtoolkit::MakeTreeFromList("Calib.TPC.CPass0.root", "dcs","dcs","cpass0.dcsTree.list",1);
2034     //Printf("MakeTreeFromList cpass1.dcsTree.list");
2035     AliXRDPROOFtoolkit::MakeTreeFromList("Calib.TPC.CPass1.root", "dcs","dcs","cpass1.dcsTree.list",1);
2036     //
2037     // Calibration status dump
2038     //
2039     TFile *fprod = TFile::Open("fproduction.root","recreate");
2040     TTree  tree0, tree1;
2041     //Printf("reading summary_pass0.tree");
2042     tree0.ReadFile("summary_pass0.tree");
2043     //Printf("reading summary_pass1.tree");
2044     tree1.ReadFile("summary_pass1.tree");
2045     tree0.Write("CPass0");
2046     tree1.Write("CPass1");
2047     fprod->Close();
2048     //
2049     //
2050     //
2051     TString stringSetup="";
2052     stringSetup+="1#QA.TPC#run#SummaryTPCQA/tpcQA#trending.root+";  //
2053     stringSetup+="1#Calib.TPC.CPass0#run#dcs#Calib.TPC.CPass0.root+";  //
2054     stringSetup+="1#Calib.TPC.CPass1#run#dcs#Calib.TPC.CPass1.root+";  //
2055     //
2056     stringSetup+="1#CPass0#runnumber#CPass0#fproduction.root+";  //
2057     stringSetup+="1#CPass1#runnumber#CPass1#fproduction.root+";  //
2058     //
2059     //Printf("stringSetup: %s", stringSetup.Data());
2060     AliXRDPROOFtoolkit::JoinTreesIndex("outAll.root","joinAll","run",stringSetup.Data(), 1);
2061   }
2062 EOF
2063
2064   aliroot -b -q "mergeTree.C" > mergeTrees.log
2065   return $?
2066 )
2067
2068 goMakeSummary()
2069 (
2070   #all the final stuff goes in here for ease of use:
2071   # summary logs
2072   # qa plot making
2073   # final file lists
2074   #some defaults:
2075   log="summary.log"
2076   productionID="qa"
2077
2078   configFile=${1}
2079   shift 1
2080   extraOpts=("$@")
2081   if ! parseConfig ${configFile} "${extraOpts[@]}"; then return 1; fi
2082
2083   #if which greadlink; then configFile=$(greadlink -f ${configFile}); fi
2084
2085   #record the working directory provided by the batch system
2086   batchWorkingDirectory=${PWD}
2087
2088   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
2089
2090   [[ ! -f ${configFile} ]] && echo "no config file ${configFile}!" && return
2091
2092   [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
2093
2094   #copy some useful stuff
2095   #and go to the commonOutputPath
2096   cp ${configFile} ${commonOutputPath}
2097   cd ${commonOutputPath}
2098
2099   exec &> >(tee ${log})
2100
2101   #summarize the global stuff
2102   echo "env script: ${alirootSource} ${alirootEnv}"
2103   echo "\$ALICE_ROOT=${ALICE_ROOT}"
2104   echo "commonOutputPath=${commonOutputPath}"
2105
2106   #summarize the stacktraces
2107   awk '
2108        BEGIN {
2109                print "frame/I:method/C:line/C:cpass/I:aliroot/I";
2110                RS="#[0-9]*";
2111                aliroot=0;
2112              }
2113              {
2114                if ($3 ~ /Ali*/) aliroot=1; else aliroot=0;
2115                gsub("#","",RT);
2116                if ($NF!="" && RT!="" && $3!="") print RT" "$3" "$NF" "0" "aliroot
2117              }
2118       ' 000*/cpass0/*/stacktrace* 2>/dev/null > stacktrace.tree
2119   awk '
2120        BEGIN {
2121                RS="#[0-9]*";
2122                aliroot=0;
2123              }
2124              {
2125                if ($3 ~ /Ali*/) aliroot=1; else aliroot=0;
2126                gsub("#","",RT);
2127                if ($NF!="" && RT!="" && $3!="") print RT" "$3" "$NF" "1" "aliroot
2128              }
2129       ' 000*/cpass1/*/stacktrace* 2>/dev/null >> stacktrace.tree
2130
2131   echo total numbers for the production:
2132   echo
2133   awk 'BEGIN {nFiles=0;nCore=0;}
2134   /^calibfile/ {nFiles++;}
2135   /core dumped/ {nCore++i;}
2136   END {print     "cpass0 produced "nFiles" calib files, "nCore" core files";}' meta/cpass0.job*done 2>/dev/null
2137   awk 'BEGIN {nOK=0; nBAD=0; }
2138   /\/rec.log OK/ {nOK++;}
2139   /\/rec.log BAD/ {nBAD++;}
2140   /stderr BAD/ {if ($0 ~ /rec.log/){nBAD++;}}
2141   END {print     "cpass0 reco:  OK: "nOK"\tBAD: "nBAD;}' meta/cpass0.job*done 2>/dev/null
2142   awk 'BEGIN {nOK=0; nBAD=0; }
2143   /\/calib.log OK/ {nOK++;}
2144   /\/calib.log BAD/ {nBAD++;}
2145   END {print "cpass0 calib: OK: "nOK"\tBAD: "nBAD;}' meta/cpass0.job*done 2>/dev/null
2146
2147   awk 'BEGIN {nOK=0; nBAD=0; }
2148   /merge.log OK/ {nOK++;}
2149   /merge.log BAD/ {nBAD++;}
2150   END {print "cpass0 merge: OK: "nOK"\tBAD: "nBAD;}' meta/merge.cpass0*done 2>/dev/null
2151   awk 'BEGIN {nOK=0; nBAD=0; }
2152   /ocdb.log OK/ {nOK++;}
2153   /ocdb.log BAD/ {nBAD++;}
2154   END {print   "cpass0 OCDB:  OK: "nOK"\tBAD: "nBAD;}' meta/merge.cpass0*done 2>/dev/null
2155
2156   echo
2157   awk 'BEGIN {nFiles=0;nCore=0;}
2158   /^calibfile/ {nFiles++;}
2159   /core dumped/ {nCore++;}
2160   END {print     "cpass1 produced "nFiles" calib files, "nCore" core files";}' meta/cpass1.job*done 2>/dev/null
2161   awk 'BEGIN {nOK=0; nBAD=0; }
2162   /\/rec.log OK/ {nOK++;}
2163   /\/rec.log BAD/ {nBAD++;}
2164   /stderr BAD/ {if ($0 ~ /rec.log/){nBAD++;}}
2165   END {print     "cpass1 reco:  OK: "nOK"\tBAD: "nBAD;}' meta/cpass1.job*done 2>/dev/null
2166   awk 'BEGIN {nOK=0; nBAD=0; }
2167   /\/calib.log OK/ {nOK++;}
2168   /\/calib.log BAD/ {nBAD++;}
2169   END {print "cpass1 calib: OK: "nOK"\tBAD: "nBAD;}' meta/cpass1.job*done 2>/dev/null
2170
2171   awk 'BEGIN {nOK=0; nBAD=0; }
2172   /merge.log OK/ {nOK++;}
2173   /merge.log BAD/ {nBAD++;}
2174   END {print "cpass1 merge: OK: "nOK"\tBAD: "nBAD;}' meta/merge.cpass1*done 2>/dev/null
2175   awk 'BEGIN {nOK=0; nBAD=0; }
2176   /ocdb.log OK/ {nOK++;}
2177   /ocdb.log BAD/ {nBAD++;}
2178   END {print   "cpass1 OCDB:  OK: "nOK"\tBAD: "nBAD;}' meta/merge.cpass1*done 2>/dev/null
2179
2180   echo
2181   echo per run stats:
2182   /bin/ls -1 meta/merge.cpass0.run*.done | while read x
2183 do
2184   dir=$(goPrintValues calibfile - ${x})
2185   runNumber=$(guessRunNumber ${dir})
2186   [[ -z ${runNumber} ]] && continue
2187
2188   if $(/bin/ls meta/cpass0.job*.run${runNumber}.done &> /dev/null); then
2189     statusCPass0=( $(
2190     awk 'BEGIN {nOKrec=0;nBADrec=0;nOKcalib=0;nBADcalib=0;nOKstderr=0;nBADstderr=0;}
2191     /\/rec.log OK/ {nOKrec++;}
2192     /\/rec.log BAD/ {nBADrec++;}
2193     /stderr BAD/ {if ($0 ~ /rec.log/) {nBADrec++;} nBADstderr++;}
2194     /stderr OK/ {nOKstderr++;}
2195     /\/calib.log OK/ {nOKcalib++;}
2196     /\/calib.log BAD/ {nBADcalib++}
2197     END {print ""nOKrec" "nBADrec" "nOKstderr" "nBADstderr" "nOKcalib" "nBADcalib;}' meta/cpass0.job*.run${runNumber}.done 2>/dev/null
2198     ) )
2199   fi
2200
2201   if $(/bin/ls meta/cpass1.job*.run${runNumber}.done &>/dev/null); then
2202     statusCPass1=( $(
2203     awk 'BEGIN {nOKrec=0;nBADrec=0;nOKcalib=0;nBADcalib=0;nOKstderr=0;nBADstderr=0;nQAbarrelOK=0;nQAbarrelBAD=0;nQAouterOK=0;nQAouterBAD=0;}
2204     /\/rec.log OK/ {nOKrec++;}
2205     /\/rec.log BAD/ {nBADrec++;}
2206     /stderr BAD/ {if ($0 ~ /rec.log/) nBADrec++;nBADstderr++;}
2207     /stderr OK/ {nOKstderr++;}
2208     /\/calib.log OK/ {nOKcalib++;}
2209     /\/calib.log BAD/ {nBADcalib++}
2210     /\/qa_barrel.log OK/ {nQAbarrelOK++;}
2211     /\/qa_barrel.log BAD/ {nQAbarrelBAD++;}
2212     /\/qa_outer.log OK/ {nQAouterOK++;}
2213     /\/qa_outer.log BAD/ {nQAouterBAD++;}
2214     END {print ""nOKrec" "nBADrec" "nOKstderr" "nBADstderr" "nOKcalib" "nBADcalib" "nQAbarrelOK" "nQAbarrelBAD" "nQAouterOK" "nQAouterBAD;}' meta/cpass1.job*.run${runNumber}.done 2>/dev/null
2215     ) )
2216   fi
2217
2218   statusOCDBcpass0=$(awk '/ocdb.log/ {print $2} ' ${x} 2>/dev/null)
2219   statusOCDBcpass1=$(awk '/ocdb.log/ {print $2}' ${x/cpass0/cpass1} 2>/dev/null)
2220   statusQA=$(awk '/mergeMakeOCDB.log/ {print $2}' ${x/cpass0/cpass1} 2>/dev/null)
2221
2222   printf "%s\t ocdb.log cpass0: %s\t ocdb.log cpass1: %s\tqa.log:%s\t| cpass0: rec:%s/%s stderr:%s/%s calib:%s/%s cpass1: rec:%s/%s stderr:%s/%s calib:%s/%s QAbarrel:%s/%s QAouter:%s/%s\n" ${runNumber} ${statusOCDBcpass0} ${statusOCDBcpass1} ${statusQA} ${statusCPass0[0]} ${statusCPass0[1]} ${statusCPass0[2]} ${statusCPass0[3]} ${statusCPass0[4]} ${statusCPass0[5]} ${statusCPass1[0]} ${statusCPass1[1]} ${statusCPass1[2]} ${statusCPass1[3]} ${statusCPass1[4]} ${statusCPass1[5]} ${statusCPass1[6]} ${statusCPass1[7]} ${statusCPass1[8]} ${statusCPass1[9]}
2223 done
2224
2225   #make lists with output files - QA, trending, filtering and calibration
2226   ### wait for the merging of all runs to be over ###
2227   rm -f qa.list
2228   goPrintValues qafile qa.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2229   rm -f calib.list
2230   goPrintValues calibfile calib.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2231   rm -f trending.list
2232   goPrintValues trendingfile trending.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2233   rm -f filtering.list
2234   goPrintValues filteredTree filtering.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2235   rm -f cpass0.dcsTree.list
2236   goPrintValues dcsTree cpass0.dcsTree.list ${commonOutputPath}/meta/merge.cpass0.run*.done &>/dev/null
2237   rm -f cpass1.dcsTree.list
2238   goPrintValues dcsTree cpass1.dcsTree.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2239
2240   #merge trending
2241   rm -f ${commonOutputPath}/trending_merged.root
2242   goMerge trending.list ${commonOutputPath}/trending.root ${configFile} "${extraOpts[@]}" &> mergeTrending.log
2243
2244   goMakeSummaryTree ${commonOutputPath} 0
2245   goMakeSummaryTree ${commonOutputPath} 1
2246
2247   goCreateQAplots "${PWD}/qa.list" "${productionID}" "QAplots" "${configFile}" "${extraOpts[@]}" filteringList="${PWD}/filtering.list" &>createQAplots.log
2248
2249   #make a merged summary tree out of the QA trending, dcs trees and log summary trees
2250   goMakeMergedSummaryTree
2251
2252   #if set, email the summary
2253   [[ -n ${MAILTO} ]] && cat ${log} | mail -s "benchmark ${productionID} done" ${MAILTO}
2254
2255   return 0
2256 )
2257
2258 goMakeSummaryTree()
2259 (
2260   if [[ $# -lt 1 ]] ; then
2261     return
2262   fi
2263   #1. define vars/arrays
2264   DIR=${1} #use input or exec in current dir
2265   pass=${2-"0"} #pass from input
2266   outfile="summary_pass${pass}.tree"
2267   Ncolumns=0
2268   test -f ${outfile} && : >${outfile}
2269   errfile=${outfile/tree/err}
2270   test -f ${errfile} && : >${errfile}
2271
2272   declare -a counterString=(TOFevents TOFtracks TPCevents TPCtracks TRDevents TRDtracks T0events SDDevents SDDtracks MeanVertexevents)
2273   Ncounter=${#counterString[@]}
2274
2275   declare -a statusString=(TRDStatus TOFStatus TPCStatus T0Status MeanVertexStatus)
2276   Nstatus=${#statusString[@]}
2277
2278
2279   declare -a ratesString=(rec stderr calib qa_barrel qa_outer)
2280   Nrates=${#ratesString[@]}
2281
2282   runs=( $(ls -1 ${DIR}/meta/merge.cpass0* | while read x; do guessRunNumber $x; done) )
2283   Nruns=${#runs[@]}
2284
2285   echo -n runnumber/I >>${outfile}
2286   echo -n :cpass${pass}status/I >>${outfile}
2287   echo -n :cpass${pass}QAstatus/I >>${outfile}
2288   for i in ${ratesString[@]}; do
2289     echo -n :${i}OK/I >>${outfile}
2290     echo -n :${i}BAD/I >>${outfile}
2291
2292   done
2293   for i in ${counterString[@]} ${statusString[@]} ; do
2294     echo -n :${i}/I >>${outfile}
2295   done
2296   Ncolumns=$((2 + 2*Nrates + Ncounter + Nstatus))
2297   echo >> ${outfile}
2298
2299   #2. loop runs
2300
2301   for runnumber in ${runs[@]} ; do
2302
2303
2304
2305     filejob="${DIR}/meta/cpass${pass}.job*.run${runnumber}.done"
2306     filemerge="${DIR}/meta/merge.cpass${pass}.run${runnumber}.done"
2307     fileOCDB=$(grep /ocdb.log ${filemerge} | awk '{print $1}')
2308     if ! $(/bin/ls ${filemerge} &>/dev/null) ; then
2309       echo "${filemerge} does not exist!" >>${errfile}
2310       continue
2311     elif ! $(/bin/ls ${filejob} &>/dev/null) ; then
2312       echo "${filejob} does not exist!" >>${errfile}
2313       echo -n ${runnumber} >> ${outfile}
2314       for i in $(seq ${Ncolumns}) ; do
2315         echo -n "-1" >> ${outfile}
2316       done
2317       echo >> ${outfile}
2318       continue
2319     fi
2320     echo -n ${runnumber} >> ${outfile}
2321     #pass0status= grep '/ocdb.log' ${filemerge} |  cut -d' ' -f2 | tr OK x1 | tr BAD xx0 | tr -d 'x'
2322     passStatus=$(grep '/ocdb.log' ${filemerge} | grep OK | wc -l)
2323     echo -n " ${passStatus}" >> ${outfile}
2324     qaStatus=$(grep '/mergeMakeOCDB.log' ${filemerge} | grep OK | wc -l)
2325     echo -n " ${qaStatus}" >> ${outfile}
2326
2327
2328     #fill OK/BAD rates
2329     for i in $(seq 0 $((${Nrates}-1))) ; do
2330       var1=$(grep "/${ratesString[${i}]}.log" ${filejob} | grep OK | wc -l)
2331       var2=$(grep "/${ratesString[${i}]}.log" ${filejob} | grep BAD | wc -l)
2332
2333       if [[ ${ratesString[${i}]} == "stderr" ]] ; then
2334         var1=$(grep "stderr" ${filejob} | grep OK | wc -l)
2335         var2=$(grep "stderr" ${filejob} | grep "rec.log" | grep BAD | wc -l)
2336       fi
2337       echo -n " ${var1}" >> ${outfile}
2338       echo -n " ${var2}" >> ${outfile}
2339     done
2340
2341     if [[ -f ${fileOCDB} ]] ; then
2342       #fill counter
2343       for i in $(seq 0 $((${Ncounter}-1))) ; do
2344         var1=$(grep Monalisa ${fileOCDB} | grep ${counterString[${i}]} | cut -f2)
2345         echo -n " ${var1:-"-1"}" >> ${outfile}
2346       done
2347
2348       #fill status
2349       for i in $(seq 0 $((${Nstatus}-1))) ; do
2350         var1=$(grep "calibration status=" ${fileOCDB} | grep ${statusString[${i}]/Status/} | cut -d'=' -f2)
2351         echo -n " ${var1:-"-1"}" >> ${outfile}
2352       done
2353     fi
2354     echo >> ${outfile}
2355   done
2356
2357   return 0
2358 )
2359
2360 parseConfig()
2361 {
2362   configFile=${1}
2363   shift
2364   args=("$@")
2365
2366
2367   #some defaults
2368   #autoOCDB=0
2369   defaultOCDB="raw://"
2370   #runNumber=167123
2371   #makeflowPath="/hera/alice/aux/cctools/bin"
2372   #makeflowOptions="-T wq -N alice -d all -C ali-copilot.cern.ch:9097"
2373   #makeflowOptions="-T wq -N alice -C ali-copilot.cern.ch:9097"
2374   makeflowOptions=""
2375   #batchCommand="/usr/bin/qsub"
2376   batchFlags="-b y -cwd -l h_rt=24:0:0,h_rss=4G "
2377   baseOutputDirectory="$PWD/output"
2378   #alirootEnv="/cvmfs/alice.cern.ch/bin/alienv setenv AliRoot/v5-04-34-AN -c"
2379   #alirootEnv="/home/mkrzewic/alisoft/balice_master.sh"
2380   #trustedQAtrainMacro='/hera/alice/mkrzewic/gsisvn/Calibration/QAtrain_duo.C'
2381   reconstructInTemporaryDir=0
2382   recoTriggerOptions="\"\""
2383   percentProcessedFilesToContinue=100
2384   maxSecondsToWait=$(( 3600*24 ))
2385   nEvents=-1
2386   nMaxChunks=0
2387   postSetUpActionCPass0=""
2388   postSetUpActionCPass1=""
2389   runCPass0reco=1
2390   runCPass0MergeMakeOCDB=1
2391   runCPass1reco=1
2392   runCPass1MergeMakeOCDB=1
2393   runESDfiltering=1
2394   filteringFactorHighPt=1e2
2395   filteringFactorV0s=1e1
2396   MAILTO=""
2397   #pretend=1
2398   #dontRedirectStdOutToLog=1
2399   logToFinalDestination=1
2400   ALIROOT_FORCE_COREDUMP=1
2401   pretendDelay=0
2402
2403   #first, source the config file
2404   if [ -f ${configFile} ]; then
2405     source ${configFile}
2406   else
2407     echo "config file ${configFile} not found!"
2408     return 1
2409   fi
2410
2411   unset encodedSpaces
2412   for opt in "${args[@]}"; do
2413     [[ "${opt}" =~ encodedSpaces=.* ]] && encodedSpaces=1 && echo "encodedSpaces!" && break
2414   done
2415
2416   #then, parse the options as they override the options from file
2417   for opt in "${args[@]}"; do
2418     [[ -z ${opt} ]] && continue
2419     [[ -n ${encodedSpaces} ]] && opt="$(decSpaces ${opt})"
2420     [[ "${opt}" =~ ^[[:space:]]*$ ]] && continue
2421     if [[ ! "${opt}" =~ .*=.* ]]; then
2422       echo "badly formatted option \"${opt}\" should be: option=value, stopping..."
2423       return 1
2424     fi
2425     local var="${opt%%=*}"
2426     local value="${opt#*=}"
2427     echo "${var}=${value}"
2428     export ${var}="${value}"
2429   done
2430
2431   #do some checking
2432   [[ -z ${alirootEnv} ]] && echo "alirootEnv not defined!" && return 1
2433
2434   #export the aliroot function if defined to override normal behaviour
2435   [[ $(type -t aliroot) =~ "function" ]] && export -f aliroot
2436
2437   return 0
2438 }
2439
2440 aliroot()
2441 {
2442   args=("$@")
2443   if [[ -n ${useProfilingCommand} ]]; then
2444     profilerLogFile="cpu.txt"
2445     [[ "${args}" =~ rec ]] && profilerLogFile="cpu_rec.txt"
2446     [[ "${args}}" =~ Calib ]] && profilerLogFile="cpu_calib.txt"
2447     echo running "${useProfilingCommand} aliroot ${args} &> ${profilerLogFile}"
2448     ${useProfilingCommand} aliroot "${args[@]}" &> ${profilerLogFile}
2449   else
2450     #to prevent an infinite recursion use "command aliroot" to disable
2451     #aliases and functions
2452     echo running command aliroot "${args[@]}"
2453     command aliroot "${args[@]}"
2454   fi
2455   return 0
2456 }
2457
2458 guessRunData()
2459 {
2460   #guess the period from the path, pick the rightmost one
2461   period=""
2462   runNumber=""
2463   year=""
2464   pass=""
2465   legoTrainRunNumber=""
2466   dataType=""
2467
2468   local shortRunNumber=""
2469   local IFS="/"
2470   declare -a path=( $1 )
2471   local dirDepth=$(( ${#path[*]}-1 ))
2472   i=0
2473   #for ((x=${dirDepth};x>=0;x--)); do
2474   for ((x=0;x<=${dirDepth};x++)); do
2475
2476     [[ $((x-1)) -ge 0 ]] && local fieldPrev=${path[$((x-1))]}
2477     local field=${path[${x}]}
2478     local fieldNext=${path[$((x+1))]}
2479
2480     [[ ${field} =~ ^[0-9]*$ && ${fieldNext} =~ (.*\.zip$|.*\.root$) ]] && legoTrainRunNumber=${field}
2481     [[ -n ${legoTrainRunNumber} && -z ${pass} ]] && pass=${fieldPrev}
2482     [[ ${field} =~ ^LHC[0-9][0-9][a-z].*$ ]] && period=${field%_*}
2483     [[ ${field} =~ ^000[0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && runNumber=${field#000}
2484     [[ ${field} =~ ^[0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && shortRunNumber=${field}
2485     [[ ${field} =~ ^20[0-9][0-9]$ ]] && year=${field}
2486     [[ ${field} =~ ^(^sim$|^data$) ]] && dataType=${field}
2487     (( i++ ))
2488   done
2489   [[ -z ${legoTrainRunNumber} ]] && pass=${path[$((dirDepth-1))]}
2490   [[ "${dataType}" =~ ^sim$ ]] && pass="passMC" && runNumber=${shortRunNumber}
2491
2492   #if [[ -z ${dataType} || -z ${year} || -z ${period} || -z ${runNumber}} || -z ${pass} ]];
2493   if [[ -z ${runNumber}} ]];
2494   then
2495     #error condition
2496     return 1
2497   else
2498     #ALL OK
2499     return 0
2500   fi
2501   return 0
2502 }
2503
2504 #these functions encode strings to and from a space-less form
2505 #use when spaces are not well handled (e.g. in arguments to
2506 #commands in makeflow files, etc.
2507 encSpaces()(echo "${1// /@@@@}")
2508 decSpaces()(echo "${1//@@@@/ }")
2509
2510 main "$@"