PWGPP/benchmark/benchmark.sh

   1 #!/bin/bash
   2 #include benchmark.config
   3
   4 # this script runs the CPass0/CPass1 train
   5 # produced OCDB updates are local
   6
   7 main()
   8 {
   9   #run in proper mode depending on the selection
  10   if [[ $# -lt 1 ]]; then
  11     if [[ ! "${0}" =~ "bash" ]]; then
  12       echo "uses makeflow:"
  13       echo " ${0} \"run\" productionID inputList configFile [extraOpts]"
  14       echo "uses a batch system (SGE):"
  15       echo " ${0} \"submit\" productionID inputList configFile [extraOpts]"
  16       echo "extraOpts if non-empty override the config file, e.g.:"
  17       echo " ${0} submit test1 benchmark.list benchmark.config runNumber=169123 nEvents=10"
  18     fi
  19     return
  20   fi
  21
  22   #define some aliases - default is to call one of the functions directly
  23   runMode=${1}
  24   umask 0002
  25   shift
  26   case ${runMode} in
  27     "CPass0") goCPass0 "$@";;
  28     "CPass1") goCPass1 "$@";;
  29     "MakeLocalOCDBaccessConfig") goMakeLocalOCDBaccessConfig "$@";;
  30     "MergeCPass0") goMergeCPass0 "$@";;
  31     "MergeCPass1") goMergeCPass1 "$@";;
  32     "MakeFilteredTrees") goMakeFilteredTrees "$@";;
  33     "MakeSummary") goMakeSummary "$@";;
  34     "run") goSubmitMakeflow "$@";;
  35     "submit") goSubmitBatch "$@";;
  36     "test") goTest "$@";;
  37     "GenerateMakeflow") goGenerateMakeflow "$@";;
  38     "PrintValues") goPrintValues "$@";;
  39     "CreateQAplots") goCreateQAplots "$@";;
  40     "WaitForOutput") goWaitForOutput "$@";;
  41     "Merge") goMerge "$@";;
  42     *)
  43       ${runMode} "$@"
  44     ;;
  45   esac
  46   return 0
  47 }
  48
  49 generateMC()
  50 {
  51   #generate one raw chunk in current directory
  52   SEED=${JOB_ID}${SGE_TASK_ID}
  53   export CONFIG_SEED=${SEED}
  54   runNumber=${1}
  55   OCDBpath=${2}
  56   nEventsim=${3}
  57   if [[ -n ${pretend} ]]; then
  58     touch galice.root
  59   else
  60     if [[ -f sim.C && -f Config.C ]] ; then
  61         time aliroot -b -q -x sim.C\(${runNumber},\"${OCDBpath}\",${nEventsim}\) >sim.log 2>&1
  62         mv syswatch.log simwatch.log
  63     fi
  64   fi
  65 }
  66
  67 goCPass0()
  68 {
  69   umask 0002
  70
  71   targetDirectory=${1}
  72   inputList=${2}
  73   nEvents=${3}
  74   ocdbPath=${4}
  75   configFile=${5}
  76   runNumber=${6}
  77   jobindex=${7}
  78   shift 7
  79   if ! parseConfig ${configFile} "$@"; then return 1; fi
  80
  81   #record the working directory provided by the batch system
  82   batchWorkingDirectory=${PWD}
  83
  84   #use the jobindex only if set and non-negative
  85   if [[ -z ${jobindex} || ${jobindex} -lt 0 ]]; then
  86     [[ -n "${LSB_JOBINDEX}" ]] && jobindex=${LSB_JOBINDEX}
  87     [[ -n "${SGE_TASK_ID}" ]] && jobindex=${SGE_TASK_ID}
  88   fi
  89
  90   [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
  91   doneFile="${commonOutputPath}/meta/cpass0.job${jobindex}.run${runNumber}.done"
  92
  93   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
  94
  95   if [[ -n ${ALIROOT_FORCE_COREDUMP} ]]; then
  96     ulimit -c unlimited
  97     export ALIROOT_FORCE_COREDUMP
  98   fi
  99
 100   #the contents of this is stored in the tree and used later (e.g. AliAnalysisTaskPIDResponse)!
 101   #at the QA stage the pass number is guessed from the path stored here.
 102   #The Format is:
 103   #Packages= ;OutputDir= ;LPMPass= ;TriggerAlias= ;LPMRunNumber= ;LPMProductionType= ;LPMInteractionType= ;LPMProductionTag= ;LPMAnchorRun= ;LPMAnchorProduction= ;LPMAnchorYear=
 104   export PRODUCTION_METADATA="OutputDir=cpass0"
 105
 106   if [[ "${inputList}" =~ \.root$ ]]; then
 107     infile=${inputList}
 108   else
 109     infile=$(sed -ne "${jobindex}p" ${inputList} | egrep '\s*\w*/\w*')
 110   fi
 111   chunkName=${infile##*/}
 112
 113   outputDir=${targetDirectory}/${jobindex}_${chunkName%.*}
 114   mkdir -p ${outputDir}
 115   [[ ! -d ${outputDir} ]] && echo "cannot make ${outputDir}" && touch ${doneFile} && return 1
 116
 117   #runpath=${PWD}/rundir_cpass0_${runNumber}_${jobindex}
 118   runpath=${outputDir}
 119   [[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
 120   [[ ${reconstructInTemporaryDir} -eq 1 && -z ${TMPDIR} ]] && runpath=$(mktemp -d)
 121   mkdir -p ${runpath}
 122   [[ ! -d ${runpath} ]] && echo "cannot make runpath ${runpath}" && touch ${doneFile} && return 1
 123   cd ${runpath}
 124   [[ ! ${PWD} =~ ${runpath} ]] && echo "PWD=$PWD is not the runpath=${runpath}" && touch ${doneFile} && return 1
 125
 126   #runCPassX/C expects the raw chunk to be linked in the run dir
 127   #despite it being accessed by the full path
 128   ln -s ${infile} ${runpath}/${chunkName}
 129
 130   #####MC
 131   if [[ -n ${generateMC} ]]; then
 132     olddir=${PWD}
 133     outputDirMC=${commonOutputPath}/000${runNumber}/sim/${jobindex}
 134     simrunpath=${outputDirMC}
 135     [[ ${simulateInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && simrunpath=${TMPDIR}
 136     [[ ${simulateInTemporaryDir} -eq 1 && -z ${TMPDIR} ]] && simrunpath=$(mktemp -d)
 137     mkdir -p ${outputDirMC}
 138     mkdir -p ${simrunpath}
 139     cd ${simrunpath}
 140
 141     filesMC=(
 142               "${batchWorkingDirectory}/sim.C"
 143               "${batchWorkingDirectory}/rec.C"
 144               "${batchWorkingDirectory}/Config.C"
 145               "${batchWorkingDirectory}/OCDB_*.root"
 146     )
 147     for file in ${filesMC[*]}; do
 148       [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
 149     done
 150
 151     generateMC ${runNumber} ${ocdbPath} ${nEvents}
 152
 153     [[ ! "${simrunpath}" =~ "${outputDirMC}" ]] && mv * ${outputDirMC} #TODO check if it works
 154     cd ${olddir}
 155
 156     ln -s ${outputDirMC}/* ${runpath}/
 157
 158     inputList=${outputDirMC}/galice.root #TODO not valid outside shell !!!
 159     infile=""
 160   fi
 161   ######
 162
 163   [[ ! -f ${inputList} && -z ${pretend} ]] && echo "input file ${inputList} not found, exiting..." && touch ${doneFile} && return 1
 164
 165   logOutputDir=${runpath}
 166   [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir}
 167   [[ -z ${dontRedirectStdOutToLog} ]] && exec 1> ${logOutputDir}/stdout
 168   [[ -z ${dontRedirectStdOutToLog} ]] && exec 2> ${logOutputDir}/stderr
 169   echo "${0} $*"
 170
 171   echo "#####################"
 172   echo CPass0:
 173   echo JOB setup
 174   echo nEvents            ${nEvents}
 175   echo runNumber          ${runNumber}
 176   echo ocdbPath           ${ocdbPath}
 177   echo infile             ${infile}
 178   echo chunkName          ${chunkName}
 179   echo jobindex           ${jobindex}
 180   echo recoTriggerOptions ${recoTriggerOptions}
 181   echo targetDirectory    ${targetDirectory}
 182   echo commonOutputPath         ${commonOutputPath}
 183   echo doneFile      ${doneFile}
 184   echo runpath            ${runpath}
 185   echo outputDir          ${outputDir}
 186   echo ALICE_ROOT         ${ALICE_ROOT}
 187   echo PWD                ${PWD}
 188   echo "########## ###########"
 189
 190   alirootInfo > ALICE_ROOT.log
 191
 192   filesCPass0=(
 193                "${batchWorkingDirectory}/runCPass0.sh"
 194                "${batchWorkingDirectory}/recCPass0.C"
 195                "${batchWorkingDirectory}/runCalibTrain.C"
 196                "${batchWorkingDirectory}/localOCDBaccessConfig.C"
 197                "${batchWorkingDirectory}/OCDB.root"
 198                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/runCPass0.sh"
 199                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/recCPass0.C"
 200                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/runCalibTrain.C"
 201   )
 202
 203   for file in ${filesCPass0[*]}; do
 204     [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
 205     [[ ${file##*/} =~ .*\.sh ]] && chmod +x ${file##*/}
 206   done
 207
 208   echo "this directory (${PWD}) contents:"
 209   /bin/ls
 210   echo
 211   chmod u+x runCPass0.sh
 212
 213   sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh
 214
 215   if [[ -n ${postSetUpActionCPass0} ]]; then
 216     echo "running ${postSetUpActionCPass0}"
 217     eval ${postSetUpActionCPass0}
 218   fi
 219
 220   #run CPass0
 221   echo "${runpath}/runCPass0.sh ${infile} ${nEvents} ${runNumber} ${ocdbPath} ${recoTriggerOptions}"
 222   if [[ -n ${pretend} ]]; then
 223     touch AliESDs.root
 224     touch AliESDfriends.root
 225     touch AliESDfriends_v1.root
 226     touch rec.log
 227     touch calib.log
 228   else
 229     echo ./runCPass0.sh "${infile}" "${nEvents}" "${runNumber}" "${ocdbPath}" "${recoTriggerOptions}"
 230     ./runCPass0.sh "${infile}" "${nEvents}" "${runNumber}" "${ocdbPath}" "${recoTriggerOptions}"
 231   fi
 232
 233   #move stuff to final destination
 234   echo "this directory (${PWD}) contents:"
 235   /bin/ls
 236   echo
 237
 238   echo rm -f ./${chunkName}
 239   rm -f ./${chunkName}
 240   echo "cp --recursive ${runpath}/* ${outputDir}"
 241   cp -p --recursive ${runpath}/* ${outputDir}
 242   echo
 243
 244   #validate CPass0
 245   cd ${outputDir}
 246   touch ${doneFile}
 247   echo "dir ${outputDir}" >> ${doneFile}
 248   if summarizeLogs >> ${doneFile}; then
 249     [[ -f ${outputDirMC}/galice.root ]] && echo "sim ${outputDirMC}/galice.root" >> ${doneFile}
 250     [[ -f AliESDfriends_v1.root ]] && echo "calibfile ${outputDir}/AliESDfriends_v1.root" >> ${doneFile}
 251     [[ -f AliESDs.root ]] && echo "esd ${outputDir}/AliESDs.root" >> ${doneFile}
 252   fi
 253
 254   [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
 255   return 0
 256 }
 257
 258 goCPass1()
 259 {
 260   umask 0002
 261
 262   targetDirectory=${1}
 263   inputList=${2}
 264   nEvents=${3}
 265   ocdbPath=${4}
 266   configFile=${5}
 267   runNumber=${6}
 268   jobindex=${7}
 269   shift 7
 270   extraOpts=("$@")
 271   if ! parseConfig ${configFile} "$@"; then return 1; fi
 272
 273   #record the working directory provided by the batch system
 274   batchWorkingDirectory=${PWD}
 275
 276   #use the jobindex only if set and non-negative
 277   if [[ -z ${jobindex} || ${jobindex} -lt 0 ]]; then
 278     [[ -n "${LSB_JOBINDEX}" ]] && jobindex=${LSB_JOBINDEX}
 279     [[ -n "${SGE_TASK_ID}" ]] && jobindex=${SGE_TASK_ID}
 280   fi
 281
 282   [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
 283   doneFile="${commonOutputPath}/meta/cpass1.job${jobindex}.run${runNumber}.done"
 284
 285   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
 286
 287   if [[ -n ${ALIROOT_FORCE_COREDUMP} ]]; then
 288     ulimit -c unlimited
 289     export ALIROOT_FORCE_COREDUMP
 290   fi
 291
 292   #the contents of this is stored in the tree and used later (e.g. AliAnalysisTaskPIDResponse)!
 293   #at the QA stage the pass number is guessed from the path stored here.
 294   #The Format is:
 295   #Packages= ;OutputDir= ;LPMPass= ;TriggerAlias= ;LPMRunNumber= ;LPMProductionType= ;LPMInteractionType= ;LPMProductionTag= ;LPMAnchorRun= ;LPMAnchorProduction= ;LPMAnchorYear=
 296   export PRODUCTION_METADATA="OutputDir=cpass1"
 297
 298   [[ ! -f ${inputList} && -z ${pretend} ]] && echo "input file ${inputList} not found, exiting..." && touch ${doneFile} && return 1
 299   if [[ "${inputList}" =~ \.root$ ]]; then
 300     infile=${inputList}
 301   else
 302     infile=$(sed -ne "${jobindex}p" ${inputList} | egrep '\s*\w*/\w*')
 303   fi
 304   chunkName=${infile##*/}
 305
 306   outputDir=${targetDirectory}/${jobindex}_${chunkName%.*}
 307   mkdir -p ${outputDir}
 308   [[ ! -d ${outputDir} ]] && echo "cannot make ${outputDir}" && touch ${doneFile} && return 1
 309
 310   #runpath=${PWD}/rundir_cpass1_${runNumber}_${jobindex}
 311   runpath=${outputDir}
 312   [[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
 313   [[ ${reconstructInTemporaryDir} -eq 1 && -z ${TMPDIR} ]] && runpath=$(mktemp -d)
 314
 315   #MC
 316   if [[ "${infile}" =~ galice\.root ]]; then
 317     ln -s ${inputList%/*}/* ${runpath}
 318     infile=""
 319   fi
 320
 321   #init the running path
 322   mkdir -p ${runpath}
 323   [[ ! -d ${runpath} ]] && echo "cannot make runpath ${runpath}" && touch ${doneFile} && return 1
 324   cd ${runpath}
 325   [[ ! ${PWD} =~ ${runpath} ]] && echo "PWD=$PWD is not the runpath=${runpath}" && touch ${doneFile} && return 1
 326
 327   #this is needed for runCPass1.sh
 328   ln -s ${infile} ${runpath}/${chunkName}
 329
 330   logOutputDir=${runpath}
 331   [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir}
 332   [[ -z ${dontRedirectStdOutToLog} ]] && exec 1> ${logOutputDir}/stdout
 333   [[ -z ${dontRedirectStdOutToLog} ]] && exec 2> ${logOutputDir}/stderr
 334   echo "${0} $*"
 335
 336   echo "#####################"
 337   echo CPass1:
 338   echo JOB setup
 339   echo nEvents            ${nEvents}
 340   echo runNumber          ${runNumber}
 341   echo ocdbPath           ${ocdbPath}
 342   echo infile             ${infile}
 343   echo chunkName          ${chunkName}
 344   echo jobindex           ${jobindex}
 345   echo recoTriggerOptions ${recoTriggerOptions}
 346   echo targetDirectory    ${targetDirectory}
 347   echo commonOutputPath         ${commonOutputPath}
 348   echo doneFile      ${doneFile}
 349   echo runpath            ${runpath}
 350   echo outputDir          ${outputDir}
 351   echo ALICE_ROOT         ${ALICE_ROOT}
 352   echo PWD                ${PWD}
 353   echo "########## ###########"
 354
 355   alirootInfo > ALICE_ROOT.log
 356
 357   filesCPass1=(
 358                "${batchWorkingDirectory}/runCPass1.sh"
 359                "${batchWorkingDirectory}/recCPass1.C"
 360                "${batchWorkingDirectory}/recCPass1_OuterDet.C"
 361                "${batchWorkingDirectory}/runCalibTrain.C"
 362                "${batchWorkingDirectory}/QAtrain_duo.C"
 363                "${batchWorkingDirectory}/localOCDBaccessConfig.C"
 364                "${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz"
 365                "${batchWorkingDirectory}/OCDB.root"
 366                "${trustedQAtrainMacro}"
 367                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/runCPass1.sh"
 368                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/recCPass1.C"
 369                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/recCPass1_OuterDet.C"
 370                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/runCalibTrain.C"
 371                "${ALICE_ROOT}/ANALYSIS/macros/QAtrain_duo.C"
 372   )
 373
 374   for file in ${filesCPass1[*]}; do
 375     [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
 376     [[ ${file##*/} =~ .*\.sh ]] && chmod +x ${file##*/}
 377   done
 378
 379   echo "this directory (${PWD}) contents:"
 380   /bin/ls
 381   echo
 382
 383   sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh
 384
 385   if [[ -n ${postSetUpActionCPass1} ]]; then
 386     echo "running ${postSetUpActionCPass1}"
 387     eval ${postSetUpActionCPass1}
 388     echo
 389   fi
 390
 391   #configure local OCDB storage from CPass0 (creates the localOCDBaccessConfig.C script)
 392   if [[ -f cpass0.localOCDB.${runNumber}.tgz ]]; then
 393     echo goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz"
 394     goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz"
 395   else
 396     echo "WARNING: file cpass0.localOCDB.${runNumber}.tgz not found!"
 397   fi
 398
 399   if [[ ! $(/bin/ls -1 OCDB/*/*/*/*.root 2>/dev/null) ]]; then
 400     echo "cpass0 produced no calibration! exiting..."
 401     touch ${doneFile}
 402     return 1
 403   fi
 404
 405   #create the Barrel and OuterDet directories for CPass1 and link the local OCDB directory
 406   #there to make the localOCDBaccessConfig.C file work, since it may point to the OCDB
 407   #entries using a relative path, e.g. local://./OCDB
 408   mkdir Barrel OuterDet
 409   ln -s ../OCDB Barrel/OCDB
 410   ln -s ../OCDB OuterDet/OCDB
 411
 412   #setup the filtering
 413   #the following option enables the filtering task inside the QAtrain_duo.C
 414   [[ -n $runESDfiltering ]] && export QA_TaskFilteredTree=1
 415   #set the downscaling factors during the filtering fro expert QA (overrides the previous values)
 416   if [[ -n ${filteringFactorHighPt} ]]; then
 417     export AliAnalysisTaskFilteredTree_fLowPtTrackDownscaligF=${filteringFactorHighPt}
 418   fi
 419   if [[ -n ${filteringFactorV0s} ]]; then
 420     export AliAnalysisTaskFilteredTree_fLowPtV0DownscaligF=${filteringFactorV0s}
 421   fi
 422
 423   #run CPass1
 424   chmod u+x runCPass1.sh
 425   echo "${runpath}/runCPass1.sh ${infile} ${nEvents} ${runNumber} ${ocdbPath} ${recoTriggerOptions}"
 426   if [[ -n ${pretend} ]]; then
 427     touch AliESDs_Barrel.root
 428     touch AliESDfriends_Barrel.root
 429     touch AliESDfriends_v1.root
 430     touch QAresults_barrel.root
 431     touch EventStat_temp_barrel.root
 432     touch AODtpITS.root
 433     touch AliESDs_Outer.root
 434     touch AliESDfriends_Outer.root
 435     touch QAresults_outer.root
 436     touch EventStat_temp_outer.root
 437     touch rec.log
 438     touch calib.log
 439     touch qa.log
 440     touch filtering.log FilterEvents_Trees.root
 441   else
 442     ./runCPass1.sh "${infile}" "${nEvents}" "${runNumber}" "${ocdbPath}" "${recoTriggerOptions}"
 443
 444     [[ ! -f AliESDs_Barrel.root && -f Barrel/AliESDs.root ]] && mv Barrel/AliESDs.root AliESDs_Barrel.root
 445     [[ ! -f AliESDfriends_Barrel.root && -f Barrel/AliESDfriends.root ]] && mv Barrel/AliESDfriends.root AliESDfriends_Barrel.root
 446     [[ ! -f AliESDfriends_v1.root && -f Barrel/AliESDfriends_v1.root ]] && mv Barrel/AliESDfriends_v1.root .
 447     [[ ! -f QAresults_barrel.root && -f Barrel/QAresults_barrel.root ]] && mv Barrel/QAresults_barrel.root .
 448     [[ ! -f AliESDs_Outer.root && -f OuterDet/AliESDs.root ]] && mv OuterDet/AliESDs.root AliESDs_Outer.root
 449     [[ ! -f AliESDfriends_Outer.root && -f OuterDet/AliESDfriends.root ]] && mv OuterDet/AliESDfriends.root AliESDfriends_Outer.root
 450     [[ ! -f QAresults_outer.root && -f OuterDet/QAresults_outer.root ]] && mv OuterDet/QAresults_outer.root .
 451     [[ ! -f FilterEvents_Trees.root && -f Barrel/FilterEvents_Trees.root ]] && mv Barrel/FilterEvents_Trees.root .
 452
 453     #make the filtered tree (if requested and not already produced by QA
 454     [[ -f AliESDs_Barrel.root ]] && echo "AliESDs_Barrel.root" > filtered.list
 455     if [[ -n ${runESDfiltering} && ! -f FilterEvents_Trees.root ]]; then
 456       goMakeFilteredTrees ${PWD} ${runNumber} "filtered.list" ${filteringFactorHighPt} ${filteringFactorV0s} ${ocdbPath} 1000000 0 10000000 0 ${configFile} AliESDs_Barrel.root "${extraOpts[@]}"
 457     fi
 458
 459   fi
 460
 461   ##handle possible crashes in QA (happens often in trunk)
 462   ##rerun QA with a trusted aliroot version
 463   #if [[ $(validateLog qa_barrel.log) ]]; then
 464   #  echo "qa_barrel.log not validated!"
 465   #fi
 466   #if [[ ! -f QAresults_barrel.root && -f ${setupTrustedAliROOTenvInCurrentShell} || $(validateLog qa_barrel.log) ]]; then
 467   #  echo "WARNING: using trusted QA aliroot ${ALICE_ROOT}"
 468   #  source ${setupTrustedAliROOTenvInCurrentShell}
 469   #  cd Barrel
 470   #  rm QAresults_barrel.root
 471   #  rm EventStat_temp_barrel.root
 472   #  rm AODtpITS.root
 473   #  [[ ! -f AliESDs.root ]] && ln -s ../AliESDs_Barrel.root AliESDs.root
 474   #  [[ ! -f AliESDfriends.root ]] && ln -s ../AliESDfriends_Barrel.root AliESDfriends.root
 475   #  if [[ -n ${trustedQAtrainMacro} ]]; then
 476   #    eval "cp ${trustedQAtrainMacro} QAtrain_duo_trusted.C"
 477   #  fi
 478   #  echo executing aliroot -b -q "QAtrain_duo_trusted.C(\"_barrel\",${runNumber},\"wn.xml\",0,\"${ocdbPath}\")"
 479   #  time aliroot -b -q "QAtrain_duo.C(\"_barrel\",${runNumber},\"wn.xml\",0,\"${ocdbPath}\")" &> ../qa_barrel_trusted.log
 480   #  cd ../
 481   #fi
 482
 483   #move stuff to final destination
 484   echo "this directory (${PWD}) contents:"
 485   /bin/ls
 486   echo rm -f ./${chunkName}
 487   rm -f ./${chunkName}
 488   echo "cp --recursive ${runpath}/* ${outputDir}"
 489   cp -pf --recursive ${runpath}/* ${outputDir}
 490   echo
 491
 492   #validate CPass1
 493   cd ${outputDir}
 494   touch ${doneFile}
 495   echo "dir ${outputDir}" >> ${doneFile}
 496   if summarizeLogs >> ${doneFile}; then
 497     [[ -f AliESDs_Barrel.root ]] && echo "esd ${outputDir}/AliESDs_Barrel.root" >> ${doneFile}
 498     [[ -f AliESDfriends_v1.root ]] && echo "calibfile ${outputDir}/AliESDfriends_v1.root" >> ${doneFile}
 499     [[ -f QAresults_Barrel.root ]] && echo "qafile ${outputDir}/QAresults_Barrel.root" >> ${doneFile}
 500     [[ -f QAresults_Outer.root ]] && echo "qafile ${outputDir}/QAresults_Outer.root" >> ${doneFile}
 501     [[ -f QAresults_barrel.root ]] && echo "qafile ${outputDir}/QAresults_barrel.root" >> ${doneFile}
 502     [[ -f QAresults_outer.root ]] && echo "qafile ${outputDir}/QAresults_outer.root" >> ${doneFile}
 503     [[ -f FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFile}
 504   fi
 505
 506   [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
 507   return 0
 508 }
 509
 510
 511 goMergeCPass0()
 512 {
 513   #
 514   # find the output files and merge them
 515   #
 516
 517   outputDir=${1}
 518   ocdbStorage=${2}
 519   configFile=${3}
 520   runNumber=${4}
 521   calibrationFilesToMerge=${5}  #can be a non-existent file, will then be produced on the fly
 522   shift 5
 523   if ! parseConfig ${configFile} "$@"; then return 1; fi
 524
 525   #record the working directory provided by the batch system
 526   batchWorkingDirectory=${PWD}
 527
 528   [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
 529   doneFile="${commonOutputPath}/meta/merge.cpass0.run${runNumber}.done"
 530
 531   umask 0002
 532   ulimit -c unlimited
 533
 534   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
 535
 536   #runpath=${PWD}/rundir_cpass0_Merge_${runNumber}
 537   runpath=${outputDir}
 538   [[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
 539   [[ ${reconstructInTemporaryDir} -eq 1 && -z ${TMPDIR} ]] && runpath=$(mktemp -d)
 540
 541   mkdir -p ${runpath}
 542   [[ ! -d ${runpath} ]] && echo "not able to make the runpath ${runpath}" && touch ${doneFile} && return 1
 543   cd ${runpath}
 544   [[ ! ${PWD} =~ ${runpath} ]] && echo "PWD=$PWD is not the runpath=${runpath}" && touch ${doneFile} && return 1
 545
 546   logOutputDir=${runpath}
 547   [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir}
 548   [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/mergeMakeOCDB.log
 549   echo "${0} $*"
 550
 551   mergingScript="mergeMakeOCDB.byComponent.sh"
 552
 553   echo goMergeCPass0 SETUP:
 554   echo runNumber=${runNumber}
 555   echo outputDir=${outputDir}
 556   echo ocdbStorage=${ocdbStorage}
 557   echo calibrationFilesToMerge=${calibrationFilesToMerge}
 558   echo mergingScript=${mergingScript}
 559   echo commonOutputPath=${commonOutputPath}
 560   echo runpath=${runpath}
 561
 562   # copy files in case they are not already there
 563   filesMergeCPass0=(
 564                     "${batchWorkingDirectory}/${calibrationFilesToMerge}"
 565                     "${batchWorkingDirectory}/OCDB.root"
 566                     "${batchWorkingDirectory}/localOCDBaccessConfig.C"
 567                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/mergeMakeOCDB.byComponent.sh"
 568                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/mergeByComponent.C"
 569                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/makeOCDB.C"
 570                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/merge.C"
 571                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/mergeMakeOCDB.sh"
 572   )
 573   for file in ${filesMergeCPass0[*]}; do
 574     [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
 575     [[ ${file##*/} =~ .*\.sh ]] && chmod +x ${file##*/}
 576   done
 577
 578   sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh
 579
 580   alirootInfo > ALICE_ROOT.log
 581
 582   #
 583   echo "PWD"
 584   /bin/ls
 585   echo "PWD/.."
 586   /bin/ls ../
 587
 588
 589   #merge calibration
 590   chmod u+x ${mergingScript}
 591   mkdir -p ./OCDB
 592   if [[ ! -f ${calibrationFilesToMerge} ]]; then
 593     echo "/bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root > ${calibrationFilesToMerge}"
 594     /bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root 2>/dev/null > ${calibrationFilesToMerge}
 595   fi
 596
 597   echo "${mergingScript} ${calibrationFilesToMerge} ${runNumber} local://./OCDB ${ocdbStorage}"
 598   if [[ -n ${pretend} ]]; then
 599     touch CalibObjects.root
 600     touch ocdb.log
 601     touch merge.log
 602     touch dcsTime.root
 603     mkdir -p ./OCDB/TPC/Calib/TimeGain/
 604     mkdir -p ./OCDB/TPC/Calib/TimeDrift/
 605     touch ./OCDB/TPC/Calib/TimeGain/someCalibObject_0-999999_cpass0.root
 606     touch ./OCDB/TPC/Calib/TimeDrift/otherCalibObject_0-999999_cpass0.root
 607   else
 608     ./${mergingScript} ${calibrationFilesToMerge} ${runNumber} "local://./OCDB" ${ocdbStorage}
 609
 610     #produce the calib trees for expert QA (dcsTime.root)
 611     goMakeLocalOCDBaccessConfig ./OCDB
 612     echo aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")"
 613     aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")"
 614   fi
 615
 616   ### produce the output
 617   #tar the produced OCDB for reuse
 618   tar czf ${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz ./OCDB
 619
 620   /bin/ls
 621
 622   #copy all to output dir
 623   cp -pf --recursive ${runpath}/* ${outputDir}
 624
 625   if [[ -n ${generateMC} ]]; then
 626     goPrintValues sim ${commonOutputPath}/meta/sim.run${runNumber}.list ${commonOutputPath}/meta/cpass0.job*.run${runNumber}.done
 627   fi
 628
 629   #validate merging cpass0
 630   cd ${outputDir}
 631   touch ${doneFile}
 632   echo "dir ${outputDir}" >> ${doneFile}
 633   if summarizeLogs >> ${doneFile}; then
 634     [[ -f CalibObjects.root ]] && echo "calibfile ${outputDir}/CalibObjects.root" >> ${doneFile}
 635     [[ -f dcsTime.root ]] && echo "dcsTree ${outputDir}/dcsTime.root" >> ${doneFile}
 636   fi
 637
 638   [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
 639   return 0
 640 }
 641
 642 goMergeCPass1()
 643 {
 644   #
 645   # find the output files and merge them
 646   #
 647
 648   outputDir=${1}
 649   ocdbStorage=${2}
 650   configFile=${3}
 651   runNumber=${4}
 652   calibrationFilesToMerge=${5}
 653   qaFilesToMerge=${6}
 654   filteredFilesToMerge=${7}
 655   shift 7
 656   if ! parseConfig ${configFile} "$@"; then return 1; fi
 657
 658   #record the working directory provided by the batch system
 659   batchWorkingDirectory=${PWD}
 660
 661   [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
 662   doneFile="${commonOutputPath}/meta/merge.cpass1.run${runNumber}.done"
 663
 664   umask 0002
 665   ulimit -c unlimited
 666
 667   #clean up first:
 668   rm -f ${outputDir}/*.log
 669   rm -f ${outputDir}/*.root
 670   rm -f ${outputDir}/*done
 671
 672   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
 673
 674   #runpath=${PWD}/rundir_cpass1_Merge_${runNumber}
 675   runpath=${outputDir}
 676   [[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
 677   [[ ${reconstructInTemporaryDir} -eq 1 && -z ${TMPDIR} ]] && runpath=$(mktemp -d)
 678
 679   mkdir -p ${runpath}
 680   [[ ! -d ${runpath} ]] && echo "not able to make the runpath ${runpath}" && touch ${doneFile} && return 1
 681   cd ${runpath}
 682   [[ ! ${PWD} =~ ${runpath} ]] && echo "PWD=$PWD is not the runpath=${runpath}" && touch ${doneFile} && return 1
 683
 684   logOutputDir=${runpath}
 685   [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir}
 686   [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/mergeMakeOCDB.log
 687   echo "${0} $*"
 688
 689   calibrationOutputFileName='AliESDfriends_v1.root'
 690   qaOutputFileName='QAresults*.root'
 691   mergingScript="mergeMakeOCDB.byComponent.sh"
 692   #important to have the string "Stage.txt" in the filename to trigger the merging
 693   #it has to be a list of directories containing the files
 694   qaMergedOutputFileName="QAresults_merged.root"
 695
 696   echo goMergeCPass1 SETUP:
 697   echo runNumber=${runNumber}
 698   echo outputDir=${outputDir}
 699   echo ocdbStorage=${ocdbStorage}
 700   echo calibrationFilesToMerge=$calibrationFilesToMerge
 701   echo qaFilesToMerge=$qaFilesToMerge
 702   echo calibrationOutputFileName=${calibrationOutputFileName}
 703   echo mergingScript=${mergingScript}
 704
 705   # copy files in case they are not already there
 706   filesMergeCPass1=(
 707                     "${batchWorkingDirectory}/${calibrationFilesToMerge}"
 708                     "${batchWorkingDirectory}/${qaFilesToMerge}"
 709                     "${batchWorkingDirectory}/OCDB.root"
 710                     "${batchWorkingDirectory}/localOCDBaccessConfig.C"
 711                     "${batchWorkingDirectory}/meta/cpass0.localOCDB.${runNumber}.tgz"
 712                     "${batchWorkingDirectory}/QAtrain_duo.C"
 713                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/mergeMakeOCDB.byComponent.sh"
 714                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/mergeByComponent.C"
 715                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/makeOCDB.C"
 716                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/merge.C"
 717                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/mergeMakeOCDB.sh"
 718                     "${trustedQAtrainMacro}"
 719                     "${ALICE_ROOT}/ANALYSIS/macros/QAtrain_duo.C"
 720   )
 721   for file in ${filesMergeCPass1[*]}; do
 722     [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
 723     [[ ${file##*/} =~ .*\.sh ]] && chmod +x ${file##*/}
 724   done
 725
 726   sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh
 727
 728   #configure local OCDB storage from CPass0 (creates the localOCDBaccessConfig.C script)
 729   if [[ -f cpass0.localOCDB.${runNumber}.tgz ]]; then
 730     echo goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz"
 731     goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz"
 732   else
 733     echo "WARNING: file cpass0.localOCDB.${runNumber}.tgz not found!"
 734   fi
 735
 736   alirootInfo > ALICE_ROOT.log
 737
 738   #
 739   /bin/ls
 740
 741   #merge calibration
 742   chmod u+x ${mergingScript}
 743   mkdir -p OCDB
 744
 745   #if not provided, create the lists of files to merge
 746   if [[ ! -f ${filteredFilesToMerge} ]]; then
 747     echo "/bin/ls -1 ${outputDir}/*/FilterEvents_Trees.root > ${filteredFilesToMerge}"
 748     /bin/ls -1 ${outputDir}/*/FilterEvents_Trees.root 2>/dev/null > ${filteredFilesToMerge}
 749   fi
 750   if [[ ! -f ${calibrationFilesToMerge} ]]; then
 751     echo "/bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root > ${calibrationFilesToMerge}"
 752     /bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root 2>/dev/null > ${calibrationFilesToMerge}
 753   fi
 754   if [[ ! -f ${qaFilesToMerge} ]]; then
 755     #find the files, but only store the directories (QAtrain_duo.C requires this)
 756     echo "/bin/ls -1 ${outputDir}/*/QAresults*.root | while read x; do echo ${x%/*}; done | sort | uniq > ${qaFilesToMerge}"
 757     /bin/ls -1 ${outputDir}/*/QAresults*.root | while read x; do echo ${x%/*}; done | sort | uniq > ${qaFilesToMerge}
 758   fi
 759
 760   echo "${mergingScript} ${calibrationFilesToMerge} ${runNumber} local://./OCDB ${ocdbStorage}"
 761   if [[ -n ${pretend} ]]; then
 762     touch ocdb.log
 763     touch cpass1.localOCDB.${runNumber}.tgz
 764     touch ${qaMergedOutputFileName}
 765     touch merge.log
 766     touch trending.root
 767     touch FilterEvents_Trees.root
 768     touch CalibObjects.root
 769     touch dcsTime.root
 770     touch ${qaMergedOutputFileName}
 771     mkdir -p OCDB
 772   else
 773     ./${mergingScript} ${calibrationFilesToMerge} ${runNumber} "local://./OCDB" ${ocdbStorage}
 774
 775     #merge QA (and filtered trees)
 776     [[ -n ${AliAnalysisTaskFilteredTree_fLowPtTrackDownscaligF} ]] && export AliAnalysisTaskFilteredTree_fLowPtTrackDownscaligF
 777     [[ -n ${AliAnalysisTaskFilteredTree_fLowPtV0DownscaligF} ]] && export AliAnalysisTaskFilteredTree_fLowPtV0DownscaligF
 778
 779     #echo aliroot -l -b -q "merge.C(\"${qaFilesToMerge}\",\"\",kFALSE,\"${qaMergedOutputFileName}\")"
 780     echo aliroot -b -q "QAtrain_duo.C(\"_barrel\",${runNumber},\"${qaFilesToMerge}\",1,\"${ocdbStorage}\")"
 781     #aliroot -l -b -q "merge.C(\"${qaFilesToMerge}\",\"\",kFALSE,\"${qaMergedOutputFileName}\")"
 782     aliroot -b -q "QAtrain_duo.C(\"_barrel\",${runNumber},\"${qaFilesToMerge}\",1,\"${ocdbStorage}\")" > mergeQA.log
 783     mv QAresults_barrel.root ${qaMergedOutputFileName}
 784     mv trending_barrel.root trending.root
 785
 786     #merge filtered trees
 787     echo aliroot -l -b -q "merge.C(\"${qaFilesToMerge}\",\"\",kFALSE,\"${qaMergedOutputFileName}\")"
 788     aliroot -l -b -q "merge.C(\"${filteredFilesToMerge}\",\"\",kFALSE,\"FilterEvents_Trees.root\")"
 789
 790     #produce the calib trees for expert QA
 791     echo aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")"
 792     aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")"
 793   fi
 794
 795   tar czf ${commonOutputPath}/meta/cpass1.localOCDB.${runNumber}.tgz ./OCDB
 796
 797   /bin/ls
 798
 799   #copy all to output dir
 800   cp -pf --recursive ${runpath}/* ${outputDir}
 801
 802   #validate merge cpass1
 803   cd ${outputDir}
 804   touch ${doneFile}
 805   echo "dir ${outputDir}" >> ${doneFile}
 806   if summarizeLogs >>  ${doneFile}; then
 807     [[ -f CalibObjects.root ]] && echo "calibfile ${outputDir}/CalibObjects.root" >> ${doneFile}
 808     [[ -f ${qaMergedOutputFileName} ]] && echo "qafile ${outputDir}/${qaMergedOutputFileName}" >> ${doneFile}
 809     [[ -f trending.root ]] && echo "trendingfile ${outputDir}/trending.root" >> ${doneFile}
 810     [[ -f dcsTime.root ]] && echo "dcsTree ${outputDir}/dcsTime.root" >> ${doneFile}
 811     [[ -f FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFile}
 812   fi
 813
 814   [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
 815   return 0
 816 }
 817
 818 goMerge()
 819 {
 820   #generic root merge using CPass1 merge.C script
 821   inputList=${1}
 822   outputFile=${2}
 823   configFile=${3-"becnhmark.config"}
 824   shift 3
 825   if ! parseConfig ${configFile} "$@"; then return 1; fi
 826
 827   #record the working directory provided by the batch system
 828   batchWorkingDirectory=${PWD}
 829
 830   [[ ! -f ${inputList} ]] && echo "inputList ${inputList} does not exist!" && return 1
 831   [[ ! -f ${configFile} ]] && echo "configFile ${configFile} does not exist!" && return 1
 832   umask 0002
 833   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
 834   rm -f ${outputFile}
 835   aliroot -b -q "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/merge.C(\"${inputList}\",\"\",kFALSE,\"${outputFile}\")" > merge_${inputList}.log
 836   return 0
 837 }
 838
 839 goSubmitMakeflow()
 840 {
 841   #run
 842   productionID=${1}
 843   inputFileList=${2}
 844   configFile=${3}
 845   shift 3
 846   extraOpts=("$@")
 847   if ! parseConfig ${configFile} "${extraOpts[@]}"; then return 1; fi
 848
 849   #record the working directory provided by the batch system
 850   batchWorkingDirectory=${PWD}
 851
 852   [[ -z ${configFile} ]] && configFile="benchmark.config"
 853   [[ ! -f ${configFile} ]] && echo "no config file found (${configFile})" && return 1
 854
 855   if [[ ! $(which makeflow &>/dev/null) && -n ${makeflowPath} ]]; then
 856     echo "setting the makflow path from the config: "
 857     echo "  export PATH=${makeflowPath}:${PATH}"
 858     export PATH=${makeflowPath}:${PATH}
 859   fi
 860
 861   #create the common output dir and the meta dir
 862   commonOutputPath=${baseOutputDirectory}/${productionID}
 863   if [[ -d ${commonOutputPath} ]]; then
 864     echo "output dir ${commonOutputPath} exists!"
 865     return 1
 866   else
 867     mkdir -p ${commonOutputPath}
 868   fi
 869   mkdir -p ${commonOutputPath}/meta
 870
 871   self=$(readlink -f "${0}")
 872
 873   cp ${self} ${commonOutputPath}
 874   cp ${configFile} ${commonOutputPath}
 875   cp ${inputList} ${commonOutputPath}
 876
 877   #submit - use makeflow if available, fall back to old stuff when makeflow not there
 878   if which makeflow; then
 879
 880     goGenerateMakeflow ${productionID} ${inputFileList} ${configFile} "${extraOpts[@]}" commonOutputPath=${commonOutputPath} > benchmark.makeflow
 881
 882     makeflow ${makeflowOptions} benchmark.makeflow
 883     cd ../
 884   else
 885     echo "no makeflow!"
 886   fi
 887   return 0
 888 }
 889
 890 goGenerateMakeflow()
 891 {
 892   #generate the makeflow file
 893   [[ $# -lt 3 ]] && echo "args: id inputFileList configFile" && return 1
 894   productionID=${1}
 895   inputFileList=${2}
 896   configFile=${3}
 897   shift 3
 898   extraOpts=("$@")
 899   if ! parseConfig ${configFile} "${extraOpts[@]}" &>/dev/null; then return 1; fi
 900
 901   #extra safety
 902   if [[ -z ${commonOutputPath} ]]; then
 903     commonOutputPath=${baseOutputDirectory}/${productionID}
 904     extraOpts=( "${extraOpts[@]}" "commonOutputPath=${commonOutputPath}" )
 905   fi
 906
 907   #record the working directory provided by the batch system
 908   batchWorkingDirectory=${PWD}
 909
 910   [[ -z ${configFile} ]] && configFile="benchmark.config"
 911   [[ ! -f ${configFile} ]] && echo "no config file found (${configFile})" && return 1
 912
 913   #these files will be made a dependency - will be copied to the working dir of the jobs
 914   declare -a copyFiles
 915   inputFiles=(
 916               "OCDB.root"
 917               "localOCDBaccessConfig.C"
 918               "QAtrain_duo.C"
 919               "runCPass1.sh"
 920               "recCPass1.C"
 921               "recCPass1_OuterDet.C"
 922               "runCalibTrain.C"
 923               "runCPass0.sh"
 924               "recCPass0.C"
 925   )
 926   for file in ${inputFiles[*]}; do
 927     [[ -f ${file} ]] && copyFiles+=("${file}")
 928   done
 929
 930   #create the makeflow file
 931   [[ -n ${batchFlags} ]] && echo "BATCH_OPTIONS = ${batchFlags}"
 932   declare -A arr_cpass0_merged arr_cpass1_merged
 933   declare -A arr_cpass0_calib_list arr_cpass1_calib_list
 934   declare -A arr_cpass1_QA_list arr_cpass1_ESD_list arr_cpass1_filtered_list
 935   declare -A arr_cpass0_profiled_outputs
 936   declare -A listOfRuns
 937   [[ -n ${runNumber} ]] && listOfRuns[${runNumber}]=1
 938   while read x; do tmpRun=$(guessRunNumber ${x}); [[ -n ${tmpRun} ]] && listOfRuns[${tmpRun}]=1; done < ${inputFileList}
 939   for runNumber in "${!listOfRuns[@]}"; do
 940     [[ -z ${runNumber} ]] && continue
 941     [[ ! ${runNumber} =~ ^[0-9]*[0-9]$ ]] && continue
 942
 943     unset arr_cpass0_outputs
 944     unset arr_cpass1_outputs
 945     declare -a arr_cpass0_outputs
 946     declare -a arr_cpass1_outputs
 947
 948     jobindex=0
 949     inputFile=""
 950     while read inputFile; do
 951       currentDefaultOCDB=${defaultOCDB}
 952       [[ -z ${autoOCDB} ]] && autoOCDB=1
 953       if [[ ${autoOCDB} -ne 0 ]]; then
 954         currentDefaultOCDB=$(setYear ${inputFile} ${defaultOCDB})
 955       fi
 956
 957       #CPass0
 958       arr_cpass0_outputs[${jobindex}]="${commonOutputPath}/meta/cpass0.job${jobindex}.run${runNumber}.done"
 959       echo "${arr_cpass0_outputs[${jobindex}]} : benchmark.sh ${configFile} ${copyFiles[@]}"
 960       echo -n " ${alirootEnv} ./benchmark.sh CPass0 ${commonOutputPath}/000${runNumber}/cpass0 ${inputFile} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} ${jobindex}"" "
 961       for extraOption in "${extraOpts[@]}"; do echo -n \'${extraOption}\'" "; done; echo
 962       echo
 963
 964       #CPass1
 965       arr_cpass1_outputs[${jobindex}]="${commonOutputPath}/meta/cpass1.job${jobindex}.run${runNumber}.done"
 966       echo "${arr_cpass1_outputs[${jobindex}]} : benchmark.sh ${configFile} ${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz ${copyFiles[@]}"
 967       echo -n " ${alirootEnv} ./benchmark.sh CPass1 ${commonOutputPath}/000${runNumber}/cpass1 ${inputFile} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} ${jobindex}"" "
 968       for extraOption in "${extraOpts[@]}"; do echo -n \'${extraOption}\'" "; done; echo
 969       echo
 970       ((jobindex++))
 971
 972     done< <(grep "/000${runNumber}/" ${inputFileList})
 973
 974     #CPass0 list of Calib files to merge
 975     arr_cpass0_calib_list[${runNumber}]="${commonOutputPath}/meta/cpass0.calib.run${runNumber}.list"
 976     echo "${arr_cpass0_calib_list[${runNumber}]} : benchmark.sh ${arr_cpass0_outputs[*]}"
 977     echo "  ./benchmark.sh PrintValues calibfile ${arr_cpass0_calib_list[${runNumber}]} ${arr_cpass0_outputs[*]}"
 978     echo
 979
 980     #CPass0 merging
 981     arr_cpass0_merged[${runNumber}]="${commonOutputPath}/meta/merge.cpass0.run${runNumber}.done"
 982     echo "${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz ${arr_cpass0_merged[${runNumber}]} : benchmark.sh ${configFile} ${arr_cpass0_calib_list[${runNumber}]} ${copyFiles[@]}"
 983     echo -n " ${alirootEnv} ./benchmark.sh MergeCPass0 ${commonOutputPath}/000${runNumber}/cpass0 ${currentDefaultOCDB} ${configFile} ${runNumber} ${arr_cpass0_calib_list[${runNumber}]}"" "
 984     for extraOption in "${extraOpts[@]}"; do echo -n \'${extraOption}\'" "; done; echo
 985     echo
 986
 987     #CPass1 list of Calib/QA/ESD/filtered files
 988     # the trick with QA is to have the string "Stage.txt" in the file name of the list of directories with QA output to trigger
 989     # the production of the QA trending tree (only then the task->Finish() will be called in QAtrain_duo.C, on the grid
 990     # this corresponds to the last merging stage)
 991     arr_cpass1_QA_list[${runNumber}]="${commonOutputPath}/meta/cpass1.QA.run${runNumber}.lastMergingStage.txt.list"
 992     echo "${arr_cpass1_QA_list[${runNumber}]}: benchmark.sh ${arr_cpass1_outputs[*]}"
 993     echo "  ./benchmark.sh PrintValues dir ${arr_cpass1_QA_list[${runNumber}]} ${arr_cpass1_outputs[*]}"
 994     echo
 995     arr_cpass1_calib_list[${runNumber}]="${commonOutputPath}/meta/cpass1.calib.run${runNumber}.list"
 996     echo "${arr_cpass1_calib_list[${runNumber}]} : benchmark.sh ${arr_cpass1_outputs[*]}"
 997     echo "  ./benchmark.sh PrintValues calibfile ${arr_cpass1_calib_list[${runNumber}]} ${arr_cpass1_outputs[*]};"
 998     echo
 999     arr_cpass1_ESD_list[${runNumber}]="${commonOutputPath}/meta/cpass1.ESD.run${runNumber}.list"
1000     echo "${arr_cpass1_ESD_list[${runNumber}]} : benchmark.sh ${arr_cpass1_outputs[*]}"
1001     echo "  ./benchmark.sh PrintValues esd ${arr_cpass1_ESD_list[${runNumber}]} ${arr_cpass1_outputs[*]}"
1002     echo
1003     arr_cpass1_filtered_list[${runNumber}]="${commonOutputPath}/meta/cpass1.filtered.run${runNumber}.list"
1004     echo "${arr_cpass1_filtered_list[${runNumber}]} : benchmark.sh ${arr_cpass1_outputs[*]}"
1005     echo "  ./benchmark.sh PrintValues filteredTree ${arr_cpass1_filtered_list[${runNumber}]} ${arr_cpass1_outputs[*]}"
1006     echo
1007
1008     #CPass1 merging
1009     arr_cpass1_merged[${runNumber}]="${commonOutputPath}/meta/merge.cpass1.run${runNumber}.done"
1010     echo "${commonOutputPath}/meta/cpass1.localOCDB.${runNumber}.tgz ${arr_cpass1_merged[${runNumber}]} :  benchmark.sh ${configFile} ${arr_cpass1_calib_list[${runNumber}]} ${arr_cpass1_QA_list[${runNumber}]} ${copyFiles[@]}"
1011     echo -n " ${alirootEnv} ./benchmark.sh MergeCPass1 ${commonOutputPath}/000${runNumber}/cpass1 ${currentDefaultOCDB} ${configFile} ${runNumber} ${arr_cpass1_calib_list[${runNumber}]} ${arr_cpass1_QA_list[${runNumber}]} ${arr_cpass1_filtered_list[${runNumber}]}"" "
1012     for extraOption in "${extraOpts[@]}"; do echo -n \'${extraOption}\'" "; done; echo
1013     echo
1014
1015     #CPass0 wrapped in a profiling tool (valgrind,....)
1016     if [[ -n ${profilingCommand} ]]; then
1017       arr_cpass0_profiled_outputs[${runNumber}]="${commonOutputPath}/meta/cpass0.jobProfiled.run${runNumber}.done"
1018       echo "${arr_cpass0_profiled_outputs[${runNumber}]} : benchmark.sh ${configFile} ${copyFiles[@]}"
1019       echo -n " ${alirootEnv} ./benchmark.sh CPass0 ${commonOutputPath}/000${runNumber}/profiled ${inputFile} ${nEventsProfiling} ${currentDefaultOCDB} ${configFile} ${runNumber} profiling"" "
1020       for extraOption in "${extraOpts[@]}"; do echo -n \'${extraOption}\'" "; done;
1021       echo "\"useProfilingCommand=${profilingCommand}\""
1022       echo
1023     fi
1024
1025   done #runs
1026
1027   #Summary
1028   echo "${commonOutputPath}/summary.log : benchmark.sh ${configFile} ${arr_cpass1_merged[*]}"
1029   echo -n "  ${alirootEnv} ./benchmark.sh MakeSummary ${configFile}"" "
1030   for extraOption in "${extraOpts[@]}"; do echo -n \'${extraOption}\'" "; done; echo
1031   echo
1032
1033   return 0
1034 }
1035
1036 goPrintValues()
1037 {
1038   #print the values given the key from any number of files (store in output file on second argument)
1039   if [[ $# -lt 3 ]]; then
1040     echo "goPrintValues key outputFile inputFiles"
1041     echo "if outputFile is \"-\" dont redirect to a file"
1042     return
1043   fi
1044   key=${1}
1045   outputFile=${2}
1046   [[ ${outputFile} =~ "-" ]] && outputFile=""
1047   shift 2 #remove 2 first arguments from arg list to only pass the input files to awk
1048   awk -v key=${key} '$0 ~ key" " {print $2}' "$@" | tee ${outputFile}
1049   return 0
1050 }
1051
1052 goCreateQAplots()
1053 {
1054   umask 0002
1055   mergedQAfileList=${1}
1056   productionID=${2}
1057   outputDir=${3}
1058   configFile=${4}
1059   shift 4
1060   if ! parseConfig ${configFile} ${@}; then return 1; fi
1061
1062   #record the working directory provided by the batch system
1063   batchWorkingDirectory=${PWD}
1064
1065   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
1066
1067   [[ -z ${logOutputDir} ]] && logOutputDir=${PWD}
1068   [[ -z ${dontRedirectStdOutToLog} ]] && exec 2>&1 > ${logOutputDir}/makeQAplots.log
1069   echo "${0} $*"
1070
1071   olddir=${PWD}
1072   mkdir -p ${outputDir}
1073   cd ${outputDir}
1074   [[ ! "${PWD}" =~ "${outputDir}" ]] && echo "PWD is not equal to outputDir=${outputDir}" && cd ${olddir} && return 1
1075
1076   ${ALICE_ROOT}/PWGPP/QA/scripts/runQA.sh inputList="${mergedQAfileList}" inputListHighPtTrees="${filteringList}"
1077
1078   cd ${olddir}
1079   return 0
1080 }
1081
1082 goTest()
1083 {
1084   umask 0002
1085   exec 2>&1
1086   exec > >(tee test.log)
1087   echo "$@"
1088   echo something
1089   return 0
1090 }
1091
1092 alirootInfo()
1093 {
1094   umask 0002
1095   # save aliroot repository info
1096   [[ -z "${ALICE_ROOT}" ]] && return 1
1097
1098   echo "\${ALICE_ROOT}=${ALICE_ROOT}"
1099   echo "\${ROOTSYS}=${ROOTSYS}"
1100   echo "\${PATH}=${PATH}"
1101   echo "\${LD_LIBRARY_PATH}=${LD_LIBRARY_PATH}"
1102
1103   pushd ${PWD}
1104   cd ${ALICE_ROOT}
1105   echo
1106
1107   currentBranch=$(git rev-parse --abbrev-ref HEAD)
1108   git status
1109   echo ""
1110   echo ""
1111   git diff ${currentBranch}
1112   popd
1113   return 0
1114 }
1115
1116 setYear()
1117 {
1118   #set the year
1119   #  ${1} - year to be set
1120   #  ${2} - where to set the year
1121   year1=$(guessYear ${1})
1122   year2=$(guessYear ${2})
1123   local path=${2}
1124   [[ ${year1} -ne ${year2} && -n ${year2} && -n ${year1} ]] && path=${2/\/${year2}\//\/${year1}\/}
1125   echo ${path}
1126   return 0
1127 }
1128
1129 guessPeriod()
1130 {
1131   #guess the period from the path, pick the rightmost one
1132   local IFS="/"
1133   declare -a path=( ${1} )
1134   local dirDepth=${#path[*]}
1135   for ((x=${dirDepth}-1;x>=0;x--)); do
1136     local field=${path[${x}]}
1137     [[ ${field} =~ ^LHC[0-9][0-9][a-z]$ ]] && period=${field} && break
1138   done
1139   echo ${period}
1140   return 0
1141 }
1142
1143 guessYear()
1144 {
1145   #guess the year from the path, pick the rightmost one
1146   local IFS="/"
1147   declare -a path=( ${1} )
1148   local dirDepth=${#path[*]}
1149   for ((x=${dirDepth}-1;x>=0;x--)); do
1150     local field=${path[${x}]}
1151     [[ ${field} =~ ^20[0-9][0-9]$ ]] && year=${field} && break
1152   done
1153   echo ${year}
1154   return 0
1155 }
1156
1157 guessRunNumber()
1158 {
1159   #guess the run number from the path, pick the rightmost one
1160   #works for /path/foo/000123456/bar/...
1161   #and       /path/foo.run123456.bar
1162   local IFS="/."
1163   declare -a path=( ${1} )
1164   local dirDepth=${#path[*]}
1165   for ((x=${dirDepth}-1;x>=0;x--)); do
1166     local field=${path[${x}]}
1167     field=${field/run/000}
1168     [[ ${field} =~ [0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && runNumber=${field#000} && break
1169   done
1170   echo ${runNumber}
1171   return 0
1172 }
1173
1174 validateLog()
1175 {
1176   log=${1}
1177   errorConditions=(
1178                     'There was a crash'
1179                     'floating'
1180                     'error while loading shared libraries'
1181                     'std::bad_alloc'
1182                     's_err_syswatch_'
1183                     'Thread [0-9]* (Thread'
1184                     'AliFatal'
1185                     'core dumped'
1186                     '\.C.*error:.*\.h: No such file'
1187                     'line.*Aborted'
1188   )
1189
1190   warningConditions=(
1191                      'This is serious !'
1192                      'rocVoltage out of range:'
1193   )
1194
1195   local logstatus=0
1196   local errorSummary=""
1197   local warningSummary=""
1198
1199   for ((i=0; i<${#errorConditions[@]};i++)); do
1200     local tmp=$(grep -m1 -e "${errorConditions[${i}]}" ${log})
1201     [[ -n ${tmp} ]] && tmp+=" : "
1202     errorSummary+=${tmp}
1203   done
1204
1205   for ((i=0; i<${#warningConditions[@]};i++)); do
1206     local tmp=$(grep -m1 -e "${warningConditions[${i}]}" ${log})
1207     [[ -n ${tmp} ]] && tmp+=" : "
1208     warningSummary+=${tmp}
1209   done
1210
1211   if [[ -n ${errorSummary} ]]; then
1212     echo "${errorSummary}"
1213     return 1
1214   fi
1215
1216   if [[ -n ${warningSummary} ]]; then
1217     echo "${warningSummary}"
1218     return 2
1219   fi
1220
1221   return 0
1222 }
1223
1224 summarizeLogs()
1225 {
1226   #print a summary of logs
1227   logFiles=(
1228             "*.log"
1229             "stdout"
1230             "stderr"
1231   )
1232
1233   #check logs
1234   local logstatus=0
1235   for log in ${logFiles[*]}; do
1236     finallog=${PWD%/}/${log}
1237     [[ ! -f ${log} ]] && continue
1238     errorSummary=$(validateLog ${log})
1239     validationStatus=$?
1240     if [[ ${validationStatus} -eq 0 ]]; then
1241       #in pretend mode randomly report an error in rec.log some cases
1242       if [[ -n ${pretend} && "${log}" == "rec.log" ]]; then
1243         #[[ $(( ${RANDOM}%2 )) -ge 1 ]] && echo "${finallog} BAD random error" || echo "${finallog} OK"
1244         echo "${finallog} OK"
1245       else
1246         echo "${finallog} OK"
1247       fi
1248     elif [[ ${validationStatus} -eq 1 ]]; then
1249       echo "${finallog} BAD ${errorSummary}"
1250       logstatus=1
1251     elif [[ ${validationStatus} -eq 2 ]]; then
1252       echo "${finallog} OK MWAH ${errorSummary}"
1253     fi
1254   done
1255
1256   #report core files
1257   while read x; do
1258     echo ${x}
1259     chmod 644 ${x}
1260     gdb --batch --quiet -ex "bt" -ex "quit" aliroot ${x} > stacktrace_${x//\//_}.log
1261   done < <(/bin/ls ${PWD}/*/core 2>/dev/null; /bin/ls ${PWD}/core 2>/dev/null)
1262
1263   return ${logstatus}
1264 }
1265
1266 spitOutLocalOCDBaccessConfig()
1267 {
1268   umask 0002
1269   #find ${1} -name "*root" | \
1270   /bin/ls -1 ${1}/*/*/*/*.root 2>/dev/null | \
1271   while read line
1272   do
1273     local tmp=${line#${1}}
1274     echo ${tmp%/*} | \
1275     awk -v ocdb=${1} '{print "  man->SetSpecificStorage(\""$1"\",\"local://"ocdb"\");"}'
1276   done
1277   return 0
1278 }
1279
1280 goMakeLocalOCDBaccessConfig()
1281 {
1282   umask 0002
1283   # make a script that sets the specific storages form all the root files produced by CPass0
1284   local localOCDBpathCPass0=${1}
1285   local OCDBpathPrefix=${2}
1286   [[ -z ${OCDBpathPrefix} ]] && OCDBpathPrefix="."
1287
1288   if [[ -f ${localOCDBpathCPass0} && ${localOCDBpathCPass0} =~ \.tgz$ ]]; then
1289     tar xzf ${localOCDBpathCPass0}
1290     local localOCDBpathCPass0="${OCDBpathPrefix}/OCDB"
1291   fi
1292
1293   echo
1294   echo creating the specific storage script
1295   echo   localOCDBaccessConfig.C
1296   echo   based on OCDB: ${localOCDBaccessConfig}
1297   echo
1298
1299   local tempLocalOCDB=""
1300   if [[ -f localOCDBaccessConfig.C ]]; then
1301     tempLocalOCDB=$(mktemp)
1302     echo "egrep "SetSpecificStorage" localOCDBaccessConfig.C > ${tempLocalOCDB}"
1303     egrep "SetSpecificStorage" localOCDBaccessConfig.C > ${tempLocalOCDB}
1304   fi
1305
1306   echo "localOCDBaccessConfig()"                               >  localOCDBaccessConfig.C
1307   echo "{"                                                     >> localOCDBaccessConfig.C
1308   echo "  AliCDBManager* man = AliCDBManager::Instance();"     >> localOCDBaccessConfig.C
1309   spitOutLocalOCDBaccessConfig ${localOCDBpathCPass0}|sort|uniq  >> localOCDBaccessConfig.C
1310   [[ -f "${tempLocalOCDB}" ]] && cat ${tempLocalOCDB}              >> localOCDBaccessConfig.C
1311   echo "}"                                                     >> localOCDBaccessConfig.C
1312
1313   [[ -f "${tempLocalOCDB}" ]] && rm -f ${tempLocalOCDB}
1314
1315   if ! grep SetSpecificStorage localOCDBaccessConfig.C; then
1316     echo
1317     echo "!!!!!!! CPass0 produced no OCDB entries"
1318     return 1
1319   fi
1320   return 0
1321 }
1322
1323 goMakeFilteredTrees()
1324 {
1325   outputDir=${1}
1326   runNumber=${2}
1327   #get path to input list
1328   inputListfiles=${3}
1329   #get scale number for tracks
1330   filterT=${4}
1331   #get scale number for V0s
1332   filterV=${5}
1333   #get OCDB path (optional)
1334   OCDBpath=${6}
1335   #get max number of files
1336   maxFiles=${7-"1000000"}
1337   #get offset of first file
1338   offsetFile=${8-"0"}
1339   #get max number of events
1340   maxEvents=${9-"30000000"}
1341   #get offset of first event
1342   offsetEvent=${10-"0"}
1343   configFile=${11-"benchmark.config"}
1344   esdFileName=${12-"AliESDs_Barrel.root"}
1345   shift 12
1346   if ! parseConfig ${configFile} "$@"; then return 1; fi
1347
1348   #record the working directory provided by the batch system
1349   batchWorkingDirectory=${PWD}
1350
1351
1352   commonOutputPath=${PWD}
1353   doneFile=${commonOutputPath}/meta/filtering.cpass1.run${runNumber}.done
1354
1355   runpath=${outputDir}
1356   [[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
1357   [[ ${reconstructInTemporaryDir} -eq 1 && -z ${TMPDIR} ]] && runpath=$(mktemp -d)
1358
1359
1360   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
1361
1362   mkdir -p ${outputDir}
1363   mkdir -p ${runpath}
1364
1365   cd ${runpath}
1366   [[ ! ${PWD} =~ ${runpath} ]] && echo "PWD=$PWD is not the runpath=${runpath}" && touch ${doneFile} && return 1
1367
1368   cat > filtering.log << EOF
1369   goMakeFilteredTrees config:
1370   runpath=${runpath}
1371   outputDir=${outputDir}
1372   commonOutputPath=${commonOutputPath}
1373   ALICE_ROOT=${ALICE_ROOT}
1374   PATH=${PATH}
1375   offsetEvent=$offsetEvent
1376   configFile=$configFile
1377   esdFileName=$esdFileName
1378 EOF
1379
1380   if [[ -z ${pretend} ]];then
1381     aliroot -l -b -q "${ALICE_ROOT}/PWGPP/macros/runFilteringTask.C(\"${inputListfiles}\",${filterT},${filterV},\"${OCDBpath}\",${maxFiles},${offsetFile},${maxEvents},${offsetEvent},\"${esdFileName}\")" &>> filtering.log
1382   else
1383     touch filtering.log FilterEvents_Trees.root
1384   fi
1385   pwd
1386   /bin/ls
1387   touch ${doneFile}
1388   summarizeLogs >>  ${doneFile}
1389   echo mv -f * ${outputDir}
1390   mv -f * ${outputDir}
1391   [[ -f ${outputDir}/FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFile}
1392   cd ${commonOutputPath}
1393   [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
1394   return 0
1395 }
1396
1397 submit()
1398 {
1399   umask 0002
1400   [[ $# -lt 5 ]] && echo "at least 5 args needed, you supplied $#" && return 1
1401   JobID=${1}
1402   startID=${2}
1403   endID=${3}
1404   waitForJOBID=${4}
1405   command=${5}
1406   shift 5
1407   commandArgs=("$@")
1408
1409   #add quote strings around the extra arguments
1410   for ((i=0; i<${#commandArgs[@]}; i++)); do
1411     commandArgs[i]=\"${commandArgs[i]}\"
1412   done
1413
1414   [[ -z ${waitForJOBID} ]] && waitForJOBID=0
1415
1416   newFarm=$(which qsub|grep "^/usr/bin/qsub")
1417
1418   batchSystem="SGE"
1419
1420   if [[ -z "${newFarm}" ]]
1421   then
1422     #old LSF
1423     # submit it (as job array)
1424     nFiles=$(( ${endID}-${startID}+1 ))
1425     while [ ${startID} -le ${nFiles}  ] ; do
1426       if [ $(expr ${nFiles} - ${startID}) -gt 999 ] ; then
1427         endID=$(expr ${startID} + 999)
1428       else
1429         endID=${nFiles}
1430       fi
1431     if [[ ${waitForJOBID} -eq 0 ]]; then
1432         echo ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}"
1433         ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}"
1434       else
1435         echo ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -w "ended(${waitForJOBID})" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}"
1436         ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -w "ended(${waitForJOBID})" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}"
1437       fi
1438       startID=$(expr ${endID} + 1)
1439     done
1440   else
1441     #new SGE farm
1442     if [[ ${waitForJOBID} =~ "000" ]]; then
1443       echo ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}"
1444       ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}"
1445     else
1446       echo ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -hold_jid "${waitForJOBID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}"
1447       ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -hold_jid "${waitForJOBID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}"
1448     fi
1449   fi
1450   return 0
1451 }
1452
1453 goSubmitBatch()
1454 {
1455   if [[ $# -lt 3 ]]; then
1456     echo "minimal use:"
1457     echo " ${0} submit fileList productionID configFile"
1458     return 0
1459   fi
1460
1461   productionID=${1}
1462   inputList=${2}
1463   configFile=${3:-"benchmark.config"}
1464   configFile=$(readlink -f ${configFile})
1465   shift 3
1466   extraOpts=("$@")
1467   if ! parseConfig ${configFile} "${extraOpts[@]}"; then return 1; fi
1468
1469   #record the working directory provided by the batch system
1470   batchWorkingDirectory=${PWD}
1471
1472   #redirect all output to submit.log
1473   echo "redirecting all output to ${PWD}/submit_${productionID//"/"/_}.log"
1474   exec 7>&1
1475   exec 1>submit_${productionID//"/"/_}.log 2>&1
1476
1477   umask 0002
1478   echo ${0}" submit $*"
1479   if [[ -z "${inputList}" || -z "${productionID}" ]]
1480   then
1481     echo
1482     echo " Usage: ${0} submit inputList productionID [configFile=benchmark.config]"
1483     echo
1484     return
1485   fi
1486
1487   # check if config file is there
1488   if [ ! -f ${configFile} ]; then
1489     echo "ERROR! Config File '${configFile}' not found" >&2
1490     return
1491   else
1492     echo "Using Config File: '${configFile}'"
1493   fi
1494
1495   [[ ! -f ${alirootEnv} ]] && echo "alirootEnv script ${alirootEnv} not found!..." && return 1
1496
1497   #move the script, config and some other stuff to ${commonOutputPath} first, then use them from there
1498   self=$(readlink -f "${0}")
1499   configPath=$(dirname ${configFile})
1500   export commonOutputPath=${baseOutputDirectory}/${productionID}
1501
1502   mkdir -p ${commonOutputPath}
1503   mkdir -p ${commonOutputPath}/logs
1504   mkdir -p ${commonOutputPath}/meta
1505
1506   cp ${self} ${commonOutputPath}
1507   cp ${configFile} ${commonOutputPath}
1508   cp ${inputList} ${commonOutputPath}
1509   self=${commonOutputPath}/${self##*/}
1510   chmod u+x ${self}
1511   configFile=${commonOutputPath}/${configFile##*/}
1512   inputList=${commonOutputPath}/${inputList##*/}
1513
1514   #convert to absolut pathnames
1515   inputList=$(readlink -f "${inputList}")
1516   #make list of runs
1517   if [[ -z ${runNumber} ]]; then
1518     listOfRuns=($(while read x; do guessRunNumber ${x}; done < ${inputList} | sort | uniq))
1519   else
1520     listOfRuns=${runNumber}
1521   fi
1522
1523   alirootSource=$(readlink -f "${alirootSource}")
1524
1525   echo ""
1526   echo "### BEGIN CONFIGURATION ###"
1527   echo ""
1528   echo "GENERAL:"
1529   echo ""
1530   echo "    productionID:    ${productionID}"
1531   echo "    batchCommand:    ${batchCommand}"
1532   echo "    batchFlags:      ${batchFlags}"
1533   echo "    alirootEnv:   ${alirootEnv}"
1534   ${alirootEnv} echo '    ALICE_ROOT:      ${ALICE_ROOT}'
1535   ${alirootEnv} echo '    ALIROOT_RELEASE: ${ALICE_RELEASE}'
1536   echo "    inputList:       ${inputList}"
1537   echo "    configPath:      ${configPath}"
1538   echo "    commonOutputPath:      ${commonOutputPath}"
1539   echo "    defaultOCDB:     ${defaultOCDB}"
1540   echo "      autoOCDB: ${autoOCDB}"
1541   echo "    recoTriggerOptions:   ${recoTriggerOptions}"
1542   echo "    runs:"
1543   echo "      ${listOfRuns[*]}"
1544   echo ""
1545   echo "THE TRAIN WILL RUN:"
1546
1547   if [ ${runCPass0reco} -eq 1 ]; then
1548     echo "    Pass0 - Recontruction"
1549   fi
1550
1551   if [ ${runCPass0MergeMakeOCDB} -eq 1 ]; then
1552     echo "    Pass0 - merging and OCDB export"
1553   fi
1554
1555   if [ ${runCPass1reco} -eq 1 ]; then
1556     echo "    Pass1 - Recontruction"
1557   fi
1558   if [ ${runCPass1MergeMakeOCDB} -eq 1 ]; then
1559     echo "    Pass1 - merging and OCDB export"
1560   fi
1561
1562   echo ""
1563   echo "LIMITS:"
1564   echo "    max. Events/Chunk:   ${nEvents}"
1565   echo "    max. Number of Chunks per Run:     ${nMaxChunks}"
1566   echo ""
1567   echo "### END CONFIGURATION ###"
1568   echo ""
1569
1570
1571   # check if input file is there
1572   if [ ! -f ${inputList} ]; then
1573     echo "ERROR! Input List '${inputList}' not found" >&2
1574     return
1575   fi
1576
1577   # define jobid (for dependent jobs)
1578   date=$(date +%Y_%m_%d_%H%M%S)
1579   #for each run we submit one jobarray:
1580   for runNumber in ${listOfRuns[*]}; do
1581
1582     [[ -z ${runNumber} ]] && continue
1583     [[ ! ${runNumber} =~ ^[0-9]*[0-9]$ ]] && continue
1584
1585     JOBpostfix="${productionID//"/"/_}_${runNumber}_${date}"
1586     JOBID1="p0_${JOBpostfix}"
1587     JOBID1wait="w0_${JOBpostfix}"
1588     JOBID2="m0_${JOBpostfix}"
1589     JOBID2wait="wm0_${JOBpostfix}"
1590     JOBID3="op0_${JOBpostfix}"
1591     JOBID3wait="wop0_${JOBpostfix}"
1592     JOBID4="p1_${JOBpostfix}"
1593     JOBID4wait="w1_${JOBpostfix}"
1594     JOBID5="m1_${JOBpostfix}"
1595     JOBID5wait="wm1_${JOBpostfix}"
1596     JOBID6="s1_${JOBpostfix}"
1597     JOBID6wait="ws1_${JOBpostfix}"
1598     JOBID7="QA_${JOBpostfix}"
1599     JOBmakeESDlistCPass1="lp1_${JOBpostfix}"
1600     JOBfilterESDcpass1="fp1_${JOBpostfix}"
1601     LASTJOB="000"
1602
1603     oneInputFile=$(egrep -m1 "${runNumber}\/" ${inputList})
1604
1605     currentDefaultOCDB=${defaultOCDB}
1606     [[ -z ${autoOCDB} ]] && autoOCDB=1
1607     if [[ ${autoOCDB} -ne 0 ]]; then
1608       currentDefaultOCDB=$(setYear ${oneInputFile} ${defaultOCDB})
1609     fi
1610
1611     echo "submitting run ${runNumber} with OCDB ${currentDefaultOCDB}"
1612
1613     ###############################################################################
1614     #run one chunk with valgrind:
1615     if [[ -n ${profilingCommand} ]]; then
1616       [[ -z ${nEventsProfiling} ]] && nEventsProfiling=2
1617       [[ -z ${profilingCommand} ]] && profilingCommand="/usr/bin/valgrind --tool=callgrind --num-callers=40 -v --trace-children=yes"
1618       submit "valgrind" 1 1 000 "${alirootEnv} ${self}" CPass0 ${commonOutputPath}/000${runNumber}/valgrind ${oneInputFile} ${nEventsProfiling} ${currentDefaultOCDB} ${configFile} ${runNumber} valgrind useProfilingCommand=${profilingCommand} "${extraOpts[@]}"
1619     fi
1620
1621     ################################################################################
1622     ################################################################################
1623     # run the CPass0 if requested
1624
1625     if [ ${runCPass0reco} -eq 1 ]; then
1626
1627       echo
1628       echo "starting CPass0... for run ${runNumber}"
1629       echo
1630
1631       # create directory and copy all files that are needed
1632       targetDirectory="${commonOutputPath}/000${runNumber}/cpass0"
1633       mkdir -p ${targetDirectory}
1634
1635       filesCPass0=(
1636                     "${configPath}/runCPass0.sh"
1637                     "${configPath}/recCPass0.C"
1638                     "${configPath}/runCalibTrain.C"
1639                     "${configPath}/localOCDBaccessConfig.C"
1640                     "${configPath}/OCDB*.root"
1641                     "${configPath}/sim.C"
1642                     "${configPath}/rec.C"
1643                     "${configPath}/Config.C"
1644       )
1645       for file in ${filesCPass0[*]}; do
1646         [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
1647       done
1648
1649       localInputList=${targetDirectory}/${inputList##*/}
1650       [[ ! -f ${localInputList} ]] && egrep "\/000${runNumber}\/" ${inputList} > ${localInputList}
1651       # limit nFiles to nMaxChunks
1652       nFiles=$(wc -l < ${localInputList})
1653       [[ ${nFiles} -eq 0 ]] && echo "list contains ZERO files! exiting..." && return 1
1654       echo "raw files in list:    ${nFiles}"
1655       if [[ ${nMaxChunks} -gt 0 && ${nMaxChunks} -le ${nFiles} ]]; then
1656         nFiles=${nMaxChunks}
1657       fi
1658       echo "raw files to process: ${nFiles}"
1659       [[ -z "${percentProcessedFilesToContinue}" ]] && percentProcessedFilesToContinue=100
1660       if [[ ${percentProcessedFilesToContinue} -eq 100 ]]; then
1661         nFilesToWaitFor=${nFiles}
1662       else
1663         nFilesToWaitFor=$(( ${nFiles}-${nFiles}/(100/(100-${percentProcessedFilesToContinue})) ))
1664       fi
1665       echo "requested success rate is ${percentProcessedFilesToContinue}%"
1666       echo "merging will start after ${nFilesToWaitFor} jobs are done"
1667
1668       submit ${JOBID1} 1 ${nFiles} 000 "${alirootEnv} ${self}" CPass0 ${targetDirectory} ${localInputList} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} -1 "${extraOpts[@]}"
1669
1670       ## submit a monitoring job that will run until a certain number of jobs are done with reconstruction
1671       submit "${JOBID1wait}" 1 1 000 "${alirootEnv} ${self}" WaitForOutput ${commonOutputPath} "meta/cpass0.job*.run${runNumber}.done" ${nFilesToWaitFor} ${maxSecondsToWait} '-maxdepth 1'
1672       LASTJOB=${JOBID1wait}
1673
1674     fi #end running CPass0
1675     ################################################################################
1676
1677
1678     ################################################################################
1679     # submit merging of CPass0, depends on the reconstruction
1680
1681     if [ ${runCPass0MergeMakeOCDB} -eq 1 ]; then
1682
1683       echo
1684       echo "submit CPass0 merging for run ${runNumber}"
1685       echo
1686
1687       targetDirectory="${commonOutputPath}/000${runNumber}/cpass0"
1688       mkdir -p ${targetDirectory}
1689
1690       #copy the scripts
1691       filesMergeCPass0=(
1692                         "${configPath}/OCDB.root"
1693                         "${configPath}/mergeMakeOCDB.byComponent.sh"
1694                         "${configPath}/mergeMakeOCDB.sh"
1695                         "${configPath}/localOCDBaccessConfig.C"
1696                         "${configPath}/mergeByComponent.C"
1697                         "${configPath}/makeOCDB.C"
1698                         "${configPath}/merge.C"
1699       )
1700       for file in ${filesMergeCPass0[*]}; do
1701         [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
1702       done
1703
1704       submit ${JOBID2} 1 1 "${LASTJOB}" "${alirootEnv} ${self}" MergeCPass0 ${targetDirectory} ${currentDefaultOCDB} ${configFile} ${runNumber} cpass0.calib.run${runNumber}.list "${extraOpts[@]}"
1705       LASTJOB=${JOBID2}
1706
1707       if [[ -n ${generateMC} ]]; then
1708         submit "mrl${JOBpostfix}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" PrintValues sim ${commonOutputPath}/meta/sim.run${runNumber}.list ${commonOutputPath}/meta/cpass0.job*.run${runNumber}.done
1709         LASTJOB="mrl${JOBpostfix}"
1710       fi
1711
1712       echo
1713     fi
1714     # end of merging CPass0
1715     ################################################################################
1716
1717     ################################################################################
1718     ################################################################################
1719     # run the CPass1 if requested
1720
1721     if [ ${runCPass1reco} -eq 1 ]; then
1722
1723       targetDirectory="${commonOutputPath}/000${runNumber}/cpass1"
1724       rm -f ${commonOutputPath}/meta/cpass1.job*.run${runNumber}.done
1725
1726       # safety feature: if we are re-running for any reason we want to delete the previous output first.
1727       [[ -d ${targetDirectory} ]] && rm -rf ${targetDirectory}/* && echo "removed old output at ${targetDirectory}/*"
1728
1729       echo
1730       echo "starting CPass1... for run ${runNumber}"
1731       echo
1732
1733       # create directory and copy all files that are needed
1734       mkdir -p ${targetDirectory}
1735
1736       filesCPass1=(
1737                     "${configPath}/runCPass1.sh"
1738                     "${configPath}/recCPass1.C"
1739                     "${configPath}/recCPass1_OuterDet.C"
1740                     "${configPath}/runCalibTrain.C"
1741                     "${configPath}/QAtrain_duo.C"
1742                     "${configPath}/localOCDBaccessConfig.C"
1743                     "${configPath}/OCDB.root"
1744       )
1745       for file in ${filesCPass1[*]}; do
1746         [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
1747       done
1748
1749       if [[ -n ${generateMC} ]]; then
1750         localInputList=${commonOutputPath}/meta/sim.run${runNumber}.list
1751       else
1752         localInputList=${targetDirectory}/${inputList##*/}
1753         [[ ! -f ${localInputList} ]] && egrep "\/000${runNumber}\/" ${inputList} > ${localInputList}
1754       fi
1755       # limit nFiles to nMaxChunks
1756       nFiles=$(wc -l < ${localInputList})
1757       [[ ${nFiles} -eq 0 ]] && echo "list contains ZERO files! continuing..." && continue
1758       echo "raw files in list:    ${nFiles}"
1759       if [[ ${nMaxChunks} -gt 0 && ${nMaxChunks} -le ${nFiles} ]]; then
1760         nFiles=${nMaxChunks}
1761       fi
1762       echo "raw files to process: ${nFiles}"
1763       [[ -z "${percentProcessedFilesToContinue}" ]] && percentProcessedFilesToContinue=100
1764       if [[ ${percentProcessedFilesToContinue} -eq 100 ]]; then
1765         nFilesToWaitFor=${nFiles}
1766       else
1767         nFilesToWaitFor=$(( ${nFiles}-${nFiles}/(100/(100-${percentProcessedFilesToContinue})) ))
1768       fi
1769       echo "requested success rate is ${percentProcessedFilesToContinue}%"
1770       echo "merging will start after ${nFilesToWaitFor} jobs are done"
1771
1772       submit ${JOBID4} 1 ${nFiles} "${LASTJOB}" "${alirootEnv} ${self}" CPass1 ${targetDirectory} ${localInputList} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} -1 "${extraOpts[@]}"
1773
1774       ################################################################################
1775       ## submit a monitoring job that will run until a certain number of jobs are done with reconstruction
1776       submit "${JOBID4wait}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" WaitForOutput ${commonOutputPath} "meta/cpass1.job*.run${runNumber}.done" ${nFilesToWaitFor} ${maxSecondsToWait} '-maxdepth 1'
1777       LASTJOB=${JOBID4wait}
1778       ################################################################################
1779
1780       echo
1781     fi #end running CPass1
1782
1783     ################################################################################
1784     # submit merging of CPass1, depends on the reconstruction
1785     if [ ${runCPass1MergeMakeOCDB} -eq 1 ]; then
1786
1787       echo
1788       echo "submit CPass1 merging for run ${runNumber}"
1789       echo
1790
1791       targetDirectory="${commonOutputPath}/000${runNumber}/cpass1"
1792       rm -f ${commonOutputPath}/meta/merge.cpass1.run${runNumber}.done
1793       mkdir -p ${targetDirectory}
1794
1795       # copy files
1796       filesMergeCPass1=(
1797                         "${configPath}/OCDB.root"
1798                         "${configPath}/localOCDBaccessConfig.C"
1799                         "${configPath}/mergeMakeOCDB.byComponent.sh"
1800                         "${configPath}/mergeByComponent.C"
1801                         "${configPath}/makeOCDB.C"
1802                         "${configPath}/merge.C"
1803                         "${configPath}/mergeMakeOCDB.sh"
1804                         "${configPath}/QAtrain_duo.C"
1805       )
1806       for file in ${filesMergeCPass1[*]}; do
1807         [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
1808       done
1809
1810       submit "${JOBID5}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" MergeCPass1 ${targetDirectory} ${currentDefaultOCDB} ${configFile} ${runNumber} cpass1.calib.run${runNumber}.list cpass1.QA.run${runNumber}.lastMergingStage.txt.list cpass1.filtered.run${runNumber}.list "${extraOpts[@]}"
1811       LASTJOB=${JOBID5}
1812       echo
1813     fi
1814
1815     ###############################
1816     #if [ ${runESDfiltering} -eq 1 ]; then
1817     #  rm -f ${commonOutputPath}/cpass1.ESD.run${runNumber}.list
1818     #  rm -f ${commonOutputPath}/meta/filtering.cpass1.run*.done
1819     #  echo
1820     #  echo submitting filtering for run ${runNumber}
1821     #  echo
1822     #  submit "${JOBmakeESDlistCPass1}" 1 1 "${LASTJOB}" "${self}" PrintValues esd ${commonOutputPath}/meta/cpass1.ESD.run${runNumber}.list ${commonOutputPath}/meta/cpass1.job*.run${runNumber}.done
1823     #  submit "${JOBfilterESDcpass1}" 1 1 "${JOBmakeESDlistCPass1}" "${alirootEnv} ${self}" MakeFilteredTrees ${commonOutputPath}/000${runNumber}/cpass1 ${runNumber} ${commonOutputPath}/meta/cpass1.ESD.run${runNumber}.list ${filteringFactorHighPt} ${filteringFactorV0s} ${currentDefaultOCDB} 1000000 0 10000000 0 ${configFile} AliESDs_Barrel.root "${extraOpts[@]}"
1824     #  LASTJOB=${JOBfilterESDcpass1}
1825     #fi
1826
1827   done
1828
1829   #################################################################################
1830   #################################################################################
1831   #if [ ${runESDfiltering} -eq 1 ]; then
1832   #  submit "${JOBID5wait}" 1 1 "${LASTJOB}" "${self}" WaitForOutput ${commonOutputPath} "meta/filtering.cpass1.run*.done" "${#listOfRuns[@]}" ${maxSecondsToWait}
1833   #else
1834     submit "${JOBID5wait}" 1 1 "${LASTJOB}" "${self}" WaitForOutput ${commonOutputPath} "meta/merge.cpass1.run*.done" ${#listOfRuns[@]} ${maxSecondsToWait}
1835   #fi
1836   LASTJOB=${JOBID5wait}
1837
1838   #################################################################################
1839   echo
1840   echo "submit make a summary"
1841   echo
1842
1843   submit "${JOBID6}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" MakeSummary ${configFile}
1844   LASTJOB=${JOBID6}
1845   #################################################################################
1846
1847   #restore stdout
1848   exec 1>&7 7>&-
1849   echo "jobs submitted."
1850   return 0
1851 }
1852
1853 goWaitForOutput()
1854 {
1855   umask 0002
1856   [[ $# -lt 3 ]] && echo "goWaitForOutput() wrong number of arguments, exiting.." && return 1
1857   echo searchPath=${1}
1858   echo fileName=${2}
1859   echo numberOfFiles=${3}
1860   echo maxSecondsToWait=${4}
1861   searchPath=${1}
1862   fileName=${2}
1863   numberOfFiles=${3}
1864   maxSecondsToWait=${4}
1865   extraFindOptions=${5}
1866   echo "command to be executed: /bin/ls -1 ${searchPath}/${fileName} ${extraFindOptions}"
1867   [[ -z "${maxSecondsToWait}" ]] && maxSecondsToWait=$(( 3600*12 ))
1868   while true; do
1869     #n=$(find ${searchPath} ${extraFindOptions} -name "${fileName}" | wc -l)
1870     n=$(/bin/ls -1 ${searchPath}/${fileName} 2>/dev/null | wc -l)
1871     [[ ${n} -gt 0 ]] && echo "found ${n} X ${fileName}"
1872     [[ ${n} -ge ${numberOfFiles} ]] && break
1873     [[ ${SECONDS} -gt ${maxSecondsToWait} ]] && echo "timeout of ${maxSecondsToWait}s!" && break
1874     sleep 60
1875   done
1876   echo "DONE! exiting..."
1877   return 0
1878 }
1879
1880 mergeSysLogs()
1881 {
1882   outputFile=${1}
1883   shift
1884   inputFiles="$@"
1885   i=0
1886   if ! ls -1 ${inputFiles} &>/dev/null; then echo "the files dont exist!: ${inputFiles}"; return 1; fi
1887   while read x; do
1888     runNumber=$(guessRunNumber ${x})
1889     [[ -z ${runNumber} ]] && echo "run number cannot be guessed for ${x}" && continue
1890     awk -v run=${runNumber} -v i=${i} 'NR > 1 {print run" "$0} NR==1 && i==0 {print "run/I:"$0}' ${x}
1891     (( i++ ))
1892   done < <(ls -1 ${inputFiles}) > ${outputFile}
1893   return 0
1894 }
1895
1896 goMakeMergedSummaryTree()
1897 {
1898   # create list of calibration entries
1899   # takes no arguments, just run it in the base output
1900   # directory with the following files in the working directory
1901   #
1902   #  Calibration file lists:
1903   #       cpass0.dcsTree.list, cpass1.dcsTree.list
1904   #  QA trending root file:
1905   #       trending.root
1906   #
1907   #  Production infoo ascii files:
1908   #       summary_pass0.tree
1909   #       summary_pass1.tree
1910   #
1911
1912   [[ ! -f cpass0.dcsTree.list ]] && echo "no cpass0.dcsTree.list" && return 1
1913   [[ ! -f cpass1.dcsTree.list ]] && echo "no cpass1.dcsTree.list" && return 1
1914   [[ ! -f trending.root ]] && echo "no trending.root" && return 1
1915   [[ ! -f summary_pass0.tree ]] && echo "no summary_pass0.tree" && return 1
1916   [[ ! -f summary_pass1.tree ]] && echo "no summary_pass1.tree" && return 1
1917
1918   #first, dump the C macro to file
1919   cat << EOF > mergeTree.C
1920   //
1921   // Merge summary information
1922   // Following files are expected to be in the working directory
1923   //
1924   // Calibration file lists:
1925   //      cpass0.dcsTree.list, cpass1.dcsTree.list
1926   // QA trending root file:
1927   //      trending.root
1928   //
1929   // Production infoo ascii files:
1930   //      summary_pass0.tree
1931   //      summary_pass1.tree
1932   //
1933   void mergeTree(){
1934     //
1935     //
1936     //
1937     // Calibration values dump
1938     //
1939     //Printf("MakeTreeFromList cpass0.dcsTree.list");
1940     AliXRDPROOFtoolkit::MakeTreeFromList("Calib.TPC.CPass0.root", "dcs","dcs","cpass0.dcsTree.list",1);
1941     //Printf("MakeTreeFromList cpass1.dcsTree.list");
1942     AliXRDPROOFtoolkit::MakeTreeFromList("Calib.TPC.CPass1.root", "dcs","dcs","cpass1.dcsTree.list",1);
1943     //
1944     // Calibration status dump
1945     //
1946     TFile *fprod = TFile::Open("fproduction.root","recreate");
1947     TTree  tree0, tree1;
1948     //Printf("reading summary_pass0.tree");
1949     tree0.ReadFile("summary_pass0.tree");
1950     //Printf("reading summary_pass1.tree");
1951     tree1.ReadFile("summary_pass1.tree");
1952     tree0.Write("CPass0");
1953     tree1.Write("CPass1");
1954     fprod->Close();
1955     //
1956     //
1957     //
1958     TString stringSetup="";
1959     stringSetup+="1#QA.TPC#run#SummaryTPCQA/tpcQA#trending.root+";  //
1960     stringSetup+="1#Calib.TPC.CPass0#run#dcs#Calib.TPC.CPass0.root+";  //
1961     stringSetup+="1#Calib.TPC.CPass1#run#dcs#Calib.TPC.CPass1.root+";  //
1962     //
1963     stringSetup+="1#CPass0#runnumber#CPass0#fproduction.root+";  //
1964     stringSetup+="1#CPass1#runnumber#CPass1#fproduction.root+";  //
1965     //
1966     //Printf("stringSetup: %s", stringSetup.Data());
1967     AliXRDPROOFtoolkit::JoinTreesIndex("outAll.root","joinAll","run",stringSetup.Data(), 1);
1968   }
1969 EOF
1970
1971   aliroot -b -q "mergeTree.C" > mergeTrees.log
1972   return $?
1973 }
1974
1975 goMakeSummary()
1976 {
1977   #all the final stuff goes in here for ease of use:
1978   # summary logs
1979   # qa plot making
1980   # final file lists
1981   #some defaults:
1982   log="summary.log"
1983   productionID="qa"
1984
1985   configFile=${1}
1986   shift 1
1987   extraOpts=("$@")
1988   if ! parseConfig ${configFile} "${extraOpts[@]}"; then return 1; fi
1989
1990   configFile=$(readlink -f ${configFile})
1991
1992   #record the working directory provided by the batch system
1993   batchWorkingDirectory=${PWD}
1994
1995   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
1996
1997   [[ ! -f ${configFile} ]] && echo "no config file ${configFile}!" && return
1998
1999   [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
2000
2001   #copy some useful stuff
2002   #and go to the commonOutputPath
2003   cp ${configFile} ${commonOutputPath}
2004   cd ${commonOutputPath}
2005
2006   exec &> >(tee ${log})
2007
2008   #summarize the global stuff
2009   echo "env script: ${alirootSource} ${alirootEnv}"
2010   echo "\$ALICE_ROOT=${ALICE_ROOT}"
2011   echo "commonOutputPath=${commonOutputPath}"
2012
2013   #summarize the stacktraces
2014   awk '
2015        BEGIN {
2016                print "frame/I:method/C:line/C:cpass/I:aliroot/I";
2017                RS="#[0-9]*";
2018                aliroot=0;
2019              }
2020              {
2021                if ($3 ~ /Ali*/) aliroot=1; else aliroot=0;
2022                gsub("#","",RT);
2023                if ($NF!="" && RT!="" && $3!="") print RT" "$3" "$NF" "0" "aliroot
2024              }
2025       ' 000*/cpass0/*/stacktrace* 2>/dev/null > stacktrace.tree
2026   awk '
2027        BEGIN {
2028                RS="#[0-9]*";
2029                aliroot=0;
2030              }
2031              {
2032                if ($3 ~ /Ali*/) aliroot=1; else aliroot=0;
2033                gsub("#","",RT);
2034                if ($NF!="" && RT!="" && $3!="") print RT" "$3" "$NF" "1" "aliroot
2035              }
2036       ' 000*/cpass1/*/stacktrace* 2>/dev/null >> stacktrace.tree
2037
2038   echo total numbers for the production:
2039   echo
2040   awk 'BEGIN {nFiles=0;nCore=0;}
2041   /^calibfile/ {nFiles++;}
2042   /core dumped/ {nCore++i;}
2043   END {print     "cpass0 produced "nFiles" calib files, "nCore" core files";}' meta/cpass0.job*done 2>/dev/null
2044   awk 'BEGIN {nOK=0; nBAD=0; }
2045   /\/rec.log OK/ {nOK++;}
2046   /\/rec.log BAD/ {nBAD++;}
2047   /stderr BAD/ {if ($0 ~ /rec.log/){nBAD++;}}
2048   END {print     "cpass0 reco:  OK: "nOK"\tBAD: "nBAD;}' meta/cpass0.job*done 2>/dev/null
2049   awk 'BEGIN {nOK=0; nBAD=0; }
2050   /\/calib.log OK/ {nOK++;}
2051   /\/calib.log BAD/ {nBAD++;}
2052   END {print "cpass0 calib: OK: "nOK"\tBAD: "nBAD;}' meta/cpass0.job*done 2>/dev/null
2053
2054   awk 'BEGIN {nOK=0; nBAD=0; }
2055   /merge.log OK/ {nOK++;}
2056   /merge.log BAD/ {nBAD++;}
2057   END {print "cpass0 merge: OK: "nOK"\tBAD: "nBAD;}' meta/merge.cpass0*done 2>/dev/null
2058   awk 'BEGIN {nOK=0; nBAD=0; }
2059   /ocdb.log OK/ {nOK++;}
2060   /ocdb.log BAD/ {nBAD++;}
2061   END {print   "cpass0 OCDB:  OK: "nOK"\tBAD: "nBAD;}' meta/merge.cpass0*done 2>/dev/null
2062
2063   echo
2064   awk 'BEGIN {nFiles=0;nCore=0;}
2065   /^calibfile/ {nFiles++;}
2066   /core dumped/ {nCore++;}
2067   END {print     "cpass1 produced "nFiles" calib files, "nCore" core files";}' meta/cpass1.job*done 2>/dev/null
2068   awk 'BEGIN {nOK=0; nBAD=0; }
2069   /\/rec.log OK/ {nOK++;}
2070   /\/rec.log BAD/ {nBAD++;}
2071   /stderr BAD/ {if ($0 ~ /rec.log/){nBAD++;}}
2072   END {print     "cpass1 reco:  OK: "nOK"\tBAD: "nBAD;}' meta/cpass1.job*done 2>/dev/null
2073   awk 'BEGIN {nOK=0; nBAD=0; }
2074   /\/calib.log OK/ {nOK++;}
2075   /\/calib.log BAD/ {nBAD++;}
2076   END {print "cpass1 calib: OK: "nOK"\tBAD: "nBAD;}' meta/cpass1.job*done 2>/dev/null
2077
2078   awk 'BEGIN {nOK=0; nBAD=0; }
2079   /merge.log OK/ {nOK++;}
2080   /merge.log BAD/ {nBAD++;}
2081   END {print "cpass1 merge: OK: "nOK"\tBAD: "nBAD;}' meta/merge.cpass1*done 2>/dev/null
2082   awk 'BEGIN {nOK=0; nBAD=0; }
2083   /ocdb.log OK/ {nOK++;}
2084   /ocdb.log BAD/ {nBAD++;}
2085   END {print   "cpass1 OCDB:  OK: "nOK"\tBAD: "nBAD;}' meta/merge.cpass1*done 2>/dev/null
2086
2087   echo
2088   echo per run stats:
2089   /bin/ls -1 meta/merge.cpass0.run*.done | while read x
2090 do
2091   dir=$(goPrintValues calibfile - ${x})
2092   runNumber=$(guessRunNumber ${dir})
2093   [[ -z ${runNumber} ]] && continue
2094
2095   if $(/bin/ls meta/cpass0.job*.run${runNumber}.done &> /dev/null); then
2096     statusCPass0=( $(
2097     awk 'BEGIN {nOKrec=0;nBADrec=0;nOKcalib=0;nBADcalib=0;nOKstderr=0;nBADstderr=0;}
2098     /\/rec.log OK/ {nOKrec++;}
2099     /\/rec.log BAD/ {nBADrec++;}
2100     /stderr BAD/ {if ($0 ~ /rec.log/) {nBADrec++;} nBADstderr++;}
2101     /stderr OK/ {nOKstderr++;}
2102     /\/calib.log OK/ {nOKcalib++;}
2103     /\/calib.log BAD/ {nBADcalib++}
2104     END {print ""nOKrec" "nBADrec" "nOKstderr" "nBADstderr" "nOKcalib" "nBADcalib;}' meta/cpass0.job*.run${runNumber}.done 2>/dev/null
2105     ) )
2106   fi
2107
2108   if $(/bin/ls meta/cpass1.job*.run${runNumber}.done &>/dev/null); then
2109     statusCPass1=( $(
2110     awk 'BEGIN {nOKrec=0;nBADrec=0;nOKcalib=0;nBADcalib=0;nOKstderr=0;nBADstderr=0;nQAbarrelOK=0;nQAbarrelBAD=0;nQAouterOK=0;nQAouterBAD=0;}
2111     /\/rec.log OK/ {nOKrec++;}
2112     /\/rec.log BAD/ {nBADrec++;}
2113     /stderr BAD/ {if ($0 ~ /rec.log/) nBADrec++;nBADstderr++;}
2114     /stderr OK/ {nOKstderr++;}
2115     /\/calib.log OK/ {nOKcalib++;}
2116     /\/calib.log BAD/ {nBADcalib++}
2117     /\/qa_barrel.log OK/ {nQAbarrelOK++;}
2118     /\/qa_barrel.log BAD/ {nQAbarrelBAD++;}
2119     /\/qa_outer.log OK/ {nQAouterOK++;}
2120     /\/qa_outer.log BAD/ {nQAouterBAD++;}
2121     END {print ""nOKrec" "nBADrec" "nOKstderr" "nBADstderr" "nOKcalib" "nBADcalib" "nQAbarrelOK" "nQAbarrelBAD" "nQAouterOK" "nQAouterBAD;}' meta/cpass1.job*.run${runNumber}.done 2>/dev/null
2122     ) )
2123   fi
2124
2125   statusOCDBcpass0=$(awk '/ocdb.log/ {print $2} ' ${x} 2>/dev/null)
2126   statusOCDBcpass1=$(awk '/ocdb.log/ {print $2}' ${x/cpass0/cpass1} 2>/dev/null)
2127   statusQA=$(awk '/mergeMakeOCDB.log/ {print $2}' ${x/cpass0/cpass1} 2>/dev/null)
2128
2129   printf "%s\t ocdb.log cpass0: %s\t ocdb.log cpass1: %s\tqa.log:%s\t| cpass0: rec:%s/%s stderr:%s/%s calib:%s/%s cpass1: rec:%s/%s stderr:%s/%s calib:%s/%s QAbarrel:%s/%s QAouter:%s/%s\n" ${runNumber} ${statusOCDBcpass0} ${statusOCDBcpass1} ${statusQA} ${statusCPass0[0]} ${statusCPass0[1]} ${statusCPass0[2]} ${statusCPass0[3]} ${statusCPass0[4]} ${statusCPass0[5]} ${statusCPass1[0]} ${statusCPass1[1]} ${statusCPass1[2]} ${statusCPass1[3]} ${statusCPass1[4]} ${statusCPass1[5]} ${statusCPass1[6]} ${statusCPass1[7]} ${statusCPass1[8]} ${statusCPass1[9]}
2130 done
2131
2132   #make lists with output files - QA, trending, filtering and calibration
2133   ### wait for the merging of all runs to be over ###
2134   rm -f qa.list
2135   goPrintValues qafile qa.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2136   rm -f calib.list
2137   goPrintValues calibfile calib.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2138   rm -f trending.list
2139   goPrintValues trendingfile trending.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2140   rm -f filtering.list
2141   goPrintValues filteredTree filtering.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2142   rm -f cpass0.dcsTree.list
2143   goPrintValues dcsTree cpass0.dcsTree.list ${commonOutputPath}/meta/merge.cpass0.run*.done &>/dev/null
2144   rm -f cpass1.dcsTree.list
2145   goPrintValues dcsTree cpass1.dcsTree.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2146
2147   #merge trending
2148   rm -f ${commonOutputPath}/trending_merged.root
2149   goMerge trending.list ${commonOutputPath}/trending.root ${configFile} "${extraOpts[@]}" &> mergeTrending.log
2150
2151   goMakeSummaryTree ${commonOutputPath} 0
2152   goMakeSummaryTree ${commonOutputPath} 1
2153
2154   goCreateQAplots "${PWD}/qa.list" "${productionID}" "QAplots" "${configFile}" "${extraOpts[@]}" filteringList="${PWD}/filtering.list" &>createQAplots.log
2155
2156   #make a merged summary tree out of the QA trending, dcs trees and log summary trees
2157   echo goMakeMergedSummaryTree PWD=$PWD
2158   goMakeMergedSummaryTree
2159
2160   #if set, email the summary
2161   [[ -n ${MAILTO} ]] && cat ${log} | mail -s "benchmark ${productionID} done" ${MAILTO}
2162
2163   return 0
2164 }
2165
2166 goMakeSummaryTree()
2167 {
2168   if [[ $# -lt 1 ]] ; then
2169     return
2170   fi
2171   #1. define vars/arrays
2172   DIR=${1} #use input or exec in current dir
2173   pass=${2-"0"} #pass from input
2174   outfile="summary_pass${pass}.tree"
2175   Ncolumns=0
2176   test -f ${outfile} && : >${outfile}
2177   errfile=${outfile/tree/err}
2178   test -f ${errfile} && : >${errfile}
2179
2180   declare -a counterString=(TOFevents TOFtracks TPCevents TPCtracks TRDevents TRDtracks T0events SDDevents SDDtracks MeanVertexevents)
2181   Ncounter=${#counterString[@]}
2182
2183   declare -a statusString=(TRDStatus TOFStatus TPCStatus T0Status MeanVertexStatus)
2184   Nstatus=${#statusString[@]}
2185
2186
2187   declare -a ratesString=(rec stderr calib qa_barrel qa_outer)
2188   Nrates=${#ratesString[@]}
2189
2190   runs=( $(ls -1 ${DIR}/meta/merge.cpass0* | while read x; do guessRunNumber $x; done) )
2191   Nruns=${#runs[@]}
2192
2193   echo -n runnumber/I >>${outfile}
2194   echo -n :cpass${pass}status/I >>${outfile}
2195   echo -n :cpass${pass}QAstatus/I >>${outfile}
2196   for i in ${ratesString[@]}; do
2197     echo -n :${i}OK/I >>${outfile}
2198     echo -n :${i}BAD/I >>${outfile}
2199
2200   done
2201   for i in ${counterString[@]} ${statusString[@]} ; do
2202     echo -n :${i}/I >>${outfile}
2203   done
2204   Ncolumns=$((2 + 2*Nrates + Ncounter + Nstatus))
2205   echo >> ${outfile}
2206
2207   #2. loop runs
2208
2209   for runnumber in ${runs[@]} ; do
2210
2211
2212
2213     filejob="${DIR}/meta/cpass${pass}.job*.run${runnumber}.done"
2214     filemerge="${DIR}/meta/merge.cpass${pass}.run${runnumber}.done"
2215     fileOCDB=$(grep /ocdb.log ${filemerge} | awk '{print $1}')
2216     if ! $(/bin/ls ${filemerge} &>/dev/null) ; then
2217       echo "${filemerge} does not exist!" >>${errfile}
2218       continue
2219     elif ! $(/bin/ls ${filejob} &>/dev/null) ; then
2220       echo "${filejob} does not exist!" >>${errfile}
2221       echo -n ${runnumber} >> ${outfile}
2222       for i in $(seq ${Ncolumns}) ; do
2223         echo -n "-1" >> ${outfile}
2224       done
2225       echo >> ${outfile}
2226       continue
2227     fi
2228     echo -n ${runnumber} >> ${outfile}
2229     #pass0status= grep '/ocdb.log' ${filemerge} |  cut -d' ' -f2 | tr OK x1 | tr BAD xx0 | tr -d 'x'
2230     passStatus=$(grep '/ocdb.log' ${filemerge} | grep OK | wc -l)
2231     echo -n " ${passStatus}" >> ${outfile}
2232     qaStatus=$(grep '/mergeMakeOCDB.log' ${filemerge} | grep OK | wc -l)
2233     echo -n " ${qaStatus}" >> ${outfile}
2234
2235
2236     #fill OK/BAD rates
2237     for i in $(seq 0 $((${Nrates}-1))) ; do
2238       var1=$(grep "/${ratesString[${i}]}.log" ${filejob} | grep OK | wc -l)
2239       var2=$(grep "/${ratesString[${i}]}.log" ${filejob} | grep BAD | wc -l)
2240
2241       if [[ ${ratesString[${i}]} == "stderr" ]] ; then
2242         var1=$(grep "stderr" ${filejob} | grep OK | wc -l)
2243         var2=$(grep "stderr" ${filejob} | grep "rec.log" | grep BAD | wc -l)
2244       fi
2245       echo -n " ${var1}" >> ${outfile}
2246       echo -n " ${var2}" >> ${outfile}
2247     done
2248
2249     if [[ -f ${fileOCDB} ]] ; then
2250       #fill counter
2251       for i in $(seq 0 $((${Ncounter}-1))) ; do
2252         var1=$(grep Monalisa ${fileOCDB} | grep ${counterString[${i}]} | cut -f2)
2253         echo -n " ${var1:-"-1"}" >> ${outfile}
2254       done
2255
2256       #fill status
2257       for i in $(seq 0 $((${Nstatus}-1))) ; do
2258         var1=$(grep "calibration status=" ${fileOCDB} | grep ${statusString[${i}]/Status/} | cut -d'=' -f2)
2259         echo -n " ${var1:-"-1"}" >> ${outfile}
2260       done
2261     fi
2262     echo >> ${outfile}
2263   done
2264
2265   return 0
2266 }
2267
2268 parseConfig()
2269 {
2270   configFile=${1}
2271   shift
2272   args=("$@")
2273
2274   #some defaults
2275   #autoOCDB=0
2276   defaultOCDB="raw://"
2277   #runNumber=167123
2278   #makeflowPath="/hera/alice/aux/cctools/bin"
2279   #makeflowOptions="-T wq -N alice -d all -C ali-copilot.cern.ch:9097"
2280   #makeflowOptions="-T wq -N alice -C ali-copilot.cern.ch:9097"
2281   makeflowOptions=""
2282   #batchCommand="/usr/bin/qsub"
2283   batchFlags="-b y -cwd -l h_rt=24:0:0,h_rss=4G "
2284   baseOutputDirectory="$PWD/output"
2285   #alirootEnv="/cvmfs/alice.cern.ch/bin/alienv setenv AliRoot/v5-04-34-AN -c"
2286   #alirootEnv="/home/mkrzewic/alisoft/balice_master.sh"
2287   #trustedQAtrainMacro='/hera/alice/mkrzewic/gsisvn/Calibration/QAtrain_duo.C'
2288   reconstructInTemporaryDir=0
2289   recoTriggerOptions="\"\""
2290   percentProcessedFilesToContinue=100
2291   maxSecondsToWait=$(( 3600*24 ))
2292   nEvents=-1
2293   nMaxChunks=0
2294   postSetUpActionCPass0=""
2295   postSetUpActionCPass1=""
2296   runCPass0reco=1
2297   runCPass0MergeMakeOCDB=1
2298   runCPass1reco=1
2299   runCPass1MergeMakeOCDB=1
2300   runESDfiltering=1
2301   filteringFactorHighPt=1e2
2302   filteringFactorV0s=1e1
2303   MAILTO=""
2304   #pretend=1
2305   #dontRedirectStdOutToLog=1
2306   logToFinalDestination=1
2307   ALIROOT_FORCE_COREDUMP=1
2308
2309   #first, source the config file
2310   if [ -f ${configFile} ]; then
2311     source ${configFile}
2312   else
2313     echo "config file ${configFile} not found!, skipping..."
2314   fi
2315
2316   unset encodedSpaces
2317   for opt in "${args[@]}"; do
2318     [[ "${opt}" =~ encodedSpaces=.* ]] && encodedSpaces=1 && break
2319   done
2320
2321   #then, parse the options as they override the options from file
2322   for opt in "${args[@]}"; do
2323     if [[ ! "${opt}" =~ .*=.* ]]; then
2324       echo "badly formatted option ${var}, should be: option=value, stopping..."
2325       return 1
2326     fi
2327     local var="${opt%%=*}"
2328     local value="${opt#*=}"
2329     [[ -n ${encodedSpaces} ]] && value=$(decSpaces "${value}")
2330     echo "${var} = ${value}"
2331     export ${var}="${value}"
2332   done
2333
2334   #do some checking
2335   [[ -z ${alirootEnv} ]] && echo "alirootEnv not defined!" && return 1
2336
2337   #export the aliroot function if defined to override normal behaviour
2338   [[ $(type -t aliroot) =~ "function" ]] && export -f aliroot
2339
2340   return 0
2341 }
2342
2343 aliroot()
2344 {
2345   args="$@"
2346   if [[ -n ${useProfilingCommand} ]]; then
2347     valgrindLogFile="cpu.txt"
2348     [[ "${args}" =~ rec ]] && valgrindLogFile="cpu_rec.txt"
2349     [[ "${args}}" =~ Calib ]] && valgrindLogFile="cpu_calib.txt"
2350     [[ -n ${useProfilingCommand} ]] && useProfilingCommand="${useProfilingCommand} --log-file=${valgrindLogFile}"
2351     echo running ${useProfilingCommand} aliroot ${args}
2352     ${useProfilingCommand} aliroot ${args}
2353   else
2354     #to prevent an infinite recursion use "command aliroot" to disable
2355     #aliases and functions
2356     echo running command aliroot ${args}
2357     command aliroot "$@"
2358   fi
2359   return 0
2360 }
2361
2362 guessRunData()
2363 {
2364   #guess the period from the path, pick the rightmost one
2365   period=""
2366   runNumber=""
2367   year=""
2368   pass=""
2369   legoTrainRunNumber=""
2370   dataType=""
2371
2372   local shortRunNumber=""
2373   local IFS="/"
2374   declare -a path=( $1 )
2375   local dirDepth=$(( ${#path[*]}-1 ))
2376   i=0
2377   #for ((x=${dirDepth};x>=0;x--)); do
2378   for ((x=0;x<=${dirDepth};x++)); do
2379
2380     [[ $((x-1)) -ge 0 ]] && local fieldPrev=${path[$((x-1))]}
2381     local field=${path[${x}]}
2382     local fieldNext=${path[$((x+1))]}
2383
2384     [[ ${field} =~ ^[0-9]*$ && ${fieldNext} =~ (.*\.zip$|.*\.root$) ]] && legoTrainRunNumber=${field}
2385     [[ -n ${legoTrainRunNumber} && -z ${pass} ]] && pass=${fieldPrev}
2386     [[ ${field} =~ ^LHC[0-9][0-9][a-z].*$ ]] && period=${field%_*}
2387     [[ ${field} =~ ^000[0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && runNumber=${field#000}
2388     [[ ${field} =~ ^[0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && shortRunNumber=${field}
2389     [[ ${field} =~ ^20[0-9][0-9]$ ]] && year=${field}
2390     [[ ${field} =~ ^(^sim$|^data$) ]] && dataType=${field}
2391     (( i++ ))
2392   done
2393   [[ -z ${legoTrainRunNumber} ]] && pass=${path[$((dirDepth-1))]}
2394   [[ "${dataType}" =~ ^sim$ ]] && pass="passMC" && runNumber=${shortRunNumber}
2395
2396   #if [[ -z ${dataType} || -z ${year} || -z ${period} || -z ${runNumber}} || -z ${pass} ]];
2397   if [[ -z ${runNumber}} ]];
2398   then
2399     #error condition
2400     return 1
2401   else
2402     #ALL OK
2403     return 0
2404   fi
2405   return 0
2406 }
2407
2408 #these functions encode strings to and from a space-less form
2409 #use when spaces are not well handled (e.g. in arguments to
2410 #commands in makeflow files, etc.
2411 encSpaces()(a="${1//,/\\,}";echo "${a// /,}")
2412 decSpaces()(a="${1//\\,/\\ }";b="${a//,/ }";echo "${b//\\ /,}")
2413
2414 main "$@"