PWGPP/benchmark/benchmark.sh

   1 #!/bin/bash
   2 #include benchmark.config
   3
   4 # this script runs the CPass0/CPass1 train
   5 # produced OCDB updates are local
   6
   7 main()
   8 {
   9   #run in proper mode depending on the selection
  10   if [[ $# -lt 1 ]]; then
  11     if [[ ! "${0}" =~ "bash" ]]; then
  12       echo "uses makeflow:"
  13       echo " ${0} \"run\" productionID inputList configFile [extraOpts]"
  14       echo "uses a batch system (SGE):"
  15       echo " ${0} \"submit\" productionID inputList configFile [extraOpts]"
  16       echo "extraOpts if non-empty override the config file, e.g.:"
  17       echo " ${0} submit test1 benchmark.list benchmark.config runNumber=169123 nEvents=10"
  18     fi
  19     return
  20   fi
  21
  22   #define some aliases - default is to call one of the functions directly
  23   runMode=${1}
  24   umask 0002
  25   shift
  26   case ${runMode} in
  27     "CPass0") goCPass0 "$@";;
  28     "CPass1") goCPass1 "$@";;
  29     "MakeLocalOCDBaccessConfig") goMakeLocalOCDBaccessConfig "$@";;
  30     "MergeCPass0") goMergeCPass0 "$@";;
  31     "MergeCPass1") goMergeCPass1 "$@";;
  32     "MakeFilteredTrees") goMakeFilteredTrees "$@";;
  33     "MakeSummary") goMakeSummary "$@";;
  34     "run") goSubmitMakeflow "$@";;
  35     "submit") goSubmitBatch "$@";;
  36     "test") goTest "$@";;
  37     "GenerateMakeflow") goGenerateMakeflow "$@";;
  38     "PrintValues") goPrintValues "$@";;
  39     "CreateQAplots") goCreateQAplots "$@";;
  40     "WaitForOutput") goWaitForOutput "$@";;
  41     "Merge") goMerge "$@";;
  42     *)
  43       ${runMode} "$@"
  44     ;;
  45   esac
  46   return 0
  47 }
  48
  49 generateMC()
  50 {
  51   #generate one raw chunk in current directory
  52   SEED=${JOB_ID}${SGE_TASK_ID}
  53   export CONFIG_SEED=${SEED}
  54   runNumber=${1}
  55   OCDBpath=${2}
  56   nEventsim=${3}
  57   if [[ -n ${pretend} ]]; then
  58     touch galice.root
  59   else
  60     if [[ -f sim.C && -f Config.C ]] ; then
  61         time aliroot -b -q -x sim.C\(${runNumber},\"${OCDBpath}\",${nEventsim}\) >sim.log 2>&1
  62         mv syswatch.log simwatch.log
  63     fi
  64   fi
  65 }
  66
  67 goCPass0()
  68 {
  69   umask 0002
  70
  71   targetDirectory=${1}
  72   inputList=${2}
  73   nEvents=${3}
  74   ocdbPath=${4}
  75   configFile=${5}
  76   runNumber=${6}
  77   jobindex=${7}
  78   shift 7
  79   if ! parseConfig ${configFile} "$@"; then return 1; fi
  80
  81   #record the working directory provided by the batch system
  82   batchWorkingDirectory=${PWD}
  83
  84   #use the jobindex only if set and non-negative
  85   if [[ -z ${jobindex} || ${jobindex} -lt 0 ]]; then
  86     [[ -n "${LSB_JOBINDEX}" ]] && jobindex=${LSB_JOBINDEX}
  87     [[ -n "${SGE_TASK_ID}" ]] && jobindex=${SGE_TASK_ID}
  88   fi
  89
  90   [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
  91   doneFile="${commonOutputPath}/meta/cpass0.job${jobindex}.run${runNumber}.done"
  92
  93   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
  94
  95   if [[ -n ${ALIROOT_FORCE_COREDUMP} ]]; then
  96     ulimit -c unlimited
  97     export ALIROOT_FORCE_COREDUMP
  98   fi
  99
 100   #the contents of this is stored in the tree and used later (e.g. AliAnalysisTaskPIDResponse)!
 101   #at the QA stage the pass number is guessed from the path stored here.
 102   #The Format is:
 103   #Packages= ;OutputDir= ;LPMPass= ;TriggerAlias= ;LPMRunNumber= ;LPMProductionType= ;LPMInteractionType= ;LPMProductionTag= ;LPMAnchorRun= ;LPMAnchorProduction= ;LPMAnchorYear=
 104   export PRODUCTION_METADATA="OutputDir=cpass0"
 105
 106   if [[ "${inputList}" =~ \.root$ ]]; then
 107     infile=${inputList}
 108   else
 109     infile=$(sed -ne "${jobindex}p" ${inputList} | egrep '\s*\w*/\w*')
 110   fi
 111   chunkName=${infile##*/}
 112
 113   outputDir=${targetDirectory}/${jobindex}_${chunkName%.*}
 114   mkdir -p ${outputDir}
 115   [[ ! -d ${outputDir} ]] && echo "cannot make ${outputDir}" && touch ${doneFile} && return 1
 116
 117   #runpath=${PWD}/rundir_cpass0_${runNumber}_${jobindex}
 118   runpath=${outputDir}
 119   [[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
 120   [[ ${reconstructInTemporaryDir} -eq 1 && -z ${TMPDIR} ]] && runpath=$(mktemp -d)
 121   mkdir -p ${runpath}
 122   [[ ! -d ${runpath} ]] && echo "cannot make runpath ${runpath}" && touch ${doneFile} && return 1
 123   cd ${runpath}
 124   [[ ! ${PWD} =~ ${runpath} ]] && echo "PWD=$PWD is not the runpath=${runpath}" && touch ${doneFile} && return 1
 125
 126   #runCPassX/C expects the raw chunk to be linked in the run dir
 127   #despite it being accessed by the full path
 128   ln -s ${infile} ${runpath}/${chunkName}
 129
 130   #####MC
 131   if [[ -n ${generateMC} ]]; then
 132     olddir=${PWD}
 133     outputDirMC=${commonOutputPath}/000${runNumber}/sim/${jobindex}
 134     simrunpath=${outputDirMC}
 135     [[ ${simulateInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && simrunpath=${TMPDIR}
 136     [[ ${simulateInTemporaryDir} -eq 1 && -z ${TMPDIR} ]] && simrunpath=$(mktemp -d)
 137     mkdir -p ${outputDirMC}
 138     mkdir -p ${simrunpath}
 139     cd ${simrunpath}
 140
 141     filesMC=(
 142               "${batchWorkingDirectory}/sim.C"
 143               "${batchWorkingDirectory}/rec.C"
 144               "${batchWorkingDirectory}/Config.C"
 145               "${batchWorkingDirectory}/OCDB_*.root"
 146     )
 147     for file in ${filesMC[*]}; do
 148       [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
 149     done
 150
 151     generateMC ${runNumber} ${ocdbPath} ${nEvents}
 152
 153     [[ ! "${simrunpath}" =~ "${outputDirMC}" ]] && mv * ${outputDirMC} #TODO check if it works
 154     cd ${olddir}
 155
 156     ln -s ${outputDirMC}/* ${runpath}/
 157
 158     inputList=${outputDirMC}/galice.root #TODO not valid outside shell !!!
 159     infile=""
 160   fi
 161   ######
 162
 163   [[ ! -f ${inputList} && -z ${pretend} ]] && echo "input file ${inputList} not found, exiting..." && touch ${doneFile} && return 1
 164
 165   logOutputDir=${runpath}
 166   [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir}
 167   [[ -z ${dontRedirectStdOutToLog} ]] && exec 1> ${logOutputDir}/stdout
 168   [[ -z ${dontRedirectStdOutToLog} ]] && exec 2> ${logOutputDir}/stderr
 169   echo "${0} $*"
 170
 171   echo "#####################"
 172   echo CPass0:
 173   echo JOB setup
 174   echo nEvents            ${nEvents}
 175   echo runNumber          ${runNumber}
 176   echo ocdbPath           ${ocdbPath}
 177   echo infile             ${infile}
 178   echo chunkName          ${chunkName}
 179   echo jobindex           ${jobindex}
 180   echo recoTriggerOptions ${recoTriggerOptions}
 181   echo targetDirectory    ${targetDirectory}
 182   echo commonOutputPath         ${commonOutputPath}
 183   echo doneFile      ${doneFile}
 184   echo runpath            ${runpath}
 185   echo outputDir          ${outputDir}
 186   echo ALICE_ROOT         ${ALICE_ROOT}
 187   echo PWD                ${PWD}
 188   echo "########## ###########"
 189
 190   alirootInfo > ALICE_ROOT.log
 191
 192   filesCPass0=(
 193                "${batchWorkingDirectory}/runCPass0.sh"
 194                "${batchWorkingDirectory}/recCPass0.C"
 195                "${batchWorkingDirectory}/runCalibTrain.C"
 196                "${batchWorkingDirectory}/localOCDBaccessConfig.C"
 197                "${batchWorkingDirectory}/OCDB.root"
 198                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/runCPass0.sh"
 199                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/recCPass0.C"
 200                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/runCalibTrain.C"
 201   )
 202
 203   for file in ${filesCPass0[*]}; do
 204     [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
 205     [[ ${file##*/} =~ .*\.sh ]] && chmod +x ${file##*/}
 206   done
 207
 208   echo "this directory (${PWD}) contents:"
 209   /bin/ls
 210   echo
 211   chmod u+x runCPass0.sh
 212
 213   sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh
 214
 215   if [[ -n ${postSetUpActionCPass0} ]]; then
 216     echo "running ${postSetUpActionCPass0}"
 217     eval ${postSetUpActionCPass0}
 218   fi
 219
 220   #run CPass0
 221   echo "${runpath}/runCPass0.sh ${infile} ${nEvents} ${runNumber} ${ocdbPath} ${recoTriggerOptions}"
 222   if [[ -n ${pretend} ]]; then
 223     touch AliESDs.root
 224     touch AliESDfriends.root
 225     touch AliESDfriends_v1.root
 226     touch rec.log
 227     touch calib.log
 228   else
 229     echo ./runCPass0.sh "${infile}" "${nEvents}" "${runNumber}" "${ocdbPath}" "${recoTriggerOptions}"
 230     ./runCPass0.sh "${infile}" "${nEvents}" "${runNumber}" "${ocdbPath}" "${recoTriggerOptions}"
 231   fi
 232
 233   #move stuff to final destination
 234   echo "this directory (${PWD}) contents:"
 235   /bin/ls
 236   echo
 237
 238   echo rm -f ./${chunkName}
 239   rm -f ./${chunkName}
 240   echo "cp --recursive ${runpath}/* ${outputDir}"
 241   cp -p --recursive ${runpath}/* ${outputDir}
 242   echo
 243
 244   #validate CPass0
 245   cd ${outputDir}
 246   touch ${doneFile}
 247   echo "dir ${outputDir}" >> ${doneFile}
 248   if summarizeLogs >> ${doneFile}; then
 249     [[ -f ${outputDirMC}/galice.root ]] && echo "sim ${outputDirMC}/galice.root" >> ${doneFile}
 250     [[ -f AliESDfriends_v1.root ]] && echo "calibfile ${outputDir}/AliESDfriends_v1.root" >> ${doneFile}
 251     [[ -f AliESDs.root ]] && echo "esd ${outputDir}/AliESDs.root" >> ${doneFile}
 252   fi
 253
 254   [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
 255   return 0
 256 }
 257
 258 goCPass1()
 259 {
 260   umask 0002
 261
 262   targetDirectory=${1}
 263   inputList=${2}
 264   nEvents=${3}
 265   ocdbPath=${4}
 266   configFile=${5}
 267   runNumber=${6}
 268   jobindex=${7}
 269   shift 7
 270   extraOpts=("$@")
 271   if ! parseConfig ${configFile} "$@"; then return 1; fi
 272
 273   #record the working directory provided by the batch system
 274   batchWorkingDirectory=${PWD}
 275
 276   #use the jobindex only if set and non-negative
 277   if [[ -z ${jobindex} || ${jobindex} -lt 0 ]]; then
 278     [[ -n "${LSB_JOBINDEX}" ]] && jobindex=${LSB_JOBINDEX}
 279     [[ -n "${SGE_TASK_ID}" ]] && jobindex=${SGE_TASK_ID}
 280   fi
 281
 282   [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
 283   doneFile="${commonOutputPath}/meta/cpass1.job${jobindex}.run${runNumber}.done"
 284
 285   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
 286
 287   if [[ -n ${ALIROOT_FORCE_COREDUMP} ]]; then
 288     ulimit -c unlimited
 289     export ALIROOT_FORCE_COREDUMP
 290   fi
 291
 292   #the contents of this is stored in the tree and used later (e.g. AliAnalysisTaskPIDResponse)!
 293   #at the QA stage the pass number is guessed from the path stored here.
 294   #The Format is:
 295   #Packages= ;OutputDir= ;LPMPass= ;TriggerAlias= ;LPMRunNumber= ;LPMProductionType= ;LPMInteractionType= ;LPMProductionTag= ;LPMAnchorRun= ;LPMAnchorProduction= ;LPMAnchorYear=
 296   export PRODUCTION_METADATA="OutputDir=cpass1"
 297
 298   [[ ! -f ${inputList} && -z ${pretend} ]] && echo "input file ${inputList} not found, exiting..." && touch ${doneFile} && return 1
 299   if [[ "${inputList}" =~ \.root$ ]]; then
 300     infile=${inputList}
 301   else
 302     infile=$(sed -ne "${jobindex}p" ${inputList} | egrep '\s*\w*/\w*')
 303   fi
 304   chunkName=${infile##*/}
 305
 306   outputDir=${targetDirectory}/${jobindex}_${chunkName%.*}
 307   mkdir -p ${outputDir}
 308   [[ ! -d ${outputDir} ]] && echo "cannot make ${outputDir}" && touch ${doneFile} && return 1
 309
 310   #runpath=${PWD}/rundir_cpass1_${runNumber}_${jobindex}
 311   runpath=${outputDir}
 312   [[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
 313   [[ ${reconstructInTemporaryDir} -eq 1 && -z ${TMPDIR} ]] && runpath=$(mktemp -d)
 314
 315   #MC
 316   if [[ "${infile}" =~ galice\.root ]]; then
 317     ln -s ${inputList%/*}/* ${runpath}
 318     infile=""
 319   fi
 320
 321   #init the running path
 322   mkdir -p ${runpath}
 323   [[ ! -d ${runpath} ]] && echo "cannot make runpath ${runpath}" && touch ${doneFile} && return 1
 324   cd ${runpath}
 325   [[ ! ${PWD} =~ ${runpath} ]] && echo "PWD=$PWD is not the runpath=${runpath}" && touch ${doneFile} && return 1
 326
 327   #this is needed for runCPass1.sh
 328   ln -s ${infile} ${runpath}/${chunkName}
 329
 330   logOutputDir=${runpath}
 331   [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir}
 332   [[ -z ${dontRedirectStdOutToLog} ]] && exec 1> ${logOutputDir}/stdout
 333   [[ -z ${dontRedirectStdOutToLog} ]] && exec 2> ${logOutputDir}/stderr
 334   echo "${0} $*"
 335
 336   echo "#####################"
 337   echo CPass1:
 338   echo JOB setup
 339   echo nEvents            ${nEvents}
 340   echo runNumber          ${runNumber}
 341   echo ocdbPath           ${ocdbPath}
 342   echo infile             ${infile}
 343   echo chunkName          ${chunkName}
 344   echo jobindex           ${jobindex}
 345   echo recoTriggerOptions ${recoTriggerOptions}
 346   echo targetDirectory    ${targetDirectory}
 347   echo commonOutputPath         ${commonOutputPath}
 348   echo doneFile      ${doneFile}
 349   echo runpath            ${runpath}
 350   echo outputDir          ${outputDir}
 351   echo ALICE_ROOT         ${ALICE_ROOT}
 352   echo PWD                ${PWD}
 353   echo "########## ###########"
 354
 355   alirootInfo > ALICE_ROOT.log
 356
 357   filesCPass1=(
 358                "${batchWorkingDirectory}/runCPass1.sh"
 359                "${batchWorkingDirectory}/recCPass1.C"
 360                "${batchWorkingDirectory}/recCPass1_OuterDet.C"
 361                "${batchWorkingDirectory}/runCalibTrain.C"
 362                "${batchWorkingDirectory}/QAtrain_duo.C"
 363                "${batchWorkingDirectory}/localOCDBaccessConfig.C"
 364                "${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz"
 365                "${batchWorkingDirectory}/OCDB.root"
 366                "${trustedQAtrainMacro}"
 367                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/runCPass1.sh"
 368                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/recCPass1.C"
 369                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/recCPass1_OuterDet.C"
 370                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/runCalibTrain.C"
 371                "${ALICE_ROOT}/ANALYSIS/macros/QAtrain_duo.C"
 372   )
 373
 374   for file in ${filesCPass1[*]}; do
 375     [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
 376     [[ ${file##*/} =~ .*\.sh ]] && chmod +x ${file##*/}
 377   done
 378
 379   echo "this directory (${PWD}) contents:"
 380   /bin/ls
 381   echo
 382
 383   sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh
 384
 385   if [[ -n ${postSetUpActionCPass1} ]]; then
 386     echo "running ${postSetUpActionCPass1}"
 387     eval ${postSetUpActionCPass1}
 388     echo
 389   fi
 390
 391   #configure local OCDB storage from CPass0 (creates the localOCDBaccessConfig.C script)
 392   if [[ -f cpass0.localOCDB.${runNumber}.tgz ]]; then
 393     echo goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz"
 394     goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz"
 395   else
 396     echo "WARNING: file cpass0.localOCDB.${runNumber}.tgz not found!"
 397   fi
 398
 399   if [[ ! $(/bin/ls -1 OCDB/*/*/*/*.root 2>/dev/null) ]]; then
 400     echo "cpass0 produced no calibration! exiting..."
 401     touch ${doneFile}
 402     return 1
 403   fi
 404
 405   #create the Barrel and OuterDet directories for CPass1 and link the local OCDB directory
 406   #there to make the localOCDBaccessConfig.C file work, since it may point to the OCDB
 407   #entries using a relative path, e.g. local://./OCDB
 408   mkdir Barrel OuterDet
 409   ln -s ../OCDB Barrel/OCDB
 410   ln -s ../OCDB OuterDet/OCDB
 411
 412   #setup the filtering
 413   #the following option enables the filtering task inside the QAtrain_duo.C
 414   [[ -n $runESDfiltering ]] && export QA_TaskFilteredTree=1
 415   #set the downscaling factors during the filtering fro expert QA (overrides the previous values)
 416   if [[ -n ${filteringFactorHighPt} ]]; then
 417     export AliAnalysisTaskFilteredTree_fLowPtTrackDownscaligF=${filteringFactorHighPt}
 418   fi
 419   if [[ -n ${filteringFactorV0s} ]]; then
 420     export AliAnalysisTaskFilteredTree_fLowPtV0DownscaligF=${filteringFactorV0s}
 421   fi
 422
 423   #run CPass1
 424   chmod u+x runCPass1.sh
 425   echo "${runpath}/runCPass1.sh ${infile} ${nEvents} ${runNumber} ${ocdbPath} ${recoTriggerOptions}"
 426   if [[ -n ${pretend} ]]; then
 427     touch AliESDs_Barrel.root
 428     touch AliESDfriends_Barrel.root
 429     touch AliESDfriends_v1.root
 430     touch QAresults_barrel.root
 431     touch EventStat_temp_barrel.root
 432     touch AODtpITS.root
 433     touch AliESDs_Outer.root
 434     touch AliESDfriends_Outer.root
 435     touch QAresults_outer.root
 436     touch EventStat_temp_outer.root
 437     touch rec.log
 438     touch calib.log
 439     touch qa.log
 440     touch filtering.log FilterEvents_Trees.root
 441   else
 442     ./runCPass1.sh "${infile}" "${nEvents}" "${runNumber}" "${ocdbPath}" "${recoTriggerOptions}"
 443
 444     [[ ! -f AliESDs_Barrel.root && -f Barrel/AliESDs.root ]] && mv Barrel/AliESDs.root AliESDs_Barrel.root
 445     [[ ! -f AliESDfriends_Barrel.root && -f Barrel/AliESDfriends.root ]] && mv Barrel/AliESDfriends.root AliESDfriends_Barrel.root
 446     [[ ! -f AliESDfriends_v1.root && -f Barrel/AliESDfriends_v1.root ]] && mv Barrel/AliESDfriends_v1.root .
 447     [[ ! -f QAresults_barrel.root && -f Barrel/QAresults_barrel.root ]] && mv Barrel/QAresults_barrel.root .
 448     [[ ! -f AliESDs_Outer.root && -f OuterDet/AliESDs.root ]] && mv OuterDet/AliESDs.root AliESDs_Outer.root
 449     [[ ! -f AliESDfriends_Outer.root && -f OuterDet/AliESDfriends.root ]] && mv OuterDet/AliESDfriends.root AliESDfriends_Outer.root
 450     [[ ! -f QAresults_outer.root && -f OuterDet/QAresults_outer.root ]] && mv OuterDet/QAresults_outer.root .
 451     [[ ! -f FilterEvents_Trees.root && -f Barrel/FilterEvents_Trees.root ]] && mv Barrel/FilterEvents_Trees.root .
 452
 453     #make the filtered tree (if requested and not already produced by QA
 454     [[ -f AliESDs_Barrel.root ]] && echo "AliESDs_Barrel.root" > filtered.list
 455     if [[ -n ${runESDfiltering} && ! -f FilterEvents_Trees.root ]]; then
 456       goMakeFilteredTrees ${PWD} ${runNumber} "filtered.list" ${filteringFactorHighPt} ${filteringFactorV0s} ${ocdbPath} 1000000 0 10000000 0 ${configFile} AliESDs_Barrel.root "${extraOpts[@]}" >filtering.log
 457     fi
 458
 459   fi
 460
 461   ##handle possible crashes in QA (happens often in trunk)
 462   ##rerun QA with a trusted aliroot version
 463   #if [[ $(validateLog qa_barrel.log) ]]; then
 464   #  echo "qa_barrel.log not validated!"
 465   #fi
 466   #if [[ ! -f QAresults_barrel.root && -f ${setupTrustedAliROOTenvInCurrentShell} || $(validateLog qa_barrel.log) ]]; then
 467   #  echo "WARNING: using trusted QA aliroot ${ALICE_ROOT}"
 468   #  source ${setupTrustedAliROOTenvInCurrentShell}
 469   #  cd Barrel
 470   #  rm QAresults_barrel.root
 471   #  rm EventStat_temp_barrel.root
 472   #  rm AODtpITS.root
 473   #  [[ ! -f AliESDs.root ]] && ln -s ../AliESDs_Barrel.root AliESDs.root
 474   #  [[ ! -f AliESDfriends.root ]] && ln -s ../AliESDfriends_Barrel.root AliESDfriends.root
 475   #  if [[ -n ${trustedQAtrainMacro} ]]; then
 476   #    eval "cp ${trustedQAtrainMacro} QAtrain_duo_trusted.C"
 477   #  fi
 478   #  echo executing aliroot -b -q "QAtrain_duo_trusted.C(\"_barrel\",${runNumber},\"wn.xml\",0,\"${ocdbPath}\")"
 479   #  time aliroot -b -q "QAtrain_duo.C(\"_barrel\",${runNumber},\"wn.xml\",0,\"${ocdbPath}\")" &> ../qa_barrel_trusted.log
 480   #  cd ../
 481   #fi
 482
 483   #move stuff to final destination
 484   echo "this directory (${PWD}) contents:"
 485   /bin/ls
 486   echo rm -f ./${chunkName}
 487   rm -f ./${chunkName}
 488   echo "cp --recursive ${runpath}/* ${outputDir}"
 489   cp -pf --recursive ${runpath}/* ${outputDir}
 490   echo
 491
 492   #validate CPass1
 493   cd ${outputDir}
 494   touch ${doneFile}
 495   echo "dir ${outputDir}" >> ${doneFile}
 496   if summarizeLogs >> ${doneFile}; then
 497     [[ -f AliESDs_Barrel.root ]] && echo "esd ${outputDir}/AliESDs_Barrel.root" >> ${doneFile}
 498     [[ -f AliESDfriends_v1.root ]] && echo "calibfile ${outputDir}/AliESDfriends_v1.root" >> ${doneFile}
 499     [[ -f QAresults_Barrel.root ]] && echo "qafile ${outputDir}/QAresults_Barrel.root" >> ${doneFile}
 500     [[ -f QAresults_Outer.root ]] && echo "qafile ${outputDir}/QAresults_Outer.root" >> ${doneFile}
 501     [[ -f QAresults_barrel.root ]] && echo "qafile ${outputDir}/QAresults_barrel.root" >> ${doneFile}
 502     [[ -f QAresults_outer.root ]] && echo "qafile ${outputDir}/QAresults_outer.root" >> ${doneFile}
 503     [[ -f FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFile}
 504   else
 505     if grep "qa_outer.log.*OK" ${doneFile} > /dev/null; then
 506       [[ -f QAresults_Outer.root ]] && echo "qafile ${outputDir}/QAresults_Outer.root" >> ${doneFile}
 507       [[ -f QAresults_outer.root ]] && echo "qafile ${outputDir}/QAresults_outer.root" >> ${doneFile}
 508     fi
 509     if grep "qa_barrel.log.*OK" ${doneFile} > /dev/null; then
 510       [[ -f QAresults_Barrel.root ]] && echo "qafile ${outputDir}/QAresults_Barrel.root" >> ${doneFile}
 511       [[ -f QAresults_barrel.root ]] && echo "qafile ${outputDir}/QAresults_barrel.root" >> ${doneFile}
 512     fi
 513     if grep "filtering.log.*OK" ${doneFile} > /dev/null; then
 514       [[ -f FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFile}
 515     fi
 516   fi
 517
 518   [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
 519   return 0
 520 }
 521
 522
 523 goMergeCPass0()
 524 {
 525   #
 526   # find the output files and merge them
 527   #
 528
 529   outputDir=${1}
 530   ocdbStorage=${2}
 531   configFile=${3}
 532   runNumber=${4}
 533   calibrationFilesToMerge=${5}  #can be a non-existent file, will then be produced on the fly
 534   shift 5
 535   if ! parseConfig ${configFile} "$@"; then return 1; fi
 536
 537   #record the working directory provided by the batch system
 538   batchWorkingDirectory=${PWD}
 539
 540   [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
 541   doneFile="${commonOutputPath}/meta/merge.cpass0.run${runNumber}.done"
 542
 543   umask 0002
 544   ulimit -c unlimited
 545
 546   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
 547
 548   #runpath=${PWD}/rundir_cpass0_Merge_${runNumber}
 549   runpath=${outputDir}
 550   [[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
 551   [[ ${reconstructInTemporaryDir} -eq 1 && -z ${TMPDIR} ]] && runpath=$(mktemp -d)
 552
 553   mkdir -p ${runpath}
 554   [[ ! -d ${runpath} ]] && echo "not able to make the runpath ${runpath}" && touch ${doneFile} && return 1
 555   cd ${runpath}
 556   [[ ! ${PWD} =~ ${runpath} ]] && echo "PWD=$PWD is not the runpath=${runpath}" && touch ${doneFile} && return 1
 557
 558   logOutputDir=${runpath}
 559   [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir}
 560   [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/mergeMakeOCDB.log
 561   echo "${0} $*"
 562
 563   mergingScript="mergeMakeOCDB.byComponent.sh"
 564
 565   echo goMergeCPass0 SETUP:
 566   echo runNumber=${runNumber}
 567   echo outputDir=${outputDir}
 568   echo ocdbStorage=${ocdbStorage}
 569   echo calibrationFilesToMerge=${calibrationFilesToMerge}
 570   echo mergingScript=${mergingScript}
 571   echo commonOutputPath=${commonOutputPath}
 572   echo runpath=${runpath}
 573
 574   # copy files in case they are not already there
 575   filesMergeCPass0=(
 576                     "${batchWorkingDirectory}/${calibrationFilesToMerge}"
 577                     "${batchWorkingDirectory}/OCDB.root"
 578                     "${batchWorkingDirectory}/localOCDBaccessConfig.C"
 579                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/mergeMakeOCDB.byComponent.sh"
 580                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/mergeByComponent.C"
 581                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/makeOCDB.C"
 582                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/merge.C"
 583                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/mergeMakeOCDB.sh"
 584   )
 585   for file in ${filesMergeCPass0[*]}; do
 586     [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
 587     [[ ${file##*/} =~ .*\.sh ]] && chmod +x ${file##*/}
 588   done
 589
 590   sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh
 591
 592   alirootInfo > ALICE_ROOT.log
 593
 594   #
 595   echo "PWD"
 596   /bin/ls
 597   echo "PWD/.."
 598   /bin/ls ../
 599
 600
 601   #merge calibration
 602   chmod u+x ${mergingScript}
 603   mkdir -p ./OCDB
 604   if [[ ! -f ${calibrationFilesToMerge} ]]; then
 605     echo "/bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root > ${calibrationFilesToMerge}"
 606     /bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root 2>/dev/null > ${calibrationFilesToMerge}
 607   fi
 608
 609   echo "${mergingScript} ${calibrationFilesToMerge} ${runNumber} local://./OCDB ${ocdbStorage}"
 610   if [[ -n ${pretend} ]]; then
 611     touch CalibObjects.root
 612     touch ocdb.log
 613     touch merge.log
 614     touch dcsTime.root
 615     mkdir -p ./OCDB/TPC/Calib/TimeGain/
 616     mkdir -p ./OCDB/TPC/Calib/TimeDrift/
 617     touch ./OCDB/TPC/Calib/TimeGain/someCalibObject_0-999999_cpass0.root
 618     touch ./OCDB/TPC/Calib/TimeDrift/otherCalibObject_0-999999_cpass0.root
 619   else
 620     ./${mergingScript} ${calibrationFilesToMerge} ${runNumber} "local://./OCDB" ${ocdbStorage}
 621
 622     #produce the calib trees for expert QA (dcsTime.root)
 623     goMakeLocalOCDBaccessConfig ./OCDB
 624     echo aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")"
 625     aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")"
 626   fi
 627
 628   ### produce the output
 629   #tar the produced OCDB for reuse
 630   tar czf ${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz ./OCDB
 631
 632   /bin/ls
 633
 634   #copy all to output dir
 635   cp -pf --recursive ${runpath}/* ${outputDir}
 636
 637   if [[ -n ${generateMC} ]]; then
 638     goPrintValues sim ${commonOutputPath}/meta/sim.run${runNumber}.list ${commonOutputPath}/meta/cpass0.job*.run${runNumber}.done
 639   fi
 640
 641   #validate merging cpass0
 642   cd ${outputDir}
 643   touch ${doneFile}
 644   echo "dir ${outputDir}" >> ${doneFile}
 645   if summarizeLogs >> ${doneFile}; then
 646     [[ -f CalibObjects.root ]] && echo "calibfile ${outputDir}/CalibObjects.root" >> ${doneFile}
 647     [[ -f dcsTime.root ]] && echo "dcsTree ${outputDir}/dcsTime.root" >> ${doneFile}
 648   fi
 649
 650   [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
 651   return 0
 652 }
 653
 654 goMergeCPass1()
 655 {
 656   #
 657   # find the output files and merge them
 658   #
 659
 660   outputDir=${1}
 661   ocdbStorage=${2}
 662   configFile=${3}
 663   runNumber=${4}
 664   calibrationFilesToMerge=${5}
 665   qaFilesToMerge=${6}
 666   filteredFilesToMerge=${7}
 667   shift 7
 668   if ! parseConfig ${configFile} "$@"; then return 1; fi
 669
 670   #record the working directory provided by the batch system
 671   batchWorkingDirectory=${PWD}
 672
 673   [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
 674   doneFile="${commonOutputPath}/meta/merge.cpass1.run${runNumber}.done"
 675
 676   umask 0002
 677   ulimit -c unlimited
 678
 679   #clean up first:
 680   rm -f ${outputDir}/*.log
 681   rm -f ${outputDir}/*.root
 682   rm -f ${outputDir}/*done
 683
 684   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
 685
 686   #runpath=${PWD}/rundir_cpass1_Merge_${runNumber}
 687   runpath=${outputDir}
 688   [[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
 689   [[ ${reconstructInTemporaryDir} -eq 1 && -z ${TMPDIR} ]] && runpath=$(mktemp -d)
 690
 691   mkdir -p ${runpath}
 692   [[ ! -d ${runpath} ]] && echo "not able to make the runpath ${runpath}" && touch ${doneFile} && return 1
 693   cd ${runpath}
 694   [[ ! ${PWD} =~ ${runpath} ]] && echo "PWD=$PWD is not the runpath=${runpath}" && touch ${doneFile} && return 1
 695
 696   logOutputDir=${runpath}
 697   [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir}
 698   [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/mergeMakeOCDB.log
 699   echo "${0} $*"
 700
 701   calibrationOutputFileName='AliESDfriends_v1.root'
 702   qaOutputFileName='QAresults*.root'
 703   mergingScript="mergeMakeOCDB.byComponent.sh"
 704   #important to have the string "Stage.txt" in the filename to trigger the merging
 705   #it has to be a list of directories containing the files
 706   qaMergedOutputFileName="QAresults_merged.root"
 707
 708   echo goMergeCPass1 SETUP:
 709   echo runNumber=${runNumber}
 710   echo outputDir=${outputDir}
 711   echo ocdbStorage=${ocdbStorage}
 712   echo calibrationFilesToMerge=$calibrationFilesToMerge
 713   echo qaFilesToMerge=$qaFilesToMerge
 714   echo calibrationOutputFileName=${calibrationOutputFileName}
 715   echo mergingScript=${mergingScript}
 716
 717   # copy files in case they are not already there
 718   filesMergeCPass1=(
 719                     "${batchWorkingDirectory}/${calibrationFilesToMerge}"
 720                     "${batchWorkingDirectory}/${qaFilesToMerge}"
 721                     "${batchWorkingDirectory}/OCDB.root"
 722                     "${batchWorkingDirectory}/localOCDBaccessConfig.C"
 723                     "${batchWorkingDirectory}/meta/cpass0.localOCDB.${runNumber}.tgz"
 724                     "${batchWorkingDirectory}/QAtrain_duo.C"
 725                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/mergeMakeOCDB.byComponent.sh"
 726                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/mergeByComponent.C"
 727                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/makeOCDB.C"
 728                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/merge.C"
 729                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/mergeMakeOCDB.sh"
 730                     "${trustedQAtrainMacro}"
 731                     "${ALICE_ROOT}/ANALYSIS/macros/QAtrain_duo.C"
 732   )
 733   for file in ${filesMergeCPass1[*]}; do
 734     [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
 735     [[ ${file##*/} =~ .*\.sh ]] && chmod +x ${file##*/}
 736   done
 737
 738   sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh
 739
 740   #configure local OCDB storage from CPass0 (creates the localOCDBaccessConfig.C script)
 741   if [[ -f cpass0.localOCDB.${runNumber}.tgz ]]; then
 742     echo goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz"
 743     goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz"
 744   else
 745     echo "WARNING: file cpass0.localOCDB.${runNumber}.tgz not found!"
 746   fi
 747
 748   alirootInfo > ALICE_ROOT.log
 749
 750   #
 751   /bin/ls
 752
 753   #merge calibration
 754   chmod u+x ${mergingScript}
 755   mkdir -p OCDB
 756
 757   #if not provided, create the lists of files to merge
 758   if [[ ! -f ${filteredFilesToMerge} ]]; then
 759     echo "/bin/ls -1 ${outputDir}/*/FilterEvents_Trees.root > ${filteredFilesToMerge}"
 760     /bin/ls -1 ${outputDir}/*/FilterEvents_Trees.root 2>/dev/null > ${filteredFilesToMerge}
 761   fi
 762   if [[ ! -f ${calibrationFilesToMerge} ]]; then
 763     echo "/bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root > ${calibrationFilesToMerge}"
 764     /bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root 2>/dev/null > ${calibrationFilesToMerge}
 765   fi
 766   if [[ ! -f ${qaFilesToMerge} ]]; then
 767     #find the files, but only store the directories (QAtrain_duo.C requires this)
 768     echo "/bin/ls -1 ${outputDir}/*/QAresults*.root | while read x; do echo ${x%/*}; done | sort | uniq > ${qaFilesToMerge}"
 769     /bin/ls -1 ${outputDir}/*/QAresults*.root | while read x; do echo ${x%/*}; done | sort | uniq > ${qaFilesToMerge}
 770   fi
 771
 772   echo "${mergingScript} ${calibrationFilesToMerge} ${runNumber} local://./OCDB ${ocdbStorage}"
 773   if [[ -n ${pretend} ]]; then
 774     touch ocdb.log
 775     touch cpass1.localOCDB.${runNumber}.tgz
 776     touch ${qaMergedOutputFileName}
 777     touch merge.log
 778     touch trending.root
 779     touch FilterEvents_Trees.root
 780     touch CalibObjects.root
 781     touch dcsTime.root
 782     touch ${qaMergedOutputFileName}
 783     mkdir -p OCDB
 784   else
 785     ./${mergingScript} ${calibrationFilesToMerge} ${runNumber} "local://./OCDB" ${ocdbStorage}
 786
 787     #merge QA (and filtered trees)
 788     [[ -n ${AliAnalysisTaskFilteredTree_fLowPtTrackDownscaligF} ]] && export AliAnalysisTaskFilteredTree_fLowPtTrackDownscaligF
 789     [[ -n ${AliAnalysisTaskFilteredTree_fLowPtV0DownscaligF} ]] && export AliAnalysisTaskFilteredTree_fLowPtV0DownscaligF
 790
 791     #echo aliroot -l -b -q "merge.C(\"${qaFilesToMerge}\",\"\",kFALSE,\"${qaMergedOutputFileName}\")"
 792     echo aliroot -b -q "QAtrain_duo.C(\"_barrel\",${runNumber},\"${qaFilesToMerge}\",1,\"${ocdbStorage}\")"
 793     #aliroot -l -b -q "merge.C(\"${qaFilesToMerge}\",\"\",kFALSE,\"${qaMergedOutputFileName}\")"
 794     aliroot -b -q "QAtrain_duo.C(\"_barrel\",${runNumber},\"${qaFilesToMerge}\",1,\"${ocdbStorage}\")" > mergeQA.log
 795     mv QAresults_barrel.root ${qaMergedOutputFileName}
 796     mv trending_barrel.root trending.root
 797
 798     #merge filtered trees
 799     echo aliroot -l -b -q "merge.C(\"${qaFilesToMerge}\",\"\",kFALSE,\"${qaMergedOutputFileName}\")"
 800     aliroot -l -b -q "merge.C(\"${filteredFilesToMerge}\",\"\",kFALSE,\"FilterEvents_Trees.root\")" > mergeFilteredTrees.log
 801
 802     #produce the calib trees for expert QA
 803     echo aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")"
 804     aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")" > calibTree.log
 805   fi
 806
 807   tar czf ${commonOutputPath}/meta/cpass1.localOCDB.${runNumber}.tgz ./OCDB
 808
 809   /bin/ls
 810
 811   #copy all to output dir
 812   cp -pf --recursive ${runpath}/* ${outputDir}
 813
 814   #validate merge cpass1
 815   cd ${outputDir}
 816   touch ${doneFile}
 817   echo "dir ${outputDir}" >> ${doneFile}
 818   if summarizeLogs >>  ${doneFile}; then
 819     [[ -f CalibObjects.root ]] && echo "calibfile ${outputDir}/CalibObjects.root" >> ${doneFile}
 820     [[ -f ${qaMergedOutputFileName} ]] && echo "qafile ${outputDir}/${qaMergedOutputFileName}" >> ${doneFile}
 821     [[ -f trending.root ]] && echo "trendingfile ${outputDir}/trending.root" >> ${doneFile}
 822     [[ -f dcsTime.root ]] && echo "dcsTree ${outputDir}/dcsTime.root" >> ${doneFile}
 823     [[ -f FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFile}
 824   else
 825     if grep "mergeQA.log.*OK" ${doneFile} > /dev/null; then
 826       [[ -f ${qaMergedOutputFileName} ]] && echo "qafile ${outputDir}/${qaMergedOutputFileName}" >> ${doneFile}
 827     fi
 828     if grep "mergeFilteredTrees.log.*OK" ${doneFile} > /dev/null; then
 829       [[ -f FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFile}
 830     fi
 831   fi
 832
 833   [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
 834   return 0
 835 }
 836
 837 goMerge()
 838 {
 839   #generic root merge using CPass1 merge.C script
 840   inputList=${1}
 841   outputFile=${2}
 842   configFile=${3-"becnhmark.config"}
 843   shift 3
 844   if ! parseConfig ${configFile} "$@"; then return 1; fi
 845
 846   #record the working directory provided by the batch system
 847   batchWorkingDirectory=${PWD}
 848
 849   [[ ! -f ${inputList} ]] && echo "inputList ${inputList} does not exist!" && return 1
 850   [[ ! -f ${configFile} ]] && echo "configFile ${configFile} does not exist!" && return 1
 851   umask 0002
 852   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
 853   rm -f ${outputFile}
 854   aliroot -b -q "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/merge.C(\"${inputList}\",\"\",kFALSE,\"${outputFile}\")" > merge_${inputList}.log
 855   return 0
 856 }
 857
 858 goSubmitMakeflow()
 859 {
 860   #run
 861   productionID=${1}
 862   inputFileList=${2}
 863   configFile=${3}
 864   shift 3
 865   extraOpts=("$@")
 866   if ! parseConfig ${configFile} "${extraOpts[@]}"; then return 1; fi
 867
 868   #record the working directory provided by the batch system
 869   batchWorkingDirectory=${PWD}
 870
 871   [[ -z ${configFile} ]] && configFile="benchmark.config"
 872   [[ ! -f ${configFile} ]] && echo "no config file found (${configFile})" && return 1
 873
 874   if [[ ! $(which makeflow &>/dev/null) && -n ${makeflowPath} ]]; then
 875     echo "setting the makflow path from the config: "
 876     echo "  export PATH=${makeflowPath}:${PATH}"
 877     export PATH=${makeflowPath}:${PATH}
 878   fi
 879
 880   #create the common output dir and the meta dir
 881   commonOutputPath=${baseOutputDirectory}/${productionID}
 882   if [[ -d ${commonOutputPath} ]]; then
 883     echo "output dir ${commonOutputPath} exists!"
 884     return 1
 885   else
 886     mkdir -p ${commonOutputPath}
 887   fi
 888   mkdir -p ${commonOutputPath}/meta
 889
 890   self=$(readlink -f "${0}")
 891
 892   cp ${self} ${commonOutputPath}
 893   cp ${configFile} ${commonOutputPath}
 894   cp ${inputList} ${commonOutputPath}
 895
 896   #submit - use makeflow if available, fall back to old stuff when makeflow not there
 897   if which makeflow; then
 898
 899     goGenerateMakeflow ${productionID} ${inputFileList} ${configFile} "${extraOpts[@]}" commonOutputPath=${commonOutputPath} > benchmark.makeflow
 900
 901     makeflow ${makeflowOptions} benchmark.makeflow
 902     cd ../
 903   else
 904     echo "no makeflow!"
 905   fi
 906   return 0
 907 }
 908
 909 goGenerateMakeflow()
 910 {
 911   #generate the makeflow file
 912   [[ $# -lt 3 ]] && echo "args: id inputFileList configFile" && return 1
 913   productionID=${1}
 914   inputFileList=${2}
 915   configFile=${3}
 916   shift 3
 917   extraOpts=("$@")
 918   if ! parseConfig ${configFile} "${extraOpts[@]}" &>/dev/null; then return 1; fi
 919
 920   #extra safety
 921   if [[ -z ${commonOutputPath} ]]; then
 922     commonOutputPath=${baseOutputDirectory}/${productionID}
 923     extraOpts=( "${extraOpts[@]}" "commonOutputPath=${commonOutputPath}" )
 924   fi
 925
 926   #record the working directory provided by the batch system
 927   batchWorkingDirectory=${PWD}
 928
 929   [[ -z ${configFile} ]] && configFile="benchmark.config"
 930   [[ ! -f ${configFile} ]] && echo "no config file found (${configFile})" && return 1
 931
 932   #these files will be made a dependency - will be copied to the working dir of the jobs
 933   declare -a copyFiles
 934   inputFiles=(
 935               "OCDB.root"
 936               "localOCDBaccessConfig.C"
 937               "QAtrain_duo.C"
 938               "runCPass1.sh"
 939               "recCPass1.C"
 940               "recCPass1_OuterDet.C"
 941               "runCalibTrain.C"
 942               "runCPass0.sh"
 943               "recCPass0.C"
 944   )
 945   for file in ${inputFiles[*]}; do
 946     [[ -f ${file} ]] && copyFiles+=("${file}")
 947   done
 948
 949   #create the makeflow file
 950   [[ -n ${batchFlags} ]] && echo "BATCH_OPTIONS = ${batchFlags}"
 951   declare -A arr_cpass0_merged arr_cpass1_merged
 952   declare -A arr_cpass0_calib_list arr_cpass1_calib_list
 953   declare -A arr_cpass1_QA_list arr_cpass1_ESD_list arr_cpass1_filtered_list
 954   declare -A arr_cpass0_profiled_outputs
 955   declare -A listOfRuns
 956   [[ -n ${runNumber} ]] && listOfRuns[${runNumber}]=1
 957   while read x; do tmpRun=$(guessRunNumber ${x}); [[ -n ${tmpRun} ]] && listOfRuns[${tmpRun}]=1; done < ${inputFileList}
 958   for runNumber in "${!listOfRuns[@]}"; do
 959     [[ -z ${runNumber} ]] && continue
 960     [[ ! ${runNumber} =~ ^[0-9]*[0-9]$ ]] && continue
 961
 962     unset arr_cpass0_outputs
 963     unset arr_cpass1_outputs
 964     declare -a arr_cpass0_outputs
 965     declare -a arr_cpass1_outputs
 966
 967     jobindex=0
 968     inputFile=""
 969     while read inputFile; do
 970       currentDefaultOCDB=${defaultOCDB}
 971       [[ -z ${autoOCDB} ]] && autoOCDB=1
 972       if [[ ${autoOCDB} -ne 0 ]]; then
 973         currentDefaultOCDB=$(setYear ${inputFile} ${defaultOCDB})
 974       fi
 975
 976       #CPass0
 977       arr_cpass0_outputs[${jobindex}]="${commonOutputPath}/meta/cpass0.job${jobindex}.run${runNumber}.done"
 978       echo "${arr_cpass0_outputs[${jobindex}]} : benchmark.sh ${configFile} ${copyFiles[@]}"
 979       echo -n " ${alirootEnv} ./benchmark.sh CPass0 ${commonOutputPath}/000${runNumber}/cpass0 ${inputFile} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} ${jobindex}"" "
 980       for extraOption in "${extraOpts[@]}"; do echo -n \'${extraOption}\'" "; done; echo
 981       echo
 982
 983       #CPass1
 984       arr_cpass1_outputs[${jobindex}]="${commonOutputPath}/meta/cpass1.job${jobindex}.run${runNumber}.done"
 985       echo "${arr_cpass1_outputs[${jobindex}]} : benchmark.sh ${configFile} ${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz ${copyFiles[@]}"
 986       echo -n " ${alirootEnv} ./benchmark.sh CPass1 ${commonOutputPath}/000${runNumber}/cpass1 ${inputFile} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} ${jobindex}"" "
 987       for extraOption in "${extraOpts[@]}"; do echo -n \'${extraOption}\'" "; done; echo
 988       echo
 989       ((jobindex++))
 990
 991     done< <(grep "/000${runNumber}/" ${inputFileList})
 992
 993     #CPass0 list of Calib files to merge
 994     arr_cpass0_calib_list[${runNumber}]="${commonOutputPath}/meta/cpass0.calib.run${runNumber}.list"
 995     echo "${arr_cpass0_calib_list[${runNumber}]} : benchmark.sh ${arr_cpass0_outputs[*]}"
 996     echo "  ./benchmark.sh PrintValues calibfile ${arr_cpass0_calib_list[${runNumber}]} ${arr_cpass0_outputs[*]}"
 997     echo
 998
 999     #CPass0 merging
1000     arr_cpass0_merged[${runNumber}]="${commonOutputPath}/meta/merge.cpass0.run${runNumber}.done"
1001     echo "${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz ${arr_cpass0_merged[${runNumber}]} : benchmark.sh ${configFile} ${arr_cpass0_calib_list[${runNumber}]} ${copyFiles[@]}"
1002     echo -n " ${alirootEnv} ./benchmark.sh MergeCPass0 ${commonOutputPath}/000${runNumber}/cpass0 ${currentDefaultOCDB} ${configFile} ${runNumber} ${arr_cpass0_calib_list[${runNumber}]}"" "
1003     for extraOption in "${extraOpts[@]}"; do echo -n \'${extraOption}\'" "; done; echo
1004     echo
1005
1006     #CPass1 list of Calib/QA/ESD/filtered files
1007     # the trick with QA is to have the string "Stage.txt" in the file name of the list of directories with QA output to trigger
1008     # the production of the QA trending tree (only then the task->Finish() will be called in QAtrain_duo.C, on the grid
1009     # this corresponds to the last merging stage)
1010     arr_cpass1_QA_list[${runNumber}]="${commonOutputPath}/meta/cpass1.QA.run${runNumber}.lastMergingStage.txt.list"
1011     echo "${arr_cpass1_QA_list[${runNumber}]}: benchmark.sh ${arr_cpass1_outputs[*]}"
1012     echo "  ./benchmark.sh PrintValues dir ${arr_cpass1_QA_list[${runNumber}]} ${arr_cpass1_outputs[*]}"
1013     echo
1014     arr_cpass1_calib_list[${runNumber}]="${commonOutputPath}/meta/cpass1.calib.run${runNumber}.list"
1015     echo "${arr_cpass1_calib_list[${runNumber}]} : benchmark.sh ${arr_cpass1_outputs[*]}"
1016     echo "  ./benchmark.sh PrintValues calibfile ${arr_cpass1_calib_list[${runNumber}]} ${arr_cpass1_outputs[*]};"
1017     echo
1018     arr_cpass1_ESD_list[${runNumber}]="${commonOutputPath}/meta/cpass1.ESD.run${runNumber}.list"
1019     echo "${arr_cpass1_ESD_list[${runNumber}]} : benchmark.sh ${arr_cpass1_outputs[*]}"
1020     echo "  ./benchmark.sh PrintValues esd ${arr_cpass1_ESD_list[${runNumber}]} ${arr_cpass1_outputs[*]}"
1021     echo
1022     arr_cpass1_filtered_list[${runNumber}]="${commonOutputPath}/meta/cpass1.filtered.run${runNumber}.list"
1023     echo "${arr_cpass1_filtered_list[${runNumber}]} : benchmark.sh ${arr_cpass1_outputs[*]}"
1024     echo "  ./benchmark.sh PrintValues filteredTree ${arr_cpass1_filtered_list[${runNumber}]} ${arr_cpass1_outputs[*]}"
1025     echo
1026
1027     #CPass1 merging
1028     arr_cpass1_merged[${runNumber}]="${commonOutputPath}/meta/merge.cpass1.run${runNumber}.done"
1029     echo "${commonOutputPath}/meta/cpass1.localOCDB.${runNumber}.tgz ${arr_cpass1_merged[${runNumber}]} :  benchmark.sh ${configFile} ${arr_cpass1_calib_list[${runNumber}]} ${arr_cpass1_QA_list[${runNumber}]} ${copyFiles[@]}"
1030     echo -n " ${alirootEnv} ./benchmark.sh MergeCPass1 ${commonOutputPath}/000${runNumber}/cpass1 ${currentDefaultOCDB} ${configFile} ${runNumber} ${arr_cpass1_calib_list[${runNumber}]} ${arr_cpass1_QA_list[${runNumber}]} ${arr_cpass1_filtered_list[${runNumber}]}"" "
1031     for extraOption in "${extraOpts[@]}"; do echo -n \'${extraOption}\'" "; done; echo
1032     echo
1033
1034     #CPass0 wrapped in a profiling tool (valgrind,....)
1035     if [[ -n ${profilingCommand} ]]; then
1036       arr_cpass0_profiled_outputs[${runNumber}]="${commonOutputPath}/meta/cpass0.jobProfiled.run${runNumber}.done"
1037       echo "${arr_cpass0_profiled_outputs[${runNumber}]} : benchmark.sh ${configFile} ${copyFiles[@]}"
1038       echo -n " ${alirootEnv} ./benchmark.sh CPass0 ${commonOutputPath}/000${runNumber}/profiled ${inputFile} ${nEventsProfiling} ${currentDefaultOCDB} ${configFile} ${runNumber} profiling"" "
1039       for extraOption in "${extraOpts[@]}"; do echo -n \'${extraOption}\'" "; done;
1040       echo "\"useProfilingCommand=${profilingCommand}\""
1041       echo
1042     fi
1043
1044   done #runs
1045
1046   #Summary
1047   echo "${commonOutputPath}/summary.log : benchmark.sh ${configFile} ${arr_cpass1_merged[*]}"
1048   echo -n "  ${alirootEnv} ./benchmark.sh MakeSummary ${configFile}"" "
1049   for extraOption in "${extraOpts[@]}"; do echo -n \'${extraOption}\'" "; done; echo
1050   echo
1051
1052   return 0
1053 }
1054
1055 goPrintValues()
1056 {
1057   #print the values given the key from any number of files (store in output file on second argument)
1058   if [[ $# -lt 3 ]]; then
1059     echo "goPrintValues key outputFile inputFiles"
1060     echo "if outputFile is \"-\" dont redirect to a file"
1061     return
1062   fi
1063   key=${1}
1064   outputFile=${2}
1065   [[ ${outputFile} =~ "-" ]] && outputFile=""
1066   shift 2 #remove 2 first arguments from arg list to only pass the input files to awk
1067   awk -v key=${key} '$0 ~ key" " {print $2}' "$@" | tee ${outputFile}
1068   return 0
1069 }
1070
1071 goCreateQAplots()
1072 {
1073   umask 0002
1074   mergedQAfileList=${1}
1075   productionID=${2}
1076   outputDir=${3}
1077   configFile=${4}
1078   shift 4
1079   if ! parseConfig ${configFile} ${@}; then return 1; fi
1080
1081   #record the working directory provided by the batch system
1082   batchWorkingDirectory=${PWD}
1083
1084   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
1085
1086   [[ -z ${logOutputDir} ]] && logOutputDir=${PWD}
1087   [[ -z ${dontRedirectStdOutToLog} ]] && exec 2>&1 > ${logOutputDir}/makeQAplots.log
1088   echo "${0} $*"
1089
1090   olddir=${PWD}
1091   mkdir -p ${outputDir}
1092   cd ${outputDir}
1093   [[ ! "${PWD}" =~ "${outputDir}" ]] && echo "PWD is not equal to outputDir=${outputDir}" && cd ${olddir} && return 1
1094
1095   ${ALICE_ROOT}/PWGPP/QA/scripts/runQA.sh inputList="${mergedQAfileList}" inputListHighPtTrees="${filteringList}"
1096
1097   cd ${olddir}
1098   return 0
1099 }
1100
1101 goTest()
1102 {
1103   umask 0002
1104   exec 2>&1
1105   exec > >(tee test.log)
1106   echo "$@"
1107   echo something
1108   return 0
1109 }
1110
1111 alirootInfo()
1112 {
1113   umask 0002
1114   # save aliroot repository info
1115   [[ -z "${ALICE_ROOT}" ]] && return 1
1116
1117   echo "\${ALICE_ROOT}=${ALICE_ROOT}"
1118   echo "\${ROOTSYS}=${ROOTSYS}"
1119   echo "\${PATH}=${PATH}"
1120   echo "\${LD_LIBRARY_PATH}=${LD_LIBRARY_PATH}"
1121
1122   pushd ${PWD}
1123   cd ${ALICE_ROOT}
1124   echo
1125
1126   currentBranch=$(git rev-parse --abbrev-ref HEAD)
1127   git status
1128   echo ""
1129   echo ""
1130   git diff ${currentBranch}
1131   popd
1132   return 0
1133 }
1134
1135 setYear()
1136 {
1137   #set the year
1138   #  ${1} - year to be set
1139   #  ${2} - where to set the year
1140   year1=$(guessYear ${1})
1141   year2=$(guessYear ${2})
1142   local path=${2}
1143   [[ ${year1} -ne ${year2} && -n ${year2} && -n ${year1} ]] && path=${2/\/${year2}\//\/${year1}\/}
1144   echo ${path}
1145   return 0
1146 }
1147
1148 guessPeriod()
1149 {
1150   #guess the period from the path, pick the rightmost one
1151   local IFS="/"
1152   declare -a path=( ${1} )
1153   local dirDepth=${#path[*]}
1154   for ((x=${dirDepth}-1;x>=0;x--)); do
1155     local field=${path[${x}]}
1156     [[ ${field} =~ ^LHC[0-9][0-9][a-z]$ ]] && period=${field} && break
1157   done
1158   echo ${period}
1159   return 0
1160 }
1161
1162 guessYear()
1163 {
1164   #guess the year from the path, pick the rightmost one
1165   local IFS="/"
1166   declare -a path=( ${1} )
1167   local dirDepth=${#path[*]}
1168   for ((x=${dirDepth}-1;x>=0;x--)); do
1169     local field=${path[${x}]}
1170     [[ ${field} =~ ^20[0-9][0-9]$ ]] && year=${field} && break
1171   done
1172   echo ${year}
1173   return 0
1174 }
1175
1176 guessRunNumber()
1177 {
1178   #guess the run number from the path, pick the rightmost one
1179   #works for /path/foo/000123456/bar/...
1180   #and       /path/foo.run123456.bar
1181   local IFS="/."
1182   declare -a path=( ${1} )
1183   local dirDepth=${#path[*]}
1184   for ((x=${dirDepth}-1;x>=0;x--)); do
1185     local field=${path[${x}]}
1186     field=${field/run/000}
1187     [[ ${field} =~ [0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && runNumber=${field#000} && break
1188   done
1189   echo ${runNumber}
1190   return 0
1191 }
1192
1193 validateLog()
1194 {
1195   log=${1}
1196   errorConditions=(
1197                     'There was a crash'
1198                     'floating'
1199                     'error while loading shared libraries'
1200                     'std::bad_alloc'
1201                     's_err_syswatch_'
1202                     'Thread [0-9]* (Thread'
1203                     'AliFatal'
1204                     'core dumped'
1205                     '\.C.*error:.*\.h: No such file'
1206                     'line.*Aborted'
1207   )
1208
1209   warningConditions=(
1210                      'This is serious !'
1211                      'rocVoltage out of range:'
1212   )
1213
1214   local logstatus=0
1215   local errorSummary=""
1216   local warningSummary=""
1217
1218   for ((i=0; i<${#errorConditions[@]};i++)); do
1219     local tmp=$(grep -m1 -e "${errorConditions[${i}]}" ${log})
1220     [[ -n ${tmp} ]] && tmp+=" : "
1221     errorSummary+=${tmp}
1222   done
1223
1224   for ((i=0; i<${#warningConditions[@]};i++)); do
1225     local tmp=$(grep -m1 -e "${warningConditions[${i}]}" ${log})
1226     [[ -n ${tmp} ]] && tmp+=" : "
1227     warningSummary+=${tmp}
1228   done
1229
1230   if [[ -n ${errorSummary} ]]; then
1231     echo "${errorSummary}"
1232     return 1
1233   fi
1234
1235   if [[ -n ${warningSummary} ]]; then
1236     echo "${warningSummary}"
1237     return 2
1238   fi
1239
1240   return 0
1241 }
1242
1243 summarizeLogs()
1244 {
1245   #print a summary of logs
1246   logFiles=(
1247             "*.log"
1248             "stdout"
1249             "stderr"
1250   )
1251
1252   #check logs
1253   local logstatus=0
1254   for log in ${logFiles[*]}; do
1255     finallog=${PWD%/}/${log}
1256     [[ ! -f ${log} ]] && continue
1257     errorSummary=$(validateLog ${log})
1258     validationStatus=$?
1259     if [[ ${validationStatus} -eq 0 ]]; then
1260       #in pretend mode randomly report an error in rec.log some cases
1261       if [[ -n ${pretend} && "${log}" == "rec.log" ]]; then
1262         #[[ $(( ${RANDOM}%2 )) -ge 1 ]] && echo "${finallog} BAD random error" || echo "${finallog} OK"
1263         echo "${finallog} OK"
1264       else
1265         echo "${finallog} OK"
1266       fi
1267     elif [[ ${validationStatus} -eq 1 ]]; then
1268       echo "${finallog} BAD ${errorSummary}"
1269       logstatus=1
1270     elif [[ ${validationStatus} -eq 2 ]]; then
1271       echo "${finallog} OK MWAH ${errorSummary}"
1272     fi
1273   done
1274
1275   #report core files
1276   while read x; do
1277     echo ${x}
1278     chmod 644 ${x}
1279     gdb --batch --quiet -ex "bt" -ex "quit" aliroot ${x} > stacktrace_${x//\//_}.log
1280   done < <(/bin/ls ${PWD}/*/core 2>/dev/null; /bin/ls ${PWD}/core 2>/dev/null)
1281
1282   return ${logstatus}
1283 }
1284
1285 spitOutLocalOCDBaccessConfig()
1286 {
1287   umask 0002
1288   #find ${1} -name "*root" | \
1289   /bin/ls -1 ${1}/*/*/*/*.root 2>/dev/null | \
1290   while read line
1291   do
1292     local tmp=${line#${1}}
1293     echo ${tmp%/*} | \
1294     awk -v ocdb=${1} '{print "  man->SetSpecificStorage(\""$1"\",\"local://"ocdb"\");"}'
1295   done
1296   return 0
1297 }
1298
1299 goMakeLocalOCDBaccessConfig()
1300 {
1301   umask 0002
1302   # make a script that sets the specific storages form all the root files produced by CPass0
1303   local localOCDBpathCPass0=${1}
1304   local OCDBpathPrefix=${2}
1305   [[ -z ${OCDBpathPrefix} ]] && OCDBpathPrefix="."
1306
1307   if [[ -f ${localOCDBpathCPass0} && ${localOCDBpathCPass0} =~ \.tgz$ ]]; then
1308     tar xzf ${localOCDBpathCPass0}
1309     local localOCDBpathCPass0="${OCDBpathPrefix}/OCDB"
1310   fi
1311
1312   echo
1313   echo creating the specific storage script
1314   echo   localOCDBaccessConfig.C
1315   echo   based on OCDB: ${localOCDBaccessConfig}
1316   echo
1317
1318   local tempLocalOCDB=""
1319   if [[ -f localOCDBaccessConfig.C ]]; then
1320     tempLocalOCDB=$(mktemp)
1321     echo "egrep "SetSpecificStorage" localOCDBaccessConfig.C > ${tempLocalOCDB}"
1322     egrep "SetSpecificStorage" localOCDBaccessConfig.C > ${tempLocalOCDB}
1323   fi
1324
1325   echo "localOCDBaccessConfig()"                               >  localOCDBaccessConfig.C
1326   echo "{"                                                     >> localOCDBaccessConfig.C
1327   echo "  AliCDBManager* man = AliCDBManager::Instance();"     >> localOCDBaccessConfig.C
1328   spitOutLocalOCDBaccessConfig ${localOCDBpathCPass0}|sort|uniq  >> localOCDBaccessConfig.C
1329   [[ -f "${tempLocalOCDB}" ]] && cat ${tempLocalOCDB}              >> localOCDBaccessConfig.C
1330   echo "}"                                                     >> localOCDBaccessConfig.C
1331
1332   [[ -f "${tempLocalOCDB}" ]] && rm -f ${tempLocalOCDB}
1333
1334   if ! grep SetSpecificStorage localOCDBaccessConfig.C; then
1335     echo
1336     echo "!!!!!!! CPass0 produced no OCDB entries"
1337     return 1
1338   fi
1339   return 0
1340 }
1341
1342 goMakeFilteredTrees()
1343 {
1344   outputDir=${1}
1345   runNumber=${2}
1346   #get path to input list
1347   inputListfiles=${3}
1348   #get scale number for tracks
1349   filterT=${4}
1350   #get scale number for V0s
1351   filterV=${5}
1352   #get OCDB path (optional)
1353   OCDBpath=${6}
1354   #get max number of files
1355   maxFiles=${7-"1000000"}
1356   #get offset of first file
1357   offsetFile=${8-"0"}
1358   #get max number of events
1359   maxEvents=${9-"30000000"}
1360   #get offset of first event
1361   offsetEvent=${10-"0"}
1362   configFile=${11-"benchmark.config"}
1363   esdFileName=${12-"AliESDs_Barrel.root"}
1364   shift 12
1365   if ! parseConfig ${configFile} "$@"; then return 1; fi
1366
1367   #record the working directory provided by the batch system
1368   batchWorkingDirectory=${PWD}
1369
1370
1371   commonOutputPath=${PWD}
1372   doneFile=${commonOutputPath}/meta/filtering.cpass1.run${runNumber}.done
1373
1374   runpath=${outputDir}
1375   [[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
1376   [[ ${reconstructInTemporaryDir} -eq 1 && -z ${TMPDIR} ]] && runpath=$(mktemp -d)
1377
1378
1379   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
1380
1381   mkdir -p ${outputDir}
1382   mkdir -p ${runpath}
1383
1384   cd ${runpath}
1385   [[ ! ${PWD} =~ ${runpath} ]] && echo "PWD=$PWD is not the runpath=${runpath}" && touch ${doneFile} && return 1
1386
1387   cat > filtering.log << EOF
1388   goMakeFilteredTrees config:
1389   runpath=${runpath}
1390   outputDir=${outputDir}
1391   commonOutputPath=${commonOutputPath}
1392   ALICE_ROOT=${ALICE_ROOT}
1393   PATH=${PATH}
1394   offsetEvent=$offsetEvent
1395   configFile=$configFile
1396   esdFileName=$esdFileName
1397 EOF
1398
1399   if [[ -z ${pretend} ]];then
1400     aliroot -l -b -q "${ALICE_ROOT}/PWGPP/macros/runFilteringTask.C(\"${inputListfiles}\",${filterT},${filterV},\"${OCDBpath}\",${maxFiles},${offsetFile},${maxEvents},${offsetEvent},\"${esdFileName}\")" &>> filtering.log
1401   else
1402     touch filtering.log FilterEvents_Trees.root
1403   fi
1404   pwd
1405   /bin/ls
1406   touch ${doneFile}
1407   summarizeLogs >>  ${doneFile}
1408   echo mv -f * ${outputDir}
1409   mv -f * ${outputDir}
1410   [[ -f ${outputDir}/FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFile}
1411   cd ${commonOutputPath}
1412   [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
1413   return 0
1414 }
1415
1416 submit()
1417 {
1418   umask 0002
1419   [[ $# -lt 5 ]] && echo "at least 5 args needed, you supplied $#" && return 1
1420   JobID=${1}
1421   startID=${2}
1422   endID=${3}
1423   waitForJOBID=${4}
1424   command=${5}
1425   shift 5
1426   commandArgs=("$@")
1427
1428   #add quote strings around the extra arguments
1429   for ((i=0; i<${#commandArgs[@]}; i++)); do
1430     commandArgs[i]=\"${commandArgs[i]}\"
1431   done
1432
1433   [[ -z ${waitForJOBID} ]] && waitForJOBID=0
1434
1435   newFarm=$(which qsub|grep "^/usr/bin/qsub")
1436
1437   batchSystem="SGE"
1438
1439   if [[ -z "${newFarm}" ]]
1440   then
1441     #old LSF
1442     # submit it (as job array)
1443     nFiles=$(( ${endID}-${startID}+1 ))
1444     while [ ${startID} -le ${nFiles}  ] ; do
1445       if [ $(expr ${nFiles} - ${startID}) -gt 999 ] ; then
1446         endID=$(expr ${startID} + 999)
1447       else
1448         endID=${nFiles}
1449       fi
1450     if [[ ${waitForJOBID} -eq 0 ]]; then
1451         echo ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}"
1452         ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}"
1453       else
1454         echo ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -w "ended(${waitForJOBID})" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}"
1455         ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -w "ended(${waitForJOBID})" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}"
1456       fi
1457       startID=$(expr ${endID} + 1)
1458     done
1459   else
1460     #new SGE farm
1461     if [[ ${waitForJOBID} =~ "000" ]]; then
1462       echo ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}"
1463       ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}"
1464     else
1465       echo ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -hold_jid "${waitForJOBID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}"
1466       ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -hold_jid "${waitForJOBID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}"
1467     fi
1468   fi
1469   return 0
1470 }
1471
1472 goSubmitBatch()
1473 {
1474   if [[ $# -lt 3 ]]; then
1475     echo "minimal use:"
1476     echo " ${0} submit fileList productionID configFile"
1477     return 0
1478   fi
1479
1480   productionID=${1}
1481   inputList=${2}
1482   configFile=${3:-"benchmark.config"}
1483   configFile=$(readlink -f ${configFile})
1484   shift 3
1485   extraOpts=("$@")
1486   if ! parseConfig ${configFile} "${extraOpts[@]}"; then return 1; fi
1487
1488   #record the working directory provided by the batch system
1489   batchWorkingDirectory=${PWD}
1490
1491   #redirect all output to submit.log
1492   echo "redirecting all output to ${PWD}/submit_${productionID//"/"/_}.log"
1493   exec 7>&1
1494   exec 1>submit_${productionID//"/"/_}.log 2>&1
1495
1496   umask 0002
1497   echo ${0}" submit $*"
1498   if [[ -z "${inputList}" || -z "${productionID}" ]]
1499   then
1500     echo
1501     echo " Usage: ${0} submit inputList productionID [configFile=benchmark.config]"
1502     echo
1503     return
1504   fi
1505
1506   # check if config file is there
1507   if [ ! -f ${configFile} ]; then
1508     echo "ERROR! Config File '${configFile}' not found" >&2
1509     return
1510   else
1511     echo "Using Config File: '${configFile}'"
1512   fi
1513
1514   [[ ! -f ${alirootEnv} ]] && echo "alirootEnv script ${alirootEnv} not found!..." && return 1
1515
1516   #move the script, config and some other stuff to ${commonOutputPath} first, then use them from there
1517   self=$(readlink -f "${0}")
1518   configPath=$(dirname ${configFile})
1519   export commonOutputPath=${baseOutputDirectory}/${productionID}
1520
1521   mkdir -p ${commonOutputPath}
1522   mkdir -p ${commonOutputPath}/logs
1523   mkdir -p ${commonOutputPath}/meta
1524
1525   cp ${self} ${commonOutputPath}
1526   cp ${configFile} ${commonOutputPath}
1527   cp ${inputList} ${commonOutputPath}
1528   self=${commonOutputPath}/${self##*/}
1529   chmod u+x ${self}
1530   configFile=${commonOutputPath}/${configFile##*/}
1531   inputList=${commonOutputPath}/${inputList##*/}
1532
1533   #convert to absolut pathnames
1534   inputList=$(readlink -f "${inputList}")
1535   #make list of runs
1536   if [[ -z ${runNumber} ]]; then
1537     listOfRuns=($(while read x; do guessRunNumber ${x}; done < ${inputList} | sort | uniq))
1538   else
1539     listOfRuns=${runNumber}
1540   fi
1541
1542   alirootSource=$(readlink -f "${alirootSource}")
1543
1544   echo ""
1545   echo "### BEGIN CONFIGURATION ###"
1546   echo ""
1547   echo "GENERAL:"
1548   echo ""
1549   echo "    productionID:    ${productionID}"
1550   echo "    batchCommand:    ${batchCommand}"
1551   echo "    batchFlags:      ${batchFlags}"
1552   echo "    alirootEnv:   ${alirootEnv}"
1553   ${alirootEnv} echo '    ALICE_ROOT:      ${ALICE_ROOT}'
1554   ${alirootEnv} echo '    ALIROOT_RELEASE: ${ALICE_RELEASE}'
1555   echo "    inputList:       ${inputList}"
1556   echo "    configPath:      ${configPath}"
1557   echo "    commonOutputPath:      ${commonOutputPath}"
1558   echo "    defaultOCDB:     ${defaultOCDB}"
1559   echo "      autoOCDB: ${autoOCDB}"
1560   echo "    recoTriggerOptions:   ${recoTriggerOptions}"
1561   echo "    runs:"
1562   echo "      ${listOfRuns[*]}"
1563   echo ""
1564   echo "THE TRAIN WILL RUN:"
1565
1566   if [ ${runCPass0reco} -eq 1 ]; then
1567     echo "    Pass0 - Recontruction"
1568   fi
1569
1570   if [ ${runCPass0MergeMakeOCDB} -eq 1 ]; then
1571     echo "    Pass0 - merging and OCDB export"
1572   fi
1573
1574   if [ ${runCPass1reco} -eq 1 ]; then
1575     echo "    Pass1 - Recontruction"
1576   fi
1577   if [ ${runCPass1MergeMakeOCDB} -eq 1 ]; then
1578     echo "    Pass1 - merging and OCDB export"
1579   fi
1580
1581   echo ""
1582   echo "LIMITS:"
1583   echo "    max. Events/Chunk:   ${nEvents}"
1584   echo "    max. Number of Chunks per Run:     ${nMaxChunks}"
1585   echo ""
1586   echo "### END CONFIGURATION ###"
1587   echo ""
1588
1589
1590   # check if input file is there
1591   if [ ! -f ${inputList} ]; then
1592     echo "ERROR! Input List '${inputList}' not found" >&2
1593     return
1594   fi
1595
1596   # define jobid (for dependent jobs)
1597   date=$(date +%Y_%m_%d_%H%M%S)
1598   #for each run we submit one jobarray:
1599   for runNumber in ${listOfRuns[*]}; do
1600
1601     [[ -z ${runNumber} ]] && continue
1602     [[ ! ${runNumber} =~ ^[0-9]*[0-9]$ ]] && continue
1603
1604     JOBpostfix="${productionID//"/"/_}_${runNumber}_${date}"
1605     JOBID1="p0_${JOBpostfix}"
1606     JOBID1wait="w0_${JOBpostfix}"
1607     JOBID2="m0_${JOBpostfix}"
1608     JOBID2wait="wm0_${JOBpostfix}"
1609     JOBID3="op0_${JOBpostfix}"
1610     JOBID3wait="wop0_${JOBpostfix}"
1611     JOBID4="p1_${JOBpostfix}"
1612     JOBID4wait="w1_${JOBpostfix}"
1613     JOBID5="m1_${JOBpostfix}"
1614     JOBID5wait="wm1_${JOBpostfix}"
1615     JOBID6="s1_${JOBpostfix}"
1616     JOBID6wait="ws1_${JOBpostfix}"
1617     JOBID7="QA_${JOBpostfix}"
1618     JOBmakeESDlistCPass1="lp1_${JOBpostfix}"
1619     JOBfilterESDcpass1="fp1_${JOBpostfix}"
1620     LASTJOB="000"
1621
1622     oneInputFile=$(egrep -m1 "${runNumber}\/" ${inputList})
1623
1624     currentDefaultOCDB=${defaultOCDB}
1625     [[ -z ${autoOCDB} ]] && autoOCDB=1
1626     if [[ ${autoOCDB} -ne 0 ]]; then
1627       currentDefaultOCDB=$(setYear ${oneInputFile} ${defaultOCDB})
1628     fi
1629
1630     echo "submitting run ${runNumber} with OCDB ${currentDefaultOCDB}"
1631
1632     ###############################################################################
1633     #run one chunk with valgrind:
1634     if [[ -n ${profilingCommand} ]]; then
1635       [[ -z ${nEventsProfiling} ]] && nEventsProfiling=2
1636       [[ -z ${profilingCommand} ]] && profilingCommand="/usr/bin/valgrind --tool=callgrind --num-callers=40 -v --trace-children=yes"
1637       submit "valgrind" 1 1 000 "${alirootEnv} ${self}" CPass0 ${commonOutputPath}/000${runNumber}/valgrind ${oneInputFile} ${nEventsProfiling} ${currentDefaultOCDB} ${configFile} ${runNumber} valgrind useProfilingCommand=${profilingCommand} "${extraOpts[@]}"
1638     fi
1639
1640     ################################################################################
1641     ################################################################################
1642     # run the CPass0 if requested
1643
1644     if [ ${runCPass0reco} -eq 1 ]; then
1645
1646       echo
1647       echo "starting CPass0... for run ${runNumber}"
1648       echo
1649
1650       # create directory and copy all files that are needed
1651       targetDirectory="${commonOutputPath}/000${runNumber}/cpass0"
1652       mkdir -p ${targetDirectory}
1653
1654       filesCPass0=(
1655                     "${configPath}/runCPass0.sh"
1656                     "${configPath}/recCPass0.C"
1657                     "${configPath}/runCalibTrain.C"
1658                     "${configPath}/localOCDBaccessConfig.C"
1659                     "${configPath}/OCDB*.root"
1660                     "${configPath}/sim.C"
1661                     "${configPath}/rec.C"
1662                     "${configPath}/Config.C"
1663       )
1664       for file in ${filesCPass0[*]}; do
1665         [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
1666       done
1667
1668       localInputList=${targetDirectory}/${inputList##*/}
1669       [[ ! -f ${localInputList} ]] && egrep "\/000${runNumber}\/" ${inputList} > ${localInputList}
1670       # limit nFiles to nMaxChunks
1671       nFiles=$(wc -l < ${localInputList})
1672       [[ ${nFiles} -eq 0 ]] && echo "list contains ZERO files! exiting..." && return 1
1673       echo "raw files in list:    ${nFiles}"
1674       if [[ ${nMaxChunks} -gt 0 && ${nMaxChunks} -le ${nFiles} ]]; then
1675         nFiles=${nMaxChunks}
1676       fi
1677       echo "raw files to process: ${nFiles}"
1678       [[ -z "${percentProcessedFilesToContinue}" ]] && percentProcessedFilesToContinue=100
1679       if [[ ${percentProcessedFilesToContinue} -eq 100 ]]; then
1680         nFilesToWaitFor=${nFiles}
1681       else
1682         nFilesToWaitFor=$(( ${nFiles}-${nFiles}/(100/(100-${percentProcessedFilesToContinue})) ))
1683       fi
1684       echo "requested success rate is ${percentProcessedFilesToContinue}%"
1685       echo "merging will start after ${nFilesToWaitFor} jobs are done"
1686
1687       submit ${JOBID1} 1 ${nFiles} 000 "${alirootEnv} ${self}" CPass0 ${targetDirectory} ${localInputList} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} -1 "${extraOpts[@]}"
1688
1689       ## submit a monitoring job that will run until a certain number of jobs are done with reconstruction
1690       submit "${JOBID1wait}" 1 1 000 "${alirootEnv} ${self}" WaitForOutput ${commonOutputPath} "meta/cpass0.job*.run${runNumber}.done" ${nFilesToWaitFor} ${maxSecondsToWait} '-maxdepth 1'
1691       LASTJOB=${JOBID1wait}
1692
1693     fi #end running CPass0
1694     ################################################################################
1695
1696
1697     ################################################################################
1698     # submit merging of CPass0, depends on the reconstruction
1699
1700     if [ ${runCPass0MergeMakeOCDB} -eq 1 ]; then
1701
1702       echo
1703       echo "submit CPass0 merging for run ${runNumber}"
1704       echo
1705
1706       targetDirectory="${commonOutputPath}/000${runNumber}/cpass0"
1707       mkdir -p ${targetDirectory}
1708
1709       #copy the scripts
1710       filesMergeCPass0=(
1711                         "${configPath}/OCDB.root"
1712                         "${configPath}/mergeMakeOCDB.byComponent.sh"
1713                         "${configPath}/mergeMakeOCDB.sh"
1714                         "${configPath}/localOCDBaccessConfig.C"
1715                         "${configPath}/mergeByComponent.C"
1716                         "${configPath}/makeOCDB.C"
1717                         "${configPath}/merge.C"
1718       )
1719       for file in ${filesMergeCPass0[*]}; do
1720         [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
1721       done
1722
1723       submit ${JOBID2} 1 1 "${LASTJOB}" "${alirootEnv} ${self}" MergeCPass0 ${targetDirectory} ${currentDefaultOCDB} ${configFile} ${runNumber} cpass0.calib.run${runNumber}.list "${extraOpts[@]}"
1724       LASTJOB=${JOBID2}
1725
1726       if [[ -n ${generateMC} ]]; then
1727         submit "mrl${JOBpostfix}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" PrintValues sim ${commonOutputPath}/meta/sim.run${runNumber}.list ${commonOutputPath}/meta/cpass0.job*.run${runNumber}.done
1728         LASTJOB="mrl${JOBpostfix}"
1729       fi
1730
1731       echo
1732     fi
1733     # end of merging CPass0
1734     ################################################################################
1735
1736     ################################################################################
1737     ################################################################################
1738     # run the CPass1 if requested
1739
1740     if [ ${runCPass1reco} -eq 1 ]; then
1741
1742       targetDirectory="${commonOutputPath}/000${runNumber}/cpass1"
1743       rm -f ${commonOutputPath}/meta/cpass1.job*.run${runNumber}.done
1744
1745       # safety feature: if we are re-running for any reason we want to delete the previous output first.
1746       [[ -d ${targetDirectory} ]] && rm -rf ${targetDirectory}/* && echo "removed old output at ${targetDirectory}/*"
1747
1748       echo
1749       echo "starting CPass1... for run ${runNumber}"
1750       echo
1751
1752       # create directory and copy all files that are needed
1753       mkdir -p ${targetDirectory}
1754
1755       filesCPass1=(
1756                     "${configPath}/runCPass1.sh"
1757                     "${configPath}/recCPass1.C"
1758                     "${configPath}/recCPass1_OuterDet.C"
1759                     "${configPath}/runCalibTrain.C"
1760                     "${configPath}/QAtrain_duo.C"
1761                     "${configPath}/localOCDBaccessConfig.C"
1762                     "${configPath}/OCDB.root"
1763       )
1764       for file in ${filesCPass1[*]}; do
1765         [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
1766       done
1767
1768       if [[ -n ${generateMC} ]]; then
1769         localInputList=${commonOutputPath}/meta/sim.run${runNumber}.list
1770       else
1771         localInputList=${targetDirectory}/${inputList##*/}
1772         [[ ! -f ${localInputList} ]] && egrep "\/000${runNumber}\/" ${inputList} > ${localInputList}
1773       fi
1774       # limit nFiles to nMaxChunks
1775       nFiles=$(wc -l < ${localInputList})
1776       [[ ${nFiles} -eq 0 ]] && echo "list contains ZERO files! continuing..." && continue
1777       echo "raw files in list:    ${nFiles}"
1778       if [[ ${nMaxChunks} -gt 0 && ${nMaxChunks} -le ${nFiles} ]]; then
1779         nFiles=${nMaxChunks}
1780       fi
1781       echo "raw files to process: ${nFiles}"
1782       [[ -z "${percentProcessedFilesToContinue}" ]] && percentProcessedFilesToContinue=100
1783       if [[ ${percentProcessedFilesToContinue} -eq 100 ]]; then
1784         nFilesToWaitFor=${nFiles}
1785       else
1786         nFilesToWaitFor=$(( ${nFiles}-${nFiles}/(100/(100-${percentProcessedFilesToContinue})) ))
1787       fi
1788       echo "requested success rate is ${percentProcessedFilesToContinue}%"
1789       echo "merging will start after ${nFilesToWaitFor} jobs are done"
1790
1791       submit ${JOBID4} 1 ${nFiles} "${LASTJOB}" "${alirootEnv} ${self}" CPass1 ${targetDirectory} ${localInputList} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} -1 "${extraOpts[@]}"
1792
1793       ################################################################################
1794       ## submit a monitoring job that will run until a certain number of jobs are done with reconstruction
1795       submit "${JOBID4wait}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" WaitForOutput ${commonOutputPath} "meta/cpass1.job*.run${runNumber}.done" ${nFilesToWaitFor} ${maxSecondsToWait} '-maxdepth 1'
1796       LASTJOB=${JOBID4wait}
1797       ################################################################################
1798
1799       echo
1800     fi #end running CPass1
1801
1802     ################################################################################
1803     # submit merging of CPass1, depends on the reconstruction
1804     if [ ${runCPass1MergeMakeOCDB} -eq 1 ]; then
1805
1806       echo
1807       echo "submit CPass1 merging for run ${runNumber}"
1808       echo
1809
1810       targetDirectory="${commonOutputPath}/000${runNumber}/cpass1"
1811       rm -f ${commonOutputPath}/meta/merge.cpass1.run${runNumber}.done
1812       mkdir -p ${targetDirectory}
1813
1814       # copy files
1815       filesMergeCPass1=(
1816                         "${configPath}/OCDB.root"
1817                         "${configPath}/localOCDBaccessConfig.C"
1818                         "${configPath}/mergeMakeOCDB.byComponent.sh"
1819                         "${configPath}/mergeByComponent.C"
1820                         "${configPath}/makeOCDB.C"
1821                         "${configPath}/merge.C"
1822                         "${configPath}/mergeMakeOCDB.sh"
1823                         "${configPath}/QAtrain_duo.C"
1824       )
1825       for file in ${filesMergeCPass1[*]}; do
1826         [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
1827       done
1828
1829       submit "${JOBID5}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" MergeCPass1 ${targetDirectory} ${currentDefaultOCDB} ${configFile} ${runNumber} cpass1.calib.run${runNumber}.list cpass1.QA.run${runNumber}.lastMergingStage.txt.list cpass1.filtered.run${runNumber}.list "${extraOpts[@]}"
1830       LASTJOB=${JOBID5}
1831       echo
1832     fi
1833
1834     ###############################
1835     #if [ ${runESDfiltering} -eq 1 ]; then
1836     #  rm -f ${commonOutputPath}/cpass1.ESD.run${runNumber}.list
1837     #  rm -f ${commonOutputPath}/meta/filtering.cpass1.run*.done
1838     #  echo
1839     #  echo submitting filtering for run ${runNumber}
1840     #  echo
1841     #  submit "${JOBmakeESDlistCPass1}" 1 1 "${LASTJOB}" "${self}" PrintValues esd ${commonOutputPath}/meta/cpass1.ESD.run${runNumber}.list ${commonOutputPath}/meta/cpass1.job*.run${runNumber}.done
1842     #  submit "${JOBfilterESDcpass1}" 1 1 "${JOBmakeESDlistCPass1}" "${alirootEnv} ${self}" MakeFilteredTrees ${commonOutputPath}/000${runNumber}/cpass1 ${runNumber} ${commonOutputPath}/meta/cpass1.ESD.run${runNumber}.list ${filteringFactorHighPt} ${filteringFactorV0s} ${currentDefaultOCDB} 1000000 0 10000000 0 ${configFile} AliESDs_Barrel.root "${extraOpts[@]}"
1843     #  LASTJOB=${JOBfilterESDcpass1}
1844     #fi
1845
1846   done
1847
1848   #################################################################################
1849   #################################################################################
1850   #if [ ${runESDfiltering} -eq 1 ]; then
1851   #  submit "${JOBID5wait}" 1 1 "${LASTJOB}" "${self}" WaitForOutput ${commonOutputPath} "meta/filtering.cpass1.run*.done" "${#listOfRuns[@]}" ${maxSecondsToWait}
1852   #else
1853     submit "${JOBID5wait}" 1 1 "${LASTJOB}" "${self}" WaitForOutput ${commonOutputPath} "meta/merge.cpass1.run*.done" ${#listOfRuns[@]} ${maxSecondsToWait}
1854   #fi
1855   LASTJOB=${JOBID5wait}
1856
1857   #################################################################################
1858   echo
1859   echo "submit make a summary"
1860   echo
1861
1862   submit "${JOBID6}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" MakeSummary ${configFile}
1863   LASTJOB=${JOBID6}
1864   #################################################################################
1865
1866   #restore stdout
1867   exec 1>&7 7>&-
1868   echo "jobs submitted."
1869   return 0
1870 }
1871
1872 goWaitForOutput()
1873 {
1874   umask 0002
1875   [[ $# -lt 3 ]] && echo "goWaitForOutput() wrong number of arguments, exiting.." && return 1
1876   echo searchPath=${1}
1877   echo fileName=${2}
1878   echo numberOfFiles=${3}
1879   echo maxSecondsToWait=${4}
1880   searchPath=${1}
1881   fileName=${2}
1882   numberOfFiles=${3}
1883   maxSecondsToWait=${4}
1884   extraFindOptions=${5}
1885   echo "command to be executed: /bin/ls -1 ${searchPath}/${fileName} ${extraFindOptions}"
1886   [[ -z "${maxSecondsToWait}" ]] && maxSecondsToWait=$(( 3600*12 ))
1887   while true; do
1888     #n=$(find ${searchPath} ${extraFindOptions} -name "${fileName}" | wc -l)
1889     n=$(/bin/ls -1 ${searchPath}/${fileName} 2>/dev/null | wc -l)
1890     [[ ${n} -gt 0 ]] && echo "found ${n} X ${fileName}"
1891     [[ ${n} -ge ${numberOfFiles} ]] && break
1892     [[ ${SECONDS} -gt ${maxSecondsToWait} ]] && echo "timeout of ${maxSecondsToWait}s!" && break
1893     sleep 60
1894   done
1895   echo "DONE! exiting..."
1896   return 0
1897 }
1898
1899 mergeSysLogs()
1900 {
1901   outputFile=${1}
1902   shift
1903   inputFiles="$@"
1904   i=0
1905   if ! ls -1 ${inputFiles} &>/dev/null; then echo "the files dont exist!: ${inputFiles}"; return 1; fi
1906   while read x; do
1907     runNumber=$(guessRunNumber ${x})
1908     [[ -z ${runNumber} ]] && echo "run number cannot be guessed for ${x}" && continue
1909     awk -v run=${runNumber} -v i=${i} 'NR > 1 {print run" "$0} NR==1 && i==0 {print "run/I:"$0}' ${x}
1910     (( i++ ))
1911   done < <(ls -1 ${inputFiles}) > ${outputFile}
1912   return 0
1913 }
1914
1915 goMakeMergedSummaryTree()
1916 {
1917   # create list of calibration entries
1918   # takes no arguments, just run it in the base output
1919   # directory with the following files in the working directory
1920   #
1921   #  Calibration file lists:
1922   #       cpass0.dcsTree.list, cpass1.dcsTree.list
1923   #  QA trending root file:
1924   #       trending.root
1925   #
1926   #  Production infoo ascii files:
1927   #       summary_pass0.tree
1928   #       summary_pass1.tree
1929   #
1930
1931   [[ ! -f cpass0.dcsTree.list ]] && echo "no cpass0.dcsTree.list" && return 1
1932   [[ ! -f cpass1.dcsTree.list ]] && echo "no cpass1.dcsTree.list" && return 1
1933   [[ ! -f trending.root ]] && echo "no trending.root" && return 1
1934   [[ ! -f summary_pass0.tree ]] && echo "no summary_pass0.tree" && return 1
1935   [[ ! -f summary_pass1.tree ]] && echo "no summary_pass1.tree" && return 1
1936
1937   #first, dump the C macro to file
1938   cat << EOF > mergeTree.C
1939   //
1940   // Merge summary information
1941   // Following files are expected to be in the working directory
1942   //
1943   // Calibration file lists:
1944   //      cpass0.dcsTree.list, cpass1.dcsTree.list
1945   // QA trending root file:
1946   //      trending.root
1947   //
1948   // Production infoo ascii files:
1949   //      summary_pass0.tree
1950   //      summary_pass1.tree
1951   //
1952   void mergeTree(){
1953     //
1954     //
1955     //
1956     // Calibration values dump
1957     //
1958     //Printf("MakeTreeFromList cpass0.dcsTree.list");
1959     AliXRDPROOFtoolkit::MakeTreeFromList("Calib.TPC.CPass0.root", "dcs","dcs","cpass0.dcsTree.list",1);
1960     //Printf("MakeTreeFromList cpass1.dcsTree.list");
1961     AliXRDPROOFtoolkit::MakeTreeFromList("Calib.TPC.CPass1.root", "dcs","dcs","cpass1.dcsTree.list",1);
1962     //
1963     // Calibration status dump
1964     //
1965     TFile *fprod = TFile::Open("fproduction.root","recreate");
1966     TTree  tree0, tree1;
1967     //Printf("reading summary_pass0.tree");
1968     tree0.ReadFile("summary_pass0.tree");
1969     //Printf("reading summary_pass1.tree");
1970     tree1.ReadFile("summary_pass1.tree");
1971     tree0.Write("CPass0");
1972     tree1.Write("CPass1");
1973     fprod->Close();
1974     //
1975     //
1976     //
1977     TString stringSetup="";
1978     stringSetup+="1#QA.TPC#run#SummaryTPCQA/tpcQA#trending.root+";  //
1979     stringSetup+="1#Calib.TPC.CPass0#run#dcs#Calib.TPC.CPass0.root+";  //
1980     stringSetup+="1#Calib.TPC.CPass1#run#dcs#Calib.TPC.CPass1.root+";  //
1981     //
1982     stringSetup+="1#CPass0#runnumber#CPass0#fproduction.root+";  //
1983     stringSetup+="1#CPass1#runnumber#CPass1#fproduction.root+";  //
1984     //
1985     //Printf("stringSetup: %s", stringSetup.Data());
1986     AliXRDPROOFtoolkit::JoinTreesIndex("outAll.root","joinAll","run",stringSetup.Data(), 1);
1987   }
1988 EOF
1989
1990   aliroot -b -q "mergeTree.C" > mergeTrees.log
1991   return $?
1992 }
1993
1994 goMakeSummary()
1995 {
1996   #all the final stuff goes in here for ease of use:
1997   # summary logs
1998   # qa plot making
1999   # final file lists
2000   #some defaults:
2001   log="summary.log"
2002   productionID="qa"
2003
2004   configFile=${1}
2005   shift 1
2006   extraOpts=("$@")
2007   if ! parseConfig ${configFile} "${extraOpts[@]}"; then return 1; fi
2008
2009   configFile=$(readlink -f ${configFile})
2010
2011   #record the working directory provided by the batch system
2012   batchWorkingDirectory=${PWD}
2013
2014   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
2015
2016   [[ ! -f ${configFile} ]] && echo "no config file ${configFile}!" && return
2017
2018   [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
2019
2020   #copy some useful stuff
2021   #and go to the commonOutputPath
2022   cp ${configFile} ${commonOutputPath}
2023   cd ${commonOutputPath}
2024
2025   exec &> >(tee ${log})
2026
2027   #summarize the global stuff
2028   echo "env script: ${alirootSource} ${alirootEnv}"
2029   echo "\$ALICE_ROOT=${ALICE_ROOT}"
2030   echo "commonOutputPath=${commonOutputPath}"
2031
2032   #summarize the stacktraces
2033   awk '
2034        BEGIN {
2035                print "frame/I:method/C:line/C:cpass/I:aliroot/I";
2036                RS="#[0-9]*";
2037                aliroot=0;
2038              }
2039              {
2040                if ($3 ~ /Ali*/) aliroot=1; else aliroot=0;
2041                gsub("#","",RT);
2042                if ($NF!="" && RT!="" && $3!="") print RT" "$3" "$NF" "0" "aliroot
2043              }
2044       ' 000*/cpass0/*/stacktrace* 2>/dev/null > stacktrace.tree
2045   awk '
2046        BEGIN {
2047                RS="#[0-9]*";
2048                aliroot=0;
2049              }
2050              {
2051                if ($3 ~ /Ali*/) aliroot=1; else aliroot=0;
2052                gsub("#","",RT);
2053                if ($NF!="" && RT!="" && $3!="") print RT" "$3" "$NF" "1" "aliroot
2054              }
2055       ' 000*/cpass1/*/stacktrace* 2>/dev/null >> stacktrace.tree
2056
2057   echo total numbers for the production:
2058   echo
2059   awk 'BEGIN {nFiles=0;nCore=0;}
2060   /^calibfile/ {nFiles++;}
2061   /core dumped/ {nCore++i;}
2062   END {print     "cpass0 produced "nFiles" calib files, "nCore" core files";}' meta/cpass0.job*done 2>/dev/null
2063   awk 'BEGIN {nOK=0; nBAD=0; }
2064   /\/rec.log OK/ {nOK++;}
2065   /\/rec.log BAD/ {nBAD++;}
2066   /stderr BAD/ {if ($0 ~ /rec.log/){nBAD++;}}
2067   END {print     "cpass0 reco:  OK: "nOK"\tBAD: "nBAD;}' meta/cpass0.job*done 2>/dev/null
2068   awk 'BEGIN {nOK=0; nBAD=0; }
2069   /\/calib.log OK/ {nOK++;}
2070   /\/calib.log BAD/ {nBAD++;}
2071   END {print "cpass0 calib: OK: "nOK"\tBAD: "nBAD;}' meta/cpass0.job*done 2>/dev/null
2072
2073   awk 'BEGIN {nOK=0; nBAD=0; }
2074   /merge.log OK/ {nOK++;}
2075   /merge.log BAD/ {nBAD++;}
2076   END {print "cpass0 merge: OK: "nOK"\tBAD: "nBAD;}' meta/merge.cpass0*done 2>/dev/null
2077   awk 'BEGIN {nOK=0; nBAD=0; }
2078   /ocdb.log OK/ {nOK++;}
2079   /ocdb.log BAD/ {nBAD++;}
2080   END {print   "cpass0 OCDB:  OK: "nOK"\tBAD: "nBAD;}' meta/merge.cpass0*done 2>/dev/null
2081
2082   echo
2083   awk 'BEGIN {nFiles=0;nCore=0;}
2084   /^calibfile/ {nFiles++;}
2085   /core dumped/ {nCore++;}
2086   END {print     "cpass1 produced "nFiles" calib files, "nCore" core files";}' meta/cpass1.job*done 2>/dev/null
2087   awk 'BEGIN {nOK=0; nBAD=0; }
2088   /\/rec.log OK/ {nOK++;}
2089   /\/rec.log BAD/ {nBAD++;}
2090   /stderr BAD/ {if ($0 ~ /rec.log/){nBAD++;}}
2091   END {print     "cpass1 reco:  OK: "nOK"\tBAD: "nBAD;}' meta/cpass1.job*done 2>/dev/null
2092   awk 'BEGIN {nOK=0; nBAD=0; }
2093   /\/calib.log OK/ {nOK++;}
2094   /\/calib.log BAD/ {nBAD++;}
2095   END {print "cpass1 calib: OK: "nOK"\tBAD: "nBAD;}' meta/cpass1.job*done 2>/dev/null
2096
2097   awk 'BEGIN {nOK=0; nBAD=0; }
2098   /merge.log OK/ {nOK++;}
2099   /merge.log BAD/ {nBAD++;}
2100   END {print "cpass1 merge: OK: "nOK"\tBAD: "nBAD;}' meta/merge.cpass1*done 2>/dev/null
2101   awk 'BEGIN {nOK=0; nBAD=0; }
2102   /ocdb.log OK/ {nOK++;}
2103   /ocdb.log BAD/ {nBAD++;}
2104   END {print   "cpass1 OCDB:  OK: "nOK"\tBAD: "nBAD;}' meta/merge.cpass1*done 2>/dev/null
2105
2106   echo
2107   echo per run stats:
2108   /bin/ls -1 meta/merge.cpass0.run*.done | while read x
2109 do
2110   dir=$(goPrintValues calibfile - ${x})
2111   runNumber=$(guessRunNumber ${dir})
2112   [[ -z ${runNumber} ]] && continue
2113
2114   if $(/bin/ls meta/cpass0.job*.run${runNumber}.done &> /dev/null); then
2115     statusCPass0=( $(
2116     awk 'BEGIN {nOKrec=0;nBADrec=0;nOKcalib=0;nBADcalib=0;nOKstderr=0;nBADstderr=0;}
2117     /\/rec.log OK/ {nOKrec++;}
2118     /\/rec.log BAD/ {nBADrec++;}
2119     /stderr BAD/ {if ($0 ~ /rec.log/) {nBADrec++;} nBADstderr++;}
2120     /stderr OK/ {nOKstderr++;}
2121     /\/calib.log OK/ {nOKcalib++;}
2122     /\/calib.log BAD/ {nBADcalib++}
2123     END {print ""nOKrec" "nBADrec" "nOKstderr" "nBADstderr" "nOKcalib" "nBADcalib;}' meta/cpass0.job*.run${runNumber}.done 2>/dev/null
2124     ) )
2125   fi
2126
2127   if $(/bin/ls meta/cpass1.job*.run${runNumber}.done &>/dev/null); then
2128     statusCPass1=( $(
2129     awk 'BEGIN {nOKrec=0;nBADrec=0;nOKcalib=0;nBADcalib=0;nOKstderr=0;nBADstderr=0;nQAbarrelOK=0;nQAbarrelBAD=0;nQAouterOK=0;nQAouterBAD=0;}
2130     /\/rec.log OK/ {nOKrec++;}
2131     /\/rec.log BAD/ {nBADrec++;}
2132     /stderr BAD/ {if ($0 ~ /rec.log/) nBADrec++;nBADstderr++;}
2133     /stderr OK/ {nOKstderr++;}
2134     /\/calib.log OK/ {nOKcalib++;}
2135     /\/calib.log BAD/ {nBADcalib++}
2136     /\/qa_barrel.log OK/ {nQAbarrelOK++;}
2137     /\/qa_barrel.log BAD/ {nQAbarrelBAD++;}
2138     /\/qa_outer.log OK/ {nQAouterOK++;}
2139     /\/qa_outer.log BAD/ {nQAouterBAD++;}
2140     END {print ""nOKrec" "nBADrec" "nOKstderr" "nBADstderr" "nOKcalib" "nBADcalib" "nQAbarrelOK" "nQAbarrelBAD" "nQAouterOK" "nQAouterBAD;}' meta/cpass1.job*.run${runNumber}.done 2>/dev/null
2141     ) )
2142   fi
2143
2144   statusOCDBcpass0=$(awk '/ocdb.log/ {print $2} ' ${x} 2>/dev/null)
2145   statusOCDBcpass1=$(awk '/ocdb.log/ {print $2}' ${x/cpass0/cpass1} 2>/dev/null)
2146   statusQA=$(awk '/mergeMakeOCDB.log/ {print $2}' ${x/cpass0/cpass1} 2>/dev/null)
2147
2148   printf "%s\t ocdb.log cpass0: %s\t ocdb.log cpass1: %s\tqa.log:%s\t| cpass0: rec:%s/%s stderr:%s/%s calib:%s/%s cpass1: rec:%s/%s stderr:%s/%s calib:%s/%s QAbarrel:%s/%s QAouter:%s/%s\n" ${runNumber} ${statusOCDBcpass0} ${statusOCDBcpass1} ${statusQA} ${statusCPass0[0]} ${statusCPass0[1]} ${statusCPass0[2]} ${statusCPass0[3]} ${statusCPass0[4]} ${statusCPass0[5]} ${statusCPass1[0]} ${statusCPass1[1]} ${statusCPass1[2]} ${statusCPass1[3]} ${statusCPass1[4]} ${statusCPass1[5]} ${statusCPass1[6]} ${statusCPass1[7]} ${statusCPass1[8]} ${statusCPass1[9]}
2149 done
2150
2151   #make lists with output files - QA, trending, filtering and calibration
2152   ### wait for the merging of all runs to be over ###
2153   rm -f qa.list
2154   goPrintValues qafile qa.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2155   rm -f calib.list
2156   goPrintValues calibfile calib.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2157   rm -f trending.list
2158   goPrintValues trendingfile trending.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2159   rm -f filtering.list
2160   goPrintValues filteredTree filtering.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2161   rm -f cpass0.dcsTree.list
2162   goPrintValues dcsTree cpass0.dcsTree.list ${commonOutputPath}/meta/merge.cpass0.run*.done &>/dev/null
2163   rm -f cpass1.dcsTree.list
2164   goPrintValues dcsTree cpass1.dcsTree.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2165
2166   #merge trending
2167   rm -f ${commonOutputPath}/trending_merged.root
2168   goMerge trending.list ${commonOutputPath}/trending.root ${configFile} "${extraOpts[@]}" &> mergeTrending.log
2169
2170   goMakeSummaryTree ${commonOutputPath} 0
2171   goMakeSummaryTree ${commonOutputPath} 1
2172
2173   goCreateQAplots "${PWD}/qa.list" "${productionID}" "QAplots" "${configFile}" "${extraOpts[@]}" filteringList="${PWD}/filtering.list" &>createQAplots.log
2174
2175   #make a merged summary tree out of the QA trending, dcs trees and log summary trees
2176   echo goMakeMergedSummaryTree PWD=$PWD
2177   goMakeMergedSummaryTree
2178
2179   #if set, email the summary
2180   [[ -n ${MAILTO} ]] && cat ${log} | mail -s "benchmark ${productionID} done" ${MAILTO}
2181
2182   return 0
2183 }
2184
2185 goMakeSummaryTree()
2186 {
2187   if [[ $# -lt 1 ]] ; then
2188     return
2189   fi
2190   #1. define vars/arrays
2191   DIR=${1} #use input or exec in current dir
2192   pass=${2-"0"} #pass from input
2193   outfile="summary_pass${pass}.tree"
2194   Ncolumns=0
2195   test -f ${outfile} && : >${outfile}
2196   errfile=${outfile/tree/err}
2197   test -f ${errfile} && : >${errfile}
2198
2199   declare -a counterString=(TOFevents TOFtracks TPCevents TPCtracks TRDevents TRDtracks T0events SDDevents SDDtracks MeanVertexevents)
2200   Ncounter=${#counterString[@]}
2201
2202   declare -a statusString=(TRDStatus TOFStatus TPCStatus T0Status MeanVertexStatus)
2203   Nstatus=${#statusString[@]}
2204
2205
2206   declare -a ratesString=(rec stderr calib qa_barrel qa_outer)
2207   Nrates=${#ratesString[@]}
2208
2209   runs=( $(ls -1 ${DIR}/meta/merge.cpass0* | while read x; do guessRunNumber $x; done) )
2210   Nruns=${#runs[@]}
2211
2212   echo -n runnumber/I >>${outfile}
2213   echo -n :cpass${pass}status/I >>${outfile}
2214   echo -n :cpass${pass}QAstatus/I >>${outfile}
2215   for i in ${ratesString[@]}; do
2216     echo -n :${i}OK/I >>${outfile}
2217     echo -n :${i}BAD/I >>${outfile}
2218
2219   done
2220   for i in ${counterString[@]} ${statusString[@]} ; do
2221     echo -n :${i}/I >>${outfile}
2222   done
2223   Ncolumns=$((2 + 2*Nrates + Ncounter + Nstatus))
2224   echo >> ${outfile}
2225
2226   #2. loop runs
2227
2228   for runnumber in ${runs[@]} ; do
2229
2230
2231
2232     filejob="${DIR}/meta/cpass${pass}.job*.run${runnumber}.done"
2233     filemerge="${DIR}/meta/merge.cpass${pass}.run${runnumber}.done"
2234     fileOCDB=$(grep /ocdb.log ${filemerge} | awk '{print $1}')
2235     if ! $(/bin/ls ${filemerge} &>/dev/null) ; then
2236       echo "${filemerge} does not exist!" >>${errfile}
2237       continue
2238     elif ! $(/bin/ls ${filejob} &>/dev/null) ; then
2239       echo "${filejob} does not exist!" >>${errfile}
2240       echo -n ${runnumber} >> ${outfile}
2241       for i in $(seq ${Ncolumns}) ; do
2242         echo -n "-1" >> ${outfile}
2243       done
2244       echo >> ${outfile}
2245       continue
2246     fi
2247     echo -n ${runnumber} >> ${outfile}
2248     #pass0status= grep '/ocdb.log' ${filemerge} |  cut -d' ' -f2 | tr OK x1 | tr BAD xx0 | tr -d 'x'
2249     passStatus=$(grep '/ocdb.log' ${filemerge} | grep OK | wc -l)
2250     echo -n " ${passStatus}" >> ${outfile}
2251     qaStatus=$(grep '/mergeMakeOCDB.log' ${filemerge} | grep OK | wc -l)
2252     echo -n " ${qaStatus}" >> ${outfile}
2253
2254
2255     #fill OK/BAD rates
2256     for i in $(seq 0 $((${Nrates}-1))) ; do
2257       var1=$(grep "/${ratesString[${i}]}.log" ${filejob} | grep OK | wc -l)
2258       var2=$(grep "/${ratesString[${i}]}.log" ${filejob} | grep BAD | wc -l)
2259
2260       if [[ ${ratesString[${i}]} == "stderr" ]] ; then
2261         var1=$(grep "stderr" ${filejob} | grep OK | wc -l)
2262         var2=$(grep "stderr" ${filejob} | grep "rec.log" | grep BAD | wc -l)
2263       fi
2264       echo -n " ${var1}" >> ${outfile}
2265       echo -n " ${var2}" >> ${outfile}
2266     done
2267
2268     if [[ -f ${fileOCDB} ]] ; then
2269       #fill counter
2270       for i in $(seq 0 $((${Ncounter}-1))) ; do
2271         var1=$(grep Monalisa ${fileOCDB} | grep ${counterString[${i}]} | cut -f2)
2272         echo -n " ${var1:-"-1"}" >> ${outfile}
2273       done
2274
2275       #fill status
2276       for i in $(seq 0 $((${Nstatus}-1))) ; do
2277         var1=$(grep "calibration status=" ${fileOCDB} | grep ${statusString[${i}]/Status/} | cut -d'=' -f2)
2278         echo -n " ${var1:-"-1"}" >> ${outfile}
2279       done
2280     fi
2281     echo >> ${outfile}
2282   done
2283
2284   return 0
2285 }
2286
2287 parseConfig()
2288 {
2289   configFile=${1}
2290   shift
2291   args=("$@")
2292
2293   #some defaults
2294   #autoOCDB=0
2295   defaultOCDB="raw://"
2296   #runNumber=167123
2297   #makeflowPath="/hera/alice/aux/cctools/bin"
2298   #makeflowOptions="-T wq -N alice -d all -C ali-copilot.cern.ch:9097"
2299   #makeflowOptions="-T wq -N alice -C ali-copilot.cern.ch:9097"
2300   makeflowOptions=""
2301   #batchCommand="/usr/bin/qsub"
2302   batchFlags="-b y -cwd -l h_rt=24:0:0,h_rss=4G "
2303   baseOutputDirectory="$PWD/output"
2304   #alirootEnv="/cvmfs/alice.cern.ch/bin/alienv setenv AliRoot/v5-04-34-AN -c"
2305   #alirootEnv="/home/mkrzewic/alisoft/balice_master.sh"
2306   #trustedQAtrainMacro='/hera/alice/mkrzewic/gsisvn/Calibration/QAtrain_duo.C'
2307   reconstructInTemporaryDir=0
2308   recoTriggerOptions="\"\""
2309   percentProcessedFilesToContinue=100
2310   maxSecondsToWait=$(( 3600*24 ))
2311   nEvents=-1
2312   nMaxChunks=0
2313   postSetUpActionCPass0=""
2314   postSetUpActionCPass1=""
2315   runCPass0reco=1
2316   runCPass0MergeMakeOCDB=1
2317   runCPass1reco=1
2318   runCPass1MergeMakeOCDB=1
2319   runESDfiltering=1
2320   filteringFactorHighPt=1e2
2321   filteringFactorV0s=1e1
2322   MAILTO=""
2323   #pretend=1
2324   #dontRedirectStdOutToLog=1
2325   logToFinalDestination=1
2326   ALIROOT_FORCE_COREDUMP=1
2327
2328   #first, source the config file
2329   if [ -f ${configFile} ]; then
2330     source ${configFile}
2331   else
2332     echo "config file ${configFile} not found!, skipping..."
2333   fi
2334
2335   unset encodedSpaces
2336   for opt in "${args[@]}"; do
2337     [[ "${opt}" =~ encodedSpaces=.* ]] && encodedSpaces=1 && break
2338   done
2339
2340   #then, parse the options as they override the options from file
2341   for opt in "${args[@]}"; do
2342     if [[ ! "${opt}" =~ .*=.* ]]; then
2343       echo "badly formatted option ${var}, should be: option=value, stopping..."
2344       return 1
2345     fi
2346     local var="${opt%%=*}"
2347     local value="${opt#*=}"
2348     [[ -n ${encodedSpaces} ]] && value=$(decSpaces "${value}")
2349     echo "${var} = ${value}"
2350     export ${var}="${value}"
2351   done
2352
2353   #do some checking
2354   [[ -z ${alirootEnv} ]] && echo "alirootEnv not defined!" && return 1
2355
2356   #export the aliroot function if defined to override normal behaviour
2357   [[ $(type -t aliroot) =~ "function" ]] && export -f aliroot
2358
2359   return 0
2360 }
2361
2362 aliroot()
2363 {
2364   args="$@"
2365   if [[ -n ${useProfilingCommand} ]]; then
2366     valgrindLogFile="cpu.txt"
2367     [[ "${args}" =~ rec ]] && valgrindLogFile="cpu_rec.txt"
2368     [[ "${args}}" =~ Calib ]] && valgrindLogFile="cpu_calib.txt"
2369     [[ -n ${useProfilingCommand} ]] && useProfilingCommand="${useProfilingCommand} --log-file=${valgrindLogFile}"
2370     echo running ${useProfilingCommand} aliroot ${args}
2371     ${useProfilingCommand} aliroot ${args}
2372   else
2373     #to prevent an infinite recursion use "command aliroot" to disable
2374     #aliases and functions
2375     echo running command aliroot ${args}
2376     command aliroot "$@"
2377   fi
2378   return 0
2379 }
2380
2381 guessRunData()
2382 {
2383   #guess the period from the path, pick the rightmost one
2384   period=""
2385   runNumber=""
2386   year=""
2387   pass=""
2388   legoTrainRunNumber=""
2389   dataType=""
2390
2391   local shortRunNumber=""
2392   local IFS="/"
2393   declare -a path=( $1 )
2394   local dirDepth=$(( ${#path[*]}-1 ))
2395   i=0
2396   #for ((x=${dirDepth};x>=0;x--)); do
2397   for ((x=0;x<=${dirDepth};x++)); do
2398
2399     [[ $((x-1)) -ge 0 ]] && local fieldPrev=${path[$((x-1))]}
2400     local field=${path[${x}]}
2401     local fieldNext=${path[$((x+1))]}
2402
2403     [[ ${field} =~ ^[0-9]*$ && ${fieldNext} =~ (.*\.zip$|.*\.root$) ]] && legoTrainRunNumber=${field}
2404     [[ -n ${legoTrainRunNumber} && -z ${pass} ]] && pass=${fieldPrev}
2405     [[ ${field} =~ ^LHC[0-9][0-9][a-z].*$ ]] && period=${field%_*}
2406     [[ ${field} =~ ^000[0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && runNumber=${field#000}
2407     [[ ${field} =~ ^[0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && shortRunNumber=${field}
2408     [[ ${field} =~ ^20[0-9][0-9]$ ]] && year=${field}
2409     [[ ${field} =~ ^(^sim$|^data$) ]] && dataType=${field}
2410     (( i++ ))
2411   done
2412   [[ -z ${legoTrainRunNumber} ]] && pass=${path[$((dirDepth-1))]}
2413   [[ "${dataType}" =~ ^sim$ ]] && pass="passMC" && runNumber=${shortRunNumber}
2414
2415   #if [[ -z ${dataType} || -z ${year} || -z ${period} || -z ${runNumber}} || -z ${pass} ]];
2416   if [[ -z ${runNumber}} ]];
2417   then
2418     #error condition
2419     return 1
2420   else
2421     #ALL OK
2422     return 0
2423   fi
2424   return 0
2425 }
2426
2427 #these functions encode strings to and from a space-less form
2428 #use when spaces are not well handled (e.g. in arguments to
2429 #commands in makeflow files, etc.
2430 encSpaces()(a="${1//,/\\,}";echo "${a// /,}")
2431 decSpaces()(a="${1//\\,/\\ }";b="${a//,/ }";echo "${b//\\ /,}")
2432
2433 main "$@"