PWGPP/benchmark/benchmark.sh

   1 #!/bin/bash
   2 #include benchmark.config
   3
   4 # this script runs the CPass0/CPass1 train
   5 # produced OCDB updates are local
   6
   7 main()
   8 {
   9   #run in proper mode depending on the selection
  10   if [[ $# -lt 1 ]]; then
  11     if [[ ! "${0}" =~ "bash" ]]; then
  12       echo "uses makeflow:"
  13       echo " ${0} \"run\" productionID inputList configFile [extraOpts]"
  14       echo "uses a batch system (SGE):"
  15       echo " ${0} \"submit\" productionID inputList configFile [extraOpts]"
  16       echo "extraOpts if non-empty override the config file, e.g.:"
  17       echo " ${0} submit test1 benchmark.list benchmark.config runNumber=169123 nEvents=10"
  18     fi
  19     return
  20   fi
  21
  22   #define some aliases - default is to call one of the functions directly
  23   runMode=${1}
  24   umask 0002
  25   shift
  26   case ${runMode} in
  27     "CPass0") goCPass0 "$@";;
  28     "CPass1") goCPass1 "$@";;
  29     "MakeLocalOCDBaccessConfig") goMakeLocalOCDBaccessConfig "$@";;
  30     "MergeCPass0") goMergeCPass0 "$@";;
  31     "MergeCPass1") goMergeCPass1 "$@";;
  32     "MakeFilteredTrees") goMakeFilteredTrees "$@";;
  33     "MakeSummary") goMakeSummary "$@";;
  34     "run") goSubmitMakeflow "$@";;
  35     "submit") goSubmitBatch "$@";;
  36     "test") goTest "$@";;
  37     "GenerateMakeflow") goGenerateMakeflow "$@";;
  38     "PrintValues") goPrintValues "$@";;
  39     "CreateQAplots") goCreateQAplots "$@";;
  40     "WaitForOutput") goWaitForOutput "$@";;
  41     "Merge") goMerge "$@";;
  42     *)
  43       ${runMode} "$@"
  44     ;;
  45   esac
  46   return 0
  47 }
  48
  49 generateMC()
  50 {
  51   #generate one raw chunk in current directory
  52   SEED=${JOB_ID}${SGE_TASK_ID}
  53   export CONFIG_SEED=${SEED}
  54   runNumber=${1}
  55   OCDBpath=${2}
  56   nEventsim=${3}
  57   if [[ -n ${pretend} ]]; then
  58     sleep ${pretendDelay}
  59     touch galice.root
  60   else
  61     if [[ -f sim.C && -f Config.C ]] ; then
  62         time aliroot -b -q -x sim.C\(${runNumber},\"${OCDBpath}\",${nEventsim}\) >sim.log 2>&1
  63         mv syswatch.log simwatch.log
  64     fi
  65   fi
  66 }
  67
  68 goCPass0()
  69 {
  70   umask 0002
  71
  72   targetDirectory=${1}
  73   inputList=${2}
  74   nEvents=${3}
  75   ocdbPath=${4}
  76   configFile=${5}
  77   runNumber=${6}
  78   jobindex=${7}
  79   shift 7
  80   if ! parseConfig ${configFile} "$@"; then return 1; fi
  81
  82   #record the working directory provided by the batch system
  83   batchWorkingDirectory=${PWD}
  84
  85   #use the jobindex only if set and non-negative
  86   if [[ -z ${jobindex} || ${jobindex} -lt 0 ]]; then
  87     [[ -n "${LSB_JOBINDEX}" ]] && jobindex=${LSB_JOBINDEX}
  88     [[ -n "${SGE_TASK_ID}" ]] && jobindex=${SGE_TASK_ID}
  89   fi
  90
  91   [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
  92   doneFile="${commonOutputPath}/meta/cpass0.job${jobindex}.run${runNumber}.done"
  93
  94   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
  95
  96   if [[ -n ${ALIROOT_FORCE_COREDUMP} ]]; then
  97     ulimit -c unlimited
  98     export ALIROOT_FORCE_COREDUMP
  99   fi
 100
 101   #the contents of this is stored in the tree and used later (e.g. AliAnalysisTaskPIDResponse)!
 102   #at the QA stage the pass number is guessed from the path stored here.
 103   #The Format is:
 104   #Packages= ;OutputDir= ;LPMPass= ;TriggerAlias= ;LPMRunNumber= ;LPMProductionType= ;LPMInteractionType= ;LPMProductionTag= ;LPMAnchorRun= ;LPMAnchorProduction= ;LPMAnchorYear=
 105   export PRODUCTION_METADATA="OutputDir=cpass0"
 106
 107   if [[ "${inputList}" =~ \.root$ ]]; then
 108     infile=${inputList}
 109   else
 110     infile=$(sed -ne "${jobindex}p" ${inputList} | egrep '\s*\w*/\w*')
 111   fi
 112   chunkName=${infile##*/}
 113
 114   outputDir=${targetDirectory}/${jobindex}_${chunkName%.*}
 115   mkdir -p ${outputDir}
 116   [[ ! -d ${outputDir} ]] && echo "cannot make ${outputDir}" && touch ${doneFile} && return 1
 117
 118   #runpath=${PWD}/rundir_cpass0_${runNumber}_${jobindex}
 119   runpath=${outputDir}
 120   #[[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
 121   [[ ${reconstructInTemporaryDir} -eq 1 ]] && runpath=$(mktemp -d -t cpass0.XXXXXX)
 122   mkdir -p ${runpath}
 123   [[ ! -d ${runpath} ]] && echo "cannot make runpath ${runpath}" && touch ${doneFile} && return 1
 124   if ! cd ${runpath}; then
 125     echo "PWD=$PWD is not the runpath=${runpath}"
 126     touch ${doneFile}
 127     return 1
 128   fi
 129
 130   #runCPassX/C expects the raw chunk to be linked in the run dir
 131   #despite it being accessed by the full path
 132   ln -s ${infile} ${runpath}/${chunkName}
 133
 134   #####MC
 135   if [[ -n ${generateMC} ]]; then
 136     olddir=${PWD}
 137     outputDirMC=${commonOutputPath}/000${runNumber}/sim/${jobindex}
 138     simrunpath=${outputDirMC}
 139     #[[ ${simulateInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && simrunpath=${TMPDIR}
 140     [[ ${simulateInTemporaryDir} -eq 1 ]] && simrunpath=$(mktemp -d -t cpass0MC.XXXXXX)
 141     mkdir -p ${outputDirMC}
 142     mkdir -p ${simrunpath}
 143     if cd ${simrunpath}; then
 144
 145       filesMC=(
 146       "${batchWorkingDirectory}/sim.C"
 147       "${batchWorkingDirectory}/rec.C"
 148       "${batchWorkingDirectory}/Config.C"
 149       "${batchWorkingDirectory}/OCDB_*.root"
 150       )
 151       for file in ${filesMC[*]}; do
 152         [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
 153       done
 154
 155       generateMC ${runNumber} ${ocdbPath} ${nEvents}
 156
 157       [[ ! "${simrunpath}" =~ "${outputDirMC}" ]] && mv * ${outputDirMC} #TODO check if it works
 158       cd ${olddir}
 159
 160       ln -s ${outputDirMC}/* ${runpath}/
 161
 162       inputList=${outputDirMC}/galice.root #TODO not valid outside shell !!!
 163       infile=""
 164     fi
 165   fi
 166   ######
 167
 168   [[ ! -f ${inputList} && -z ${pretend} ]] && echo "input file ${inputList} not found, exiting..." && touch ${doneFile} && return 1
 169
 170   logOutputDir=${runpath}
 171   [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir}
 172   [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/stdout
 173   #[[ -z ${dontRedirectStdOutToLog} ]] && exec 2> ${logOutputDir}/stderr
 174   echo "${0} $*"
 175
 176   echo "#####################"
 177   echo CPass0:
 178   echo JOB setup
 179   echo nEvents            ${nEvents}
 180   echo runNumber          ${runNumber}
 181   echo ocdbPath           ${ocdbPath}
 182   echo infile             ${infile}
 183   echo chunkName          ${chunkName}
 184   echo jobindex           ${jobindex}
 185   echo recoTriggerOptions ${recoTriggerOptions}
 186   echo targetDirectory    ${targetDirectory}
 187   echo commonOutputPath         ${commonOutputPath}
 188   echo doneFile      ${doneFile}
 189   echo runpath            ${runpath}
 190   echo outputDir          ${outputDir}
 191   echo ALICE_ROOT         ${ALICE_ROOT}
 192   echo PWD                ${PWD}
 193   echo "########## ###########"
 194
 195   alirootInfo > ALICE_ROOT.log
 196
 197   filesCPass0=(
 198                "${batchWorkingDirectory}/runCPass0.sh"
 199                "${batchWorkingDirectory}/recCPass0.C"
 200                "${batchWorkingDirectory}/runCalibTrain.C"
 201                "${batchWorkingDirectory}/localOCDBaccessConfig.C"
 202                "${batchWorkingDirectory}/OCDB.root"
 203                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/runCPass0.sh"
 204                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/recCPass0.C"
 205                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/runCalibTrain.C"
 206   )
 207
 208   for file in ${filesCPass0[*]}; do
 209     [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
 210     [[ ${file##*/} =~ .*\.sh ]] && chmod +x ${file##*/}
 211   done
 212
 213   echo "this directory (${PWD}) contents:"
 214   /bin/ls
 215   echo
 216   chmod u+x runCPass0.sh
 217
 218   sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh
 219
 220   if [[ -n ${postSetUpActionCPass0} ]]; then
 221     echo "running ${postSetUpActionCPass0}"
 222     eval ${postSetUpActionCPass0}
 223   fi
 224
 225   #run CPass0
 226   echo "${runpath}/runCPass0.sh ${infile} ${nEvents} ${runNumber} ${ocdbPath} ${recoTriggerOptions}"
 227   if [[ -n ${pretend} ]]; then
 228     sleep ${pretendDelay}
 229     touch AliESDs.root
 230     touch AliESDfriends.root
 231     touch AliESDfriends_v1.root
 232     touch rec.log
 233     touch calib.log
 234   else
 235     echo ./runCPass0.sh "${infile}" "${nEvents}" "${runNumber}" "${ocdbPath}" "${recoTriggerOptions}"
 236     ./runCPass0.sh "${infile}" "${nEvents}" "${runNumber}" "${ocdbPath}" "${recoTriggerOptions}"
 237   fi
 238
 239   #move stuff to final destination
 240   echo "this directory (${PWD}) contents:"
 241   /bin/ls
 242   echo
 243
 244   echo rm -f ./${chunkName}
 245   rm -f ./${chunkName}
 246   echo "cp -R ${runpath}/* ${outputDir}"
 247   cp -p -R ${runpath}/* ${outputDir}
 248   echo
 249
 250   #validate CPass0
 251   cd ${outputDir}
 252   touch ${doneFile}
 253   echo "dir ${outputDir}" >> ${doneFile}
 254   if summarizeLogs >> ${doneFile}; then
 255     [[ -f ${outputDirMC}/galice.root ]] && echo "sim ${outputDirMC}/galice.root" >> ${doneFile}
 256     [[ -f AliESDfriends_v1.root ]] && echo "calibfile ${outputDir}/AliESDfriends_v1.root" >> ${doneFile}
 257     [[ -f AliESDs.root ]] && echo "esd ${outputDir}/AliESDs.root" >> ${doneFile}
 258   fi
 259
 260   [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath} && echo "removing ${runpath}"
 261   return 0
 262 }
 263
 264 goCPass1()
 265 {
 266   umask 0002
 267
 268   targetDirectory=${1}
 269   inputList=${2}
 270   nEvents=${3}
 271   ocdbPath=${4}
 272   configFile=${5}
 273   runNumber=${6}
 274   jobindex=${7}
 275   shift 7
 276   extraOpts=("$@")
 277   if ! parseConfig ${configFile} "$@"; then return 1; fi
 278
 279   #record the working directory provided by the batch system
 280   batchWorkingDirectory=${PWD}
 281
 282   #use the jobindex only if set and non-negative
 283   if [[ -z ${jobindex} || ${jobindex} -lt 0 ]]; then
 284     [[ -n "${LSB_JOBINDEX}" ]] && jobindex=${LSB_JOBINDEX}
 285     [[ -n "${SGE_TASK_ID}" ]] && jobindex=${SGE_TASK_ID}
 286   fi
 287
 288   [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
 289   doneFile="${commonOutputPath}/meta/cpass1.job${jobindex}.run${runNumber}.done"
 290
 291   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
 292
 293   if [[ -n ${ALIROOT_FORCE_COREDUMP} ]]; then
 294     ulimit -c unlimited
 295     export ALIROOT_FORCE_COREDUMP
 296   fi
 297
 298   #the contents of this is stored in the tree and used later (e.g. AliAnalysisTaskPIDResponse)!
 299   #at the QA stage the pass number is guessed from the path stored here.
 300   #The Format is:
 301   #Packages= ;OutputDir= ;LPMPass= ;TriggerAlias= ;LPMRunNumber= ;LPMProductionType= ;LPMInteractionType= ;LPMProductionTag= ;LPMAnchorRun= ;LPMAnchorProduction= ;LPMAnchorYear=
 302   export PRODUCTION_METADATA="OutputDir=cpass1"
 303
 304   [[ ! -f ${inputList} && -z ${pretend} ]] && echo "input file ${inputList} not found, exiting..." && touch ${doneFile} && return 1
 305   if [[ "${inputList}" =~ \.root$ ]]; then
 306     infile=${inputList}
 307   else
 308     infile=$(sed -ne "${jobindex}p" ${inputList} | egrep '\s*\w*/\w*')
 309   fi
 310   chunkName=${infile##*/}
 311
 312   outputDir=${targetDirectory}/${jobindex}_${chunkName%.*}
 313   mkdir -p ${outputDir}
 314   [[ ! -d ${outputDir} ]] && echo "cannot make ${outputDir}" && touch ${doneFile} && return 1
 315
 316   #runpath=${PWD}/rundir_cpass1_${runNumber}_${jobindex}
 317   runpath=${outputDir}
 318   #[[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
 319   [[ ${reconstructInTemporaryDir} -eq 1 ]] && runpath=$(mktemp -d -t cpass1.XXXXXX)
 320
 321   #MC
 322   if [[ "${infile}" =~ galice\.root ]]; then
 323     ln -s ${inputList%/*}/* ${runpath}
 324     infile=""
 325   fi
 326
 327   #init the running path
 328   mkdir -p ${runpath}
 329   [[ ! -d ${runpath} ]] && echo "cannot make runpath ${runpath}" && touch ${doneFile} && return 1
 330   if ! cd ${runpath}; then
 331     echo "PWD=$PWD is not the runpath=${runpath}"
 332     touch ${doneFile}
 333     return 1
 334   fi
 335
 336   #this is needed for runCPass1.sh
 337   ln -s ${infile} ${runpath}/${chunkName}
 338
 339   logOutputDir=${runpath}
 340   [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir}
 341   [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/stdout
 342   #[[ -z ${dontRedirectStdOutToLog} ]] && exec 2> ${logOutputDir}/stderr
 343   echo "${0} $*"
 344
 345   echo "#####################"
 346   echo CPass1:
 347   echo JOB setup
 348   echo nEvents            ${nEvents}
 349   echo runNumber          ${runNumber}
 350   echo ocdbPath           ${ocdbPath}
 351   echo infile             ${infile}
 352   echo chunkName          ${chunkName}
 353   echo jobindex           ${jobindex}
 354   echo recoTriggerOptions ${recoTriggerOptions}
 355   echo targetDirectory    ${targetDirectory}
 356   echo commonOutputPath         ${commonOutputPath}
 357   echo doneFile      ${doneFile}
 358   echo runpath            ${runpath}
 359   echo outputDir          ${outputDir}
 360   echo ALICE_ROOT         ${ALICE_ROOT}
 361   echo PWD                ${PWD}
 362   echo "########## ###########"
 363
 364   alirootInfo > ALICE_ROOT.log
 365
 366   filesCPass1=(
 367                "${batchWorkingDirectory}/runCPass1.sh"
 368                "${batchWorkingDirectory}/recCPass1.C"
 369                "${batchWorkingDirectory}/recCPass1_OuterDet.C"
 370                "${batchWorkingDirectory}/runCalibTrain.C"
 371                "${batchWorkingDirectory}/QAtrain_duo.C"
 372                "${batchWorkingDirectory}/localOCDBaccessConfig.C"
 373                "${batchWorkingDirectory}/${configFile}"
 374                "${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz"
 375                "${batchWorkingDirectory}/OCDB.root"
 376                "${trustedQAtrainMacro}"
 377                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/runCPass1.sh"
 378                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/recCPass1.C"
 379                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/recCPass1_OuterDet.C"
 380                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/runCalibTrain.C"
 381                "${ALICE_ROOT}/ANALYSIS/macros/QAtrain_duo.C"
 382   )
 383
 384   for file in ${filesCPass1[*]}; do
 385     [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
 386     [[ ${file##*/} =~ .*\.sh ]] && chmod +x ${file##*/}
 387   done
 388
 389   echo "this directory (${PWD}) contents:"
 390   /bin/ls
 391   echo
 392
 393   sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh
 394
 395   if [[ -n ${postSetUpActionCPass1} ]]; then
 396     echo "running ${postSetUpActionCPass1}"
 397     eval ${postSetUpActionCPass1}
 398     echo
 399   fi
 400
 401   #configure local OCDB storage from CPass0 (creates the localOCDBaccessConfig.C script)
 402   if [[ -f cpass0.localOCDB.${runNumber}.tgz ]]; then
 403     echo goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz"
 404     goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz"
 405   else
 406     echo "WARNING: file cpass0.localOCDB.${runNumber}.tgz not found!"
 407   fi
 408
 409   if [[ ! $(/bin/ls -1 OCDB/*/*/*/*.root 2>/dev/null) ]]; then
 410     echo "cpass0 produced no calibration! exiting..."
 411     touch ${doneFile}
 412     return 1
 413   fi
 414
 415   #create the Barrel and OuterDet directories for CPass1 and link the local OCDB directory
 416   #there to make the localOCDBaccessConfig.C file work, since it may point to the OCDB
 417   #entries using a relative path, e.g. local://./OCDB
 418   mkdir Barrel OuterDet
 419   ln -s ../OCDB Barrel/OCDB
 420   ln -s ../OCDB OuterDet/OCDB
 421
 422   #setup the filtering
 423   #the following option enables the filtering task inside the QAtrain_duo.C
 424   [[ -n $runESDfiltering ]] && export QA_TaskFilteredTree=1
 425   #set the downscaling factors during the filtering fro expert QA (overrides the previous values)
 426   if [[ -n ${filteringFactorHighPt} ]]; then
 427     export AliAnalysisTaskFilteredTree_fLowPtTrackDownscaligF=${filteringFactorHighPt}
 428   fi
 429   if [[ -n ${filteringFactorV0s} ]]; then
 430     export AliAnalysisTaskFilteredTree_fLowPtV0DownscaligF=${filteringFactorV0s}
 431   fi
 432
 433   #run CPass1
 434   chmod u+x runCPass1.sh
 435   echo "${runpath}/runCPass1.sh ${infile} ${nEvents} ${runNumber} ${ocdbPath} ${recoTriggerOptions}"
 436   if [[ -n ${pretend} ]]; then
 437     sleep ${pretendDelay}
 438     touch AliESDs_Barrel.root
 439     touch AliESDfriends_Barrel.root
 440     touch AliESDfriends_v1.root
 441     touch QAresults_barrel.root
 442     touch EventStat_temp_barrel.root
 443     touch AODtpITS.root
 444     touch AliESDs_Outer.root
 445     touch AliESDfriends_Outer.root
 446     touch QAresults_outer.root
 447     touch EventStat_temp_outer.root
 448     touch rec.log
 449     touch calib.log
 450     touch qa.log
 451     touch filtering.log FilterEvents_Trees.root
 452   else
 453     ./runCPass1.sh "${infile}" "${nEvents}" "${runNumber}" "${ocdbPath}" "${recoTriggerOptions}"
 454
 455     [[ ! -f AliESDs_Barrel.root && -f Barrel/AliESDs.root ]] && mv Barrel/AliESDs.root AliESDs_Barrel.root
 456     [[ ! -f AliESDfriends_Barrel.root && -f Barrel/AliESDfriends.root ]] && mv Barrel/AliESDfriends.root AliESDfriends_Barrel.root
 457     [[ ! -f AliESDfriends_v1.root && -f Barrel/AliESDfriends_v1.root ]] && mv Barrel/AliESDfriends_v1.root .
 458     [[ ! -f QAresults_barrel.root && -f Barrel/QAresults_barrel.root ]] && mv Barrel/QAresults_barrel.root .
 459     [[ ! -f AliESDs_Outer.root && -f OuterDet/AliESDs.root ]] && mv OuterDet/AliESDs.root AliESDs_Outer.root
 460     [[ ! -f AliESDfriends_Outer.root && -f OuterDet/AliESDfriends.root ]] && mv OuterDet/AliESDfriends.root AliESDfriends_Outer.root
 461     [[ ! -f QAresults_outer.root && -f OuterDet/QAresults_outer.root ]] && mv OuterDet/QAresults_outer.root .
 462     [[ ! -f FilterEvents_Trees.root && -f Barrel/FilterEvents_Trees.root ]] && mv Barrel/FilterEvents_Trees.root .
 463
 464     #make the filtered tree (if requested and not already produced by QA
 465     [[ -f AliESDs_Barrel.root ]] && echo "AliESDs_Barrel.root" > filtered.list
 466     if [[ -n ${runESDfiltering} && ! -f FilterEvents_Trees.root ]]; then
 467       goMakeFilteredTrees ${PWD} ${runNumber} "filtered.list" ${filteringFactorHighPt} ${filteringFactorV0s} ${ocdbPath} 1000000 0 10000000 0 ${configFile} AliESDs_Barrel.root "${extraOpts[@]}" >filtering.log
 468     fi
 469
 470   fi
 471
 472   ##handle possible crashes in QA (happens often in trunk)
 473   ##rerun QA with a trusted aliroot version
 474   #if [[ $(validateLog qa_barrel.log) ]]; then
 475   #  echo "qa_barrel.log not validated!"
 476   #fi
 477   #if [[ ! -f QAresults_barrel.root && -f ${setupTrustedAliROOTenvInCurrentShell} || $(validateLog qa_barrel.log) ]]; then
 478   #  echo "WARNING: using trusted QA aliroot ${ALICE_ROOT}"
 479   #  source ${setupTrustedAliROOTenvInCurrentShell}
 480   #  cd Barrel
 481   #  rm QAresults_barrel.root
 482   #  rm EventStat_temp_barrel.root
 483   #  rm AODtpITS.root
 484   #  [[ ! -f AliESDs.root ]] && ln -s ../AliESDs_Barrel.root AliESDs.root
 485   #  [[ ! -f AliESDfriends.root ]] && ln -s ../AliESDfriends_Barrel.root AliESDfriends.root
 486   #  if [[ -n ${trustedQAtrainMacro} ]]; then
 487   #    eval "cp ${trustedQAtrainMacro} QAtrain_duo_trusted.C"
 488   #  fi
 489   #  echo executing aliroot -b -q "QAtrain_duo_trusted.C(\"_barrel\",${runNumber},\"wn.xml\",0,\"${ocdbPath}\")"
 490   #  time aliroot -b -q "QAtrain_duo.C(\"_barrel\",${runNumber},\"wn.xml\",0,\"${ocdbPath}\")" &> ../qa_barrel_trusted.log
 491   #  cd ../
 492   #fi
 493
 494   #move stuff to final destination
 495   echo "this directory (${PWD}) contents:"
 496   /bin/ls
 497   echo rm -f ./${chunkName}
 498   rm -f ./${chunkName}
 499   echo "cp -R ${runpath}/* ${outputDir}"
 500   cp -pf -R ${runpath}/* ${outputDir}
 501   echo
 502
 503   #validate CPass1
 504   cd ${outputDir}
 505   touch ${doneFile}
 506   echo "dir ${outputDir}" >> ${doneFile}
 507   if summarizeLogs >> ${doneFile}; then
 508     [[ -f AliESDs_Barrel.root ]] && echo "esd ${outputDir}/AliESDs_Barrel.root" >> ${doneFile}
 509     [[ -f AliESDfriends_v1.root ]] && echo "calibfile ${outputDir}/AliESDfriends_v1.root" >> ${doneFile}
 510     [[ -f QAresults_Barrel.root ]] && echo "qafile ${outputDir}/QAresults_Barrel.root" >> ${doneFile}
 511     [[ -f QAresults_Outer.root ]] && echo "qafile ${outputDir}/QAresults_Outer.root" >> ${doneFile}
 512     [[ -f QAresults_barrel.root ]] && echo "qafile ${outputDir}/QAresults_barrel.root" >> ${doneFile}
 513     [[ -f QAresults_outer.root ]] && echo "qafile ${outputDir}/QAresults_outer.root" >> ${doneFile}
 514     [[ -f FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFile}
 515   else
 516     if grep "qa_outer.log.*OK" ${doneFile} > /dev/null; then
 517       [[ -f QAresults_Outer.root ]] && echo "qafile ${outputDir}/QAresults_Outer.root" >> ${doneFile}
 518       [[ -f QAresults_outer.root ]] && echo "qafile ${outputDir}/QAresults_outer.root" >> ${doneFile}
 519     fi
 520     if grep "qa_barrel.log.*OK" ${doneFile} > /dev/null; then
 521       [[ -f QAresults_Barrel.root ]] && echo "qafile ${outputDir}/QAresults_Barrel.root" >> ${doneFile}
 522       [[ -f QAresults_barrel.root ]] && echo "qafile ${outputDir}/QAresults_barrel.root" >> ${doneFile}
 523     fi
 524     if grep "filtering.log.*OK" ${doneFile} > /dev/null; then
 525       [[ -f FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFile}
 526     fi
 527   fi
 528
 529   [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
 530   return 0
 531 }
 532
 533
 534 goMergeCPass0()
 535 {
 536   #
 537   # find the output files and merge them
 538   #
 539
 540   outputDir=${1}
 541   ocdbStorage=${2}
 542   configFile=${3}
 543   runNumber=${4}
 544   calibrationFilesToMerge=${5}  #can be a non-existent file, will then be produced on the fly
 545   shift 5
 546   if ! parseConfig ${configFile} "$@"; then return 1; fi
 547
 548   #record the working directory provided by the batch system
 549   batchWorkingDirectory=${PWD}
 550
 551   [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
 552   doneFile="${commonOutputPath}/meta/merge.cpass0.run${runNumber}.done"
 553
 554   umask 0002
 555   ulimit -c unlimited
 556
 557   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
 558
 559   #runpath=${PWD}/rundir_cpass0_Merge_${runNumber}
 560   runpath=${outputDir}
 561   #[[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
 562   [[ ${reconstructInTemporaryDir} -eq 1 ]] && runpath=$(mktemp -d -t mergeCPass0.XXXXXX)
 563
 564   mkdir -p ${runpath}
 565   [[ ! -d ${runpath} ]] && echo "not able to make the runpath ${runpath}" && touch ${doneFile} && return 1
 566   if ! cd ${runpath}; then
 567     echo "PWD=$PWD is not the runpath=${runpath}"
 568     touch ${doneFile}
 569     return 1
 570   fi
 571
 572   logOutputDir=${runpath}
 573   [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir}
 574   [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/mergeMakeOCDB.log
 575   echo "${0} $*"
 576
 577   mergingScript="mergeMakeOCDB.byComponent.sh"
 578
 579   echo goMergeCPass0 SETUP:
 580   echo runNumber=${runNumber}
 581   echo outputDir=${outputDir}
 582   echo ocdbStorage=${ocdbStorage}
 583   echo calibrationFilesToMerge=${calibrationFilesToMerge}
 584   echo mergingScript=${mergingScript}
 585   echo commonOutputPath=${commonOutputPath}
 586   echo runpath=${runpath}
 587
 588   # copy files in case they are not already there
 589   filesMergeCPass0=(
 590                     "${batchWorkingDirectory}/${calibrationFilesToMerge}"
 591                     "${batchWorkingDirectory}/OCDB.root"
 592                     "${batchWorkingDirectory}/localOCDBaccessConfig.C"
 593                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/mergeMakeOCDB.byComponent.sh"
 594                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/mergeByComponent.C"
 595                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/makeOCDB.C"
 596                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/merge.C"
 597                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/mergeMakeOCDB.sh"
 598   )
 599   for file in ${filesMergeCPass0[*]}; do
 600     [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
 601     [[ ${file##*/} =~ .*\.sh ]] && chmod +x ${file##*/}
 602   done
 603
 604   sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh
 605
 606   alirootInfo > ALICE_ROOT.log
 607
 608   #
 609   echo "PWD"
 610   /bin/ls
 611   echo "PWD/.."
 612   /bin/ls ../
 613
 614
 615   #merge calibration
 616   chmod u+x ${mergingScript}
 617   mkdir -p ./OCDB
 618   if [[ ! -f ${calibrationFilesToMerge} ]]; then
 619     echo "/bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root > ${calibrationFilesToMerge}"
 620     /bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root 2>/dev/null > ${calibrationFilesToMerge}
 621   fi
 622
 623   echo "${mergingScript} ${calibrationFilesToMerge} ${runNumber} local://./OCDB ${ocdbStorage}"
 624   if [[ -n ${pretend} ]]; then
 625     sleep ${pretendDelay}
 626     touch CalibObjects.root
 627     touch ocdb.log
 628     touch merge.log
 629     touch dcsTime.root
 630     mkdir -p ./OCDB/TPC/Calib/TimeGain/
 631     mkdir -p ./OCDB/TPC/Calib/TimeDrift/
 632     touch ./OCDB/TPC/Calib/TimeGain/someCalibObject_0-999999_cpass0.root
 633     touch ./OCDB/TPC/Calib/TimeDrift/otherCalibObject_0-999999_cpass0.root
 634   else
 635     ./${mergingScript} ${calibrationFilesToMerge} ${runNumber} "local://./OCDB" ${ocdbStorage}
 636
 637     #produce the calib trees for expert QA (dcsTime.root)
 638     goMakeLocalOCDBaccessConfig ./OCDB
 639     echo aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")"
 640     aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")"
 641   fi
 642
 643   ### produce the output
 644   #tar the produced OCDB for reuse
 645   tar czf ${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz ./OCDB
 646
 647   /bin/ls
 648
 649   #copy all to output dir
 650   cp -pf -R ${runpath}/* ${outputDir}
 651
 652   if [[ -n ${generateMC} ]]; then
 653     goPrintValues sim ${commonOutputPath}/meta/sim.run${runNumber}.list ${commonOutputPath}/meta/cpass0.job*.run${runNumber}.done
 654   fi
 655
 656   #validate merging cpass0
 657   cd ${outputDir}
 658   touch ${doneFile}
 659   echo "dir ${outputDir}" >> ${doneFile}
 660   if summarizeLogs >> ${doneFile}; then
 661     [[ -f CalibObjects.root ]] && echo "calibfile ${outputDir}/CalibObjects.root" >> ${doneFile}
 662     [[ -f dcsTime.root ]] && echo "dcsTree ${outputDir}/dcsTime.root" >> ${doneFile}
 663   fi
 664
 665   [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
 666   return 0
 667 }
 668
 669 goMergeCPass1()
 670 {
 671   #
 672   # find the output files and merge them
 673   #
 674
 675   outputDir=${1}
 676   ocdbStorage=${2}
 677   configFile=${3}
 678   runNumber=${4}
 679   calibrationFilesToMerge=${5}
 680   qaFilesToMerge=${6}
 681   filteredFilesToMerge=${7}
 682   shift 7
 683   if ! parseConfig ${configFile} "$@"; then return 1; fi
 684
 685   #record the working directory provided by the batch system
 686   batchWorkingDirectory=${PWD}
 687
 688   [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
 689   doneFile="${commonOutputPath}/meta/merge.cpass1.run${runNumber}.done"
 690
 691   umask 0002
 692   ulimit -c unlimited
 693
 694   #clean up first:
 695   rm -f ${outputDir}/*.log
 696   rm -f ${outputDir}/*.root
 697   rm -f ${outputDir}/*done
 698
 699   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
 700
 701   #runpath=${PWD}/rundir_cpass1_Merge_${runNumber}
 702   runpath=${outputDir}
 703   #[[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
 704   [[ ${reconstructInTemporaryDir} -eq 1 ]] && runpath=$(mktemp -d -t mergeCPass1.XXXXXX)
 705
 706   mkdir -p ${runpath}
 707   [[ ! -d ${runpath} ]] && echo "not able to make the runpath ${runpath}" && touch ${doneFile} && return 1
 708   if ! cd ${runpath}; then
 709     echo "PWD=$PWD is not the runpath=${runpath}"
 710     touch ${doneFile}
 711     return 1
 712   fi
 713
 714   logOutputDir=${runpath}
 715   [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir}
 716   [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/mergeMakeOCDB.log
 717   echo "${0} $*"
 718
 719   calibrationOutputFileName='AliESDfriends_v1.root'
 720   qaOutputFileName='QAresults*.root'
 721   mergingScript="mergeMakeOCDB.byComponent.sh"
 722   #important to have the string "Stage.txt" in the filename to trigger the merging
 723   #it has to be a list of directories containing the files
 724   qaMergedOutputFileName="QAresults_merged.root"
 725
 726   echo goMergeCPass1 SETUP:
 727   echo runNumber=${runNumber}
 728   echo outputDir=${outputDir}
 729   echo ocdbStorage=${ocdbStorage}
 730   echo calibrationFilesToMerge=$calibrationFilesToMerge
 731   echo qaFilesToMerge=$qaFilesToMerge
 732   echo calibrationOutputFileName=${calibrationOutputFileName}
 733   echo mergingScript=${mergingScript}
 734
 735   # copy files in case they are not already there
 736   filesMergeCPass1=(
 737                     "${batchWorkingDirectory}/${calibrationFilesToMerge}"
 738                     "${batchWorkingDirectory}/${qaFilesToMerge}"
 739                     "${batchWorkingDirectory}/OCDB.root"
 740                     "${batchWorkingDirectory}/localOCDBaccessConfig.C"
 741                     "${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz"
 742                     "${batchWorkingDirectory}/QAtrain_duo.C"
 743                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/mergeMakeOCDB.byComponent.sh"
 744                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/mergeByComponent.C"
 745                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/makeOCDB.C"
 746                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/merge.C"
 747                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/mergeMakeOCDB.sh"
 748                     "${trustedQAtrainMacro}"
 749                     "${ALICE_ROOT}/ANALYSIS/macros/QAtrain_duo.C"
 750   )
 751   for file in ${filesMergeCPass1[*]}; do
 752     [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
 753     [[ ${file##*/} =~ .*\.sh ]] && chmod +x ${file##*/}
 754   done
 755
 756   sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh
 757
 758   #configure local OCDB storage from CPass0 (creates the localOCDBaccessConfig.C script)
 759   if [[ -f cpass0.localOCDB.${runNumber}.tgz ]]; then
 760     echo goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz"
 761     goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz"
 762   else
 763     echo "WARNING: file cpass0.localOCDB.${runNumber}.tgz not found!"
 764   fi
 765
 766   alirootInfo > ALICE_ROOT.log
 767
 768   #
 769   /bin/ls
 770
 771   #merge calibration
 772   chmod u+x ${mergingScript}
 773   mkdir -p OCDB
 774
 775   #if not provided, create the lists of files to merge
 776   if [[ ! -f ${filteredFilesToMerge} ]]; then
 777     echo "/bin/ls -1 ${outputDir}/*/FilterEvents_Trees.root > ${filteredFilesToMerge}"
 778     /bin/ls -1 ${outputDir}/*/FilterEvents_Trees.root 2>/dev/null > ${filteredFilesToMerge}
 779   fi
 780   if [[ ! -f ${calibrationFilesToMerge} ]]; then
 781     echo "/bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root > ${calibrationFilesToMerge}"
 782     /bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root 2>/dev/null > ${calibrationFilesToMerge}
 783   fi
 784   if [[ ! -f ${qaFilesToMerge} ]]; then
 785     #find the files, but only store the directories (QAtrain_duo.C requires this)
 786     echo "/bin/ls -1 ${outputDir}/*/QAresults*.root | while read x; do echo ${x%/*}; done | sort | uniq > ${qaFilesToMerge}"
 787     /bin/ls -1 ${outputDir}/*/QAresults*.root | while read x; do echo ${x%/*}; done | sort | uniq > ${qaFilesToMerge}
 788   fi
 789
 790   echo "${mergingScript} ${calibrationFilesToMerge} ${runNumber} local://./OCDB ${ocdbStorage}"
 791   if [[ -n ${pretend} ]]; then
 792     sleep ${pretendDelay}
 793     touch ocdb.log
 794     touch cpass1.localOCDB.${runNumber}.tgz
 795     touch ${qaMergedOutputFileName}
 796     touch merge.log
 797     touch trending.root
 798     touch FilterEvents_Trees.root
 799     touch CalibObjects.root
 800     touch dcsTime.root
 801     touch ${qaMergedOutputFileName}
 802     mkdir -p OCDB
 803   else
 804     ./${mergingScript} ${calibrationFilesToMerge} ${runNumber} "local://./OCDB" ${ocdbStorage}
 805
 806     #merge QA (and filtered trees)
 807     [[ -n ${AliAnalysisTaskFilteredTree_fLowPtTrackDownscaligF} ]] && export AliAnalysisTaskFilteredTree_fLowPtTrackDownscaligF
 808     [[ -n ${AliAnalysisTaskFilteredTree_fLowPtV0DownscaligF} ]] && export AliAnalysisTaskFilteredTree_fLowPtV0DownscaligF
 809
 810     #echo aliroot -l -b -q "merge.C(\"${qaFilesToMerge}\",\"\",kFALSE,\"${qaMergedOutputFileName}\")"
 811     echo aliroot -b -q "QAtrain_duo.C(\"_barrel\",${runNumber},\"${qaFilesToMerge}\",1,\"${ocdbStorage}\")"
 812     #aliroot -l -b -q "merge.C(\"${qaFilesToMerge}\",\"\",kFALSE,\"${qaMergedOutputFileName}\")"
 813     aliroot -b -q "QAtrain_duo.C(\"_barrel\",${runNumber},\"${qaFilesToMerge}\",1,\"${ocdbStorage}\")" > mergeQA.log
 814     mv QAresults_barrel.root ${qaMergedOutputFileName}
 815     mv trending_barrel.root trending.root
 816
 817     #merge filtered trees
 818     echo aliroot -l -b -q "merge.C(\"${qaFilesToMerge}\",\"\",kFALSE,\"${qaMergedOutputFileName}\")"
 819     aliroot -l -b -q "merge.C(\"${filteredFilesToMerge}\",\"\",kFALSE,\"FilterEvents_Trees.root\")" > mergeFilteredTrees.log
 820
 821     #produce the calib trees for expert QA
 822     echo aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")"
 823     aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")" > calibTree.log
 824   fi
 825
 826   tar czf ${commonOutputPath}/meta/cpass1.localOCDB.${runNumber}.tgz ./OCDB
 827
 828   /bin/ls
 829
 830   #copy all to output dir
 831   cp -pf -R ${runpath}/* ${outputDir}
 832
 833   #validate merge cpass1
 834   cd ${outputDir}
 835   touch ${doneFile}
 836   echo "dir ${outputDir}" >> ${doneFile}
 837   if summarizeLogs >>  ${doneFile}; then
 838     [[ -f CalibObjects.root ]] && echo "calibfile ${outputDir}/CalibObjects.root" >> ${doneFile}
 839     [[ -f ${qaMergedOutputFileName} ]] && echo "qafile ${outputDir}/${qaMergedOutputFileName}" >> ${doneFile}
 840     [[ -f trending.root ]] && echo "trendingfile ${outputDir}/trending.root" >> ${doneFile}
 841     [[ -f dcsTime.root ]] && echo "dcsTree ${outputDir}/dcsTime.root" >> ${doneFile}
 842     [[ -f FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFile}
 843   else
 844     if grep "mergeQA.log.*OK" ${doneFile} > /dev/null; then
 845       [[ -f ${qaMergedOutputFileName} ]] && echo "qafile ${outputDir}/${qaMergedOutputFileName}" >> ${doneFile}
 846     fi
 847     if grep "mergeFilteredTrees.log.*OK" ${doneFile} > /dev/null; then
 848       [[ -f FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFile}
 849     fi
 850   fi
 851
 852   [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
 853   return 0
 854 }
 855
 856 goMerge()
 857 {
 858   #generic root merge using CPass1 merge.C script
 859   inputList=${1}
 860   outputFile=${2}
 861   configFile=${3-"benchmark.config"}
 862   shift 3
 863   if ! parseConfig ${configFile} "$@"; then return 1; fi
 864
 865   #record the working directory provided by the batch system
 866   batchWorkingDirectory=${PWD}
 867
 868   [[ ! -f ${inputList} ]] && echo "inputList ${inputList} does not exist!" && return 1
 869   [[ ! -f ${configFile} ]] && echo "configFile ${configFile} does not exist!" && return 1
 870   umask 0002
 871   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
 872   rm -f ${outputFile}
 873   aliroot -b -q "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/merge.C(\"${inputList}\",\"\",kFALSE,\"${outputFile}\")" > merge_${inputList}.log
 874   return 0
 875 }
 876
 877 goSubmitMakeflow()
 878 {
 879   #run
 880   productionID=${1}
 881   inputFileList=${2}
 882   configFile=${3}
 883   shift 3
 884   extraOpts=("$@")
 885   if ! parseConfig ${configFile} "${extraOpts[@]}"; then return 1; fi
 886
 887   #record the working directory provided by the batch system
 888   batchWorkingDirectory=${PWD}
 889
 890   [[ -z ${configFile} ]] && configFile="benchmark.config"
 891   [[ ! -f ${configFile} ]] && echo "no config file found (${configFile})" && return 1
 892
 893   if [[ ! $(which makeflow &>/dev/null) && -n ${makeflowPath} ]]; then
 894     echo "setting the makflow path from the config: "
 895     echo "  export PATH=${makeflowPath}:${PATH}"
 896     export PATH=${makeflowPath}:${PATH}
 897   fi
 898
 899   #create the common output dir and the meta dir
 900   commonOutputPath=${baseOutputDirectory}/${productionID}
 901   if [[ -d ${commonOutputPath} ]]; then
 902     echo "output dir ${commonOutputPath} exists!"
 903     #return 1
 904   else
 905     mkdir -p ${commonOutputPath}
 906   fi
 907   mkdir -p ${commonOutputPath}/meta
 908
 909   self=${0}
 910   #if which greadlink; then self=$(greadlink -f "${0}"); fi
 911
 912   #for reference copy the setup to the output dir
 913   cp ${self} ${commonOutputPath}
 914   cp ${configFile} ${commonOutputPath}
 915   cp ${inputList} ${commonOutputPath}
 916
 917   #submit - use makeflow if available, fall back to old stuff when makeflow not there
 918   if which makeflow; then
 919
 920     goGenerateMakeflow ${productionID} ${inputFileList} ${configFile} "${extraOpts[@]}" commonOutputPath=${commonOutputPath} > benchmark.makeflow
 921
 922     makeflow ${makeflowOptions} benchmark.makeflow
 923     cd ../
 924   else
 925     echo "no makeflow!"
 926   fi
 927   return 0
 928 }
 929
 930 goGenerateMakeflow()
 931 {
 932   #generate the makeflow file
 933   [[ $# -lt 3 ]] && echo "args: id inputFileList configFile" && return 1
 934   productionID=${1}
 935   inputFileList=${2}
 936   configFile=${3}
 937   shift 3
 938   extraOpts=("$@")
 939
 940   #batch systems/makeflow sometimes handle spaces in arguments poorly, so encode them
 941   for (( i=0;i<${#extraOpts[@]};i++ )); do
 942     extraOpts[i]=$(encSpaces "${extraOpts[i]}")
 943   done
 944   extraOpts+=("encodedSpaces=1")
 945
 946   if ! parseConfig ${configFile} "${extraOpts[@]}" &>/dev/null; then return 1; fi
 947
 948   #extra safety
 949   if [[ -z ${commonOutputPath} ]]; then
 950     commonOutputPath=${baseOutputDirectory}/${productionID}
 951     extraOpts=( "${extraOpts[@]}" "commonOutputPath=${commonOutputPath}" )
 952   fi
 953
 954   #record the working directory provided by the batch system
 955   batchWorkingDirectory=${PWD}
 956
 957   [[ -z ${configFile} ]] && configFile="benchmark.config"
 958   [[ ! -f ${configFile} ]] && echo "no config file found (${configFile})" && return 1
 959
 960   #these files will be made a dependency - will be copied to the working dir of the jobs
 961   declare -a copyFiles
 962   inputFiles=(
 963               "OCDB.root"
 964               "localOCDBaccessConfig.C"
 965               "QAtrain_duo.C"
 966               "runCPass1.sh"
 967               "recCPass1.C"
 968               "recCPass1_OuterDet.C"
 969               "runCalibTrain.C"
 970               "runCPass0.sh"
 971               "recCPass0.C"
 972   )
 973   for file in ${inputFiles[*]}; do
 974     [[ -f ${file} ]] && copyFiles+=("${file}")
 975   done
 976
 977   #create the makeflow file
 978   [[ -n ${batchFlags} ]] && echo "BATCH_OPTIONS = ${batchFlags}"
 979   declare -A arr_cpass0_merged arr_cpass1_merged
 980   declare -A arr_cpass0_calib_list arr_cpass1_calib_list
 981   declare -A arr_cpass1_QA_list arr_cpass1_ESD_list arr_cpass1_filtered_list
 982   declare -A arr_cpass0_profiled_outputs
 983   declare -A listOfRuns
 984   [[ -n ${runNumber} ]] && listOfRuns[${runNumber}]=1
 985   while read x; do tmpRun=$(guessRunNumber ${x}); [[ -n ${tmpRun} ]] && listOfRuns[${tmpRun}]=1; done < ${inputFileList}
 986   for runNumber in "${!listOfRuns[@]}"; do
 987     [[ -z ${runNumber} ]] && continue
 988     [[ ! ${runNumber} =~ ^[0-9]*[0-9]$ ]] && continue
 989
 990     unset arr_cpass0_outputs
 991     unset arr_cpass1_outputs
 992     declare -a arr_cpass0_outputs
 993     declare -a arr_cpass1_outputs
 994
 995     jobindex=0
 996     inputFile=""
 997     while read inputFile; do
 998       currentDefaultOCDB=${defaultOCDB}
 999       [[ -z ${autoOCDB} ]] && autoOCDB=1
1000       if [[ ${autoOCDB} -ne 0 ]]; then
1001         currentDefaultOCDB=$(setYear ${inputFile} ${defaultOCDB})
1002       fi
1003
1004       #CPass0
1005       arr_cpass0_outputs[${jobindex}]="${commonOutputPath}/meta/cpass0.job${jobindex}.run${runNumber}.done"
1006       echo "${arr_cpass0_outputs[${jobindex}]} : benchmark.sh ${configFile} ${copyFiles[@]}"
1007       echo " ${alirootEnv} ./benchmark.sh CPass0 ${commonOutputPath}/000${runNumber}/cpass0 ${inputFile} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} ${jobindex} ${extraOpts[@]}"" "
1008
1009       #CPass1
1010       arr_cpass1_outputs[${jobindex}]="${commonOutputPath}/meta/cpass1.job${jobindex}.run${runNumber}.done"
1011       echo "${arr_cpass1_outputs[${jobindex}]} : benchmark.sh ${configFile} ${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz ${copyFiles[@]}"
1012       echo " ${alirootEnv} ./benchmark.sh CPass1 ${commonOutputPath}/000${runNumber}/cpass1 ${inputFile} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} ${jobindex} ${extraOpts[@]}"" "
1013       ((jobindex++))
1014
1015     done< <(grep "/000${runNumber}/" ${inputFileList})
1016
1017     #CPass0 list of Calib files to merge
1018     arr_cpass0_calib_list[${runNumber}]="${commonOutputPath}/meta/cpass0.calib.run${runNumber}.list"
1019     echo "${arr_cpass0_calib_list[${runNumber}]} : benchmark.sh ${arr_cpass0_outputs[*]}"
1020     echo "  ./benchmark.sh PrintValues calibfile ${arr_cpass0_calib_list[${runNumber}]} ${arr_cpass0_outputs[*]}"
1021     echo
1022
1023     #CPass0 merging
1024     arr_cpass0_merged[${runNumber}]="${commonOutputPath}/meta/merge.cpass0.run${runNumber}.done"
1025     echo "${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz ${arr_cpass0_merged[${runNumber}]} : benchmark.sh ${configFile} ${arr_cpass0_calib_list[${runNumber}]} ${copyFiles[@]}"
1026     echo " ${alirootEnv} ./benchmark.sh MergeCPass0 ${commonOutputPath}/000${runNumber}/cpass0 ${currentDefaultOCDB} ${configFile} ${runNumber} ${arr_cpass0_calib_list[${runNumber}]} ${extraOpts[@]}"" "
1027
1028     #CPass1 list of Calib/QA/ESD/filtered files
1029     # the trick with QA is to have the string "Stage.txt" in the file name of the list of directories with QA output to trigger
1030     # the production of the QA trending tree (only then the task->Finish() will be called in QAtrain_duo.C, on the grid
1031     # this corresponds to the last merging stage)
1032     arr_cpass1_QA_list[${runNumber}]="${commonOutputPath}/meta/cpass1.QA.run${runNumber}.lastMergingStage.txt.list"
1033     echo "${arr_cpass1_QA_list[${runNumber}]}: benchmark.sh ${arr_cpass1_outputs[*]}"
1034     echo "  ./benchmark.sh PrintValues dir ${arr_cpass1_QA_list[${runNumber}]} ${arr_cpass1_outputs[*]}"
1035     echo
1036     arr_cpass1_calib_list[${runNumber}]="${commonOutputPath}/meta/cpass1.calib.run${runNumber}.list"
1037     echo "${arr_cpass1_calib_list[${runNumber}]} : benchmark.sh ${arr_cpass1_outputs[*]}"
1038     echo "  ./benchmark.sh PrintValues calibfile ${arr_cpass1_calib_list[${runNumber}]} ${arr_cpass1_outputs[*]};"
1039     echo
1040     arr_cpass1_ESD_list[${runNumber}]="${commonOutputPath}/meta/cpass1.ESD.run${runNumber}.list"
1041     echo "${arr_cpass1_ESD_list[${runNumber}]} : benchmark.sh ${arr_cpass1_outputs[*]}"
1042     echo "  ./benchmark.sh PrintValues esd ${arr_cpass1_ESD_list[${runNumber}]} ${arr_cpass1_outputs[*]}"
1043     echo
1044     arr_cpass1_filtered_list[${runNumber}]="${commonOutputPath}/meta/cpass1.filtered.run${runNumber}.list"
1045     echo "${arr_cpass1_filtered_list[${runNumber}]} : benchmark.sh ${arr_cpass1_outputs[*]}"
1046     echo "  ./benchmark.sh PrintValues filteredTree ${arr_cpass1_filtered_list[${runNumber}]} ${arr_cpass1_outputs[*]}"
1047     echo
1048
1049     #CPass1 merging
1050     arr_cpass1_merged[${runNumber}]="${commonOutputPath}/meta/merge.cpass1.run${runNumber}.done"
1051     echo "${commonOutputPath}/meta/cpass1.localOCDB.${runNumber}.tgz ${arr_cpass1_merged[${runNumber}]} :  benchmark.sh ${configFile} ${arr_cpass1_calib_list[${runNumber}]} ${arr_cpass1_QA_list[${runNumber}]} ${copyFiles[@]}"
1052     echo " ${alirootEnv} ./benchmark.sh MergeCPass1 ${commonOutputPath}/000${runNumber}/cpass1 ${currentDefaultOCDB} ${configFile} ${runNumber} ${arr_cpass1_calib_list[${runNumber}]} ${arr_cpass1_QA_list[${runNumber}]} ${arr_cpass1_filtered_list[${runNumber}]} ${extraOpts[@]}"" "
1053
1054     #CPass0 wrapped in a profiling tool (valgrind,....)
1055     if [[ -n ${profilingCommand} ]]; then
1056       inputFile=$(grep -m1 "${runNumber}/" ${inputFileList})
1057       [[ -z ${nEventsProfiling} ]] && nEventsProfiling=2
1058       currentDefaultOCDB=$(setYear ${inputFile} ${defaultOCDB})
1059       jobindex="profiling"
1060
1061       arr_cpass0_profiled_outputs[${runNumber}]="${commonOutputPath}/meta/cpass0.job${jobindex}.run${runNumber}.done"
1062       echo "${arr_cpass0_profiled_outputs[${runNumber}]} : benchmark.sh ${configFile} ${copyFiles[@]}"
1063       profilingCommand=$(encSpaces "${profilingCommand}")
1064       echo " ${alirootEnv} ./benchmark.sh CPass0 ${commonOutputPath}/000${runNumber}/${jobindex} ${inputFile} ${nEventsProfiling} ${currentDefaultOCDB} ${configFile} ${runNumber} ${jobindex} ${extraOpts[@]} useProfilingCommand=${profilingCommand}"
1065     fi
1066
1067   done #runs
1068
1069   #Summary
1070   echo "${commonOutputPath}/summary.log : benchmark.sh ${configFile} ${arr_cpass1_merged[*]}"
1071   echo "  ${alirootEnv} ./benchmark.sh MakeSummary ${configFile} ${extraOpts[@]}"
1072   echo
1073
1074   return 0
1075 }
1076
1077 goPrintValues()
1078 {
1079   #print the values given the key from any number of files (store in output file on second argument)
1080   if [[ $# -lt 3 ]]; then
1081     echo "goPrintValues key outputFile inputFiles"
1082     echo "if outputFile is \"-\" dont redirect to a file"
1083     return
1084   fi
1085   key=${1}
1086   outputFile=${2}
1087   [[ ${outputFile} =~ "-" ]] && outputFile=""
1088   shift 2 #remove 2 first arguments from arg list to only pass the input files to awk
1089   awk -v key=${key} '$0 ~ key" " {print $2}' "$@" | tee ${outputFile}
1090   return 0
1091 }
1092
1093 goCreateQAplots()
1094 {
1095   umask 0002
1096   mergedQAfileList=${1}
1097   productionID=${2}
1098   outputDir=${3}
1099   configFile=${4}
1100   shift 4
1101   if ! parseConfig ${configFile} ${@}; then return 1; fi
1102
1103   #record the working directory provided by the batch system
1104   batchWorkingDirectory=${PWD}
1105
1106   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
1107
1108   [[ -z ${logOutputDir} ]] && logOutputDir=${PWD}
1109   [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/makeQAplots.log
1110   echo "${0} $*"
1111
1112   olddir=${PWD}
1113   mkdir -p ${outputDir}
1114   cd ${outputDir}
1115   [[ ! "${PWD}" =~ "${outputDir}" ]] && echo "PWD is not equal to outputDir=${outputDir}" && cd ${olddir} && return 1
1116
1117   ${ALICE_ROOT}/PWGPP/QA/scripts/runQA.sh inputList="${mergedQAfileList}" inputListHighPtTrees="${filteringList}"
1118
1119   cd ${olddir}
1120   return 0
1121 }
1122
1123 goTest()
1124 {
1125   umask 0002
1126   exec 2>&1
1127   exec > >(tee test.log)
1128   echo "$@"
1129   echo something
1130   return 0
1131 }
1132
1133 alirootInfo()
1134 {
1135   (
1136   umask 0002
1137   # save aliroot repository info
1138   [[ -z "${ALICE_ROOT}" ]] && return 1
1139
1140   echo "\${ALICE_ROOT}=${ALICE_ROOT}"
1141   echo "\${ROOTSYS}=${ROOTSYS}"
1142   echo "\${PATH}=${PATH}"
1143   echo "\${LD_LIBRARY_PATH}=${LD_LIBRARY_PATH}"
1144   echo
1145
1146   pushd ${PWD}
1147   cd ${ALICE_ROOT}
1148
1149   currentBranch=$(git rev-parse --abbrev-ref HEAD)
1150   git status
1151   echo ""
1152   echo ""
1153   git diff ${currentBranch}
1154   popd
1155   return 0
1156   )
1157 }
1158
1159 setYear()
1160 {
1161   #set the year
1162   #  ${1} - year to be set
1163   #  ${2} - where to set the year
1164   year1=$(guessYear ${1})
1165   year2=$(guessYear ${2})
1166   local path=${2}
1167   [[ ${year1} -ne ${year2} && -n ${year2} && -n ${year1} ]] && path=${2/\/${year2}\//\/${year1}\/}
1168   echo ${path}
1169   return 0
1170 }
1171
1172 guessPeriod()
1173 {
1174   #guess the period from the path, pick the rightmost one
1175   local IFS="/"
1176   declare -a path=( ${1} )
1177   local dirDepth=${#path[*]}
1178   for ((x=${dirDepth}-1;x>=0;x--)); do
1179     local field=${path[${x}]}
1180     [[ ${field} =~ ^LHC[0-9][0-9][a-z]$ ]] && period=${field} && break
1181   done
1182   echo ${period}
1183   return 0
1184 }
1185
1186 guessYear()
1187 {
1188   #guess the year from the path, pick the rightmost one
1189   local IFS="/"
1190   declare -a path=( ${1} )
1191   local dirDepth=${#path[*]}
1192   for ((x=${dirDepth}-1;x>=0;x--)); do
1193     local field=${path[${x}]}
1194     [[ ${field} =~ ^20[0-9][0-9]$ ]] && year=${field} && break
1195   done
1196   echo ${year}
1197   return 0
1198 }
1199
1200 guessRunNumber()
1201 {
1202   #guess the run number from the path, pick the rightmost one
1203   #works for /path/foo/000123456/bar/...
1204   #and       /path/foo.run123456.bar
1205   local IFS="/."
1206   declare -a path=( ${1} )
1207   local dirDepth=${#path[*]}
1208   for ((x=${dirDepth}-1;x>=0;x--)); do
1209     local field=${path[${x}]}
1210     field=${field/run/000}
1211     [[ ${field} =~ [0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && runNumber=${field#000} && break
1212   done
1213   echo ${runNumber}
1214   return 0
1215 }
1216
1217 validateLog()
1218 {
1219   log=${1}
1220   errorConditions=(
1221                     'There was a crash'
1222                     'floating'
1223                     'error while loading shared libraries'
1224                     'std::bad_alloc'
1225                     's_err_syswatch_'
1226                     'Thread [0-9]* (Thread'
1227                     'AliFatal'
1228                     'core dumped'
1229                     '\.C.*error:.*\.h: No such file'
1230                     'line.*Aborted'
1231   )
1232
1233   warningConditions=(
1234                      'This is serious !'
1235                      'rocVoltage out of range:'
1236   )
1237
1238   local logstatus=0
1239   local errorSummary=""
1240   local warningSummary=""
1241
1242   for ((i=0; i<${#errorConditions[@]};i++)); do
1243     local tmp=$(grep -m1 -e "${errorConditions[${i}]}" ${log})
1244     [[ -n ${tmp} ]] && tmp+=" : "
1245     errorSummary+=${tmp}
1246   done
1247
1248   for ((i=0; i<${#warningConditions[@]};i++)); do
1249     local tmp=$(grep -m1 -e "${warningConditions[${i}]}" ${log})
1250     [[ -n ${tmp} ]] && tmp+=" : "
1251     warningSummary+=${tmp}
1252   done
1253
1254   if [[ -n ${errorSummary} ]]; then
1255     echo "${errorSummary}"
1256     return 1
1257   fi
1258
1259   if [[ -n ${warningSummary} ]]; then
1260     echo "${warningSummary}"
1261     return 2
1262   fi
1263
1264   return 0
1265 }
1266
1267 summarizeLogs()
1268 {
1269   #print a summary of logs
1270   logFiles=(
1271             "*.log"
1272             "stdout"
1273             "stderr"
1274   )
1275
1276   #check logs
1277   local logstatus=0
1278   for log in ${logFiles[*]}; do
1279     finallog=${PWD%/}/${log}
1280     [[ ! -f ${log} ]] && continue
1281     errorSummary=$(validateLog ${log})
1282     validationStatus=$?
1283     if [[ ${validationStatus} -eq 0 ]]; then
1284       #in pretend mode randomly report an error in rec.log some cases
1285       if [[ -n ${pretend} && "${log}" == "rec.log" ]]; then
1286         #[[ $(( ${RANDOM}%2 )) -ge 1 ]] && echo "${finallog} BAD random error" || echo "${finallog} OK"
1287         echo "${finallog} OK"
1288       else
1289         echo "${finallog} OK"
1290       fi
1291     elif [[ ${validationStatus} -eq 1 ]]; then
1292       echo "${finallog} BAD ${errorSummary}"
1293       logstatus=1
1294     elif [[ ${validationStatus} -eq 2 ]]; then
1295       echo "${finallog} OK MWAH ${errorSummary}"
1296     fi
1297   done
1298
1299   #report core files
1300   while read x; do
1301     echo ${x}
1302     chmod 644 ${x}
1303     gdb --batch --quiet -ex "bt" -ex "quit" aliroot ${x} > stacktrace_${x//\//_}.log
1304   done < <(/bin/ls ${PWD}/*/core 2>/dev/null; /bin/ls ${PWD}/core 2>/dev/null)
1305
1306   return ${logstatus}
1307 }
1308
1309 spitOutLocalOCDBaccessConfig()
1310 {
1311   umask 0002
1312   #find ${1} -name "*root" | \
1313   /bin/ls -1 ${1}/*/*/*/*.root 2>/dev/null | \
1314   while read line
1315   do
1316     local tmp=${line#${1}}
1317     echo ${tmp%/*} | \
1318     awk -v ocdb=${1} '{print "  man->SetSpecificStorage(\""$1"\",\"local://"ocdb"\");"}'
1319   done
1320   return 0
1321 }
1322
1323 goMakeLocalOCDBaccessConfig()
1324 {
1325   umask 0002
1326   # make a script that sets the specific storages form all the root files produced by CPass0
1327   local localOCDBpathCPass0=${1}
1328   local OCDBpathPrefix=${2}
1329   [[ -z ${OCDBpathPrefix} ]] && OCDBpathPrefix="."
1330
1331   if [[ -f ${localOCDBpathCPass0} && ${localOCDBpathCPass0} =~ \.tgz$ ]]; then
1332     tar xzf ${localOCDBpathCPass0}
1333     local localOCDBpathCPass0="${OCDBpathPrefix}/OCDB"
1334   fi
1335
1336   echo
1337   echo creating the specific storage script
1338   echo   localOCDBaccessConfig.C
1339   echo   based on OCDB: ${localOCDBaccessConfig}
1340   echo
1341
1342   local tempLocalOCDB=""
1343   if [[ -f localOCDBaccessConfig.C ]]; then
1344     tempLocalOCDB=$(mktemp -t tempLocalOCDB.XXXXXX)
1345     echo "egrep "SetSpecificStorage" localOCDBaccessConfig.C > ${tempLocalOCDB}"
1346     egrep "SetSpecificStorage" localOCDBaccessConfig.C > ${tempLocalOCDB}
1347   fi
1348
1349   echo "localOCDBaccessConfig()"                               >  localOCDBaccessConfig.C
1350   echo "{"                                                     >> localOCDBaccessConfig.C
1351   echo "  AliCDBManager* man = AliCDBManager::Instance();"     >> localOCDBaccessConfig.C
1352   spitOutLocalOCDBaccessConfig ${localOCDBpathCPass0}|sort|uniq  >> localOCDBaccessConfig.C
1353   [[ -f "${tempLocalOCDB}" ]] && cat ${tempLocalOCDB}              >> localOCDBaccessConfig.C
1354   echo "}"                                                     >> localOCDBaccessConfig.C
1355
1356   [[ -f "${tempLocalOCDB}" ]] && rm -f ${tempLocalOCDB}
1357
1358   if ! grep SetSpecificStorage localOCDBaccessConfig.C; then
1359     echo
1360     echo "!!!!!!! CPass0 produced no OCDB entries"
1361     return 1
1362   fi
1363   return 0
1364 }
1365
1366 goMakeFilteredTrees()
1367 {
1368   outputDir=${1}
1369   runNumber=${2}
1370   #get path to input list
1371   inputListfiles=${3}
1372   #get scale number for tracks
1373   filterT=${4}
1374   #get scale number for V0s
1375   filterV=${5}
1376   #get OCDB path (optional)
1377   OCDBpath=${6}
1378   #get max number of files
1379   maxFiles=${7-"1000000"}
1380   #get offset of first file
1381   offsetFile=${8-"0"}
1382   #get max number of events
1383   maxEvents=${9-"30000000"}
1384   #get offset of first event
1385   offsetEvent=${10-"0"}
1386   configFile=${11-"benchmark.config"}
1387   esdFileName=${12-"AliESDs_Barrel.root"}
1388   shift 12
1389   if ! parseConfig ${configFile} "$@"; then return 1; fi
1390
1391   #record the working directory provided by the batch system
1392   batchWorkingDirectory=${PWD}
1393
1394
1395   commonOutputPath=${PWD}
1396   doneFile=${commonOutputPath}/meta/filtering.cpass1.run${runNumber}.done
1397
1398   runpath=${outputDir}
1399   #[[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
1400   [[ ${reconstructInTemporaryDir} -eq 1 ]] && runpath=$(mktemp -d -t goMakeFilteredTrees.XXXXXX)
1401
1402
1403   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
1404
1405   mkdir -p ${outputDir}
1406   mkdir -p ${runpath}
1407
1408   if ! cd ${runpath}; then
1409     echo "PWD=$PWD is not the runpath=${runpath}"
1410     touch ${doneFile}
1411     return 1
1412   fi
1413
1414   cat > filtering.log << EOF
1415   goMakeFilteredTrees config:
1416   runpath=${runpath}
1417   outputDir=${outputDir}
1418   commonOutputPath=${commonOutputPath}
1419   ALICE_ROOT=${ALICE_ROOT}
1420   PATH=${PATH}
1421   offsetEvent=$offsetEvent
1422   configFile=$configFile
1423   esdFileName=$esdFileName
1424 EOF
1425
1426   if [[ -z ${pretend} ]];then
1427     aliroot -l -b -q "${ALICE_ROOT}/PWGPP/macros/runFilteringTask.C(\"${inputListfiles}\",${filterT},${filterV},\"${OCDBpath}\",${maxFiles},${offsetFile},${maxEvents},${offsetEvent},\"${esdFileName}\")" &>> filtering.log
1428   else
1429     sleep ${pretendDelay}
1430     touch filtering.log FilterEvents_Trees.root
1431   fi
1432   pwd
1433   /bin/ls
1434   touch ${doneFile}
1435   summarizeLogs >>  ${doneFile}
1436   echo mv -f * ${outputDir}
1437   mv -f * ${outputDir}
1438   [[ -f ${outputDir}/FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFile}
1439   cd ${commonOutputPath}
1440   [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
1441   return 0
1442 }
1443
1444 submit()
1445 {
1446   umask 0002
1447   [[ $# -lt 5 ]] && echo "at least 5 args needed, you supplied $#" && return 1
1448   JobID=${1}
1449   startID=${2}
1450   endID=${3}
1451   waitForJOBID=${4}
1452   command=${5}
1453   shift 5
1454   commandArgs=("$@")
1455
1456   #add quote strings around the extra arguments
1457   for ((i=0; i<${#commandArgs[@]}; i++)); do
1458     commandArgs[i]=\"${commandArgs[i]}\"
1459   done
1460
1461   [[ -z ${waitForJOBID} ]] && waitForJOBID=0
1462
1463   newFarm=$(which qsub|grep "^/usr/bin/qsub")
1464
1465   batchSystem="SGE"
1466
1467   if [[ -z "${newFarm}" ]]
1468   then
1469     #old LSF
1470     # submit it (as job array)
1471     nFiles=$(( ${endID}-${startID}+1 ))
1472     while [ ${startID} -le ${nFiles}  ] ; do
1473       if [ $(expr ${nFiles} - ${startID}) -gt 999 ] ; then
1474         endID=$(expr ${startID} + 999)
1475       else
1476         endID=${nFiles}
1477       fi
1478     if [[ ${waitForJOBID} -eq 0 ]]; then
1479         echo ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}"
1480         ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}"
1481       else
1482         echo ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -w "ended(${waitForJOBID})" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}"
1483         ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -w "ended(${waitForJOBID})" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}"
1484       fi
1485       startID=$(expr ${endID} + 1)
1486     done
1487   else
1488     #new SGE farm
1489     if [[ ${waitForJOBID} =~ "000" ]]; then
1490       echo ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}"
1491       ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}"
1492     else
1493       echo ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -hold_jid "${waitForJOBID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}"
1494       ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -hold_jid "${waitForJOBID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}"
1495     fi
1496   fi
1497   return 0
1498 }
1499
1500 goSubmitBatch()
1501 {
1502   if [[ $# -lt 3 ]]; then
1503     echo "minimal use:"
1504     echo " ${0} submit fileList productionID configFile"
1505     return 0
1506   fi
1507
1508   productionID=${1}
1509   inputList=${2}
1510   configFile=${3:-"benchmark.config"}
1511   #if which greadlink; then configFile=$(greadlink -f ${configFile}); fi
1512   shift 3
1513   extraOpts=("$@")
1514   if ! parseConfig ${configFile} "${extraOpts[@]}"; then return 1; fi
1515
1516   #batch systems/makeflow sometimes handle spaces in arguments poorly, so encode them
1517   for (( i=0;i<${#extraOpts[@]};i++ )); do
1518     extraOpts[i]=$(encSpaces "${extraOpts[i]}")
1519   done
1520   extraOpts+=("encodedSpaces=1")
1521
1522   #record the working directory provided by the batch system
1523   batchWorkingDirectory=${PWD}
1524
1525   #redirect all output to submit.log
1526   echo "redirecting all output to ${PWD}/submit_${productionID//"/"/_}.log"
1527   exec 7>&1
1528   exec 1>submit_${productionID//"/"/_}.log 2>&1
1529
1530   umask 0002
1531   echo ${0}" submit $*"
1532   if [[ -z "${inputList}" || -z "${productionID}" ]]
1533   then
1534     echo
1535     echo " Usage: ${0} submit inputList productionID [configFile=benchmark.config]"
1536     echo
1537     return
1538   fi
1539
1540   # check if config file is there
1541   if [ ! -f ${configFile} ]; then
1542     echo "ERROR! Config File '${configFile}' not found" >&2
1543     return
1544   else
1545     echo "Using Config File: '${configFile}'"
1546   fi
1547
1548   [[ ! -f ${alirootEnv} ]] && echo "alirootEnv script ${alirootEnv} not found!..." && return 1
1549
1550   #move the script, config and some other stuff to ${commonOutputPath} first, then use them from there
1551   self=${0}
1552   #if which greadlink; then self=$(greadlink -f "${0}"); fi
1553   configPath=$(dirname ${configFile})
1554   export commonOutputPath=${baseOutputDirectory}/${productionID}
1555
1556   mkdir -p ${commonOutputPath}
1557   mkdir -p ${commonOutputPath}/logs
1558   mkdir -p ${commonOutputPath}/meta
1559
1560   cp ${self} ${commonOutputPath}
1561   cp ${configFile} ${commonOutputPath}
1562   cp ${inputList} ${commonOutputPath}
1563   self=${commonOutputPath}/${self##*/}
1564   chmod u+x ${self}
1565   configFile=${commonOutputPath}/${configFile##*/}
1566   inputList=${commonOutputPath}/${inputList##*/}
1567
1568   #convert to absolut pathnames
1569   #if which greadlink; then inputList=$(greadlink -f "${inputList}"); fi
1570   #make list of runs
1571   if [[ -z ${runNumber} ]]; then
1572     listOfRuns=($(while read x; do guessRunNumber ${x}; done < ${inputList} | sort | uniq))
1573   else
1574     listOfRuns=${runNumber}
1575   fi
1576
1577   #if which greadlink; then alirootSource=$(greadlink -f "${alirootSource}"); fi
1578
1579   echo ""
1580   echo "### BEGIN CONFIGURATION ###"
1581   echo ""
1582   echo "GENERAL:"
1583   echo ""
1584   echo "    productionID:    ${productionID}"
1585   echo "    batchCommand:    ${batchCommand}"
1586   echo "    batchFlags:      ${batchFlags}"
1587   echo "    alirootEnv:   ${alirootEnv}"
1588   ${alirootEnv} echo '    ALICE_ROOT:      ${ALICE_ROOT}'
1589   ${alirootEnv} echo '    ALIROOT_RELEASE: ${ALICE_RELEASE}'
1590   echo "    inputList:       ${inputList}"
1591   echo "    configPath:      ${configPath}"
1592   echo "    commonOutputPath:      ${commonOutputPath}"
1593   echo "    defaultOCDB:     ${defaultOCDB}"
1594   echo "      autoOCDB: ${autoOCDB}"
1595   echo "    recoTriggerOptions:   ${recoTriggerOptions}"
1596   echo "    runs:"
1597   echo "      ${listOfRuns[*]}"
1598   echo ""
1599   echo "THE TRAIN WILL RUN:"
1600
1601   if [ ${runCPass0reco} -eq 1 ]; then
1602     echo "    Pass0 - Recontruction"
1603   fi
1604
1605   if [ ${runCPass0MergeMakeOCDB} -eq 1 ]; then
1606     echo "    Pass0 - merging and OCDB export"
1607   fi
1608
1609   if [ ${runCPass1reco} -eq 1 ]; then
1610     echo "    Pass1 - Recontruction"
1611   fi
1612   if [ ${runCPass1MergeMakeOCDB} -eq 1 ]; then
1613     echo "    Pass1 - merging and OCDB export"
1614   fi
1615
1616   echo ""
1617   echo "LIMITS:"
1618   echo "    max. Events/Chunk:   ${nEvents}"
1619   echo "    max. Number of Chunks per Run:     ${nMaxChunks}"
1620   echo ""
1621   echo "### END CONFIGURATION ###"
1622   echo ""
1623
1624
1625   # check if input file is there
1626   if [ ! -f ${inputList} ]; then
1627     echo "ERROR! Input List '${inputList}' not found" >&2
1628     return
1629   fi
1630
1631   # define jobid (for dependent jobs)
1632   date=$(date +%Y_%m_%d_%H%M%S)
1633   #for each run we submit one jobarray:
1634   for runNumber in ${listOfRuns[*]}; do
1635
1636     [[ -z ${runNumber} ]] && continue
1637     [[ ! ${runNumber} =~ ^[0-9]*[0-9]$ ]] && continue
1638
1639     JOBpostfix="${productionID//"/"/_}_${runNumber}_${date}"
1640     JOBID1="p0_${JOBpostfix}"
1641     JOBID1wait="w0_${JOBpostfix}"
1642     JOBID2="m0_${JOBpostfix}"
1643     JOBID2wait="wm0_${JOBpostfix}"
1644     JOBID3="op0_${JOBpostfix}"
1645     JOBID3wait="wop0_${JOBpostfix}"
1646     JOBID4="p1_${JOBpostfix}"
1647     JOBID4wait="w1_${JOBpostfix}"
1648     JOBID5="m1_${JOBpostfix}"
1649     JOBID5wait="wm1_${JOBpostfix}"
1650     JOBID6="s1_${JOBpostfix}"
1651     JOBID6wait="ws1_${JOBpostfix}"
1652     JOBID7="QA_${JOBpostfix}"
1653     JOBmakeESDlistCPass1="lp1_${JOBpostfix}"
1654     JOBfilterESDcpass1="fp1_${JOBpostfix}"
1655     LASTJOB="000"
1656
1657     oneInputFile=$(egrep -m1 "${runNumber}/" ${inputList})
1658
1659     currentDefaultOCDB=${defaultOCDB}
1660     [[ -z ${autoOCDB} ]] && autoOCDB=1
1661     if [[ ${autoOCDB} -ne 0 ]]; then
1662       currentDefaultOCDB=$(setYear ${oneInputFile} ${defaultOCDB})
1663     fi
1664
1665     echo "submitting run ${runNumber} with OCDB ${currentDefaultOCDB}"
1666
1667     ###############################################################################
1668     #run one chunk with valgrind:
1669     if [[ -n ${profilingCommand} ]]; then
1670       [[ -z ${nEventsProfiling} ]] && nEventsProfiling=2
1671       [[ -z ${profilingCommand} ]] && profilingCommand="/usr/bin/valgrind --tool=callgrind --num-callers=40 -v --trace-children=yes"
1672       submit "valgrind" 1 1 000 "${alirootEnv} ${self}" CPass0 ${commonOutputPath}/000${runNumber}/valgrind ${oneInputFile} ${nEventsProfiling} ${currentDefaultOCDB} ${configFile} ${runNumber} valgrind useProfilingCommand=${profilingCommand} "${extraOpts[@]}"
1673     fi
1674
1675     ################################################################################
1676     ################################################################################
1677     # run the CPass0 if requested
1678
1679     if [ ${runCPass0reco} -eq 1 ]; then
1680
1681       echo
1682       echo "starting CPass0... for run ${runNumber}"
1683       echo
1684
1685       # create directory and copy all files that are needed
1686       targetDirectory="${commonOutputPath}/000${runNumber}/cpass0"
1687       mkdir -p ${targetDirectory}
1688
1689       filesCPass0=(
1690                     "${configPath}/runCPass0.sh"
1691                     "${configPath}/recCPass0.C"
1692                     "${configPath}/runCalibTrain.C"
1693                     "${configPath}/localOCDBaccessConfig.C"
1694                     "${configPath}/OCDB*.root"
1695                     "${configPath}/sim.C"
1696                     "${configPath}/rec.C"
1697                     "${configPath}/Config.C"
1698       )
1699       for file in ${filesCPass0[*]}; do
1700         [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
1701       done
1702
1703       localInputList=${targetDirectory}/${inputList##*/}
1704       [[ ! -f ${localInputList} ]] && egrep "\/000${runNumber}\/" ${inputList} > ${localInputList}
1705       # limit nFiles to nMaxChunks
1706       nFiles=$(wc -l < ${localInputList})
1707       [[ ${nFiles} -eq 0 ]] && echo "list contains ZERO files! exiting..." && return 1
1708       echo "raw files in list:    ${nFiles}"
1709       if [[ ${nMaxChunks} -gt 0 && ${nMaxChunks} -le ${nFiles} ]]; then
1710         nFiles=${nMaxChunks}
1711       fi
1712       echo "raw files to process: ${nFiles}"
1713       [[ -z "${percentProcessedFilesToContinue}" ]] && percentProcessedFilesToContinue=100
1714       if [[ ${percentProcessedFilesToContinue} -eq 100 ]]; then
1715         nFilesToWaitFor=${nFiles}
1716       else
1717         nFilesToWaitFor=$(( ${nFiles}-${nFiles}/(100/(100-${percentProcessedFilesToContinue})) ))
1718       fi
1719       echo "requested success rate is ${percentProcessedFilesToContinue}%"
1720       echo "merging will start after ${nFilesToWaitFor} jobs are done"
1721
1722       submit ${JOBID1} 1 ${nFiles} 000 "${alirootEnv} ${self}" CPass0 ${targetDirectory} ${localInputList} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} -1 "${extraOpts[@]}"
1723
1724       ## submit a monitoring job that will run until a certain number of jobs are done with reconstruction
1725       submit "${JOBID1wait}" 1 1 000 "${alirootEnv} ${self}" WaitForOutput ${commonOutputPath} "meta/cpass0.job*.run${runNumber}.done" ${nFilesToWaitFor} ${maxSecondsToWait} '-maxdepth 1'
1726       LASTJOB=${JOBID1wait}
1727
1728     fi #end running CPass0
1729     ################################################################################
1730
1731
1732     ################################################################################
1733     # submit merging of CPass0, depends on the reconstruction
1734
1735     if [ ${runCPass0MergeMakeOCDB} -eq 1 ]; then
1736
1737       echo
1738       echo "submit CPass0 merging for run ${runNumber}"
1739       echo
1740
1741       targetDirectory="${commonOutputPath}/000${runNumber}/cpass0"
1742       mkdir -p ${targetDirectory}
1743
1744       #copy the scripts
1745       filesMergeCPass0=(
1746                         "${configPath}/OCDB.root"
1747                         "${configPath}/mergeMakeOCDB.byComponent.sh"
1748                         "${configPath}/mergeMakeOCDB.sh"
1749                         "${configPath}/localOCDBaccessConfig.C"
1750                         "${configPath}/mergeByComponent.C"
1751                         "${configPath}/makeOCDB.C"
1752                         "${configPath}/merge.C"
1753       )
1754       for file in ${filesMergeCPass0[*]}; do
1755         [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
1756       done
1757
1758       submit ${JOBID2} 1 1 "${LASTJOB}" "${alirootEnv} ${self}" MergeCPass0 ${targetDirectory} ${currentDefaultOCDB} ${configFile} ${runNumber} cpass0.calib.run${runNumber}.list "${extraOpts[@]}"
1759       LASTJOB=${JOBID2}
1760
1761       if [[ -n ${generateMC} ]]; then
1762         submit "mrl${JOBpostfix}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" PrintValues sim ${commonOutputPath}/meta/sim.run${runNumber}.list ${commonOutputPath}/meta/cpass0.job*.run${runNumber}.done
1763         LASTJOB="mrl${JOBpostfix}"
1764       fi
1765
1766       echo
1767     fi
1768     # end of merging CPass0
1769     ################################################################################
1770
1771     ################################################################################
1772     ################################################################################
1773     # run the CPass1 if requested
1774
1775     if [ ${runCPass1reco} -eq 1 ]; then
1776
1777       targetDirectory="${commonOutputPath}/000${runNumber}/cpass1"
1778       rm -f ${commonOutputPath}/meta/cpass1.job*.run${runNumber}.done
1779
1780       # safety feature: if we are re-running for any reason we want to delete the previous output first.
1781       [[ -d ${targetDirectory} ]] && rm -rf ${targetDirectory}/* && echo "removed old output at ${targetDirectory}/*"
1782
1783       echo
1784       echo "starting CPass1... for run ${runNumber}"
1785       echo
1786
1787       # create directory and copy all files that are needed
1788       mkdir -p ${targetDirectory}
1789
1790       filesCPass1=(
1791                     "${configPath}/runCPass1.sh"
1792                     "${configPath}/recCPass1.C"
1793                     "${configPath}/recCPass1_OuterDet.C"
1794                     "${configPath}/runCalibTrain.C"
1795                     "${configPath}/QAtrain_duo.C"
1796                     "${configPath}/localOCDBaccessConfig.C"
1797                     "${configPath}/OCDB.root"
1798       )
1799       for file in ${filesCPass1[*]}; do
1800         [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
1801       done
1802
1803       if [[ -n ${generateMC} ]]; then
1804         localInputList=${commonOutputPath}/meta/sim.run${runNumber}.list
1805       else
1806         localInputList=${targetDirectory}/${inputList##*/}
1807         [[ ! -f ${localInputList} ]] && egrep "\/000${runNumber}\/" ${inputList} > ${localInputList}
1808       fi
1809       # limit nFiles to nMaxChunks
1810       nFiles=$(wc -l < ${localInputList})
1811       [[ ${nFiles} -eq 0 ]] && echo "list contains ZERO files! continuing..." && continue
1812       echo "raw files in list:    ${nFiles}"
1813       if [[ ${nMaxChunks} -gt 0 && ${nMaxChunks} -le ${nFiles} ]]; then
1814         nFiles=${nMaxChunks}
1815       fi
1816       echo "raw files to process: ${nFiles}"
1817       [[ -z "${percentProcessedFilesToContinue}" ]] && percentProcessedFilesToContinue=100
1818       if [[ ${percentProcessedFilesToContinue} -eq 100 ]]; then
1819         nFilesToWaitFor=${nFiles}
1820       else
1821         nFilesToWaitFor=$(( ${nFiles}-${nFiles}/(100/(100-${percentProcessedFilesToContinue})) ))
1822       fi
1823       echo "requested success rate is ${percentProcessedFilesToContinue}%"
1824       echo "merging will start after ${nFilesToWaitFor} jobs are done"
1825
1826       submit ${JOBID4} 1 ${nFiles} "${LASTJOB}" "${alirootEnv} ${self}" CPass1 ${targetDirectory} ${localInputList} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} -1 "${extraOpts[@]}"
1827
1828       ################################################################################
1829       ## submit a monitoring job that will run until a certain number of jobs are done with reconstruction
1830       submit "${JOBID4wait}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" WaitForOutput ${commonOutputPath} "meta/cpass1.job*.run${runNumber}.done" ${nFilesToWaitFor} ${maxSecondsToWait} '-maxdepth 1'
1831       LASTJOB=${JOBID4wait}
1832       ################################################################################
1833
1834       echo
1835     fi #end running CPass1
1836
1837     ################################################################################
1838     # submit merging of CPass1, depends on the reconstruction
1839     if [ ${runCPass1MergeMakeOCDB} -eq 1 ]; then
1840
1841       echo
1842       echo "submit CPass1 merging for run ${runNumber}"
1843       echo
1844
1845       targetDirectory="${commonOutputPath}/000${runNumber}/cpass1"
1846       rm -f ${commonOutputPath}/meta/merge.cpass1.run${runNumber}.done
1847       mkdir -p ${targetDirectory}
1848
1849       # copy files
1850       filesMergeCPass1=(
1851                         "${configPath}/OCDB.root"
1852                         "${configPath}/localOCDBaccessConfig.C"
1853                         "${configPath}/mergeMakeOCDB.byComponent.sh"
1854                         "${configPath}/mergeByComponent.C"
1855                         "${configPath}/makeOCDB.C"
1856                         "${configPath}/merge.C"
1857                         "${configPath}/mergeMakeOCDB.sh"
1858                         "${configPath}/QAtrain_duo.C"
1859       )
1860       for file in ${filesMergeCPass1[*]}; do
1861         [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
1862       done
1863
1864       submit "${JOBID5}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" MergeCPass1 ${targetDirectory} ${currentDefaultOCDB} ${configFile} ${runNumber} cpass1.calib.run${runNumber}.list cpass1.QA.run${runNumber}.lastMergingStage.txt.list cpass1.filtered.run${runNumber}.list "${extraOpts[@]}"
1865       LASTJOB=${JOBID5}
1866       echo
1867     fi
1868
1869     ###############################
1870     #if [ ${runESDfiltering} -eq 1 ]; then
1871     #  rm -f ${commonOutputPath}/cpass1.ESD.run${runNumber}.list
1872     #  rm -f ${commonOutputPath}/meta/filtering.cpass1.run*.done
1873     #  echo
1874     #  echo submitting filtering for run ${runNumber}
1875     #  echo
1876     #  submit "${JOBmakeESDlistCPass1}" 1 1 "${LASTJOB}" "${self}" PrintValues esd ${commonOutputPath}/meta/cpass1.ESD.run${runNumber}.list ${commonOutputPath}/meta/cpass1.job*.run${runNumber}.done
1877     #  submit "${JOBfilterESDcpass1}" 1 1 "${JOBmakeESDlistCPass1}" "${alirootEnv} ${self}" MakeFilteredTrees ${commonOutputPath}/000${runNumber}/cpass1 ${runNumber} ${commonOutputPath}/meta/cpass1.ESD.run${runNumber}.list ${filteringFactorHighPt} ${filteringFactorV0s} ${currentDefaultOCDB} 1000000 0 10000000 0 ${configFile} AliESDs_Barrel.root "${extraOpts[@]}"
1878     #  LASTJOB=${JOBfilterESDcpass1}
1879     #fi
1880
1881   done
1882
1883   #################################################################################
1884   #################################################################################
1885   #if [ ${runESDfiltering} -eq 1 ]; then
1886   #  submit "${JOBID5wait}" 1 1 "${LASTJOB}" "${self}" WaitForOutput ${commonOutputPath} "meta/filtering.cpass1.run*.done" "${#listOfRuns[@]}" ${maxSecondsToWait}
1887   #else
1888     submit "${JOBID5wait}" 1 1 "${LASTJOB}" "${self}" WaitForOutput ${commonOutputPath} "meta/merge.cpass1.run*.done" ${#listOfRuns[@]} ${maxSecondsToWait}
1889   #fi
1890   LASTJOB=${JOBID5wait}
1891
1892   #################################################################################
1893   echo
1894   echo "submit make a summary"
1895   echo
1896
1897   submit "${JOBID6}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" MakeSummary ${configFile}
1898   LASTJOB=${JOBID6}
1899   #################################################################################
1900
1901   #restore stdout
1902   exec 1>&7 7>&-
1903   echo "jobs submitted."
1904   return 0
1905 }
1906
1907 goWaitForOutput()
1908 {
1909   umask 0002
1910   [[ $# -lt 3 ]] && echo "goWaitForOutput() wrong number of arguments, exiting.." && return 1
1911   echo searchPath=${1}
1912   echo fileName=${2}
1913   echo numberOfFiles=${3}
1914   echo maxSecondsToWait=${4}
1915   searchPath=${1}
1916   fileName=${2}
1917   numberOfFiles=${3}
1918   maxSecondsToWait=${4}
1919   extraFindOptions=${5}
1920   echo "command to be executed: /bin/ls -1 ${searchPath}/${fileName} ${extraFindOptions}"
1921   [[ -z "${maxSecondsToWait}" ]] && maxSecondsToWait=$(( 3600*12 ))
1922   while true; do
1923     #n=$(find ${searchPath} ${extraFindOptions} -name "${fileName}" | wc -l)
1924     n=$(/bin/ls -1 ${searchPath}/${fileName} 2>/dev/null | wc -l)
1925     [[ ${n} -gt 0 ]] && echo "found ${n} X ${fileName}"
1926     [[ ${n} -ge ${numberOfFiles} ]] && break
1927     [[ ${SECONDS} -gt ${maxSecondsToWait} ]] && echo "timeout of ${maxSecondsToWait}s!" && break
1928     sleep 60
1929   done
1930   echo "DONE! exiting..."
1931   return 0
1932 }
1933
1934 mergeSysLogs()
1935 {
1936   outputFile=${1}
1937   shift
1938   inputFiles="$@"
1939   i=0
1940   if ! ls -1 ${inputFiles} &>/dev/null; then echo "the files dont exist!: ${inputFiles}"; return 1; fi
1941   while read x; do
1942     runNumber=$(guessRunNumber ${x})
1943     [[ -z ${runNumber} ]] && echo "run number cannot be guessed for ${x}" && continue
1944     awk -v run=${runNumber} -v i=${i} 'NR > 1 {print run" "$0} NR==1 && i==0 {print "run/I:"$0}' ${x}
1945     (( i++ ))
1946   done < <(ls -1 ${inputFiles}) > ${outputFile}
1947   return 0
1948 }
1949
1950 goMakeMergedSummaryTree()
1951 {
1952   # create list of calibration entries
1953   # takes no arguments, just run it in the base output
1954   # directory with the following files in the working directory
1955   #
1956   #  Calibration file lists:
1957   #       cpass0.dcsTree.list, cpass1.dcsTree.list
1958   #  QA trending root file:
1959   #       trending.root
1960   #
1961   #  Production infoo ascii files:
1962   #       summary_pass0.tree
1963   #       summary_pass1.tree
1964   #
1965
1966   [[ ! -f cpass0.dcsTree.list ]] && echo "no cpass0.dcsTree.list" && return 1
1967   [[ ! -f cpass1.dcsTree.list ]] && echo "no cpass1.dcsTree.list" && return 1
1968   [[ ! -f trending.root ]] && echo "no trending.root" && return 1
1969   [[ ! -f summary_pass0.tree ]] && echo "no summary_pass0.tree" && return 1
1970   [[ ! -f summary_pass1.tree ]] && echo "no summary_pass1.tree" && return 1
1971
1972   #first, dump the C macro to file
1973   cat << EOF > mergeTree.C
1974   //
1975   // Merge summary information
1976   // Following files are expected to be in the working directory
1977   //
1978   // Calibration file lists:
1979   //      cpass0.dcsTree.list, cpass1.dcsTree.list
1980   // QA trending root file:
1981   //      trending.root
1982   //
1983   // Production infoo ascii files:
1984   //      summary_pass0.tree
1985   //      summary_pass1.tree
1986   //
1987   void mergeTree(){
1988     //
1989     //
1990     //
1991     // Calibration values dump
1992     //
1993     //Printf("MakeTreeFromList cpass0.dcsTree.list");
1994     AliXRDPROOFtoolkit::MakeTreeFromList("Calib.TPC.CPass0.root", "dcs","dcs","cpass0.dcsTree.list",1);
1995     //Printf("MakeTreeFromList cpass1.dcsTree.list");
1996     AliXRDPROOFtoolkit::MakeTreeFromList("Calib.TPC.CPass1.root", "dcs","dcs","cpass1.dcsTree.list",1);
1997     //
1998     // Calibration status dump
1999     //
2000     TFile *fprod = TFile::Open("fproduction.root","recreate");
2001     TTree  tree0, tree1;
2002     //Printf("reading summary_pass0.tree");
2003     tree0.ReadFile("summary_pass0.tree");
2004     //Printf("reading summary_pass1.tree");
2005     tree1.ReadFile("summary_pass1.tree");
2006     tree0.Write("CPass0");
2007     tree1.Write("CPass1");
2008     fprod->Close();
2009     //
2010     //
2011     //
2012     TString stringSetup="";
2013     stringSetup+="1#QA.TPC#run#SummaryTPCQA/tpcQA#trending.root+";  //
2014     stringSetup+="1#Calib.TPC.CPass0#run#dcs#Calib.TPC.CPass0.root+";  //
2015     stringSetup+="1#Calib.TPC.CPass1#run#dcs#Calib.TPC.CPass1.root+";  //
2016     //
2017     stringSetup+="1#CPass0#runnumber#CPass0#fproduction.root+";  //
2018     stringSetup+="1#CPass1#runnumber#CPass1#fproduction.root+";  //
2019     //
2020     //Printf("stringSetup: %s", stringSetup.Data());
2021     AliXRDPROOFtoolkit::JoinTreesIndex("outAll.root","joinAll","run",stringSetup.Data(), 1);
2022   }
2023 EOF
2024
2025   aliroot -b -q "mergeTree.C" > mergeTrees.log
2026   return $?
2027 }
2028
2029 goMakeSummary()
2030 {
2031   #all the final stuff goes in here for ease of use:
2032   # summary logs
2033   # qa plot making
2034   # final file lists
2035   #some defaults:
2036   log="summary.log"
2037   productionID="qa"
2038
2039   configFile=${1}
2040   shift 1
2041   extraOpts=("$@")
2042   if ! parseConfig ${configFile} "${extraOpts[@]}"; then return 1; fi
2043
2044   #if which greadlink; then configFile=$(greadlink -f ${configFile}); fi
2045
2046   #record the working directory provided by the batch system
2047   batchWorkingDirectory=${PWD}
2048
2049   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
2050
2051   [[ ! -f ${configFile} ]] && echo "no config file ${configFile}!" && return
2052
2053   [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
2054
2055   #copy some useful stuff
2056   #and go to the commonOutputPath
2057   cp ${configFile} ${commonOutputPath}
2058   cd ${commonOutputPath}
2059
2060   exec &> >(tee ${log})
2061
2062   #summarize the global stuff
2063   echo "env script: ${alirootSource} ${alirootEnv}"
2064   echo "\$ALICE_ROOT=${ALICE_ROOT}"
2065   echo "commonOutputPath=${commonOutputPath}"
2066
2067   #summarize the stacktraces
2068   awk '
2069        BEGIN {
2070                print "frame/I:method/C:line/C:cpass/I:aliroot/I";
2071                RS="#[0-9]*";
2072                aliroot=0;
2073              }
2074              {
2075                if ($3 ~ /Ali*/) aliroot=1; else aliroot=0;
2076                gsub("#","",RT);
2077                if ($NF!="" && RT!="" && $3!="") print RT" "$3" "$NF" "0" "aliroot
2078              }
2079       ' 000*/cpass0/*/stacktrace* 2>/dev/null > stacktrace.tree
2080   awk '
2081        BEGIN {
2082                RS="#[0-9]*";
2083                aliroot=0;
2084              }
2085              {
2086                if ($3 ~ /Ali*/) aliroot=1; else aliroot=0;
2087                gsub("#","",RT);
2088                if ($NF!="" && RT!="" && $3!="") print RT" "$3" "$NF" "1" "aliroot
2089              }
2090       ' 000*/cpass1/*/stacktrace* 2>/dev/null >> stacktrace.tree
2091
2092   echo total numbers for the production:
2093   echo
2094   awk 'BEGIN {nFiles=0;nCore=0;}
2095   /^calibfile/ {nFiles++;}
2096   /core dumped/ {nCore++i;}
2097   END {print     "cpass0 produced "nFiles" calib files, "nCore" core files";}' meta/cpass0.job*done 2>/dev/null
2098   awk 'BEGIN {nOK=0; nBAD=0; }
2099   /\/rec.log OK/ {nOK++;}
2100   /\/rec.log BAD/ {nBAD++;}
2101   /stderr BAD/ {if ($0 ~ /rec.log/){nBAD++;}}
2102   END {print     "cpass0 reco:  OK: "nOK"\tBAD: "nBAD;}' meta/cpass0.job*done 2>/dev/null
2103   awk 'BEGIN {nOK=0; nBAD=0; }
2104   /\/calib.log OK/ {nOK++;}
2105   /\/calib.log BAD/ {nBAD++;}
2106   END {print "cpass0 calib: OK: "nOK"\tBAD: "nBAD;}' meta/cpass0.job*done 2>/dev/null
2107
2108   awk 'BEGIN {nOK=0; nBAD=0; }
2109   /merge.log OK/ {nOK++;}
2110   /merge.log BAD/ {nBAD++;}
2111   END {print "cpass0 merge: OK: "nOK"\tBAD: "nBAD;}' meta/merge.cpass0*done 2>/dev/null
2112   awk 'BEGIN {nOK=0; nBAD=0; }
2113   /ocdb.log OK/ {nOK++;}
2114   /ocdb.log BAD/ {nBAD++;}
2115   END {print   "cpass0 OCDB:  OK: "nOK"\tBAD: "nBAD;}' meta/merge.cpass0*done 2>/dev/null
2116
2117   echo
2118   awk 'BEGIN {nFiles=0;nCore=0;}
2119   /^calibfile/ {nFiles++;}
2120   /core dumped/ {nCore++;}
2121   END {print     "cpass1 produced "nFiles" calib files, "nCore" core files";}' meta/cpass1.job*done 2>/dev/null
2122   awk 'BEGIN {nOK=0; nBAD=0; }
2123   /\/rec.log OK/ {nOK++;}
2124   /\/rec.log BAD/ {nBAD++;}
2125   /stderr BAD/ {if ($0 ~ /rec.log/){nBAD++;}}
2126   END {print     "cpass1 reco:  OK: "nOK"\tBAD: "nBAD;}' meta/cpass1.job*done 2>/dev/null
2127   awk 'BEGIN {nOK=0; nBAD=0; }
2128   /\/calib.log OK/ {nOK++;}
2129   /\/calib.log BAD/ {nBAD++;}
2130   END {print "cpass1 calib: OK: "nOK"\tBAD: "nBAD;}' meta/cpass1.job*done 2>/dev/null
2131
2132   awk 'BEGIN {nOK=0; nBAD=0; }
2133   /merge.log OK/ {nOK++;}
2134   /merge.log BAD/ {nBAD++;}
2135   END {print "cpass1 merge: OK: "nOK"\tBAD: "nBAD;}' meta/merge.cpass1*done 2>/dev/null
2136   awk 'BEGIN {nOK=0; nBAD=0; }
2137   /ocdb.log OK/ {nOK++;}
2138   /ocdb.log BAD/ {nBAD++;}
2139   END {print   "cpass1 OCDB:  OK: "nOK"\tBAD: "nBAD;}' meta/merge.cpass1*done 2>/dev/null
2140
2141   echo
2142   echo per run stats:
2143   /bin/ls -1 meta/merge.cpass0.run*.done | while read x
2144 do
2145   dir=$(goPrintValues calibfile - ${x})
2146   runNumber=$(guessRunNumber ${dir})
2147   [[ -z ${runNumber} ]] && continue
2148
2149   if $(/bin/ls meta/cpass0.job*.run${runNumber}.done &> /dev/null); then
2150     statusCPass0=( $(
2151     awk 'BEGIN {nOKrec=0;nBADrec=0;nOKcalib=0;nBADcalib=0;nOKstderr=0;nBADstderr=0;}
2152     /\/rec.log OK/ {nOKrec++;}
2153     /\/rec.log BAD/ {nBADrec++;}
2154     /stderr BAD/ {if ($0 ~ /rec.log/) {nBADrec++;} nBADstderr++;}
2155     /stderr OK/ {nOKstderr++;}
2156     /\/calib.log OK/ {nOKcalib++;}
2157     /\/calib.log BAD/ {nBADcalib++}
2158     END {print ""nOKrec" "nBADrec" "nOKstderr" "nBADstderr" "nOKcalib" "nBADcalib;}' meta/cpass0.job*.run${runNumber}.done 2>/dev/null
2159     ) )
2160   fi
2161
2162   if $(/bin/ls meta/cpass1.job*.run${runNumber}.done &>/dev/null); then
2163     statusCPass1=( $(
2164     awk 'BEGIN {nOKrec=0;nBADrec=0;nOKcalib=0;nBADcalib=0;nOKstderr=0;nBADstderr=0;nQAbarrelOK=0;nQAbarrelBAD=0;nQAouterOK=0;nQAouterBAD=0;}
2165     /\/rec.log OK/ {nOKrec++;}
2166     /\/rec.log BAD/ {nBADrec++;}
2167     /stderr BAD/ {if ($0 ~ /rec.log/) nBADrec++;nBADstderr++;}
2168     /stderr OK/ {nOKstderr++;}
2169     /\/calib.log OK/ {nOKcalib++;}
2170     /\/calib.log BAD/ {nBADcalib++}
2171     /\/qa_barrel.log OK/ {nQAbarrelOK++;}
2172     /\/qa_barrel.log BAD/ {nQAbarrelBAD++;}
2173     /\/qa_outer.log OK/ {nQAouterOK++;}
2174     /\/qa_outer.log BAD/ {nQAouterBAD++;}
2175     END {print ""nOKrec" "nBADrec" "nOKstderr" "nBADstderr" "nOKcalib" "nBADcalib" "nQAbarrelOK" "nQAbarrelBAD" "nQAouterOK" "nQAouterBAD;}' meta/cpass1.job*.run${runNumber}.done 2>/dev/null
2176     ) )
2177   fi
2178
2179   statusOCDBcpass0=$(awk '/ocdb.log/ {print $2} ' ${x} 2>/dev/null)
2180   statusOCDBcpass1=$(awk '/ocdb.log/ {print $2}' ${x/cpass0/cpass1} 2>/dev/null)
2181   statusQA=$(awk '/mergeMakeOCDB.log/ {print $2}' ${x/cpass0/cpass1} 2>/dev/null)
2182
2183   printf "%s\t ocdb.log cpass0: %s\t ocdb.log cpass1: %s\tqa.log:%s\t| cpass0: rec:%s/%s stderr:%s/%s calib:%s/%s cpass1: rec:%s/%s stderr:%s/%s calib:%s/%s QAbarrel:%s/%s QAouter:%s/%s\n" ${runNumber} ${statusOCDBcpass0} ${statusOCDBcpass1} ${statusQA} ${statusCPass0[0]} ${statusCPass0[1]} ${statusCPass0[2]} ${statusCPass0[3]} ${statusCPass0[4]} ${statusCPass0[5]} ${statusCPass1[0]} ${statusCPass1[1]} ${statusCPass1[2]} ${statusCPass1[3]} ${statusCPass1[4]} ${statusCPass1[5]} ${statusCPass1[6]} ${statusCPass1[7]} ${statusCPass1[8]} ${statusCPass1[9]}
2184 done
2185
2186   #make lists with output files - QA, trending, filtering and calibration
2187   ### wait for the merging of all runs to be over ###
2188   rm -f qa.list
2189   goPrintValues qafile qa.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2190   rm -f calib.list
2191   goPrintValues calibfile calib.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2192   rm -f trending.list
2193   goPrintValues trendingfile trending.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2194   rm -f filtering.list
2195   goPrintValues filteredTree filtering.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2196   rm -f cpass0.dcsTree.list
2197   goPrintValues dcsTree cpass0.dcsTree.list ${commonOutputPath}/meta/merge.cpass0.run*.done &>/dev/null
2198   rm -f cpass1.dcsTree.list
2199   goPrintValues dcsTree cpass1.dcsTree.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2200
2201   #merge trending
2202   rm -f ${commonOutputPath}/trending_merged.root
2203   goMerge trending.list ${commonOutputPath}/trending.root ${configFile} "${extraOpts[@]}" &> mergeTrending.log
2204
2205   goMakeSummaryTree ${commonOutputPath} 0
2206   goMakeSummaryTree ${commonOutputPath} 1
2207
2208   goCreateQAplots "${PWD}/qa.list" "${productionID}" "QAplots" "${configFile}" "${extraOpts[@]}" filteringList="${PWD}/filtering.list" &>createQAplots.log
2209
2210   #make a merged summary tree out of the QA trending, dcs trees and log summary trees
2211   goMakeMergedSummaryTree
2212
2213   #if set, email the summary
2214   [[ -n ${MAILTO} ]] && cat ${log} | mail -s "benchmark ${productionID} done" ${MAILTO}
2215
2216   return 0
2217 }
2218
2219 goMakeSummaryTree()
2220 {
2221   if [[ $# -lt 1 ]] ; then
2222     return
2223   fi
2224   #1. define vars/arrays
2225   DIR=${1} #use input or exec in current dir
2226   pass=${2-"0"} #pass from input
2227   outfile="summary_pass${pass}.tree"
2228   Ncolumns=0
2229   test -f ${outfile} && : >${outfile}
2230   errfile=${outfile/tree/err}
2231   test -f ${errfile} && : >${errfile}
2232
2233   declare -a counterString=(TOFevents TOFtracks TPCevents TPCtracks TRDevents TRDtracks T0events SDDevents SDDtracks MeanVertexevents)
2234   Ncounter=${#counterString[@]}
2235
2236   declare -a statusString=(TRDStatus TOFStatus TPCStatus T0Status MeanVertexStatus)
2237   Nstatus=${#statusString[@]}
2238
2239
2240   declare -a ratesString=(rec stderr calib qa_barrel qa_outer)
2241   Nrates=${#ratesString[@]}
2242
2243   runs=( $(ls -1 ${DIR}/meta/merge.cpass0* | while read x; do guessRunNumber $x; done) )
2244   Nruns=${#runs[@]}
2245
2246   echo -n runnumber/I >>${outfile}
2247   echo -n :cpass${pass}status/I >>${outfile}
2248   echo -n :cpass${pass}QAstatus/I >>${outfile}
2249   for i in ${ratesString[@]}; do
2250     echo -n :${i}OK/I >>${outfile}
2251     echo -n :${i}BAD/I >>${outfile}
2252
2253   done
2254   for i in ${counterString[@]} ${statusString[@]} ; do
2255     echo -n :${i}/I >>${outfile}
2256   done
2257   Ncolumns=$((2 + 2*Nrates + Ncounter + Nstatus))
2258   echo >> ${outfile}
2259
2260   #2. loop runs
2261
2262   for runnumber in ${runs[@]} ; do
2263
2264
2265
2266     filejob="${DIR}/meta/cpass${pass}.job*.run${runnumber}.done"
2267     filemerge="${DIR}/meta/merge.cpass${pass}.run${runnumber}.done"
2268     fileOCDB=$(grep /ocdb.log ${filemerge} | awk '{print $1}')
2269     if ! $(/bin/ls ${filemerge} &>/dev/null) ; then
2270       echo "${filemerge} does not exist!" >>${errfile}
2271       continue
2272     elif ! $(/bin/ls ${filejob} &>/dev/null) ; then
2273       echo "${filejob} does not exist!" >>${errfile}
2274       echo -n ${runnumber} >> ${outfile}
2275       for i in $(seq ${Ncolumns}) ; do
2276         echo -n "-1" >> ${outfile}
2277       done
2278       echo >> ${outfile}
2279       continue
2280     fi
2281     echo -n ${runnumber} >> ${outfile}
2282     #pass0status= grep '/ocdb.log' ${filemerge} |  cut -d' ' -f2 | tr OK x1 | tr BAD xx0 | tr -d 'x'
2283     passStatus=$(grep '/ocdb.log' ${filemerge} | grep OK | wc -l)
2284     echo -n " ${passStatus}" >> ${outfile}
2285     qaStatus=$(grep '/mergeMakeOCDB.log' ${filemerge} | grep OK | wc -l)
2286     echo -n " ${qaStatus}" >> ${outfile}
2287
2288
2289     #fill OK/BAD rates
2290     for i in $(seq 0 $((${Nrates}-1))) ; do
2291       var1=$(grep "/${ratesString[${i}]}.log" ${filejob} | grep OK | wc -l)
2292       var2=$(grep "/${ratesString[${i}]}.log" ${filejob} | grep BAD | wc -l)
2293
2294       if [[ ${ratesString[${i}]} == "stderr" ]] ; then
2295         var1=$(grep "stderr" ${filejob} | grep OK | wc -l)
2296         var2=$(grep "stderr" ${filejob} | grep "rec.log" | grep BAD | wc -l)
2297       fi
2298       echo -n " ${var1}" >> ${outfile}
2299       echo -n " ${var2}" >> ${outfile}
2300     done
2301
2302     if [[ -f ${fileOCDB} ]] ; then
2303       #fill counter
2304       for i in $(seq 0 $((${Ncounter}-1))) ; do
2305         var1=$(grep Monalisa ${fileOCDB} | grep ${counterString[${i}]} | cut -f2)
2306         echo -n " ${var1:-"-1"}" >> ${outfile}
2307       done
2308
2309       #fill status
2310       for i in $(seq 0 $((${Nstatus}-1))) ; do
2311         var1=$(grep "calibration status=" ${fileOCDB} | grep ${statusString[${i}]/Status/} | cut -d'=' -f2)
2312         echo -n " ${var1:-"-1"}" >> ${outfile}
2313       done
2314     fi
2315     echo >> ${outfile}
2316   done
2317
2318   return 0
2319 }
2320
2321 parseConfig()
2322 {
2323   configFile=${1}
2324   shift
2325   args=("$@")
2326
2327
2328   #some defaults
2329   #autoOCDB=0
2330   defaultOCDB="raw://"
2331   #runNumber=167123
2332   #makeflowPath="/hera/alice/aux/cctools/bin"
2333   #makeflowOptions="-T wq -N alice -d all -C ali-copilot.cern.ch:9097"
2334   #makeflowOptions="-T wq -N alice -C ali-copilot.cern.ch:9097"
2335   makeflowOptions=""
2336   #batchCommand="/usr/bin/qsub"
2337   batchFlags="-b y -cwd -l h_rt=24:0:0,h_rss=4G "
2338   baseOutputDirectory="$PWD/output"
2339   #alirootEnv="/cvmfs/alice.cern.ch/bin/alienv setenv AliRoot/v5-04-34-AN -c"
2340   #alirootEnv="/home/mkrzewic/alisoft/balice_master.sh"
2341   #trustedQAtrainMacro='/hera/alice/mkrzewic/gsisvn/Calibration/QAtrain_duo.C'
2342   reconstructInTemporaryDir=0
2343   recoTriggerOptions="\"\""
2344   percentProcessedFilesToContinue=100
2345   maxSecondsToWait=$(( 3600*24 ))
2346   nEvents=-1
2347   nMaxChunks=0
2348   postSetUpActionCPass0=""
2349   postSetUpActionCPass1=""
2350   runCPass0reco=1
2351   runCPass0MergeMakeOCDB=1
2352   runCPass1reco=1
2353   runCPass1MergeMakeOCDB=1
2354   runESDfiltering=1
2355   filteringFactorHighPt=1e2
2356   filteringFactorV0s=1e1
2357   MAILTO=""
2358   #pretend=1
2359   #dontRedirectStdOutToLog=1
2360   logToFinalDestination=1
2361   ALIROOT_FORCE_COREDUMP=1
2362   pretendDelay=0
2363
2364   #first, source the config file
2365   if [ -f ${configFile} ]; then
2366     source ${configFile}
2367   else
2368     echo "config file ${configFile} not found!"
2369     return 1
2370   fi
2371
2372   unset encodedSpaces
2373   for opt in "${args[@]}"; do
2374     [[ "${opt}" =~ encodedSpaces=.* ]] && encodedSpaces=1 && echo "encodedSpaces!" && break
2375   done
2376
2377   #then, parse the options as they override the options from file
2378   for opt in "${args[@]}"; do
2379     [[ -z ${opt} ]] && continue
2380     [[ -n ${encodedSpaces} ]] && opt="$(decSpaces ${opt})"
2381     [[ "${opt}" =~ ^[[:space:]]*$ ]] && continue
2382     if [[ ! "${opt}" =~ .*=.* ]]; then
2383       echo "badly formatted option \"${opt}\" should be: option=value, stopping..."
2384       return 1
2385     fi
2386     local var="${opt%%=*}"
2387     local value="${opt#*=}"
2388     echo "${var} = ${value}"
2389     export ${var}="${value}"
2390   done
2391
2392   #do some checking
2393   [[ -z ${alirootEnv} ]] && echo "alirootEnv not defined!" && return 1
2394
2395   #export the aliroot function if defined to override normal behaviour
2396   [[ $(type -t aliroot) =~ "function" ]] && export -f aliroot
2397
2398   return 0
2399 }
2400
2401 aliroot()
2402 {
2403   args="$@"
2404   if [[ -n ${useProfilingCommand} ]]; then
2405     valgrindLogFile="cpu.txt"
2406     [[ "${args}" =~ rec ]] && valgrindLogFile="cpu_rec.txt"
2407     [[ "${args}}" =~ Calib ]] && valgrindLogFile="cpu_calib.txt"
2408     [[ -n ${useProfilingCommand} ]] && useProfilingCommand="${useProfilingCommand} --log-file=${valgrindLogFile}"
2409     echo running ${useProfilingCommand} aliroot ${args}
2410     ${useProfilingCommand} aliroot ${args}
2411   else
2412     #to prevent an infinite recursion use "command aliroot" to disable
2413     #aliases and functions
2414     echo running command aliroot ${args}
2415     command aliroot "$@"
2416   fi
2417   return 0
2418 }
2419
2420 guessRunData()
2421 {
2422   #guess the period from the path, pick the rightmost one
2423   period=""
2424   runNumber=""
2425   year=""
2426   pass=""
2427   legoTrainRunNumber=""
2428   dataType=""
2429
2430   local shortRunNumber=""
2431   local IFS="/"
2432   declare -a path=( $1 )
2433   local dirDepth=$(( ${#path[*]}-1 ))
2434   i=0
2435   #for ((x=${dirDepth};x>=0;x--)); do
2436   for ((x=0;x<=${dirDepth};x++)); do
2437
2438     [[ $((x-1)) -ge 0 ]] && local fieldPrev=${path[$((x-1))]}
2439     local field=${path[${x}]}
2440     local fieldNext=${path[$((x+1))]}
2441
2442     [[ ${field} =~ ^[0-9]*$ && ${fieldNext} =~ (.*\.zip$|.*\.root$) ]] && legoTrainRunNumber=${field}
2443     [[ -n ${legoTrainRunNumber} && -z ${pass} ]] && pass=${fieldPrev}
2444     [[ ${field} =~ ^LHC[0-9][0-9][a-z].*$ ]] && period=${field%_*}
2445     [[ ${field} =~ ^000[0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && runNumber=${field#000}
2446     [[ ${field} =~ ^[0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && shortRunNumber=${field}
2447     [[ ${field} =~ ^20[0-9][0-9]$ ]] && year=${field}
2448     [[ ${field} =~ ^(^sim$|^data$) ]] && dataType=${field}
2449     (( i++ ))
2450   done
2451   [[ -z ${legoTrainRunNumber} ]] && pass=${path[$((dirDepth-1))]}
2452   [[ "${dataType}" =~ ^sim$ ]] && pass="passMC" && runNumber=${shortRunNumber}
2453
2454   #if [[ -z ${dataType} || -z ${year} || -z ${period} || -z ${runNumber}} || -z ${pass} ]];
2455   if [[ -z ${runNumber}} ]];
2456   then
2457     #error condition
2458     return 1
2459   else
2460     #ALL OK
2461     return 0
2462   fi
2463   return 0
2464 }
2465
2466 #these functions encode strings to and from a space-less form
2467 #use when spaces are not well handled (e.g. in arguments to
2468 #commands in makeflow files, etc.
2469 encSpaces()(echo "${1// /@@@@}")
2470 decSpaces()(echo "${1//@@@@/ }")
2471
2472 main "$@"