PWGPP/benchmark/benchmark.sh

   1 #!/bin/bash
   2 #include benchmark.config
   3
   4 # blame: Mikolaj Krzewicki, mkrzewic@cern.ch
   5 # this script runs the CPass0/CPass1 train
   6 # produced OCDB updates are local
   7
   8 main()
   9 {
  10   #run in proper mode depending on the selection
  11   if [[ $# -lt 1 ]]; then
  12     if [[ ! "${0}" =~ "bash" ]]; then
  13       echo "uses makeflow:"
  14       echo " ${0} \"run\" productionID inputList configFile [extraOpts]"
  15       echo "uses a batch system (SGE):"
  16       echo " ${0} \"submit\" productionID inputList configFile [extraOpts]"
  17       echo "extraOpts if non-empty override the config file, e.g.:"
  18       echo " ${0} submit test1 benchmark.list benchmark.config runNumber=169123 nEvents=10"
  19     fi
  20     return
  21   fi
  22
  23   #define some aliases - default is to call one of the functions directly
  24   runMode=${1}
  25   umask 0002
  26   shift
  27   case ${runMode} in
  28     "CPass0") goCPass0 "$@";;
  29     "CPass1") goCPass1 "$@";;
  30     "MakeLocalOCDBaccessConfig") goMakeLocalOCDBaccessConfig "$@";;
  31     "MergeCPass0") goMergeCPass0 "$@";;
  32     "MergeCPass1") goMergeCPass1 "$@";;
  33     "MakeFilteredTrees") goMakeFilteredTrees "$@";;
  34     "MakeSummary") goMakeSummary "$@";;
  35     "run") goSubmitMakeflow "$@";;
  36     "submit") goSubmitBatch "$@";;
  37     "test") goTest "$@";;
  38     "GenerateMakeflow") goGenerateMakeflow "$@";;
  39     "PrintValues") goPrintValues "$@";;
  40     "CreateQAplots") goCreateQAplots "$@";;
  41     "WaitForOutput") goWaitForOutput "$@";;
  42     "Merge") goMerge "$@";;
  43     *)
  44       ${runMode} "$@"
  45     ;;
  46   esac
  47   return 0
  48 }
  49
  50 generateMC()
  51 {
  52   #generate one raw chunk in current directory
  53   SEED=${JOB_ID}${SGE_TASK_ID}
  54   export CONFIG_SEED=${SEED}
  55   runNumber=${1}
  56   OCDBpath=${2}
  57   nEventsim=${3}
  58   if [[ -n ${pretend} ]]; then
  59     sleep ${pretendDelay}
  60     touch galice.root
  61   else
  62     if [[ -f sim.C && -f Config.C ]] ; then
  63         time aliroot -b -q -x sim.C\(${runNumber},\"${OCDBpath}\",${nEventsim}\) >sim.log 2>&1
  64         mv syswatch.log simwatch.log
  65     fi
  66   fi
  67 }
  68
  69 goCPass0()
  70 (
  71   umask 0002
  72
  73   targetDirectory=${1}
  74   inputList=${2}
  75   nEvents=${3}
  76   ocdbPath=${4}
  77   configFile=${5}
  78   runNumber=${6}
  79   jobindex=${7}
  80   shift 7
  81   if ! parseConfig ${configFile} "$@"; then return 1; fi
  82
  83   #record the working directory provided by the batch system
  84   batchWorkingDirectory=${PWD}
  85
  86   #use the jobindex only if set and non-negative
  87   if [[ -z ${jobindex} || ${jobindex} -lt 0 ]]; then
  88     [[ -n "${LSB_JOBINDEX}" ]] && jobindex=${LSB_JOBINDEX}
  89     [[ -n "${SGE_TASK_ID}" ]] && jobindex=${SGE_TASK_ID}
  90     if [[ -z ${jobindex} ]]; then
  91       echo "no jobindex!"
  92       return 1
  93     fi
  94   fi
  95
  96   [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
  97
  98   # This file signals that/if everything went fine
  99   doneFileBase="cpass0.job${jobindex}.run${runNumber}.done"
 100   [[ -n ${useProfilingCommand} ]] && doneFileBase="profiling.cpass0.job${jobindex}.run${runNumber}.done"
 101
 102   # We will have two copies of the file
 103   mkdir -p "${commonOutputPath}/meta" || return 1
 104   doneFileTmp="${batchWorkingDirectory}/${doneFileBase}"
 105   doneFile="${commonOutputPath}/meta/${doneFileBase}"
 106
 107   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
 108
 109   if [[ -n ${ALIROOT_FORCE_COREDUMP} ]]; then
 110     ulimit -c unlimited
 111     export ALIROOT_FORCE_COREDUMP
 112   fi
 113
 114   #the contents of this is stored in the tree and used later (e.g. AliAnalysisTaskPIDResponse)!
 115   #at the QA stage the pass number is guessed from the path stored here.
 116   #The Format is:
 117   #Packages= ;OutputDir= ;LPMPass= ;TriggerAlias= ;LPMRunNumber= ;LPMProductionType= ;LPMInteractionType= ;LPMProductionTag= ;LPMAnchorRun= ;LPMAnchorProduction= ;LPMAnchorYear=
 118   export PRODUCTION_METADATA="OutputDir=cpass0"
 119
 120   if [[ "${inputList}" =~ \.root$ ]]; then
 121     infile=${inputList}
 122   else
 123     infile=$(sed -ne "${jobindex}p" ${inputList} | egrep '\s*\w*/\w*')
 124   fi
 125   chunkName=${infile##*/}
 126
 127   outputDir=${targetDirectory}/${jobindex}_${chunkName%.*}
 128   mkdir -p ${outputDir}
 129   if [[ ! -d ${outputDir} ]]; then
 130     touch ${doneFileTmp}
 131     echo "cannot make ${outputDir}" >> ${doneFileTmp}
 132     cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
 133     [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
 134     return 1
 135   fi
 136
 137   runpath=${outputDir}
 138   [[ ${reconstructInTemporaryDir} -eq 1 ]] && runpath=$(mktemp -d -t cpass0.XXXXXX)
 139   [[ ${reconstructInTemporaryDir} -eq 2 ]] && runpath=${PWD}/rundir_cpass0_${runNumber}_${jobindex}
 140   mkdir -p ${runpath}
 141   if [[ ! -d ${runpath} ]]; then
 142     touch ${doneFileTmp}
 143     echo "cannot make runpath ${runpath}" >> ${doneFileTmp}
 144     cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
 145     [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
 146     return 1
 147   fi
 148   if ! cd ${runpath}; then
 149     touch ${doneFileTmp}
 150     echo "PWD=$PWD is not the runpath=${runpath}" >> ${doneFileTmp}
 151     cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
 152     [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
 153     return 1
 154   fi
 155
 156   #runCPassX/C expects the raw chunk to be linked in the run dir
 157   #despite it being accessed by the full path
 158   if [[ $copyInputData == 0 ]]; then
 159     ln -s ${infile} ${runpath}/${chunkName}
 160   else
 161     copyFileToLocal ${infile} ${runpath}/${chunkName}
 162   fi
 163
 164   #####MC
 165   if [[ -n ${generateMC} ]]; then
 166     olddir=${PWD}
 167     outputDirMC=${commonOutputPath}/000${runNumber}/sim/${jobindex}
 168     simrunpath=${outputDirMC}
 169     #[[ ${simulateInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && simrunpath=${TMPDIR}
 170     [[ ${simulateInTemporaryDir} -eq 1 ]] && simrunpath=$(mktemp -d -t cpass0MC.XXXXXX)
 171     mkdir -p ${outputDirMC}
 172     mkdir -p ${simrunpath}
 173     if cd ${simrunpath}; then
 174
 175       filesMC=(
 176       "${batchWorkingDirectory}/sim.C"
 177       "${batchWorkingDirectory}/rec.C"
 178       "${batchWorkingDirectory}/Config.C"
 179       "${batchWorkingDirectory}/OCDB_*.root"
 180       )
 181       for file in ${filesMC[*]}; do
 182         [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
 183       done
 184
 185       generateMC ${runNumber} ${ocdbPath} ${nEvents}
 186
 187       [[ ! "${simrunpath}" =~ "${outputDirMC}" ]] && mv * ${outputDirMC} #TODO check if it works
 188       cd ${olddir}
 189
 190       ln -s ${outputDirMC}/* ${runpath}/
 191
 192       inputList=${outputDirMC}/galice.root #TODO not valid outside shell !!!
 193       infile=""
 194     fi
 195   fi
 196   ######
 197
 198   if [[ "${inputList}" == "${inputList%%://*}" && ! -f "${inputList}" && -z ${pretend} ]]; then
 199     touch ${doneFileTmp}
 200     echo "input file ${inputList} not found, exiting..." >> ${doneFileTmp}
 201     cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
 202     [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
 203     return 1
 204   fi
 205
 206   logOutputDir=${runpath}
 207   [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir}
 208   [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/stdout
 209   #[[ -z ${dontRedirectStdOutToLog} ]] && exec 2> ${logOutputDir}/stderr
 210   echo "${0} $*"
 211
 212   echo "#####################"
 213   echo CPass0:
 214   echo JOB setup
 215   echo nEvents               ${nEvents}
 216   echo runNumber             ${runNumber}
 217   echo ocdbPath              ${ocdbPath}
 218   echo infile                ${infile}
 219   echo chunkName             ${chunkName}
 220   echo jobindex              ${jobindex}
 221   echo recoTriggerOptions    ${recoTriggerOptions}
 222   echo targetDirectory       ${targetDirectory}
 223   echo commonOutputPath      ${commonOutputPath}
 224   echo doneFile              ${doneFile}
 225   echo batchWorkingDirectory ${batchWorkingDirectory}
 226   echo runpath               ${runpath}
 227   echo outputDir             ${outputDir}
 228   echo PWD                   ${PWD}
 229   echo ALICE_ROOT            ${ALICE_ROOT}
 230   echo "########## ###########"
 231
 232   alirootInfo > ALICE_ROOT.log
 233
 234   filesCPass0=(
 235                "${batchWorkingDirectory}/runCPass0.sh"
 236                "${batchWorkingDirectory}/recCPass0.C"
 237                "${batchWorkingDirectory}/runCalibTrain.C"
 238                "${batchWorkingDirectory}/localOCDBaccessConfig.C"
 239                "${batchWorkingDirectory}/OCDB.root"
 240                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/runCPass0.sh"
 241                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/recCPass0.C"
 242                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/runCalibTrain.C"
 243   )
 244
 245   for file in ${filesCPass0[*]}; do
 246     [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
 247     [[ ${file##*/} =~ .*\.sh ]] && chmod +x ${file##*/}
 248   done
 249
 250   echo "this directory (${PWD}) contents:"
 251   /bin/ls
 252   echo
 253   chmod u+x runCPass0.sh
 254
 255   #remove spaces from around arguments to root macros
 256   #for example this sometimes fails:
 257   #  root 'macro.C(argument1, argument2)'
 258   sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh
 259
 260   if [[ -n ${postSetUpActionCPass0} ]]; then
 261     echo "running ${postSetUpActionCPass0}"
 262     eval ${postSetUpActionCPass0}
 263   fi
 264
 265   #run CPass0
 266   echo "${runpath}/runCPass0.sh /${infile} ${nEvents} ${runNumber} ${ocdbPath} ${recoTriggerOptions}"
 267   if [[ -n ${pretend} ]]; then
 268     sleep ${pretendDelay}
 269     touch AliESDs.root
 270     touch AliESDfriends.root
 271     touch AliESDfriends_v1.root
 272     touch rec.log
 273     touch calib.log
 274   else
 275     #caveat: in the local case, first arg must start with a slash
 276     ./runCPass0.sh "/${infile}" "${nEvents}" "${runNumber}" "${ocdbPath}" "${recoTriggerOptions}"
 277   fi
 278
 279   #move stuff to final destination
 280   echo "this directory (${PWD}) contents:"
 281   /bin/ls
 282   echo
 283
 284   # [dberzano] OK this is fine!
 285   echo rm -f ./${chunkName}
 286   rm -f ./${chunkName}
 287   echo "paranoidCp ${runpath}/* ${outputDir}"
 288   paranoidCp ${runpath}/* ${outputDir}
 289   echo
 290
 291   #validate CPass0
 292   cd ${outputDir}
 293   if summarizeLogs >> ${doneFileTmp}; then
 294     [[ -f ${outputDirMC}/galice.root ]] && echo "sim ${outputDirMC}/galice.root" >> ${doneFileTmp}
 295     [[ -f AliESDfriends_v1.root ]] && echo "calibfile ${outputDir}/AliESDfriends_v1.root" >> ${doneFileTmp}
 296     [[ -f AliESDs.root ]] && echo "esd ${outputDir}/AliESDs.root" >> ${doneFileTmp}
 297   fi
 298
 299   [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath} && echo "removing ${runpath}"
 300   cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
 301   [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
 302   return 0
 303 )
 304
 305 goCPass1()
 306 (
 307   umask 0002
 308
 309   targetDirectory=${1}
 310   inputList=${2}
 311   nEvents=${3}
 312   ocdbPath=${4}
 313   configFile=${5}
 314   runNumber=${6}
 315   jobindex=${7}
 316   shift 7
 317   extraOpts=("$@")
 318   if ! parseConfig ${configFile} "$@"; then return 1; fi
 319
 320   #record the working directory provided by the batch system
 321   batchWorkingDirectory=${PWD}
 322
 323   #use the jobindex only if set and non-negative
 324   if [[ -z ${jobindex} || ${jobindex} -lt 0 ]]; then
 325     [[ -n "${LSB_JOBINDEX}" ]] && jobindex=${LSB_JOBINDEX}
 326     [[ -n "${SGE_TASK_ID}" ]] && jobindex=${SGE_TASK_ID}
 327     if [[ -z ${jobindex} ]]; then
 328       echo "no jobindex!"
 329       return 1
 330     fi
 331   fi
 332
 333   [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
 334
 335   # This file signals that/if everything went fine
 336   doneFileBase="cpass1.job${jobindex}.run${runNumber}.done"
 337   [[ -n ${useProfilingCommand} ]] && doneFileBase="profiling.cpass0.job${jobindex}.run${runNumber}.done"
 338
 339   # We will have two copies of the file
 340   mkdir -p "${commonOutputPath}/meta" || return 1
 341   doneFileTmp="${batchWorkingDirectory}/${doneFileBase}"
 342   doneFile="${commonOutputPath}/meta/${doneFileBase}"
 343
 344   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
 345
 346   if [[ -n ${ALIROOT_FORCE_COREDUMP} ]]; then
 347     ulimit -c unlimited
 348     export ALIROOT_FORCE_COREDUMP
 349   fi
 350
 351   #the contents of this is stored in the tree and used later (e.g. AliAnalysisTaskPIDResponse)!
 352   #at the QA stage the pass number is guessed from the path stored here.
 353   #The Format is:
 354   #Packages= ;OutputDir= ;LPMPass= ;TriggerAlias= ;LPMRunNumber= ;LPMProductionType= ;LPMInteractionType= ;LPMProductionTag= ;LPMAnchorRun= ;LPMAnchorProduction= ;LPMAnchorYear=
 355   export PRODUCTION_METADATA="OutputDir=cpass1"
 356
 357   if [[ "${inputList}" == "${inputList%%://*}" && ! -f "${inputList}" && -z ${pretend} ]]; then
 358     touch ${doneFileTmp}
 359     echo "input file ${inputList} not found, exiting..." >> ${doneFileTmp}
 360     cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
 361     [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
 362     return 1
 363   fi
 364   if [[ "${inputList}" =~ \.root$ ]]; then
 365     infile=${inputList}
 366   else
 367     infile=$(sed -ne "${jobindex}p" ${inputList} | egrep '\s*\w*/\w*')
 368   fi
 369   chunkName=${infile##*/}
 370
 371   outputDir=${targetDirectory}/${jobindex}_${chunkName%.*}
 372   mkdir -p ${outputDir}
 373   if [[ ! -d ${outputDir} ]];then
 374     touch ${doneFileTmp}
 375     echo "cannot make ${outputDir}" >> ${doneFileTmp}
 376     cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
 377     [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
 378     return 1
 379   fi
 380
 381   runpath=${outputDir}
 382   [[ ${reconstructInTemporaryDir} -eq 1 ]] && runpath=$(mktemp -d -t cpass1.XXXXXX)
 383   [[ ${reconstructInTemporaryDir} -eq 2 ]] && runpath=${PWD}/rundir_cpass1_${runNumber}_${jobindex}
 384
 385   #MC
 386   if [[ "${infile}" =~ galice\.root ]]; then
 387     ln -s ${inputList%/*}/* ${runpath}
 388     infile=""
 389   fi
 390
 391   #init the running path
 392   mkdir -p ${runpath}
 393   if [[ ! -d ${runpath} ]]; then
 394    touch ${doneFileTmp}
 395    echo "cannot make runpath ${runpath}" >> ${doneFileTmp}
 396    cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
 397     [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
 398    return 1
 399  fi
 400   if ! cd ${runpath}; then
 401     touch ${doneFileTmp}
 402     echo "PWD=$PWD is not the runpath=${runpath}" >> ${doneFileTmp}
 403     cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
 404     [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
 405     return 1
 406   fi
 407
 408   #this is needed for runCPass1.sh
 409   if [[ $copyInputData == 0 ]]; then
 410     ln -s ${infile} ${runpath}/${chunkName}
 411   else
 412     copyFileToLocal ${infile} ${runpath}/${chunkName}
 413   fi
 414
 415   logOutputDir=${runpath}
 416   [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir}
 417   [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/stdout
 418   #[[ -z ${dontRedirectStdOutToLog} ]] && exec 2> ${logOutputDir}/stderr
 419   echo "${0} $*"
 420
 421   echo "#####################"
 422   echo CPass1:
 423   echo JOB setup
 424   echo nEvents               ${nEvents}
 425   echo runNumber             ${runNumber}
 426   echo ocdbPath              ${ocdbPath}
 427   echo infile                ${infile}
 428   echo chunkName             ${chunkName}
 429   echo jobindex              ${jobindex}
 430   echo recoTriggerOptions    ${recoTriggerOptions}
 431   echo targetDirectory       ${targetDirectory}
 432   echo commonOutputPath      ${commonOutputPath}
 433   echo doneFile              ${doneFile}
 434   echo runpath               ${runpath}
 435   echo outputDir             ${outputDir}
 436   echo batchWorkingDirectory ${batchWorkingDirectory}
 437   echo ALICE_ROOT            ${ALICE_ROOT}
 438   echo PWD                   ${PWD}
 439   echo "#####################"
 440
 441   alirootInfo > ALICE_ROOT.log
 442
 443   filesCPass1=(
 444                "${batchWorkingDirectory}/runCPass1.sh"
 445                "${batchWorkingDirectory}/recCPass1.C"
 446                "${batchWorkingDirectory}/recCPass1_OuterDet.C"
 447                "${batchWorkingDirectory}/runCalibTrain.C"
 448                "${batchWorkingDirectory}/QAtrain_duo.C"
 449                "${batchWorkingDirectory}/localOCDBaccessConfig.C"
 450                "${batchWorkingDirectory}/${configFile}"
 451                "${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz"
 452                "${batchWorkingDirectory}/OCDB.root"
 453                "${trustedQAtrainMacro}"
 454                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/runCPass1.sh"
 455                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/recCPass1.C"
 456                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/recCPass1_OuterDet.C"
 457                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/runCalibTrain.C"
 458                "${ALICE_ROOT}/ANALYSIS/macros/QAtrain_duo.C"
 459   )
 460
 461   for file in "${filesCPass1[@]}"; do
 462     [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
 463     [[ ${file##*/} =~ .*\.sh ]] && echo "making ${file##*/} executable" && chmod +x ${file##*/}
 464   done
 465
 466   echo "this directory (${PWD}) contents:"
 467   /bin/ls
 468   echo
 469
 470   #remove spaces from around arguments to root macros
 471   #for example this sometimes fails:
 472   #  root 'macro.C(argument1, argument2)'
 473   sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh
 474
 475   if [[ -n ${postSetUpActionCPass1} ]]; then
 476     echo "running ${postSetUpActionCPass1}"
 477     eval ${postSetUpActionCPass1}
 478     echo
 479   fi
 480
 481   #configure local OCDB storage from CPass0 (creates the localOCDBaccessConfig.C script)
 482   if [[ -f cpass0.localOCDB.${runNumber}.tgz ]]; then
 483     echo goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz"
 484     goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz"
 485   else
 486     echo "WARNING: file cpass0.localOCDB.${runNumber}.tgz not found!"
 487   fi
 488
 489   if [[ ! $(/bin/ls -1 OCDB/*/*/*/*.root 2>/dev/null) ]]; then
 490     touch ${doneFileTmp}
 491     echo "cpass0 produced no calibration! exiting..." >> ${doneFileTmp}
 492     cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
 493     [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
 494     return 1
 495   fi
 496
 497   #create the Barrel and OuterDet directories for CPass1 and link the local OCDB directory
 498   #there to make the localOCDBaccessConfig.C file work, since it may point to the OCDB
 499   #entries using a relative path, e.g. local://./OCDB
 500   echo "linking the OCDB/ for Barrel and OuterDet directories"
 501   mkdir Barrel OuterDet
 502   ls -l
 503   ln -s ../OCDB Barrel/OCDB
 504   ln -s ../OCDB OuterDet/OCDB
 505
 506   #setup the filtering
 507   #the following option enables the filtering task inside the QAtrain_duo.C
 508   [[ -n $runESDfiltering ]] && export QA_TaskFilteredTree=1
 509   #set the downscaling factors during the filtering fro expert QA (overrides the previous values)
 510   if [[ -n ${filteringFactorHighPt} ]]; then
 511     export AliAnalysisTaskFilteredTree_fLowPtTrackDownscaligF=${filteringFactorHighPt}
 512   fi
 513   if [[ -n ${filteringFactorV0s} ]]; then
 514     export AliAnalysisTaskFilteredTree_fLowPtV0DownscaligF=${filteringFactorV0s}
 515   fi
 516
 517   #run CPass1
 518   chmod u+x runCPass1.sh
 519   echo "${runpath}/runCPass1.sh /${infile} ${nEvents} ${runNumber} ${ocdbPath} ${recoTriggerOptions}"
 520   if [[ -n ${pretend} ]]; then
 521     sleep ${pretendDelay}
 522     touch AliESDs_Barrel.root
 523     touch AliESDfriends_Barrel.root
 524     touch AliESDfriends_v1.root
 525     touch QAresults_barrel.root
 526     touch EventStat_temp_barrel.root
 527     touch AODtpITS.root
 528     touch AliESDs_Outer.root
 529     touch AliESDfriends_Outer.root
 530     touch QAresults_outer.root
 531     touch EventStat_temp_outer.root
 532     touch rec.log
 533     touch calib.log
 534     touch qa.log
 535     touch filtering.log FilterEvents_Trees.root
 536   else
 537     #caveat: in the local case, first arg must start with a slash
 538     ./runCPass1.sh "/${infile}" "${nEvents}" "${runNumber}" "${ocdbPath}" "${recoTriggerOptions}"
 539
 540     [[ ! -f AliESDs_Barrel.root && -f Barrel/AliESDs.root ]] && mv Barrel/AliESDs.root AliESDs_Barrel.root
 541     [[ ! -f AliESDfriends_Barrel.root && -f Barrel/AliESDfriends.root ]] && mv Barrel/AliESDfriends.root AliESDfriends_Barrel.root
 542     [[ ! -f AliESDfriends_v1.root && -f Barrel/AliESDfriends_v1.root ]] && mv Barrel/AliESDfriends_v1.root .
 543     [[ ! -f QAresults_barrel.root && -f Barrel/QAresults_barrel.root ]] && mv Barrel/QAresults_barrel.root .
 544     [[ ! -f AliESDs_Outer.root && -f OuterDet/AliESDs.root ]] && mv OuterDet/AliESDs.root AliESDs_Outer.root
 545     [[ ! -f AliESDfriends_Outer.root && -f OuterDet/AliESDfriends.root ]] && mv OuterDet/AliESDfriends.root AliESDfriends_Outer.root
 546     [[ ! -f QAresults_outer.root && -f OuterDet/QAresults_outer.root ]] && mv OuterDet/QAresults_outer.root .
 547     [[ ! -f FilterEvents_Trees.root && -f Barrel/FilterEvents_Trees.root ]] && mv Barrel/FilterEvents_Trees.root .
 548
 549     #make the filtered tree (if requested and not already produced by QA
 550     [[ -f AliESDs_Barrel.root ]] && echo "AliESDs_Barrel.root" > filtered.list
 551     if [[ -n ${runESDfiltering} && ! -f FilterEvents_Trees.root && -f filtered.list ]]; then
 552       goMakeFilteredTrees ${PWD} ${runNumber} "${PWD}/filtered.list" ${filteringFactorHighPt} ${filteringFactorV0s} ${ocdbPath} 1000000 0 10000000 0 ${configFile} AliESDs_Barrel.root "${extraOpts[@]}" >filtering.log
 553     else
 554       echo ""
 555     fi
 556
 557   fi
 558
 559   ##handle possible crashes in QA (happens often in trunk)
 560   ##rerun QA with a trusted aliroot version
 561   #if [[ $(validateLog qa_barrel.log) ]]; then
 562   #  echo "qa_barrel.log not validated!"
 563   #fi
 564   #if [[ ! -f QAresults_barrel.root && -f ${setupTrustedAliROOTenvInCurrentShell} || $(validateLog qa_barrel.log) ]]; then
 565   #  echo "WARNING: using trusted QA aliroot ${ALICE_ROOT}"
 566   #  source ${setupTrustedAliROOTenvInCurrentShell}
 567   #  cd Barrel
 568   #  rm QAresults_barrel.root
 569   #  rm EventStat_temp_barrel.root
 570   #  rm AODtpITS.root
 571   #  [[ ! -f AliESDs.root ]] && ln -s ../AliESDs_Barrel.root AliESDs.root
 572   #  [[ ! -f AliESDfriends.root ]] && ln -s ../AliESDfriends_Barrel.root AliESDfriends.root
 573   #  if [[ -n ${trustedQAtrainMacro} ]]; then
 574   #    eval "cp ${trustedQAtrainMacro} QAtrain_duo_trusted.C"
 575   #  fi
 576   #  echo executing aliroot -b -q "QAtrain_duo_trusted.C(\"_barrel\",${runNumber},\"wn.xml\",0,\"${ocdbPath}\")"
 577   #  time aliroot -b -q "QAtrain_duo.C(\"_barrel\",${runNumber},\"wn.xml\",0,\"${ocdbPath}\")" &> ../qa_barrel_trusted.log
 578   #  cd ../
 579   #fi
 580
 581   #move stuff to final destination
 582   echo "this directory (${PWD}) contents:"
 583   /bin/ls
 584   echo rm -f ./${chunkName}
 585   rm -f ./${chunkName}
 586   echo "paranoidCp ${runpath}/* ${outputDir}"
 587   paranoidCp ${runpath}/* ${outputDir}
 588   echo
 589
 590   #validate CPass1
 591   cd ${outputDir}
 592   if summarizeLogs >> ${doneFileTmp}; then
 593     [[ -f AliESDs_Barrel.root ]] && echo "esd ${outputDir}/AliESDs_Barrel.root" >> ${doneFileTmp}
 594     [[ -f AliESDfriends_v1.root ]] && echo "calibfile ${outputDir}/AliESDfriends_v1.root" >> ${doneFileTmp}
 595     [[ -f QAresults_Barrel.root ]] && echo "qafile ${outputDir}/QAresults_Barrel.root" >> ${doneFileTmp}
 596     [[ -f QAresults_Outer.root ]] && echo "qafile ${outputDir}/QAresults_Outer.root" >> ${doneFileTmp}
 597     [[ -f QAresults_barrel.root ]] && echo "qafile ${outputDir}/QAresults_barrel.root" >> ${doneFileTmp}
 598     [[ -f QAresults_outer.root ]] && echo "qafile ${outputDir}/QAresults_outer.root" >> ${doneFileTmp}
 599     [[ -f FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFileTmp}
 600   else
 601     if grep "qa_outer.log.*OK" ${doneFileTmp} > /dev/null; then
 602       [[ -f QAresults_Outer.root ]] && echo "qafile ${outputDir}/QAresults_Outer.root" >> ${doneFileTmp}
 603       [[ -f QAresults_outer.root ]] && echo "qafile ${outputDir}/QAresults_outer.root" >> ${doneFileTmp}
 604     fi
 605     if grep "qa_barrel.log.*OK" ${doneFileTmp} > /dev/null; then
 606       [[ -f QAresults_Barrel.root ]] && echo "qafile ${outputDir}/QAresults_Barrel.root" >> ${doneFileTmp}
 607       [[ -f QAresults_barrel.root ]] && echo "qafile ${outputDir}/QAresults_barrel.root" >> ${doneFileTmp}
 608     fi
 609     if grep "filtering.log.*OK" ${doneFileTmp} > /dev/null; then
 610       [[ -f FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFileTmp}
 611     fi
 612   fi
 613
 614   [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
 615   cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
 616   [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
 617   return 0
 618 )
 619
 620
 621 goMergeCPass0()
 622 (
 623   #
 624   # find the output files and merge them
 625   #
 626
 627   outputDir=${1}
 628   ocdbStorage=${2}
 629   configFile=${3}
 630   runNumber=${4}
 631   calibrationFilesToMerge=${5}  #can be a non-existent file, will then be produced on the fly
 632   shift 5
 633   if ! parseConfig ${configFile} "$@"; then return 1; fi
 634
 635   #record the working directory provided by the batch system
 636   batchWorkingDirectory=${PWD}
 637
 638   [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
 639
 640   # This file signals that everything went fine
 641   doneFileBase="merge.cpass0.run${runNumber}.done"
 642
 643   # We will have two copies of the file
 644   mkdir -p "${commonOutputPath}/meta" || return 1
 645   doneFileTmp="${batchWorkingDirectory}/${doneFileBase}"
 646   doneFile="${commonOutputPath}/meta/${doneFileBase}"
 647
 648   umask 0002
 649   ulimit -c unlimited
 650
 651   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
 652
 653   runpath=${outputDir}
 654   [[ ${reconstructInTemporaryDir} -eq 1 ]] && runpath=$(mktemp -d -t mergeCPass0.XXXXXX)
 655   [[ ${reconstructInTemporaryDir} -eq 2 ]] && runpath=${PWD}/rundir_mergeCPass0_${runNumber}
 656
 657   mkdir -p ${runpath}
 658   if [[ ! -d ${runpath} ]]; then
 659     touch ${doneFileTmp}
 660     echo "not able to make the runpath ${runpath}" >> ${doneFileTmp}
 661     cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
 662     [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
 663     return 1
 664   fi
 665   if ! cd ${runpath}; then
 666     touch ${doneFileTmp}
 667     echo "PWD=$PWD is not the runpath=${runpath}" >> ${doneFileTmp}
 668     cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
 669     [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
 670     return 1
 671   fi
 672
 673   logOutputDir=${runpath}
 674   [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir}
 675   [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/stdout
 676   echo "${0} $*"
 677
 678   mergingScript="mergeMakeOCDB.byComponent.sh"
 679
 680   echo goMergeCPass0 SETUP:
 681   echo runNumber=${runNumber}
 682   echo outputDir=${outputDir}
 683   echo ocdbStorage=${ocdbStorage}
 684   echo calibrationFilesToMerge=${calibrationFilesToMerge}
 685   echo mergingScript=${mergingScript}
 686   echo commonOutputPath=${commonOutputPath}
 687   echo runpath=${runpath}
 688
 689   # copy files in case they are not already there
 690   filesMergeCPass0=(
 691                     "${batchWorkingDirectory}/${calibrationFilesToMerge}"
 692                     "${batchWorkingDirectory}/OCDB.root"
 693                     "${batchWorkingDirectory}/localOCDBaccessConfig.C"
 694                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/mergeMakeOCDB.byComponent.sh"
 695                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/mergeByComponent.C"
 696                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/makeOCDB.C"
 697                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/merge.C"
 698                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/mergeMakeOCDB.sh"
 699   )
 700   for file in ${filesMergeCPass0[*]}; do
 701     [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
 702     [[ ${file##*/} =~ .*\.sh ]] && chmod +x ${file##*/}
 703   done
 704
 705   #remove spaces from around arguments to root macros
 706   #for example this sometimes fails:
 707   #  root 'macro.C(argument1, argument2)'
 708   sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh
 709
 710   alirootInfo > ALICE_ROOT.log
 711
 712   #
 713   echo "PWD"
 714   /bin/ls
 715   echo "PWD/.."
 716   /bin/ls ../
 717
 718
 719   #merge calibration
 720   chmod u+x ${mergingScript}
 721   mkdir -p ./OCDB
 722   if [[ ! -f ${calibrationFilesToMerge} ]]; then
 723     echo "/bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root > ${calibrationFilesToMerge}"
 724     /bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root 2>/dev/null > ${calibrationFilesToMerge}
 725   fi
 726
 727   echo "${mergingScript} ${calibrationFilesToMerge} ${runNumber} local://./OCDB defaultOCDB=${ocdbStorage} fileAccessMethod=nocopy"
 728   if [[ -n ${pretend} ]]; then
 729     sleep ${pretendDelay}
 730     touch CalibObjects.root
 731     touch ocdb.log
 732     touch merge.log
 733     touch dcsTime.root
 734     mkdir -p ./OCDB/TPC/Calib/TimeGain/
 735     mkdir -p ./OCDB/TPC/Calib/TimeDrift/
 736     echo "some calibration" >> ./OCDB/TPC/Calib/TimeGain/someCalibObject_0-999999_cpass0.root
 737     echo "some calibration" >> ./OCDB/TPC/Calib/TimeDrift/otherCalibObject_0-999999_cpass0.root
 738   else
 739     ./${mergingScript} ${calibrationFilesToMerge} ${runNumber} "local://./OCDB" defaultOCDB=${ocdbStorage} fileAccessMethod=nocopy >> "mergeMakeOCDB.log"
 740
 741     #produce the calib trees for expert QA (dcsTime.root)
 742     goMakeLocalOCDBaccessConfig ./OCDB
 743     echo aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")"
 744     aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")"
 745   fi
 746
 747   ### produce the output
 748   #tar the produced OCDB for reuse
 749   #tar czf ${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz ./OCDB
 750
 751   # Create tarball with OCDB, store on the shared directory, create signal file on batch directory
 752   mkdir -p ${commonOutputPath}/meta
 753   baseTar="cpass0.localOCDB.${runNumber}.tgz"
 754   tar czf ${batchWorkingDirectory}/${baseTar} ./OCDB && \
 755     mv ${batchWorkingDirectory}/${baseTar} ${commonOutputPath}/meta/${baseTar} && \
 756     touch ${batchWorkingDirectory}/${baseTar}.done
 757
 758   /bin/ls
 759
 760   #copy all to output dir
 761   echo "paranoidCp ${runpath}/* ${outputDir}"
 762   paranoidCp ${runpath}/* ${outputDir}
 763
 764   if [[ -n ${generateMC} ]]; then
 765     goPrintValues sim ${commonOutputPath}/meta/sim.run${runNumber}.list ${commonOutputPath}/meta/cpass0.job*.run${runNumber}.done
 766   fi
 767
 768   #validate merging cpass0
 769   cd ${outputDir}
 770   if summarizeLogs >> ${doneFileTmp}; then
 771     [[ -f CalibObjects.root ]] && echo "calibfile ${outputDir}/CalibObjects.root" >> ${doneFileTmp}
 772     [[ -f dcsTime.root ]] && echo "dcsTree ${outputDir}/dcsTime.root" >> ${doneFileTmp}
 773   fi
 774
 775   [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
 776   cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
 777   [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
 778   return 0
 779 )
 780
 781 goMergeCPass1()
 782 (
 783   #
 784   # find the output files and merge them
 785   #
 786
 787   outputDir=${1}
 788   ocdbStorage=${2}
 789   configFile=${3}
 790   runNumber=${4}
 791   calibrationFilesToMerge=${5}
 792   qaFilesToMerge=${6}
 793   filteredFilesToMerge=${7}
 794   shift 7
 795   if ! parseConfig ${configFile} "$@"; then return 1; fi
 796
 797   #record the working directory provided by the batch system
 798   batchWorkingDirectory=${PWD}
 799
 800   [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
 801
 802   # This file signals that everything went fine
 803   doneFileBase="merge.cpass1.run${runNumber}.done"
 804
 805   # We will have two copies of the file
 806   mkdir -p "${commonOutputPath}/meta" || return 1
 807   doneFileTmp="${batchWorkingDirectory}/${doneFileBase}"
 808   doneFile="${commonOutputPath}/meta/${doneFileBase}"
 809
 810   umask 0002
 811   ulimit -c unlimited
 812
 813   #clean up first:
 814   rm -f ${outputDir}/*.log
 815   rm -f ${outputDir}/*.root
 816   rm -f ${outputDir}/*done
 817
 818   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
 819
 820   runpath=${outputDir}
 821   [[ ${reconstructInTemporaryDir} -eq 1 ]] && runpath=$(mktemp -d -t mergeCPass1.XXXXXX)
 822   [[ ${reconstructInTemporaryDir} -eq 2 ]] && runpath=${PWD}/rundir_mergeCPass1_${runNumber}
 823
 824   mkdir -p ${runpath}
 825   if [[ ! -d ${runpath} ]]; then
 826     touch ${doneFileTmp}
 827     echo "not able to make the runpath ${runpath}" >> ${doneFileTmp}
 828     cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
 829     [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
 830     return 1
 831   fi
 832   if ! cd ${runpath}; then
 833     touch ${doneFileTmp}
 834     echo "PWD=$PWD is not the runpath=${runpath}" >> ${doneFileTmp}
 835     cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
 836     [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
 837     return 1
 838   fi
 839
 840   logOutputDir=${runpath}
 841   [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir}
 842   [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/mergeMakeOCDB.log
 843   echo "${0} $*"
 844
 845   calibrationOutputFileName='AliESDfriends_v1.root'
 846   qaOutputFileName='QAresults*.root'
 847   mergingScript="mergeMakeOCDB.byComponent.sh"
 848   #important to have the string "Stage.txt" in the filename to trigger the merging
 849   #it has to be a list of directories containing the files
 850   qaMergedOutputFileName="QAresults_merged.root"
 851
 852   echo goMergeCPass1 SETUP:
 853   echo runNumber=${runNumber}
 854   echo outputDir=${outputDir}
 855   echo ocdbStorage=${ocdbStorage}
 856   echo calibrationFilesToMerge=$calibrationFilesToMerge
 857   echo qaFilesToMerge=$qaFilesToMerge
 858   echo calibrationOutputFileName=${calibrationOutputFileName}
 859   echo mergingScript=${mergingScript}
 860
 861   # copy files in case they are not already there
 862   filesMergeCPass1=(
 863                     "${batchWorkingDirectory}/${calibrationFilesToMerge}"
 864                     "${batchWorkingDirectory}/${qaFilesToMerge}"
 865                     "${batchWorkingDirectory}/OCDB.root"
 866                     "${batchWorkingDirectory}/localOCDBaccessConfig.C"
 867                     "${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz"
 868                     "${batchWorkingDirectory}/QAtrain_duo.C"
 869                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/mergeMakeOCDB.byComponent.sh"
 870                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/mergeByComponent.C"
 871                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/makeOCDB.C"
 872                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/merge.C"
 873                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/mergeMakeOCDB.sh"
 874                     "${trustedQAtrainMacro}"
 875                     "${ALICE_ROOT}/ANALYSIS/macros/QAtrain_duo.C"
 876   )
 877   for file in ${filesMergeCPass1[*]}; do
 878     [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
 879     [[ ${file##*/} =~ .*\.sh ]] && chmod +x ${file##*/}
 880   done
 881
 882   #remove spaces from around arguments to root macros
 883   #for example this sometimes fails:
 884   #  root 'macro.C(argument1, argument2)'
 885   sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh
 886
 887   #configure local OCDB storage from CPass0 (creates the localOCDBaccessConfig.C script)
 888   if [[ -f cpass0.localOCDB.${runNumber}.tgz ]]; then
 889     echo goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz"
 890     goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz"
 891   else
 892     echo "WARNING: file cpass0.localOCDB.${runNumber}.tgz not found!"
 893   fi
 894
 895   alirootInfo > ALICE_ROOT.log
 896
 897   #
 898   /bin/ls
 899
 900   #merge calibration
 901   chmod u+x ${mergingScript}
 902   mkdir -p OCDB
 903
 904   #if not provided, create the lists of files to merge
 905   if [[ ! -f ${filteredFilesToMerge} ]]; then
 906     echo "/bin/ls -1 ${outputDir}/*/FilterEvents_Trees.root > ${filteredFilesToMerge}"
 907     /bin/ls -1 ${outputDir}/*/FilterEvents_Trees.root 2>/dev/null > ${filteredFilesToMerge}
 908   fi
 909   if [[ ! -f ${calibrationFilesToMerge} ]]; then
 910     echo "/bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root > ${calibrationFilesToMerge}"
 911     /bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root 2>/dev/null > ${calibrationFilesToMerge}
 912   fi
 913   if [[ ! -f ${qaFilesToMerge} ]]; then
 914     #find the files, but only store the directories (QAtrain_duo.C requires this)
 915     echo "/bin/ls -1 ${outputDir}/*/QAresults*.root | while read x; do echo ${x%/*}; done | sort | uniq > ${qaFilesToMerge}"
 916     /bin/ls -1 ${outputDir}/*/QAresults*.root | while read x; do echo ${x%/*}; done | sort | uniq > ${qaFilesToMerge}
 917   fi
 918
 919   echo "${mergingScript} ${calibrationFilesToMerge} ${runNumber} local://./OCDB defaultOCDB=${ocdbStorage} fileAccessMethod=nocopy"
 920   if [[ -n ${pretend} ]]; then
 921     sleep ${pretendDelay}
 922     touch ocdb.log
 923     touch cpass1.localOCDB.${runNumber}.tgz
 924     touch ${qaMergedOutputFileName}
 925     touch merge.log
 926     touch trending.root
 927     touch FilterEvents_Trees.root
 928     touch CalibObjects.root
 929     touch dcsTime.root
 930     touch ${qaMergedOutputFileName}
 931     mkdir -p OCDB
 932   else
 933     ./${mergingScript} ${calibrationFilesToMerge} ${runNumber} "local://./OCDB" defaultOCDB=${ocdbStorage} fileAccessMethod=nocopy
 934
 935     #merge QA (and filtered trees)
 936     [[ -n ${AliAnalysisTaskFilteredTree_fLowPtTrackDownscaligF} ]] && export AliAnalysisTaskFilteredTree_fLowPtTrackDownscaligF
 937     [[ -n ${AliAnalysisTaskFilteredTree_fLowPtV0DownscaligF} ]] && export AliAnalysisTaskFilteredTree_fLowPtV0DownscaligF
 938
 939     #echo aliroot -l -b -q "merge.C(\"${qaFilesToMerge}\",\"\",kFALSE,\"${qaMergedOutputFileName}\")"
 940     echo aliroot -b -q "QAtrain_duo.C(\"_barrel\",${runNumber},\"${qaFilesToMerge}\",1,\"${ocdbStorage}\")"
 941     #aliroot -l -b -q "merge.C(\"${qaFilesToMerge}\",\"\",kFALSE,\"${qaMergedOutputFileName}\")"
 942     aliroot -b -q "QAtrain_duo.C(\"_barrel\",${runNumber},\"${qaFilesToMerge}\",1,\"${ocdbStorage}\")" > mergeQA.log
 943     mv QAresults_barrel.root ${qaMergedOutputFileName}
 944     mv trending_barrel.root trending.root
 945
 946     #merge filtered trees
 947     echo aliroot -l -b -q "merge.C(\"${qaFilesToMerge}\",\"\",kFALSE,\"${qaMergedOutputFileName}\")"
 948     aliroot -l -b -q "merge.C(\"${filteredFilesToMerge}\",\"\",kFALSE,\"FilterEvents_Trees.root\")" > mergeFilteredTrees.log
 949
 950     #produce the calib trees for expert QA
 951     echo aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")"
 952     aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")" > calibTree.log
 953   fi
 954
 955   # Create tarball with OCDB, store on the shared directory, create signal file on batch directory
 956   mkdir -p ${commonOutputPath}/meta
 957   baseTar="cpass1.localOCDB.${runNumber}.tgz"
 958   tar czf ${batchWorkingDirectory}/${baseTar} ./OCDB && \
 959     mv ${batchWorkingDirectory}/${baseTar} ${commonOutputPath}/meta/${baseTar} && \
 960     touch ${batchWorkingDirectory}/${baseTar}.done
 961
 962   /bin/ls
 963
 964   #copy all to output dir
 965   echo "paranoidCp ${runpath}/* ${outputDir}"
 966   paranoidCp ${runpath}/* ${outputDir}
 967
 968   #validate merge cpass1
 969   cd ${outputDir}
 970   if summarizeLogs >>  ${doneFileTmp}; then
 971     [[ -f CalibObjects.root ]] && echo "calibfile ${outputDir}/CalibObjects.root" >> ${doneFileTmp}
 972     [[ -f ${qaMergedOutputFileName} ]] && echo "qafile ${outputDir}/${qaMergedOutputFileName}" >> ${doneFileTmp}
 973     [[ -f trending.root ]] && echo "trendingfile ${outputDir}/trending.root" >> ${doneFileTmp}
 974     [[ -f dcsTime.root ]] && echo "dcsTree ${outputDir}/dcsTime.root" >> ${doneFileTmp}
 975     [[ -f FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFileTmp}
 976   else
 977     if grep "mergeQA.log.*OK" ${doneFileTmp} > /dev/null; then
 978       [[ -f ${qaMergedOutputFileName} ]] && echo "qafile ${outputDir}/${qaMergedOutputFileName}" >> ${doneFileTmp}
 979     fi
 980     if grep "mergeFilteredTrees.log.*OK" ${doneFileTmp} > /dev/null; then
 981       [[ -f FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFileTmp}
 982     fi
 983   fi
 984
 985   [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
 986   cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
 987   [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
 988   return 0
 989 )
 990
 991 goMerge()
 992 (
 993   #generic root merge using CPass1 merge.C script
 994   inputList=${1}
 995   outputFile=${2}
 996   configFile=${3-"benchmark.config"}
 997   shift 3
 998   if ! parseConfig ${configFile} "$@"; then return 1; fi
 999
1000   #record the working directory provided by the batch system
1001   batchWorkingDirectory=${PWD}
1002
1003   [[ ! -f ${inputList} ]] && echo "inputList ${inputList} does not exist!" && return 1
1004   [[ ! -f ${configFile} ]] && echo "configFile ${configFile} does not exist!" && return 1
1005   umask 0002
1006   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
1007   rm -f ${outputFile}
1008   aliroot -b -q "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/merge.C(\"${inputList}\",\"\",kFALSE,\"${outputFile}\")" > merge_${inputList}.log
1009   return 0
1010 )
1011
1012 goSubmitMakeflow()
1013 {
1014   #run
1015   productionID=${1}
1016   inputFileList=${2}
1017   configFile=${3}
1018   shift 3
1019   extraOpts=("$@")
1020   if ! parseConfig ${configFile} "${extraOpts[@]}"; then return 1; fi
1021
1022   #record the working directory provided by the batch system
1023   batchWorkingDirectory=${PWD}
1024
1025   [[ -z ${configFile} ]] && configFile="benchmark.config"
1026   [[ ! -f ${configFile} ]] && echo "no config file found (${configFile})" && return 1
1027
1028   if [[ ! $(which makeflow &>/dev/null) && -n ${makeflowPath} ]]; then
1029     echo "setting the makflow path from the config: "
1030     echo "  export PATH=${makeflowPath}:${PATH}"
1031     export PATH=${makeflowPath}:${PATH}
1032   fi
1033
1034   #create the common output dir and the meta dir
1035   commonOutputPath=${baseOutputDirectory}/${productionID}
1036   if [[ -d ${commonOutputPath} ]]; then
1037     echo "output dir ${commonOutputPath} exists!"
1038     #return 1
1039   else
1040     mkdir -p ${commonOutputPath}
1041   fi
1042   mkdir -p ${commonOutputPath}/meta
1043
1044   self=${0}
1045   #if which greadlink; then self=$(greadlink -f "${0}"); fi
1046
1047   #for reference copy the setup to the output dir
1048   paranoidCp ${self} ${commonOutputPath}
1049   paranoidCp ${configFile} ${commonOutputPath}
1050   paranoidCp ${inputFileList} ${commonOutputPath}
1051
1052   #submit - use makeflow if available, fall back to old stuff when makeflow not there
1053   if which makeflow; then
1054     goGenerateMakeflow ${productionID} ${inputFileList} ${configFile} "${extraOpts[@]}" commonOutputPath=${commonOutputPath} > benchmark.makeflow
1055     cp benchmark.makeflow ${commonOutputPath}
1056     makeflow ${makeflowOptions} benchmark.makeflow
1057   else
1058     echo "no makeflow!"
1059   fi
1060
1061   #summarize the run based on the makeflow log
1062   #and add it to the end of summary log
1063   awk '/STARTED/   {startTime=$3}
1064        /COMPLETED/ {endTime=$3}
1065        END         {print "makeflow running time: "(endTime-startTime)/1000000/3600" hours"}' \
1066       benchmark.makeflow.makeflowlog | tee -a summary.log
1067   paranoidCp summary.log ${commonOutputPath}
1068
1069   return 0
1070 }
1071
1072 goGenerateMakeflow()
1073 (
1074   #generate the makeflow file
1075   [[ $# -lt 3 ]] && echo "args: id inputFileList configFile" && return 1
1076   productionID=${1}
1077   inputFileList=${2}
1078   configFile=${3}
1079   shift 3
1080   extraOpts=("$@")
1081
1082   #batch systems/makeflow sometimes handle spaces in arguments poorly, so encode them
1083   for (( i=0;i<${#extraOpts[@]};i++ )); do
1084     extraOpts[i]=$(encSpaces "${extraOpts[i]}")
1085   done
1086   extraOpts+=("encodedSpaces=1")
1087
1088   if ! parseConfig ${configFile} "${extraOpts[@]}" &>/dev/null; then return 1; fi
1089
1090   #extra safety
1091   if [[ -z ${commonOutputPath} ]]; then
1092     commonOutputPath=${baseOutputDirectory}/${productionID}
1093     extraOpts=( "${extraOpts[@]}" "commonOutputPath=${commonOutputPath}" )
1094   fi
1095
1096   #record the working directory provided by the batch system
1097   batchWorkingDirectory=${PWD}
1098
1099   [[ -z ${configFile} ]] && configFile="benchmark.config"
1100   [[ ! -f ${configFile} ]] && echo "no config file found (${configFile})" && return 1
1101
1102   #these files will be made a dependency - will be copied to the working dir of the jobs
1103   declare -a copyFiles
1104   inputFiles=(
1105               "OCDB.root"
1106               "localOCDBaccessConfig.C"
1107               "QAtrain_duo.C"
1108               "runCPass1.sh"
1109               "recCPass1.C"
1110               "recCPass1_OuterDet.C"
1111               "runCalibTrain.C"
1112               "runCPass0.sh"
1113               "recCPass0.C"
1114               "runQA.sh"
1115   )
1116   for file in ${inputFiles[*]}; do
1117     [[ -f ${file} ]] && copyFiles+=("${file}")
1118   done
1119
1120   #create the makeflow file
1121   [[ -n ${batchFlags} ]] && echo "BATCH_OPTIONS = ${batchFlags}"
1122   declare -A arr_cpass0_merged arr_cpass1_merged
1123   declare -A arr_cpass0_calib_list arr_cpass1_calib_list
1124   declare -A arr_cpass1_QA_list arr_cpass1_ESD_list arr_cpass1_filtered_list
1125   declare -A arr_cpass0_profiled_outputs
1126   declare -A listOfRuns
1127   [[ -n ${runNumber} ]] && listOfRuns[${runNumber}]=1
1128   while read x; do tmpRun=$(guessRunNumber ${x}); [[ -n ${tmpRun} ]] && listOfRuns[${tmpRun}]=1; done < ${inputFileList}
1129   for runNumber in "${!listOfRuns[@]}"; do
1130     [[ -z ${runNumber} ]] && continue
1131     [[ ! ${runNumber} =~ ^[0-9]*[0-9]$ ]] && continue
1132
1133     unset arr_cpass0_outputs
1134     unset arr_cpass1_outputs
1135     declare -a arr_cpass0_outputs
1136     declare -a arr_cpass1_outputs
1137
1138     #Header
1139     echo "### Automatically generated on $(LANG=C date) ###"
1140     echo ; echo
1141
1142     jobindex=0
1143     inputFile=""
1144     while read inputFile; do
1145       currentDefaultOCDB=${defaultOCDB}
1146       [[ -z ${autoOCDB} ]] && autoOCDB=1
1147       if [[ ${autoOCDB} -ne 0 ]]; then
1148         currentDefaultOCDB=$(setYear ${inputFile} ${defaultOCDB})
1149       fi
1150       guessRunData ${inputFile}
1151
1152       #Set variables
1153       echo "### Variables ###"
1154       echo "OUTPATH=\"${commonOutputPath}/${year}/${period}\""
1155       echo ; echo
1156
1157       #CPass0
1158       #arr_cpass0_outputs[${jobindex}]="${commonOutputPath}/meta/cpass0.job${jobindex}.run${runNumber}.done"
1159       arr_cpass0_outputs[${jobindex}]="cpass0.job${jobindex}.run${runNumber}.done"
1160       echo "### CPass0 ###"
1161       echo "${arr_cpass0_outputs[${jobindex}]}: benchmark.sh ${configFile} ${copyFiles[@]}"
1162       echo "    ${alirootEnv} ./benchmark.sh CPass0 \$OUTPATH/000${runNumber}/cpass0 ${inputFile} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} ${jobindex} ${extraOpts[@]}"" "
1163       echo ; echo
1164
1165       #CPass1
1166       #arr_cpass1_outputs[${jobindex}]="${commonOutputPath}/meta/cpass1.job${jobindex}.run${runNumber}.done"
1167       arr_cpass1_outputs[${jobindex}]="cpass1.job${jobindex}.run${runNumber}.done"
1168       echo "### CPass1 ###"
1169       echo "${arr_cpass1_outputs[${jobindex}]}: benchmark.sh ${configFile} cpass0.localOCDB.${runNumber}.tgz.done ${copyFiles[@]}"
1170       echo "    ${alirootEnv} ./benchmark.sh CPass1 \$OUTPATH/000${runNumber}/cpass1 ${inputFile} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} ${jobindex} ${extraOpts[@]}"" "
1171       echo ; echo
1172       ((jobindex++))
1173
1174     done< <(grep "/000${runNumber}/" ${inputFileList})
1175
1176     #CPass0 list of Calib files to merge
1177     #arr_cpass0_calib_list[${runNumber}]="${commonOutputPath}/meta/cpass0.calib.run${runNumber}.list"
1178     arr_cpass0_calib_list[${runNumber}]="cpass0.calib.run${runNumber}.list"
1179     echo "### Produces the list of CPass0 files to merge (executes locally) ###"
1180     echo "${arr_cpass0_calib_list[${runNumber}]}: benchmark.sh ${arr_cpass0_outputs[*]}"
1181     echo "    LOCAL ./benchmark.sh PrintValues calibfile ${arr_cpass0_calib_list[${runNumber}]} ${arr_cpass0_outputs[*]} && mkdir -p \$OUTPATH/meta && cp ${arr_cpass0_calib_list[${runNumber}]} \$OUTPATH/meta/${arr_cpass0_calib_list[${runNumber}]}"
1182     echo ; echo
1183
1184     #CPass0 merging
1185     echo "### Merges CPass0 files ###"
1186     #arr_cpass0_merged[${runNumber}]="${commonOutputPath}/meta/merge.cpass0.run${runNumber}.done"
1187     arr_cpass0_merged[${runNumber}]="merge.cpass0.run${runNumber}.done"
1188     echo "cpass0.localOCDB.${runNumber}.tgz.done ${arr_cpass0_merged[${runNumber}]}: benchmark.sh ${configFile} ${arr_cpass0_calib_list[${runNumber}]} ${copyFiles[@]}"
1189     echo "    ${alirootEnv} ./benchmark.sh MergeCPass0 \$OUTPATH/000${runNumber}/cpass0 ${currentDefaultOCDB} ${configFile} ${runNumber} ${arr_cpass0_calib_list[${runNumber}]} ${extraOpts[@]}"" "
1190     echo ; echo
1191
1192     #CPass1 list of Calib/QA/ESD/filtered files
1193     # the trick with QA is to have the string "Stage.txt" in the file name of the list of directories with QA output to trigger
1194     # the production of the QA trending tree (only then the task->Finish() will be called in QAtrain_duo.C, on the grid
1195     # this corresponds to the last merging stage)
1196     #arr_cpass1_QA_list[${runNumber}]="${commonOutputPath}/meta/cpass1.QA.run${runNumber}.lastMergingStage.txt.list"
1197     arr_cpass1_QA_list[${runNumber}]="cpass1.QA.run${runNumber}.lastMergingStage.txt.list"
1198     echo "### Lists CPass1 QA ###"
1199     echo "${arr_cpass1_QA_list[${runNumber}]}: benchmark.sh ${arr_cpass1_outputs[*]}"
1200     echo "    LOCAL ./benchmark.sh PrintValues dir ${arr_cpass1_QA_list[${runNumber}]} ${arr_cpass1_outputs[*]} && mkdir -p \$OUTPATH/meta && cp ${arr_cpass1_QA_list[${runNumber}]} \$OUTPATH/meta/${arr_cpass1_QA_list[${runNumber}]}"
1201     echo ; echo
1202
1203     #arr_cpass1_calib_list[${runNumber}]="${commonOutputPath}/meta/cpass1.calib.run${runNumber}.list"
1204     arr_cpass1_calib_list[${runNumber}]="cpass1.calib.run${runNumber}.list"
1205     echo "### Lists CPass1 Calib ###"
1206     echo "${arr_cpass1_calib_list[${runNumber}]}: benchmark.sh ${arr_cpass1_outputs[*]}"
1207     echo "    LOCAL ./benchmark.sh PrintValues calibfile ${arr_cpass1_calib_list[${runNumber}]} ${arr_cpass1_outputs[*]} && mkdir -p \$OUTPATH/meta && cp ${arr_cpass1_calib_list[${runNumber}]} \$OUTPATH/meta/${arr_cpass1_calib_list[${runNumber}]}"
1208     echo ; echo
1209
1210     #arr_cpass1_ESD_list[${runNumber}]="${commonOutputPath}/meta/cpass1.ESD.run${runNumber}.list"
1211     arr_cpass1_ESD_list[${runNumber}]="cpass1.ESD.run${runNumber}.list"
1212     echo "### Lists CPass1 ESDs ###"
1213     echo "${arr_cpass1_ESD_list[${runNumber}]}: benchmark.sh ${arr_cpass1_outputs[*]}"
1214     echo "    LOCAL ./benchmark.sh PrintValues esd ${arr_cpass1_ESD_list[${runNumber}]} ${arr_cpass1_outputs[*]} && mkdir -p \$OUTPATH/meta && cp ${arr_cpass1_ESD_list[${runNumber}]} \$OUTPATH/meta/${arr_cpass1_ESD_list[${runNumber}]}"
1215     echo ; echo
1216
1217     #arr_cpass1_filtered_list[${runNumber}]="${commonOutputPath}/meta/cpass1.filtered.run${runNumber}.list"
1218     arr_cpass1_filtered_list[${runNumber}]="cpass1.filtered.run${runNumber}.list"
1219     echo "### Lists CPass1 filtered ###"
1220     echo "${arr_cpass1_filtered_list[${runNumber}]}: benchmark.sh ${arr_cpass1_outputs[*]}"
1221     echo "    LOCAL ./benchmark.sh PrintValues filteredTree ${arr_cpass1_filtered_list[${runNumber}]} ${arr_cpass1_outputs[*]} && mkdir -p \$OUTPATH/meta && cp ${arr_cpass1_filtered_list[${runNumber}]} \$OUTPATH/meta/${arr_cpass1_filtered_list[${runNumber}]}"
1222     echo ; echo
1223
1224     #CPass1 merging
1225     #arr_cpass1_merged[${runNumber}]="${commonOutputPath}/meta/merge.cpass1.run${runNumber}.done"
1226     arr_cpass1_merged[${runNumber}]="merge.cpass1.run${runNumber}.done"
1227     echo "### Merges CPass1 files ###"
1228     echo "cpass1.localOCDB.${runNumber}.tgz.done ${arr_cpass1_merged[${runNumber}]}: benchmark.sh ${configFile} ${arr_cpass1_calib_list[${runNumber}]} ${arr_cpass1_QA_list[${runNumber}]} ${copyFiles[@]}"
1229     echo "    ${alirootEnv} ./benchmark.sh MergeCPass1 \$OUTPATH/000${runNumber}/cpass1 ${currentDefaultOCDB} ${configFile} ${runNumber} ${arr_cpass1_calib_list[${runNumber}]} ${arr_cpass1_QA_list[${runNumber}]} ${arr_cpass1_filtered_list[${runNumber}]} ${extraOpts[@]}"" "
1230     echo ; echo
1231
1232     #CPass0 wrapped in a profiling tool (valgrind,....)
1233     if [[ -n ${profilingCommand} ]]; then
1234       inputFile=$(grep -m1 "${runNumber}/" ${inputFileList})
1235       [[ -z ${nEventsProfiling} ]] && nEventsProfiling=2
1236       currentDefaultOCDB=$(setYear ${inputFile} ${defaultOCDB})
1237       jobindex="profiling"
1238
1239       #arr_cpass0_profiled_outputs[${runNumber}]="${commonOutputPath}/meta/profiling.cpass0.job${jobindex}.run${runNumber}.done"
1240       arr_cpass0_profiled_outputs[${runNumber}]="profiling.cpass0.job${jobindex}.run${runNumber}.done"
1241       echo "### CPass0 in a profiler ###"
1242       echo "${arr_cpass0_profiled_outputs[${runNumber}]}: benchmark.sh ${configFile} ${copyFiles[@]}"
1243       profilingCommand=$(encSpaces "${profilingCommand}")
1244       echo "    ${alirootEnv} ./benchmark.sh CPass0 \$OUTPATH/000${runNumber}/${jobindex} ${inputFile} ${nEventsProfiling} ${currentDefaultOCDB} ${configFile} ${runNumber} ${jobindex} ${extraOpts[@]} useProfilingCommand=${profilingCommand}"
1245       echo ; echo
1246     fi
1247
1248   done #runs
1249
1250   #Summary
1251   echo "### Summary ###"
1252   echo "summary.log: benchmark.sh ${configFile} ${arr_cpass1_merged[*]}"
1253   echo "     ${alirootEnv} ./benchmark.sh MakeSummary ${configFile} ${extraOpts[@]}"
1254   echo ; echo
1255
1256   return 0
1257 )
1258
1259 goPrintValues()
1260 (
1261   #print the values given the key from any number of files (store in output file on second argument)
1262   if [[ $# -lt 3 ]]; then
1263     echo "goPrintValues key outputFile inputFiles"
1264     echo "if outputFile is \"-\" don't redirect to a file"
1265     return
1266   fi
1267   key=${1}
1268   outputFile=${2}
1269   [[ ${outputFile} =~ "-" ]] && outputFile=""
1270   shift 2 #remove 2 first arguments from arg list to only pass the input files to awk
1271   awk -v key=${key} '$0 ~ key" " {print $2}' "$@" | tee ${outputFile}
1272   return 0
1273 )
1274
1275 goCreateQAplots()
1276 (
1277   umask 0002
1278   mergedQAfileList=${1}
1279   productionID=${2}
1280   outputDir=${3}
1281   configFile=${4}
1282   shift 4
1283   if ! parseConfig ${configFile} "$@"; then return 1; fi
1284
1285   #record the working directory provided by the batch system
1286   batchWorkingDirectory=${PWD}
1287
1288   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
1289
1290   [[ -z ${logOutputDir} ]] && logOutputDir=${PWD}
1291   [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/makeQAplots.log
1292   echo "${0} $*"
1293
1294   olddir=${PWD}
1295   mkdir -p ${outputDir}
1296   cd ${outputDir}
1297   [[ ! "${PWD}" =~ "${outputDir}" ]] && echo "PWD is not equal to outputDir=${outputDir}" && cd ${olddir} && return 1
1298
1299   inputFiles=(
1300               "${batchWorkingDirectory}/runQA.sh"
1301               "${ALICE_ROOT}/PWGPP/QA/scripts/runQA.sh"
1302   )
1303   for file in ${inputFiles[*]}; do
1304     [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
1305   done
1306
1307   echo "running QA with command:"
1308   echo ./runQA.sh inputList="${mergedQAfileList}" inputListHighPtTrees="${filteringList}" ocdbStorage="${defaultOCDB}"
1309   ./runQA.sh inputList="${mergedQAfileList}" inputListHighPtTrees="${filteringList}" ocdbStorage="${defaultOCDB}"
1310   cd ${olddir}
1311   return 0
1312 )
1313
1314 goTest()
1315 (
1316   echo AA
1317 )
1318
1319 alirootInfo()
1320 (
1321   umask 0002
1322   # save aliroot repository info
1323   [[ -z "${ALICE_ROOT}" ]] && return 1
1324
1325   echo "\${ALICE_ROOT}=${ALICE_ROOT}"
1326   echo "\${ROOTSYS}=${ROOTSYS}"
1327   echo "\${PATH}=${PATH}"
1328   echo "\${LD_LIBRARY_PATH}=${LD_LIBRARY_PATH}"
1329   echo
1330
1331   pushd ${PWD}
1332   cd ${ALICE_ROOT}
1333
1334   currentBranch=$(git rev-parse --abbrev-ref HEAD)
1335   git status
1336   echo ""
1337   echo ""
1338   git diff ${currentBranch}
1339   popd
1340   return 0
1341 )
1342
1343 setYear()
1344 (
1345   #set the year
1346   #  ${1} - year to be set
1347   #  ${2} - where to set the year
1348   year1=$(guessYear ${1})
1349   year2=$(guessYear ${2})
1350   local path=${2}
1351   [[ ${year1} -ne ${year2} && -n ${year2} && -n ${year1} ]] && path=${2/\/${year2}\//\/${year1}\/}
1352   echo ${path}
1353   return 0
1354 )
1355
1356 guessPeriod()
1357 (
1358   #guess the period from the path, pick the rightmost one
1359   local IFS="/"
1360   declare -a path=( ${1} )
1361   local dirDepth=${#path[*]}
1362   for ((x=${dirDepth}-1;x>=0;x--)); do
1363     local field=${path[${x}]}
1364     [[ ${field} =~ ^LHC[0-9][0-9][a-z]$ ]] && period=${field} && break
1365   done
1366   echo ${period}
1367   return 0
1368 )
1369
1370 guessYear()
1371 (
1372   #guess the year from the path, pick the rightmost one
1373   local IFS="/"
1374   declare -a path=( ${1} )
1375   local dirDepth=${#path[*]}
1376   for ((x=${dirDepth}-1;x>=0;x--)); do
1377     local field=${path[${x}]}
1378     [[ ${field} =~ ^20[0-9][0-9]$ ]] && year=${field} && break
1379   done
1380   echo ${year}
1381   return 0
1382 )
1383
1384 guessRunNumber()
1385 (
1386   #guess the run number from the path, pick the rightmost one
1387   #works for /path/foo/000123456/bar/...
1388   #and       /path/foo.run123456.bar
1389   local IFS="/."
1390   declare -a path=( ${1} )
1391   local dirDepth=${#path[*]}
1392   for ((x=${dirDepth}-1;x>=0;x--)); do
1393     local field=${path[${x}]}
1394     field=${field/run/000}
1395     [[ ${field} =~ [0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && runNumber=${field#000} && break
1396   done
1397   echo ${runNumber}
1398   return 0
1399 )
1400
1401 validateLog()
1402 (
1403   log=${1}
1404   errorConditions=(
1405                     'There was a crash'
1406                     'floating'
1407                     'error while loading shared libraries'
1408                     'std::bad_alloc'
1409                     's_err_syswatch_'
1410                     'Thread [0-9]* (Thread'
1411                     'AliFatal'
1412                     'core dumped'
1413                     '\.C.*error:.*\.h: No such file'
1414                     'line.*Aborted'
1415   )
1416
1417   warningConditions=(
1418                      'This is serious !'
1419                      'rocVoltage out of range:'
1420   )
1421
1422   local logstatus=0
1423   local errorSummary=""
1424   local warningSummary=""
1425
1426   for ((i=0; i<${#errorConditions[@]};i++)); do
1427     local tmp=$(grep -m1 -e "${errorConditions[${i}]}" ${log})
1428     [[ -n ${tmp} ]] && tmp+=" : "
1429     errorSummary+=${tmp}
1430   done
1431
1432   for ((i=0; i<${#warningConditions[@]};i++)); do
1433     local tmp=$(grep -m1 -e "${warningConditions[${i}]}" ${log})
1434     [[ -n ${tmp} ]] && tmp+=" : "
1435     warningSummary+=${tmp}
1436   done
1437
1438   if [[ -n ${errorSummary} ]]; then
1439     echo "${errorSummary}"
1440     return 1
1441   fi
1442
1443   if [[ -n ${warningSummary} ]]; then
1444     echo "${warningSummary}"
1445     return 2
1446   fi
1447
1448   return 0
1449 )
1450
1451 summarizeLogs()
1452 (
1453   #print a summary of logs
1454   logFiles=(
1455             "*.log"
1456             "stdout"
1457             "stderr"
1458   )
1459
1460   #put dir information in the output
1461   echo "dir $PWD"
1462
1463   #check logs
1464   local logstatus=0
1465   for log in ${logFiles[*]}; do
1466     finallog=${PWD%/}/${log}
1467     [[ ! -f ${log} ]] && continue
1468     errorSummary=$(validateLog ${log})
1469     validationStatus=$?
1470     if [[ ${validationStatus} -eq 0 ]]; then
1471       #in pretend mode randomly report an error in rec.log some cases
1472       if [[ -n ${pretend} && "${log}" == "rec.log" ]]; then
1473         #[[ $(( ${RANDOM}%2 )) -ge 1 ]] && echo "${finallog} BAD random error" || echo "${finallog} OK"
1474         echo "${finallog} OK"
1475       else
1476         echo "${finallog} OK"
1477       fi
1478     elif [[ ${validationStatus} -eq 1 ]]; then
1479       echo "${finallog} BAD ${errorSummary}"
1480       logstatus=1
1481     elif [[ ${validationStatus} -eq 2 ]]; then
1482       echo "${finallog} OK MWAH ${errorSummary}"
1483     fi
1484   done
1485
1486   #report core files
1487   while read x; do
1488     echo ${x}
1489     chmod 644 ${x}
1490     gdb --batch --quiet -ex "bt" -ex "quit" aliroot ${x} > stacktrace_${x//\//_}.log
1491   done < <(/bin/ls ${PWD}/*/core 2>/dev/null; /bin/ls ${PWD}/core 2>/dev/null)
1492
1493   return ${logstatus}
1494 )
1495
1496 spitOutLocalOCDBaccessConfig()
1497 {
1498   umask 0002
1499   #find ${1} -name "*root" | \
1500   /bin/ls -1 ${1}/*/*/*/*.root 2>/dev/null | \
1501   while read line
1502   do
1503     local tmp=${line#${1}}
1504     echo ${tmp%/*} | \
1505     awk -v ocdb=${1} '{print "  man->SetSpecificStorage(\""$1"\",\"local://"ocdb"\");"}'
1506   done
1507   return 0
1508 }
1509
1510 goMakeLocalOCDBaccessConfig()
1511 {
1512   umask 0002
1513   # make a script that sets the specific storages form all the root files produced by CPass0
1514   local localOCDBpathCPass0=${1}
1515   local OCDBpathPrefix=${2}
1516   [[ -z ${OCDBpathPrefix} ]] && OCDBpathPrefix="."
1517
1518   if [[ -f ${localOCDBpathCPass0} && ${localOCDBpathCPass0} =~ \.tgz$ ]]; then
1519     tar xzf ${localOCDBpathCPass0}
1520     local localOCDBpathCPass0="${OCDBpathPrefix}/OCDB"
1521   fi
1522
1523   echo
1524   echo creating the specific storage script
1525   echo   localOCDBaccessConfig.C
1526   echo   based on OCDB: ${localOCDBaccessConfig}
1527   echo
1528
1529   local tempLocalOCDB=""
1530   if [[ -f localOCDBaccessConfig.C ]]; then
1531     tempLocalOCDB=$(mktemp -t tempLocalOCDB.XXXXXX)
1532     echo "egrep "SetSpecificStorage" localOCDBaccessConfig.C > ${tempLocalOCDB}"
1533     egrep "SetSpecificStorage" localOCDBaccessConfig.C > ${tempLocalOCDB}
1534   fi
1535
1536   echo "localOCDBaccessConfig()"                               >  localOCDBaccessConfig.C
1537   echo "{"                                                     >> localOCDBaccessConfig.C
1538   echo "  AliCDBManager* man = AliCDBManager::Instance();"     >> localOCDBaccessConfig.C
1539   spitOutLocalOCDBaccessConfig ${localOCDBpathCPass0}|sort|uniq  >> localOCDBaccessConfig.C
1540   [[ -f "${tempLocalOCDB}" ]] && cat ${tempLocalOCDB}              >> localOCDBaccessConfig.C
1541   echo "}"                                                     >> localOCDBaccessConfig.C
1542
1543   [[ -f "${tempLocalOCDB}" ]] && rm -f ${tempLocalOCDB}
1544
1545   if ! grep SetSpecificStorage localOCDBaccessConfig.C; then
1546     echo
1547     echo "!!!!!!! CPass0 produced no OCDB entries"
1548     return 1
1549   fi
1550   return 0
1551 }
1552
1553 goMakeFilteredTrees()
1554 (
1555   outputDir=${1}
1556   runNumber=${2}
1557   #get path to input list
1558   inputListfiles=${3}
1559   #get scale number for tracks
1560   filterT=${4}
1561   #get scale number for V0s
1562   filterV=${5}
1563   #get OCDB path (optional)
1564   OCDBpath=${6}
1565   #get max number of files
1566   maxFiles=${7-"1000000"}
1567   #get offset of first file
1568   offsetFile=${8-"0"}
1569   #get max number of events
1570   maxEvents=${9-"30000000"}
1571   #get offset of first event
1572   offsetEvent=${10-"0"}
1573   configFile=${11-"benchmark.config"}
1574   esdFileName=${12-"AliESDs_Barrel.root"}
1575   shift 12
1576   if ! parseConfig ${configFile} "$@"; then return 1; fi
1577
1578   #record the working directory provided by the batch system
1579   batchWorkingDirectory=${PWD}
1580
1581   [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
1582   doneFileBase=filtering.cpass1.run${runNumber}.done
1583   doneFileTmp=${batchWorkingDirectory}/${doneFileBase}
1584   doneFile=${commonOutputPath}/meta/${doneFileBase}
1585
1586   cat > filtering.log << EOF
1587   goMakeFilteredTrees config:
1588   runpath=${runpath}
1589   outputDir=${outputDir}
1590   commonOutputPath=${commonOutputPath}
1591   ALICE_ROOT=${ALICE_ROOT}
1592   PATH=${PATH}
1593   offsetEvent=$offsetEvent
1594   configFile=$configFile
1595   esdFileName=$esdFileName
1596   inputListfiles=$inputListfiles
1597   doneFile=$doneFile
1598 EOF
1599
1600   #runpath=${outputDir}
1601   #[[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
1602   #[[ ${reconstructInTemporaryDir} -eq 1 ]] && runpath=$(mktemp -d -t goMakeFilteredTrees.XXXXXX)
1603   #mkdir -p ${outputDir}
1604   #mkdir -p ${runpath}
1605   #if ! cd ${runpath}; then
1606   #  echo "PWD=$PWD is not the runpath=${runpath}"
1607   #  touch ${doneFile}
1608   #  return 1
1609   #fi
1610
1611   if [[ -z ${pretend} ]];then
1612     aliroot -l -b -q "${ALICE_ROOT}/PWGPP/macros/runFilteringTask.C(\"${inputListfiles}\",${filterT},${filterV},\"${OCDBpath}\",${maxFiles},${offsetFile},${maxEvents},${offsetEvent},\"${esdFileName}\")" &>> filtering.log
1613   else
1614     sleep ${pretendDelay}
1615     touch filtering.log FilterEvents_Trees.root
1616   fi
1617   pwd
1618   /bin/ls
1619   summarizeLogs >>  ${doneFile}
1620
1621   #echo mv -f * ${outputDir}
1622   #mv -f * ${outputDir}
1623   #[[ -f ${outputDir}/FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFile}
1624   #cd ${commonOutputPath}
1625   #[[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
1626
1627   cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
1628   [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
1629
1630   return 0
1631 )
1632
1633 submit()
1634 {
1635   umask 0002
1636   [[ $# -lt 5 ]] && echo "at least 5 args needed, you supplied $#" && return 1
1637   JobID=${1}
1638   startID=${2}
1639   endID=${3}
1640   waitForJOBID=${4}
1641   command=${5}
1642   shift 5
1643   local commandArgs=("$@")
1644
1645   #add quote strings around the extra arguments
1646   for ((i=0; i<${#commandArgs[@]}; i++)); do
1647     commandArgs[i]=\"${commandArgs[i]}\"
1648   done
1649
1650   [[ -z ${waitForJOBID} ]] && waitForJOBID=0
1651
1652   newFarm=$(which qsub|grep "^/usr/bin/qsub")
1653
1654   batchSystem="SGE"
1655
1656   if [[ -z "${newFarm}" ]]
1657   then
1658     #old LSF
1659     # submit it (as job array)
1660     nFiles=$(( ${endID}-${startID}+1 ))
1661     while [ ${startID} -le ${nFiles}  ] ; do
1662       if [ $(expr ${nFiles} - ${startID}) -gt 999 ] ; then
1663         endID=$(expr ${startID} + 999)
1664       else
1665         endID=${nFiles}
1666       fi
1667     if [[ ${waitForJOBID} -eq 0 ]]; then
1668         echo ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}"
1669         ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}"
1670       else
1671         echo ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -w "ended(${waitForJOBID})" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}"
1672         ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -w "ended(${waitForJOBID})" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}"
1673       fi
1674       startID=$(expr ${endID} + 1)
1675     done
1676   else
1677     #new SGE farm
1678     if [[ ${waitForJOBID} =~ "000" ]]; then
1679       echo ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}"
1680       ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}"
1681     else
1682       echo ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -hold_jid "${waitForJOBID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}"
1683       ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -hold_jid "${waitForJOBID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}"
1684     fi
1685   fi
1686   return 0
1687 }
1688
1689 goSubmitBatch()
1690 {
1691   if [[ $# -lt 3 ]]; then
1692     echo "minimal use:"
1693     echo " ${0} submit fileList productionID configFile"
1694     return 0
1695   fi
1696
1697   productionID=${1}
1698   inputList=${2}
1699   configFile=${3:-"benchmark.config"}
1700   #if which greadlink; then configFile=$(greadlink -f ${configFile}); fi
1701   shift 3
1702   extraOpts=("$@")
1703   if ! parseConfig ${configFile} "${extraOpts[@]}"; then return 1; fi
1704
1705   #batch systems/makeflow sometimes handle spaces in arguments poorly, so encode them
1706   for (( i=0;i<${#extraOpts[@]};i++ )); do
1707     extraOpts[i]=$(encSpaces "${extraOpts[i]}")
1708   done
1709   extraOpts+=("encodedSpaces=1")
1710   #this removes the copy of the done file used by makeflow (in the running dir)
1711   extraOpts+=("removeTMPdoneFile=1")
1712
1713   #record the working directory provided by the batch system
1714   batchWorkingDirectory=${PWD}
1715
1716   #redirect all output to submit.log
1717   echo "redirecting all output to ${PWD}/submit_${productionID//"/"/_}.log"
1718   exec 7>&1
1719   exec 1>submit_${productionID//"/"/_}.log 2>&1
1720
1721   umask 0002
1722   echo ${0}" submit $*"
1723   if [[ -z "${inputList}" || -z "${productionID}" ]]
1724   then
1725     echo
1726     echo " Usage: ${0} submit inputList productionID [configFile=benchmark.config]"
1727     echo
1728     return
1729   fi
1730
1731   # check if config file is there
1732   if [ ! -f ${configFile} ]; then
1733     echo "ERROR! Config File '${configFile}' not found" >&2
1734     return
1735   else
1736     echo "Using Config File: '${configFile}'"
1737   fi
1738
1739   [[ ! -f ${alirootEnv} ]] && echo "alirootEnv script ${alirootEnv} not found!..." && return 1
1740
1741   #move the script, config and some other stuff to ${commonOutputPath} first, then use them from there
1742   self=${0}
1743   #if which greadlink; then self=$(greadlink -f "${0}"); fi
1744   configPath=$(dirname ${configFile})
1745   export commonOutputPath=${baseOutputDirectory}/${productionID}
1746
1747   mkdir -p ${commonOutputPath}
1748   mkdir -p ${commonOutputPath}/logs
1749   mkdir -p ${commonOutputPath}/meta
1750
1751   cp ${self} ${commonOutputPath}
1752   cp ${configFile} ${commonOutputPath}
1753   cp ${inputList} ${commonOutputPath}
1754   self=${commonOutputPath}/${self##*/}
1755   chmod u+x ${self}
1756   configFile=${commonOutputPath}/${configFile##*/}
1757   inputList=${commonOutputPath}/${inputList##*/}
1758
1759   #convert to absolut pathnames
1760   #if which greadlink; then inputList=$(greadlink -f "${inputList}"); fi
1761   #make list of runs
1762   if [[ -z ${runNumber} ]]; then
1763     listOfRuns=($(while read x; do guessRunNumber ${x}; done < ${inputList} | sort | uniq))
1764   else
1765     listOfRuns=${runNumber}
1766   fi
1767
1768   #if which greadlink; then alirootSource=$(greadlink -f "${alirootSource}"); fi
1769
1770   echo ""
1771   echo "### BEGIN CONFIGURATION ###"
1772   echo ""
1773   echo "GENERAL:"
1774   echo ""
1775   echo "    productionID:    ${productionID}"
1776   echo "    batchCommand:    ${batchCommand}"
1777   echo "    batchFlags:      ${batchFlags}"
1778   echo "    alirootEnv:   ${alirootEnv}"
1779   ${alirootEnv} echo '    ALICE_ROOT:      ${ALICE_ROOT}'
1780   ${alirootEnv} echo '    ALIROOT_RELEASE: ${ALICE_RELEASE}'
1781   echo "    inputList:       ${inputList}"
1782   echo "    configPath:      ${configPath}"
1783   echo "    commonOutputPath:      ${commonOutputPath}"
1784   echo "    defaultOCDB:     ${defaultOCDB}"
1785   echo "      autoOCDB: ${autoOCDB}"
1786   echo "    recoTriggerOptions:   ${recoTriggerOptions}"
1787   echo "    runs:"
1788   echo "      ${listOfRuns[*]}"
1789   echo ""
1790   echo "THE TRAIN WILL RUN:"
1791
1792   if [ ${runCPass0reco} -eq 1 ]; then
1793     echo "    Pass0 - Recontruction"
1794   fi
1795
1796   if [ ${runCPass0MergeMakeOCDB} -eq 1 ]; then
1797     echo "    Pass0 - merging and OCDB export"
1798   fi
1799
1800   if [ ${runCPass1reco} -eq 1 ]; then
1801     echo "    Pass1 - Recontruction"
1802   fi
1803   if [ ${runCPass1MergeMakeOCDB} -eq 1 ]; then
1804     echo "    Pass1 - merging and OCDB export"
1805   fi
1806
1807   echo ""
1808   echo "LIMITS:"
1809   echo "    max. Events/Chunk:   ${nEvents}"
1810   echo "    max. Number of Chunks per Run:     ${nMaxChunks}"
1811   echo ""
1812   echo "### END CONFIGURATION ###"
1813   echo ""
1814
1815
1816   # check if input file is there
1817   if [ ! -f ${inputList} ]; then
1818     echo "ERROR! Input List '${inputList}' not found" >&2
1819     return
1820   fi
1821
1822   # define jobid (for dependent jobs)
1823   date=$(date +%Y_%m_%d_%H%M%S)
1824   #for each run we submit one jobarray:
1825   for runNumber in ${listOfRuns[*]}; do
1826
1827     [[ -z ${runNumber} ]] && continue
1828     [[ ! ${runNumber} =~ ^[0-9]*[0-9]$ ]] && continue
1829
1830     JOBpostfix="${productionID//"/"/_}_${runNumber}_${date}"
1831     JOBID1="p0_${JOBpostfix}"
1832     JOBID1wait="w0_${JOBpostfix}"
1833     JOBID2="m0_${JOBpostfix}"
1834     JOBID2wait="wm0_${JOBpostfix}"
1835     JOBID3="op0_${JOBpostfix}"
1836     JOBID3wait="wop0_${JOBpostfix}"
1837     JOBID4="p1_${JOBpostfix}"
1838     JOBID4wait="w1_${JOBpostfix}"
1839     JOBID5="m1_${JOBpostfix}"
1840     JOBID5wait="wm1_${JOBpostfix}"
1841     JOBID6="s1_${JOBpostfix}"
1842     JOBID6wait="ws1_${JOBpostfix}"
1843     JOBID7="QA_${JOBpostfix}"
1844     JOBmakeESDlistCPass1="lp1_${JOBpostfix}"
1845     JOBfilterESDcpass1="fp1_${JOBpostfix}"
1846     LASTJOB="000"
1847
1848     oneInputFile=$(egrep -m1 "${runNumber}/" ${inputList})
1849
1850     currentDefaultOCDB=${defaultOCDB}
1851     [[ -z ${autoOCDB} ]] && autoOCDB=1
1852     if [[ ${autoOCDB} -ne 0 ]]; then
1853       currentDefaultOCDB=$(setYear ${oneInputFile} ${defaultOCDB})
1854     fi
1855     period=$(guessPeriod ${oneInputFile})
1856     year=$(guessYear ${oneInputFile})
1857
1858     echo "submitting run ${runNumber} with OCDB ${currentDefaultOCDB}"
1859
1860     ###############################################################################
1861     #run one chunk with valgrind:
1862     if [[ -n ${profilingCommand} ]]; then
1863       [[ -z ${nEventsProfiling} ]] && nEventsProfiling=2
1864       [[ -z ${profilingCommand} ]] && profilingCommand="/usr/bin/valgrind --tool=callgrind --num-callers=40 -v --trace-children=yes"
1865       submit "profile-${JOBpostfix}" 1 1 000 "${alirootEnv} ${self}" CPass0 ${commonOutputPath}/${year}/${period}/000${runNumber}/${jobindex} ${oneInputFile} ${nEventsProfiling} ${currentDefaultOCDB} ${configFile} ${runNumber} ${jobindex} useProfilingCommand=$(encSpaces "${profilingCommand}") "${extraOpts[@]}"
1866     fi
1867
1868     ################################################################################
1869     ################################################################################
1870     # run the CPass0 if requested
1871
1872     if [ ${runCPass0reco} -eq 1 ]; then
1873
1874       echo
1875       echo "starting CPass0... for run ${runNumber}"
1876       echo
1877
1878       # create directory and copy all files that are needed
1879       targetDirectory="${commonOutputPath}/${year}/${period}/000${runNumber}/cpass0"
1880       mkdir -p ${targetDirectory}
1881
1882       filesCPass0=(
1883                     "${configPath}/runCPass0.sh"
1884                     "${configPath}/recCPass0.C"
1885                     "${configPath}/runCalibTrain.C"
1886                     "${configPath}/localOCDBaccessConfig.C"
1887                     "${configPath}/OCDB*.root"
1888                     "${configPath}/sim.C"
1889                     "${configPath}/rec.C"
1890                     "${configPath}/Config.C"
1891       )
1892       for file in ${filesCPass0[*]}; do
1893         [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
1894       done
1895
1896       localInputList=${targetDirectory}/${inputList##*/}
1897       [[ ! -f ${localInputList} ]] && egrep "\/000${runNumber}\/" ${inputList} > ${localInputList}
1898       # limit nFiles to nMaxChunks
1899       nFiles=$(wc -l < ${localInputList})
1900       [[ ${nFiles} -eq 0 ]] && echo "list contains ZERO files! exiting..." && return 1
1901       echo "raw files in list:    ${nFiles}"
1902       if [[ ${nMaxChunks} -gt 0 && ${nMaxChunks} -le ${nFiles} ]]; then
1903         nFiles=${nMaxChunks}
1904       fi
1905       echo "raw files to process: ${nFiles}"
1906       [[ -z "${percentProcessedFilesToContinue}" ]] && percentProcessedFilesToContinue=100
1907       if [[ ${percentProcessedFilesToContinue} -eq 100 ]]; then
1908         nFilesToWaitFor=${nFiles}
1909       else
1910         nFilesToWaitFor=$(( ${nFiles}-${nFiles}/(100/(100-${percentProcessedFilesToContinue})) ))
1911       fi
1912       echo "requested success rate is ${percentProcessedFilesToContinue}%"
1913       echo "merging will start after ${nFilesToWaitFor} jobs are done"
1914
1915       submit ${JOBID1} 1 ${nFiles} 000 "${alirootEnv} ${self}" CPass0 ${targetDirectory} ${localInputList} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} -1 "${extraOpts[@]}"
1916
1917       ## submit a monitoring job that will run until a certain number of jobs are done with reconstruction
1918       submit "${JOBID1wait}" 1 1 000 "${alirootEnv} ${self}" WaitForOutput ${commonOutputPath} "meta/cpass0.job*.run${runNumber}.done" ${nFilesToWaitFor} ${maxSecondsToWait}
1919       LASTJOB=${JOBID1wait}
1920
1921     fi #end running CPass0
1922     ################################################################################
1923
1924
1925     ################################################################################
1926     # submit merging of CPass0, depends on the reconstruction
1927
1928     if [ ${runCPass0MergeMakeOCDB} -eq 1 ]; then
1929
1930       echo
1931       echo "submit CPass0 merging for run ${runNumber}"
1932       echo
1933
1934       targetDirectory="${commonOutputPath}/${year}/${period}/000${runNumber}/cpass0"
1935       mkdir -p ${targetDirectory}
1936
1937       #copy the scripts
1938       filesMergeCPass0=(
1939                         "${configPath}/OCDB.root"
1940                         "${configPath}/mergeMakeOCDB.byComponent.sh"
1941                         "${configPath}/mergeMakeOCDB.sh"
1942                         "${configPath}/localOCDBaccessConfig.C"
1943                         "${configPath}/mergeByComponent.C"
1944                         "${configPath}/makeOCDB.C"
1945                         "${configPath}/merge.C"
1946       )
1947       for file in ${filesMergeCPass0[*]}; do
1948         [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
1949       done
1950
1951       submit ${JOBID2} 1 1 "${LASTJOB}" "${alirootEnv} ${self}" MergeCPass0 ${targetDirectory} ${currentDefaultOCDB} ${configFile} ${runNumber} cpass0.calib.run${runNumber}.list "${extraOpts[@]}"
1952       LASTJOB=${JOBID2}
1953
1954       if [[ -n ${generateMC} ]]; then
1955         submit "mrl${JOBpostfix}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" PrintValues sim ${commonOutputPath}/meta/sim.run${runNumber}.list ${commonOutputPath}/meta/cpass0.job*.run${runNumber}.done
1956         LASTJOB="mrl${JOBpostfix}"
1957       fi
1958
1959       echo
1960     fi
1961     # end of merging CPass0
1962     ################################################################################
1963
1964     ################################################################################
1965     ################################################################################
1966     # run the CPass1 if requested
1967
1968     if [ ${runCPass1reco} -eq 1 ]; then
1969
1970       targetDirectory="${commonOutputPath}/${year}/${period}/000${runNumber}/cpass1"
1971       rm -f ${commonOutputPath}/meta/cpass1.job*.run${runNumber}.done
1972
1973       # safety feature: if we are re-running for any reason we want to delete the previous output first.
1974       [[ -d ${targetDirectory} ]] && rm -rf ${targetDirectory}/* && echo "removed old output at ${targetDirectory}/*"
1975
1976       echo
1977       echo "starting CPass1... for run ${runNumber}"
1978       echo
1979
1980       # create directory and copy all files that are needed
1981       mkdir -p ${targetDirectory}
1982
1983       filesCPass1=(
1984                     "${configPath}/runCPass1.sh"
1985                     "${configPath}/recCPass1.C"
1986                     "${configPath}/recCPass1_OuterDet.C"
1987                     "${configPath}/runCalibTrain.C"
1988                     "${configPath}/QAtrain_duo.C"
1989                     "${configPath}/localOCDBaccessConfig.C"
1990                     "${configPath}/OCDB.root"
1991       )
1992       for file in ${filesCPass1[*]}; do
1993         [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
1994       done
1995
1996       if [[ -n ${generateMC} ]]; then
1997         localInputList=${commonOutputPath}/meta/sim.run${runNumber}.list
1998       else
1999         localInputList=${targetDirectory}/${inputList##*/}
2000         [[ ! -f ${localInputList} ]] && egrep "\/000${runNumber}\/" ${inputList} > ${localInputList}
2001       fi
2002       # limit nFiles to nMaxChunks
2003       nFiles=$(wc -l < ${localInputList})
2004       [[ ${nFiles} -eq 0 ]] && echo "list contains ZERO files! continuing..." && continue
2005       echo "raw files in list:    ${nFiles}"
2006       if [[ ${nMaxChunks} -gt 0 && ${nMaxChunks} -le ${nFiles} ]]; then
2007         nFiles=${nMaxChunks}
2008       fi
2009       echo "raw files to process: ${nFiles}"
2010       [[ -z "${percentProcessedFilesToContinue}" ]] && percentProcessedFilesToContinue=100
2011       if [[ ${percentProcessedFilesToContinue} -eq 100 ]]; then
2012         nFilesToWaitFor=${nFiles}
2013       else
2014         nFilesToWaitFor=$(( ${nFiles}-${nFiles}/(100/(100-${percentProcessedFilesToContinue})) ))
2015       fi
2016       echo "requested success rate is ${percentProcessedFilesToContinue}%"
2017       echo "merging will start after ${nFilesToWaitFor} jobs are done"
2018
2019       submit ${JOBID4} 1 ${nFiles} "${LASTJOB}" "${alirootEnv} ${self}" CPass1 ${targetDirectory} ${localInputList} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} -1 "${extraOpts[@]}"
2020
2021       ################################################################################
2022       ## submit a monitoring job that will run until a certain number of jobs are done with reconstruction
2023       submit "${JOBID4wait}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" WaitForOutput ${commonOutputPath} "meta/cpass1.job*.run${runNumber}.done" ${nFilesToWaitFor} ${maxSecondsToWait}
2024       LASTJOB=${JOBID4wait}
2025       ################################################################################
2026
2027       echo
2028     fi #end running CPass1
2029
2030     ################################################################################
2031     # submit merging of CPass1, depends on the reconstruction
2032     if [ ${runCPass1MergeMakeOCDB} -eq 1 ]; then
2033
2034       echo
2035       echo "submit CPass1 merging for run ${runNumber}"
2036       echo
2037
2038       targetDirectory="${commonOutputPath}/${year}/${period}/000${runNumber}/cpass1"
2039       rm -f ${commonOutputPath}/meta/merge.cpass1.run${runNumber}.done
2040       mkdir -p ${targetDirectory}
2041
2042       # copy files
2043       filesMergeCPass1=(
2044                         "${configPath}/OCDB.root"
2045                         "${configPath}/localOCDBaccessConfig.C"
2046                         "${configPath}/mergeMakeOCDB.byComponent.sh"
2047                         "${configPath}/mergeByComponent.C"
2048                         "${configPath}/makeOCDB.C"
2049                         "${configPath}/merge.C"
2050                         "${configPath}/mergeMakeOCDB.sh"
2051                         "${configPath}/QAtrain_duo.C"
2052       )
2053       for file in ${filesMergeCPass1[*]}; do
2054         [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
2055       done
2056
2057       submit "${JOBID5}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" MergeCPass1 ${targetDirectory} ${currentDefaultOCDB} ${configFile} ${runNumber} cpass1.calib.run${runNumber}.list cpass1.QA.run${runNumber}.lastMergingStage.txt.list cpass1.filtered.run${runNumber}.list "${extraOpts[@]}"
2058       LASTJOB=${JOBID5}
2059       echo
2060     fi
2061
2062     ###############################
2063     #if [ ${runESDfiltering} -eq 1 ]; then
2064     #  rm -f ${commonOutputPath}/cpass1.ESD.run${runNumber}.list
2065     #  rm -f ${commonOutputPath}/meta/filtering.cpass1.run*.done
2066     #  echo
2067     #  echo submitting filtering for run ${runNumber}
2068     #  echo
2069     #  submit "${JOBmakeESDlistCPass1}" 1 1 "${LASTJOB}" "${self}" PrintValues esd ${commonOutputPath}/meta/cpass1.ESD.run${runNumber}.list ${commonOutputPath}/meta/cpass1.job*.run${runNumber}.done
2070     #  submit "${JOBfilterESDcpass1}" 1 1 "${JOBmakeESDlistCPass1}" "${alirootEnv} ${self}" MakeFilteredTrees ${commonOutputPath}/${year}/${period}/000${runNumber}/cpass1 ${runNumber} ${commonOutputPath}/meta/cpass1.ESD.run${runNumber}.list ${filteringFactorHighPt} ${filteringFactorV0s} ${currentDefaultOCDB} 1000000 0 10000000 0 ${configFile} AliESDs_Barrel.root "${extraOpts[@]}"
2071     #  LASTJOB=${JOBfilterESDcpass1}
2072     #fi
2073
2074   done
2075
2076   #################################################################################
2077   #################################################################################
2078   #if [ ${runESDfiltering} -eq 1 ]; then
2079   #  submit "${JOBID5wait}" 1 1 "${LASTJOB}" "${self}" WaitForOutput ${commonOutputPath} "meta/filtering.cpass1.run*.done" "${#listOfRuns[@]}" ${maxSecondsToWait}
2080   #else
2081     submit "${JOBID5wait}" 1 1 "${LASTJOB}" "${self}" WaitForOutput ${commonOutputPath} "meta/merge.cpass1.run*.done" ${#listOfRuns[@]} ${maxSecondsToWait}
2082   #fi
2083   LASTJOB=${JOBID5wait}
2084
2085   #################################################################################
2086   echo
2087   echo "submit make a summary"
2088   echo
2089
2090   [[ -z ${alirootEnvQA} ]] && alirootEnvQA=$(encSpaces "${alirootEnv}")
2091   submit "${JOBID6}" 1 1 "${LASTJOB}" "${alirootEnvQA} ${self}" MakeSummary ${configFile} "commonOutputPath=${commonOutputPath}"
2092   LASTJOB=${JOBID6}
2093   #################################################################################
2094
2095   #restore stdout
2096   exec 1>&7 7>&-
2097   echo "jobs submitted."
2098   return 0
2099 }
2100
2101 goWaitForOutput()
2102 (
2103   umask 0002
2104   [[ $# -lt 3 ]] && echo "goWaitForOutput() wrong number of arguments, exiting.." && return 1
2105   echo searchPath=${1}
2106   echo fileName=${2}
2107   echo numberOfFiles=${3}
2108   echo maxSecondsToWait=${4}
2109   searchPath=${1}
2110   fileName=${2}
2111   numberOfFiles=${3}
2112   maxSecondsToWait=${4}
2113   echo "command to be executed: /bin/ls -1 ${searchPath}/${fileName}"
2114   [[ -z "${maxSecondsToWait}" ]] && maxSecondsToWait=$(( 3600*12 ))
2115   while true; do
2116     n=$(/bin/ls -1 ${searchPath}/${fileName} 2>/dev/null | wc -l)
2117     [[ ${n} -gt 0 ]] && echo "found ${n} X ${fileName}"
2118     [[ ${n} -ge ${numberOfFiles} ]] && break
2119     [[ ${SECONDS} -gt ${maxSecondsToWait} ]] && echo "timeout of ${maxSecondsToWait}s!" && break
2120     sleep 60
2121   done
2122   echo "DONE! exiting..."
2123   return 0
2124 )
2125
2126 mergeSysLogs()
2127 (
2128   outputFile=${1}
2129   shift
2130   inputFiles="$@"
2131   i=0
2132   if ! ls -1 ${inputFiles} &>/dev/null; then echo "the files dont exist!: ${inputFiles}"; return 1; fi
2133   while read x; do
2134     runNumber=$(guessRunNumber ${x})
2135     [[ -z ${runNumber} ]] && echo "run number cannot be guessed for ${x}" && continue
2136     awk -v run=${runNumber} -v i=${i} 'NR > 1 {print run" "$0} NR==1 && i==0 {print "run/I:"$0}' ${x}
2137     (( i++ ))
2138   done < <(ls -1 ${inputFiles}) > ${outputFile}
2139   return 0
2140 )
2141
2142 goMakeMergedSummaryTree()
2143 (
2144   # create list of calibration entries
2145   # takes no arguments, just run it in the base output
2146   # directory with the following files in the working directory
2147   #
2148   #  Calibration file lists:
2149   #       cpass0.dcsTree.list, cpass1.dcsTree.list
2150   #  QA trending root file:
2151   #       trending.root
2152   #
2153   #  Production infoo ascii files:
2154   #       summary_pass0.tree
2155   #       summary_pass1.tree
2156   #
2157
2158   [[ ! -f cpass0.dcsTree.list ]] && echo "no cpass0.dcsTree.list" && return 1
2159   [[ ! -f cpass1.dcsTree.list ]] && echo "no cpass1.dcsTree.list" && return 1
2160   [[ ! -f trending.root ]] && echo "no trending.root" && return 1
2161   [[ ! -f summary_pass0.tree ]] && echo "no summary_pass0.tree" && return 1
2162   [[ ! -f summary_pass1.tree ]] && echo "no summary_pass1.tree" && return 1
2163
2164   #first, dump the C macro to file
2165   cat << EOF > mergeTree.C
2166   //
2167   // Merge summary information
2168   // Following files are expected to be in the working directory
2169   //
2170   // Calibration file lists:
2171   //      cpass0.dcsTree.list, cpass1.dcsTree.list
2172   // QA trending root file:
2173   //      trending.root
2174   //
2175   // Production infoo ascii files:
2176   //      summary_pass0.tree
2177   //      summary_pass1.tree
2178   //
2179   void mergeTree(){
2180     //
2181     //
2182     //
2183     // Calibration values dump
2184     //
2185     //Printf("MakeTreeFromList cpass0.dcsTree.list");
2186     AliXRDPROOFtoolkit::MakeTreeFromList("Calib.TPC.CPass0.root", "dcs","dcs","cpass0.dcsTree.list",1);
2187     //Printf("MakeTreeFromList cpass1.dcsTree.list");
2188     AliXRDPROOFtoolkit::MakeTreeFromList("Calib.TPC.CPass1.root", "dcs","dcs","cpass1.dcsTree.list",1);
2189     //
2190     // Calibration status dump
2191     //
2192     TFile *fprod = TFile::Open("fproduction.root","recreate");
2193     TTree  tree0, tree1;
2194     //Printf("reading summary_pass0.tree");
2195     tree0.ReadFile("summary_pass0.tree");
2196     //Printf("reading summary_pass1.tree");
2197     tree1.ReadFile("summary_pass1.tree");
2198     tree0.Write("CPass0");
2199     tree1.Write("CPass1");
2200     fprod->Close();
2201     //
2202     //
2203     //
2204     TString stringSetup="";
2205     stringSetup+="1#QA.TPC#run#SummaryTPCQA/tpcQA#trending.root+";  //
2206     stringSetup+="1#Calib.TPC.CPass0#run#dcs#Calib.TPC.CPass0.root+";  //
2207     stringSetup+="1#Calib.TPC.CPass1#run#dcs#Calib.TPC.CPass1.root+";  //
2208     //
2209     stringSetup+="1#CPass0#runnumber#CPass0#fproduction.root+";  //
2210     stringSetup+="1#CPass1#runnumber#CPass1#fproduction.root+";  //
2211     //
2212     //Printf("stringSetup: %s", stringSetup.Data());
2213     AliXRDPROOFtoolkit::JoinTreesIndex("outAll.root","joinAll","run",stringSetup.Data(), 1);
2214   }
2215 EOF
2216
2217   aliroot -b -q "mergeTree.C" > mergeTrees.log
2218   return $?
2219 )
2220
2221 stackTraceTree()
2222 (
2223   if [[ $# -lt 1 ]]; then
2224     echo 'make stacktrace processing  in case of standard root crash log'
2225     echo 'input is a (list of) text files with the stack trace (either gdb aoutput'
2226     echo 'produced with e.g. gdb --batch --quiet -ex "bt" -ex "quit" aliroot core,'
2227     echo 'or the root crash log), output is a TTree formatted table.'
2228     echo 'example usage:'
2229     echo 'benchmark.sh stackTraceTree /foo/*/rec.log'
2230     echo 'benchmark.sh stackTraceTree $(cat file.list)'
2231     echo 'benchmark.sh stackTraceTree `cat file.list`'
2232     return 0
2233   fi
2234   gawk '
2235        BEGIN {
2236                print "frame/I:method/C:line/C:cpass/I:aliroot/I:file/C";
2237                RS="#[0-9]*";
2238                aliroot=0;
2239                read=1;
2240              }
2241       /There was a crash/ {read=1;}
2242       /The lines below might hint at the cause of the crash/ {read=0;}
2243       read==1 {
2244                if ($3 ~ /Ali*/) aliroot=1; else aliroot=0;
2245                gsub("#","",RT);
2246                if ($NF!="" && RT!="" && $3!="") print RT" "$3" "$NF" "0" "aliroot" "FILENAME
2247              }
2248       ' "$@" 2>/dev/null
2249 )
2250
2251 goMakeSummary()
2252 (
2253   #all the final stuff goes in here for ease of use:
2254   # summary logs
2255   # qa plot making
2256   # final file lists
2257   # runs in current dir - in makeflow mode it can run LOCAL, then the QA plots and summaries
2258   # will appear in the submission dir.
2259   #some defaults:
2260   log="summary.log"
2261   jsonLog="summary.json"
2262   productionID="qa"
2263
2264   configFile=${1}
2265   shift 1
2266   extraOpts=("$@")
2267   if ! parseConfig ${configFile} "${extraOpts[@]}"; then return 1; fi
2268
2269   #if which greadlink; then configFile=$(greadlink -f ${configFile}); fi
2270
2271   #record the working directory provided by the batch system
2272   batchWorkingDirectory=${PWD}
2273
2274   logTmp="${batchWorkingDirectory}/${log}"
2275   jsonLogTmp="${batchWorkingDirectory}/${jsonLog}"
2276
2277   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
2278
2279   [[ ! -f ${configFile} ]] && echo "no config file ${configFile}!" && return
2280
2281   [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
2282
2283   #copy some useful stuff
2284   [ -f "${commonOutputPath}/${configFile}" ] || paranoidCp "${configFile}" "${commonOutputPath}"
2285
2286   exec &> >(tee ${logTmp})
2287
2288   #summarize the global stuff
2289   echo "env script: ${alirootSource} ${alirootEnv}"
2290   echo "\$ALICE_ROOT=${ALICE_ROOT}"
2291   echo "commonOutputPath=${commonOutputPath}"
2292
2293   #summarize the stacktraces
2294   stackTraceTree ${commonOutputPath}/*/*/000*/cpass0/*/stacktrace* > stacktrace_cpass0.tree
2295   stackTraceTree ${commonOutputPath}/*/*/000*/cpass1/*/stacktrace* > stacktrace_cpass1.tree
2296
2297   # json header: open array of objects
2298   echo '[' > "${jsonLogTmp}"
2299
2300   echo total numbers for the production:
2301   echo
2302   awk 'BEGIN {nFiles=0;nCore=0;}
2303   /^calibfile/ {nFiles++;}
2304   /core dumped/ {nCore++i;}
2305   END {print     "cpass0 produced "nFiles" calib files, "nCore" core files";}' ${commonOutputPath}/meta/cpass0.job*done 2>/dev/null
2306   awk 'BEGIN {nOK=0; nBAD=0; }
2307   /\/rec.log OK/ {nOK++;}
2308   /\/rec.log BAD/ {nBAD++;}
2309   /stderr BAD/ {if ($0 ~ /rec.log/){nBAD++;}}
2310   END {print     "cpass0 reco:  OK: "nOK"\tBAD: "nBAD;}' ${commonOutputPath}/meta/cpass0.job*done 2>/dev/null
2311   awk 'BEGIN {nOK=0; nBAD=0; }
2312   /\/calib.log OK/ {nOK++;}
2313   /\/calib.log BAD/ {nBAD++;}
2314   END {print "cpass0 calib: OK: "nOK"\tBAD: "nBAD;}' ${commonOutputPath}/meta/cpass0.job*done 2>/dev/null
2315
2316   awk 'BEGIN {nOK=0; nBAD=0; }
2317   /merge.log OK/ {nOK++;}
2318   /merge.log BAD/ {nBAD++;}
2319   END {print "cpass0 merge: OK: "nOK"\tBAD: "nBAD;}' ${commonOutputPath}/meta/merge.cpass0*done 2>/dev/null
2320   awk 'BEGIN {nOK=0; nBAD=0; }
2321   /ocdb.log OK/ {nOK++;}
2322   /ocdb.log BAD/ {nBAD++;}
2323   END {print   "cpass0 OCDB:  OK: "nOK"\tBAD: "nBAD;}' ${commonOutputPath}/meta/merge.cpass0*done 2>/dev/null
2324
2325   echo
2326   awk 'BEGIN {nFiles=0;nCore=0;}
2327   /^calibfile/ {nFiles++;}
2328   /core dumped/ {nCore++;}
2329   END {print     "cpass1 produced "nFiles" calib files, "nCore" core files";}' ${commonOutputPath}/meta/cpass1.job*done 2>/dev/null
2330   awk 'BEGIN {nOK=0; nBAD=0; }
2331   /\/rec.log OK/ {nOK++;}
2332   /\/rec.log BAD/ {nBAD++;}
2333   /stderr BAD/ {if ($0 ~ /rec.log/){nBAD++;}}
2334   END {print     "cpass1 reco:  OK: "nOK"\tBAD: "nBAD;}' ${commonOutputPath}/meta/cpass1.job*done 2>/dev/null
2335   awk 'BEGIN {nOK=0; nBAD=0; }
2336   /\/calib.log OK/ {nOK++;}
2337   /\/calib.log BAD/ {nBAD++;}
2338   END {print "cpass1 calib: OK: "nOK"\tBAD: "nBAD;}' ${commonOutputPath}/meta/cpass1.job*done 2>/dev/null
2339
2340   awk 'BEGIN {nOK=0; nBAD=0; }
2341   /merge.log OK/ {nOK++;}
2342   /merge.log BAD/ {nBAD++;}
2343   END {print "cpass1 merge: OK: "nOK"\tBAD: "nBAD;}' ${commonOutputPath}/meta/merge.cpass1*done 2>/dev/null
2344   awk 'BEGIN {nOK=0; nBAD=0; }
2345   /ocdb.log OK/ {nOK++;}
2346   /ocdb.log BAD/ {nBAD++;}
2347   END {print   "cpass1 OCDB:  OK: "nOK"\tBAD: "nBAD;}' ${commonOutputPath}/meta/merge.cpass1*done 2>/dev/null
2348
2349   echo
2350   echo per run stats:
2351   /bin/ls -1 ${commonOutputPath}/meta/merge.cpass0.run*.done | while read x
2352 do
2353   dir=$(goPrintValues dir - ${x})
2354   runNumber=$(guessRunNumber ${dir})
2355   [[ -z ${runNumber} ]] && continue
2356
2357   if $(/bin/ls ${commonOutputPath}/meta/cpass0.job*.run${runNumber}.done &> /dev/null); then
2358     statusCPass0=( $(
2359     awk 'BEGIN {nOKrec=0;nBADrec=0;nOKcalib=0;nBADcalib=0;nOKstderr=0;nBADstderr=0;}
2360     /\/rec.log OK/ {nOKrec++;}
2361     /\/rec.log BAD/ {nBADrec++;}
2362     /stderr BAD/ {if ($0 ~ /rec.log/) {nBADrec++;} nBADstderr++;}
2363     /stderr OK/ {nOKstderr++;}
2364     /\/calib.log OK/ {nOKcalib++;}
2365     /\/calib.log BAD/ {nBADcalib++}
2366     END {print ""nOKrec" "nBADrec" "nOKstderr" "nBADstderr" "nOKcalib" "nBADcalib;}' ${commonOutputPath}/meta/cpass0.job*.run${runNumber}.done 2>/dev/null
2367     ) )
2368   fi
2369
2370   if $(/bin/ls ${commonOutputPath}/meta/cpass1.job*.run${runNumber}.done &>/dev/null); then
2371     statusCPass1=( $(
2372     awk 'BEGIN {nOKrec=0;nBADrec=0;nOKcalib=0;nBADcalib=0;nOKstderr=0;nBADstderr=0;nQAbarrelOK=0;nQAbarrelBAD=0;nQAouterOK=0;nQAouterBAD=0;}
2373     /\/rec.log OK/ {nOKrec++;}
2374     /\/rec.log BAD/ {nBADrec++;}
2375     /stderr BAD/ {if ($0 ~ /rec.log/) nBADrec++;nBADstderr++;}
2376     /stderr OK/ {nOKstderr++;}
2377     /\/calib.log OK/ {nOKcalib++;}
2378     /\/calib.log BAD/ {nBADcalib++}
2379     /\/qa_barrel.log OK/ {nQAbarrelOK++;}
2380     /\/qa_barrel.log BAD/ {nQAbarrelBAD++;}
2381     /\/qa_outer.log OK/ {nQAouterOK++;}
2382     /\/qa_outer.log BAD/ {nQAouterBAD++;}
2383     END {print ""nOKrec" "nBADrec" "nOKstderr" "nBADstderr" "nOKcalib" "nBADcalib" "nQAbarrelOK" "nQAbarrelBAD" "nQAouterOK" "nQAouterBAD;}' ${commonOutputPath}/meta/cpass1.job*.run${runNumber}.done 2>/dev/null
2384     ) )
2385   fi
2386
2387   statusOCDBcpass0=$(awk '/ocdb.log/ {print $2} ' ${x} 2>/dev/null)
2388   statusOCDBcpass1=$(awk '/ocdb.log/ {print $2}' ${x/cpass0/cpass1} 2>/dev/null)
2389   statusQA=$(awk '/mergeMakeOCDB.log/ {print $2}' ${x/cpass0/cpass1} 2>/dev/null)
2390
2391   printf "%s\t ocdb.log cpass0: %s\t ocdb.log cpass1: %s\tqa.log:%s\t| cpass0: rec:%s/%s stderr:%s/%s calib:%s/%s cpass1: rec:%s/%s stderr:%s/%s calib:%s/%s QAbarrel:%s/%s QAouter:%s/%s\n" ${runNumber} ${statusOCDBcpass0} ${statusOCDBcpass1} ${statusQA} ${statusCPass0[0]} ${statusCPass0[1]} ${statusCPass0[2]} ${statusCPass0[3]} ${statusCPass0[4]} ${statusCPass0[5]} ${statusCPass1[0]} ${statusCPass1[1]} ${statusCPass1[2]} ${statusCPass1[3]} ${statusCPass1[4]} ${statusCPass1[5]} ${statusCPass1[6]} ${statusCPass1[7]} ${statusCPass1[8]} ${statusCPass1[9]}
2392
2393   # produce json summary
2394   statusOCDBcpass0json=false
2395   statusOCDBcpass1json=false
2396   statusQAjson=false
2397   [[ "$statusOCDBcpass0" == 'OK' ]] && statusOCDBcpass0json=true
2398   [[ "$statusOCDBcpass1" == 'OK' ]] && statusOCDBcpass1json=true
2399   [[ "$statusQA" == 'OK' ]] && statusQAjson=true
2400   cat >> "$jsonLogTmp" <<EOF
2401   {
2402     run: ${runNumber},
2403     status: { ocdb_pass0: ${statusOCDBcpass0json}, ocdb_pass1: ${statusOCDBcpass1json}, qa: ${statusQAjson} },
2404     cpass0: {
2405       reco: { n_ok: ${statusCPass0[0]}, n_bad: ${statusCPass0[1]} },
2406       stderr: { n_ok: ${statusCPass0[2]}, n_bad: ${statusCPass0[3]} },
2407       calib: { n_ok: ${statusCPass0[4]}, n_bad: ${statusCPass0[5]} }
2408     },
2409     cpass1: {
2410       reco: { n_ok: ${statusCPass1[0]}, n_bad: ${statusCPass1[1]} },
2411       stderr: { n_ok: ${statusCPass1[2]}, n_bad: ${statusCPass1[3]} },
2412       calib: { n_ok: ${statusCPass1[4]}, n_bad: ${statusCPass1[5]} },
2413       qabarrel: { n_ok: ${statusCPass1[6]}, n_bad: ${statusCPass1[7]} },
2414       qarouter: { n_ok: ${statusCPass1[8]}, n_bad: ${statusCPass1[9]} }
2415     }
2416   },
2417 EOF
2418
2419 done
2420
2421   # json footer: close array of objects
2422   echo ']' >> "${jsonLogTmp}"
2423
2424   #make lists with output files - QA, trending, filtering and calibration
2425   ### wait for the merging of all runs to be over ###
2426   rm -f qa.list
2427   goPrintValues qafile qa.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2428   rm -f calib.list
2429   goPrintValues calibfile calib.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2430   rm -f trending.list
2431   goPrintValues trendingfile trending.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2432   rm -f filtering.list
2433   goPrintValues filteredTree filtering.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2434   rm -f cpass0.dcsTree.list
2435   goPrintValues dcsTree cpass0.dcsTree.list ${commonOutputPath}/meta/merge.cpass0.run*.done &>/dev/null
2436   rm -f cpass1.dcsTree.list
2437   goPrintValues dcsTree cpass1.dcsTree.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2438
2439   #merge trending
2440   rm -f trending.root
2441   goMerge trending.list trending.root ${configFile} "${extraOpts[@]}" &> mergeTrending.log
2442
2443   goMakeSummaryTree ${commonOutputPath} 0
2444   goMakeSummaryTree ${commonOutputPath} 1
2445
2446   goCreateQAplots "${PWD}/qa.list" "${productionID}" "QAplots" "${configFile}" "${extraOpts[@]}" filteringList="${PWD}/filtering.list" &>createQAplots.log
2447
2448   #make a merged summary tree out of the QA trending, dcs trees and log summary trees
2449   goMakeMergedSummaryTree
2450
2451   #if set, email the summary
2452   [[ -n ${MAILTO} ]] && cat ${logTmp} | mail -s "benchmark ${productionID} done" ${MAILTO}
2453
2454   #copy logs to destination
2455   paranoidCp "$logTmp" "${commonOutputPath}"
2456   paranoidCp "$jsonLogTmp" "${commonOutputPath}"
2457
2458   #copy output files
2459   exec &> >(tee fileCopy.log)
2460   paranoidCp QAplots ${commonOutputPath}
2461   paranoidCp *.list ${commonOutputPath}
2462   paranoidCp *.root ${commonOutputPath}
2463   paranoidCp *.log ${commonOutputPath}
2464   paranoidCp fileCopy.log ${commonOutputPath}
2465
2466   return 0
2467 )
2468
2469 goMakeSummaryTree()
2470 (
2471   if [[ $# -lt 1 ]] ; then
2472     return
2473   fi
2474   #1. define vars/arrays
2475   DIR=${1} #use input or exec in current dir
2476   pass=${2-"0"} #pass from input
2477   outfile="summary_pass${pass}.tree"
2478   Ncolumns=0
2479   test -f ${outfile} && : >${outfile}
2480   errfile=${outfile/tree/err}
2481   test -f ${errfile} && : >${errfile}
2482
2483   declare -a counterString=(TOFevents TOFtracks TPCevents TPCtracks TRDevents TRDtracks T0events SDDevents SDDtracks MeanVertexevents)
2484   Ncounter=${#counterString[@]}
2485
2486   declare -a statusString=(TRDStatus TOFStatus TPCStatus T0Status MeanVertexStatus)
2487   Nstatus=${#statusString[@]}
2488
2489
2490   declare -a ratesString=(rec stderr calib qa_barrel qa_outer)
2491   Nrates=${#ratesString[@]}
2492
2493   runs=( $(ls -1 ${DIR}/meta/merge.cpass0* | while read x; do guessRunNumber $x; done) )
2494   Nruns=${#runs[@]}
2495
2496   echo -n runnumber/I >>${outfile}
2497   echo -n :cpass${pass}status/I >>${outfile}
2498   echo -n :cpass${pass}QAstatus/I >>${outfile}
2499   for i in ${ratesString[@]}; do
2500     echo -n :${i}OK/I >>${outfile}
2501     echo -n :${i}BAD/I >>${outfile}
2502
2503   done
2504   for i in ${counterString[@]} ${statusString[@]} ; do
2505     echo -n :${i}/I >>${outfile}
2506   done
2507   Ncolumns=$((2 + 2*Nrates + Ncounter + Nstatus))
2508   echo >> ${outfile}
2509
2510   #2. loop runs
2511
2512   for runnumber in ${runs[@]} ; do
2513
2514
2515
2516     filejob="${DIR}/meta/cpass${pass}.job*.run${runnumber}.done"
2517     filemerge="${DIR}/meta/merge.cpass${pass}.run${runnumber}.done"
2518     fileOCDB=$(grep /ocdb.log ${filemerge} | awk '{print $1}')
2519     if ! $(/bin/ls ${filemerge} &>/dev/null) ; then
2520       echo "${filemerge} does not exist!" >>${errfile}
2521       continue
2522     elif ! $(/bin/ls ${filejob} &>/dev/null) ; then
2523       echo "${filejob} does not exist!" >>${errfile}
2524       echo -n ${runnumber} >> ${outfile}
2525       for i in $(seq ${Ncolumns}) ; do
2526         echo -n "-1" >> ${outfile}
2527       done
2528       echo >> ${outfile}
2529       continue
2530     fi
2531     echo -n ${runnumber} >> ${outfile}
2532     #pass0status= grep '/ocdb.log' ${filemerge} |  cut -d' ' -f2 | tr OK x1 | tr BAD xx0 | tr -d 'x'
2533     passStatus=$(grep '/ocdb.log' ${filemerge} | grep OK | wc -l)
2534     echo -n " ${passStatus}" >> ${outfile}
2535     qaStatus=$(grep '/mergeMakeOCDB.log' ${filemerge} | grep OK | wc -l)
2536     echo -n " ${qaStatus}" >> ${outfile}
2537
2538
2539     #fill OK/BAD rates
2540     for i in $(seq 0 $((${Nrates}-1))) ; do
2541       var1=$(grep "/${ratesString[${i}]}.log" ${filejob} | grep OK | wc -l)
2542       var2=$(grep "/${ratesString[${i}]}.log" ${filejob} | grep BAD | wc -l)
2543
2544       if [[ ${ratesString[${i}]} == "stderr" ]] ; then
2545         var1=$(grep "stderr" ${filejob} | grep OK | wc -l)
2546         var2=$(grep "stderr" ${filejob} | grep "rec.log" | grep BAD | wc -l)
2547       fi
2548       echo -n " ${var1}" >> ${outfile}
2549       echo -n " ${var2}" >> ${outfile}
2550     done
2551
2552     if [[ -f ${fileOCDB} ]] ; then
2553       #fill counter
2554       for i in $(seq 0 $((${Ncounter}-1))) ; do
2555         var1=$(grep Monalisa ${fileOCDB} | grep ${counterString[${i}]} | cut -f2)
2556         echo -n " ${var1:-"-1"}" >> ${outfile}
2557       done
2558
2559       #fill status
2560       for i in $(seq 0 $((${Nstatus}-1))) ; do
2561         var1=$(grep "calibration status=" ${fileOCDB} | grep ${statusString[${i}]/Status/} | cut -d'=' -f2)
2562         echo -n " ${var1:-"-1"}" >> ${outfile}
2563       done
2564     fi
2565     echo >> ${outfile}
2566   done
2567
2568   return 0
2569 )
2570
2571 parseConfig()
2572 {
2573   configFile=${1}
2574   shift
2575   args=("$@")
2576
2577
2578   #some defaults
2579   #autoOCDB=0
2580   defaultOCDB="raw://"
2581   #runNumber=167123
2582   #makeflowPath="/hera/alice/aux/cctools/bin"
2583   #makeflowOptions="-T wq -N alice -d all -C ali-copilot.cern.ch:9097"
2584   #makeflowOptions="-T wq -N alice -C ali-copilot.cern.ch:9097"
2585   makeflowOptions=""
2586   #batchCommand="/usr/bin/qsub"
2587   batchFlags=""
2588   baseOutputDirectory="$PWD/output"
2589   #alirootEnv="/cvmfs/alice.cern.ch/bin/alienv setenv AliRoot/v5-04-34-AN -c"
2590   #alirootEnv="/home/mkrzewic/alisoft/balice_master.sh"
2591   #trustedQAtrainMacro='/hera/alice/mkrzewic/gsisvn/Calibration/QAtrain_duo.C'
2592   reconstructInTemporaryDir=0
2593   recoTriggerOptions="\"\""
2594   percentProcessedFilesToContinue=100
2595   maxSecondsToWait=$(( 3600*24 ))
2596   nEvents=-1
2597   nMaxChunks=0
2598   postSetUpActionCPass0=""
2599   postSetUpActionCPass1=""
2600   runCPass0reco=1
2601   runCPass0MergeMakeOCDB=1
2602   runCPass1reco=1
2603   runCPass1MergeMakeOCDB=1
2604   runESDfiltering=1
2605   filteringFactorHighPt=1e2
2606   filteringFactorV0s=1e1
2607   MAILTO=""
2608   #pretend=1
2609   #dontRedirectStdOutToLog=1
2610   logToFinalDestination=1
2611   ALIROOT_FORCE_COREDUMP=1
2612   pretendDelay=0
2613   copyInputData=0
2614
2615   #first, source the config file
2616   if [ -f ${configFile} ]; then
2617     source ${configFile}
2618   else
2619     echo "config file ${configFile} not found!"
2620     return 1
2621   fi
2622
2623   unset encodedSpaces
2624   for opt in "${args[@]}"; do
2625     [[ "${opt}" =~ encodedSpaces=.* ]] && encodedSpaces=1 && echo "encodedSpaces!" && break
2626   done
2627
2628   #then, parse the options as they override the options from file
2629   for opt in "${args[@]}"; do
2630     [[ -z ${opt} ]] && continue
2631     [[ -n ${encodedSpaces} ]] && opt="$(decSpaces ${opt})"
2632     [[ "${opt}" =~ ^[[:space:]]*$ ]] && continue
2633     if [[ ! "${opt}" =~ .*=.* ]]; then
2634       echo "badly formatted option \"${opt}\" should be: option=value, stopping..."
2635       return 1
2636     fi
2637     local var="${opt%%=*}"
2638     local value="${opt#*=}"
2639     echo "${var}=${value}"
2640     export ${var}="${value}"
2641   done
2642
2643   #do some checking
2644   [[ -z ${alirootEnv} ]] && echo "alirootEnv not defined!" && return 1
2645
2646   #export the aliroot function if defined to override normal behaviour
2647   [[ $(type -t aliroot) =~ "function" ]] && export -f aliroot && echo "exporting aliroot() function..."
2648
2649   return 0
2650 }
2651
2652 aliroot()
2653 {
2654   args=("$@")
2655   if [[ -n ${useProfilingCommand} ]]; then
2656     profilerLogFile="cpu.txt"
2657     [[ "${args[@]}" =~ rec ]] && profilerLogFile="cpu_rec.txt"
2658     [[ "${args[@]}" =~ Calib ]] && profilerLogFile="cpu_calib.txt"
2659     echo running "${useProfilingCommand} aliroot ${args[@]} &> ${profilerLogFile}"
2660     ${useProfilingCommand} aliroot "${args[@]}" &> ${profilerLogFile}
2661   else
2662     #to prevent an infinite recursion use "command aliroot" to disable
2663     #aliases and functions
2664     echo running command aliroot "${args[@]}"
2665     command aliroot "${args[@]}"
2666   fi
2667   return 0
2668 }
2669
2670 copyFileToLocal()
2671 (
2672   #copies a single file to a local destination: the file may either come from
2673   #a local filesystem or from a remote location (whose protocol must be
2674   #supported)
2675   #copy is "robust" and it is repeated some times in case of failure before
2676   #giving up (1 is returned in that case)
2677   src="$1"
2678   dst="$2"
2679   ok=0
2680   [[ -z "${maxCopyTries}" ]] && maxCopyTries=10
2681
2682   proto="${src%%://*}"
2683
2684   echo "copy file to local dest started: $src -> $dst"
2685
2686   for (( i=1 ; i<=maxCopyTries ; i++ )) ; do
2687
2688     echo "...attempt $i of $maxCopyTries"
2689     rm -f "$dst"
2690
2691     if [[ "$proto" == "$src" ]]; then
2692       cp "$src" "$dst"
2693     else
2694       case "$proto" in
2695         root)
2696           xrdcp -f "$src" "$dst"
2697         ;;
2698         http)
2699           curl -L "$src" -O "$dst"
2700         ;;
2701         *)
2702           echo "protocol not supported: $proto"
2703           return 2
2704         ;;
2705       esac
2706     fi
2707
2708     if [ $? == 0 ] ; then
2709       ok=1
2710       break
2711     fi
2712
2713   done
2714
2715   if [[ "$ok" == 1 ]] ; then
2716     echo "copy file to local dest OK after $i attempt(s): $src -> $dst"
2717     return 0
2718   fi
2719
2720   echo "copy file to local dest FAILED after $maxCopyTries attempt(s): $src -> $dst"
2721   return 1
2722 )
2723
2724 paranoidCp()
2725 (
2726   #recursively copy files and directories
2727   #to avoid using find and the like as they kill
2728   #the performance on some cluster file systems
2729   #does not copy links to avoid problems
2730   sourceFiles=("${@}")
2731   destination="${sourceFiles[@]:(-1)}" #last element
2732   unset sourceFiles[${#sourceFiles[@]}-1] #remove last element (dst)
2733   for src in "${sourceFiles[@]}"; do
2734     if [[ -f "${src}" && ! -h  "${src}" ]]; then
2735       paranoidCopyFile "${src}" "${destination}"
2736     elif [[ -d "${src}" && ! -h "${src}" ]]; then
2737       src="${src%/}"
2738       dst="${destination}/${src##*/}"
2739       mkdir -p "${dst}"
2740       paranoidCp "${src}"/* "${dst}"
2741     fi
2742   done
2743 )
2744
2745 paranoidCopyFile()
2746 (
2747   #copy a single file to a target in an existing dir
2748   #repeat a few times if copy fails
2749   #returns 1 on failure, 0 on success
2750   src="${1}"
2751   dst="${2}"
2752   ok=0
2753   [[ -d "${dst}" ]] && dst="${dst}/${src##*/}"
2754   [[ -z "${maxCopyTries}" ]] && maxCopyTries=10
2755
2756   echo "paranoid copy started: $src -> $dst"
2757
2758   for (( i=1 ; i<=maxCopyTries ; i++ )) ; do
2759
2760     echo "...attempt $i of $maxCopyTries"
2761     rm -f "$dst"
2762     cp "$src" "$dst"
2763
2764     cmp -s "$src" "$dst"
2765     if [ $? == 0 ] ; then
2766       ok=1
2767       break
2768     fi
2769
2770   done
2771
2772   if [[ "$ok" == 1 ]] ; then
2773     echo "paranoid copy OK after $i attempt(s): $src -> $dst"
2774     return 0
2775   fi
2776
2777   echo "paranoid copy FAILED after $maxCopyTries attempt(s): $src -> $dst"
2778   return 1
2779 )
2780
2781 guessRunData()
2782 {
2783   #guess the period from the path, pick the rightmost one
2784   period=""
2785   runNumber=""
2786   year=""
2787   pass=""
2788   legoTrainRunNumber=""
2789   dataType=""
2790
2791   local shortRunNumber=""
2792   local IFS="/"
2793   declare -a path=( $1 )
2794   local dirDepth=$(( ${#path[*]}-1 ))
2795   i=0
2796   #for ((x=${dirDepth};x>=0;x--)); do
2797   for ((x=0;x<=${dirDepth};x++)); do
2798
2799     [[ $((x-1)) -ge 0 ]] && local fieldPrev=${path[$((x-1))]}
2800     local field=${path[${x}]}
2801     local fieldNext=${path[$((x+1))]}
2802
2803     [[ ${field} =~ ^[0-9]*$ && ${fieldNext} =~ (.*\.zip$|.*\.root$) ]] && legoTrainRunNumber=${field}
2804     [[ -n ${legoTrainRunNumber} && -z ${pass} ]] && pass=${fieldPrev}
2805     [[ ${field} =~ ^LHC[0-9][0-9][a-z].*$ ]] && period=${field%_*}
2806     [[ ${field} =~ ^000[0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && runNumber=${field#000}
2807     [[ ${field} =~ ^[0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && shortRunNumber=${field}
2808     [[ ${field} =~ ^20[0-9][0-9]$ ]] && year=${field}
2809     [[ ${field} =~ ^(^sim$|^data$) ]] && dataType=${field}
2810     (( i++ ))
2811   done
2812   [[ -z ${legoTrainRunNumber} ]] && pass=${path[$((dirDepth-1))]}
2813   [[ "${dataType}" =~ ^sim$ ]] && pass="passMC" && runNumber=${shortRunNumber}
2814
2815   #if [[ -z ${dataType} || -z ${year} || -z ${period} || -z ${runNumber}} || -z ${pass} ]];
2816   if [[ -z ${runNumber}} ]];
2817   then
2818     #error condition
2819     return 1
2820   else
2821     #ALL OK
2822     return 0
2823   fi
2824   return 0
2825 }
2826
2827 #these functions encode strings to and from a space-less form
2828 #use when spaces are not well handled (e.g. in arguments to
2829 #commands in makeflow files, etc.
2830 encSpaces()(echo "${1// /@@@@}")
2831 decSpaces()(echo "${1//@@@@/ }")
2832
2833 main "$@"