PWGPP/benchmark/benchmark.sh

   1 #!/bin/bash
   2 #include benchmark.config
   3
   4 # blame: Mikolaj Krzewicki, mkrzewic@cern.ch
   5 # this script runs the CPass0/CPass1 train
   6 # produced OCDB updates are local
   7
   8 main()
   9 {
  10   #run in proper mode depending on the selection
  11   if [[ $# -lt 1 ]]; then
  12     if [[ ! "${0}" =~ "bash" ]]; then
  13       echo "uses makeflow:"
  14       echo " ${0} \"run\" productionID inputList configFile [extraOpts]"
  15       echo "uses a batch system (SGE):"
  16       echo " ${0} \"submit\" productionID inputList configFile [extraOpts]"
  17       echo "extraOpts if non-empty override the config file, e.g.:"
  18       echo " ${0} submit test1 benchmark.list benchmark.config runNumber=169123 nEvents=10"
  19     fi
  20     return
  21   fi
  22
  23   #define some aliases - default is to call one of the functions directly
  24   runMode=${1}
  25   umask 0002
  26   shift
  27   case ${runMode} in
  28     "CPass0") goCPass0 "$@";;
  29     "CPass1") goCPass1 "$@";;
  30     "MakeLocalOCDBaccessConfig") goMakeLocalOCDBaccessConfig "$@";;
  31     "MergeCPass0") goMergeCPass0 "$@";;
  32     "MergeCPass1") goMergeCPass1 "$@";;
  33     "MakeFilteredTrees") goMakeFilteredTrees "$@";;
  34     "MakeSummary") goMakeSummary "$@";;
  35     "run") goSubmitMakeflow "$@";;
  36     "submit") goSubmitBatch "$@";;
  37     "test") goTest "$@";;
  38     "GenerateMakeflow") goGenerateMakeflow "$@";;
  39     "PrintValues") goPrintValues "$@";;
  40     "CreateQAplots") goCreateQAplots "$@";;
  41     "WaitForOutput") goWaitForOutput "$@";;
  42     "Merge") goMerge "$@";;
  43     *)
  44       ${runMode} "$@"
  45     ;;
  46   esac
  47   return 0
  48 }
  49
  50 generateMC()
  51 {
  52   #generate one raw chunk in current directory
  53   SEED=${JOB_ID}${SGE_TASK_ID}
  54   export CONFIG_SEED=${SEED}
  55   runNumber=${1}
  56   OCDBpath=${2}
  57   nEventsim=${3}
  58   if [[ -n ${pretend} ]]; then
  59     sleep ${pretendDelay}
  60     touch galice.root
  61   else
  62     if [[ -f sim.C && -f Config.C ]] ; then
  63         time aliroot -b -q -x sim.C\(${runNumber},\"${OCDBpath}\",${nEventsim}\) >sim.log 2>&1
  64         mv syswatch.log simwatch.log
  65     fi
  66   fi
  67 }
  68
  69 goCPass0()
  70 (
  71   umask 0002
  72
  73   targetDirectory=${1}
  74   inputList=${2}
  75   nEvents=${3}
  76   ocdbPath=${4}
  77   configFile=${5}
  78   runNumber=${6}
  79   jobindex=${7}
  80   shift 7
  81   if ! parseConfig ${configFile} "$@"; then return 1; fi
  82
  83   #record the working directory provided by the batch system
  84   batchWorkingDirectory=${PWD}
  85
  86   #use the jobindex only if set and non-negative
  87   if [[ -z ${jobindex} || ${jobindex} -lt 0 ]]; then
  88     [[ -n "${LSB_JOBINDEX}" ]] && jobindex=${LSB_JOBINDEX}
  89     [[ -n "${SGE_TASK_ID}" ]] && jobindex=${SGE_TASK_ID}
  90     if [[ -z ${jobindex} ]]; then
  91       echo "no jobindex!"
  92       return 1
  93     fi
  94   fi
  95
  96   [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
  97
  98   # This file signals that/if everything went fine
  99   doneFileBase="cpass0.job${jobindex}.run${runNumber}.done"
 100   [[ -n ${useProfilingCommand} ]] && doneFileBase="profiling.cpass0.job${jobindex}.run${runNumber}.done"
 101
 102   # We will have two copies of the file
 103   mkdir -p "${commonOutputPath}/meta" || return 1
 104   doneFileTmp="${batchWorkingDirectory}/${doneFileBase}"
 105   doneFile="${commonOutputPath}/meta/${doneFileBase}"
 106
 107   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
 108
 109   if [[ -n ${ALIROOT_FORCE_COREDUMP} ]]; then
 110     ulimit -c unlimited
 111     export ALIROOT_FORCE_COREDUMP
 112   fi
 113
 114   #the contents of this is stored in the tree and used later (e.g. AliAnalysisTaskPIDResponse)!
 115   #at the QA stage the pass number is guessed from the path stored here.
 116   #The Format is:
 117   #Packages= ;OutputDir= ;LPMPass= ;TriggerAlias= ;LPMRunNumber= ;LPMProductionType= ;LPMInteractionType= ;LPMProductionTag= ;LPMAnchorRun= ;LPMAnchorProduction= ;LPMAnchorYear=
 118   export PRODUCTION_METADATA="OutputDir=cpass0"
 119
 120   if [[ "${inputList}" =~ \.root$ ]]; then
 121     infile=${inputList}
 122   else
 123     infile=$(sed -ne "${jobindex}p" ${inputList} | egrep '\s*\w*/\w*')
 124   fi
 125   chunkName=${infile##*/}
 126
 127   outputDir=${targetDirectory}/${jobindex}_${chunkName%.*}
 128   mkdir -p ${outputDir}
 129   if [[ ! -d ${outputDir} ]]; then
 130     touch ${doneFileTmp}
 131     echo "cannot make ${outputDir}" >> ${doneFileTmp}
 132     cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
 133     [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
 134     return 1
 135   fi
 136
 137   runpath=${outputDir}
 138   [[ ${reconstructInTemporaryDir} -eq 1 ]] && runpath=$(mktemp -d -t cpass0.XXXXXX)
 139   [[ ${reconstructInTemporaryDir} -eq 2 ]] && runpath=${PWD}/rundir_cpass0_${runNumber}_${jobindex}
 140   mkdir -p ${runpath}
 141   if [[ ! -d ${runpath} ]]; then
 142     touch ${doneFileTmp}
 143     echo "cannot make runpath ${runpath}" >> ${doneFileTmp}
 144     cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
 145     [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
 146     return 1
 147   fi
 148   if ! cd ${runpath}; then
 149     touch ${doneFileTmp}
 150     echo "PWD=$PWD is not the runpath=${runpath}" >> ${doneFileTmp}
 151     cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
 152     [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
 153     return 1
 154   fi
 155
 156   #runCPassX/C expects the raw chunk to be linked in the run dir
 157   #despite it being accessed by the full path
 158   ln -s ${infile} ${runpath}/${chunkName}
 159
 160   #####MC
 161   if [[ -n ${generateMC} ]]; then
 162     olddir=${PWD}
 163     outputDirMC=${commonOutputPath}/000${runNumber}/sim/${jobindex}
 164     simrunpath=${outputDirMC}
 165     #[[ ${simulateInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && simrunpath=${TMPDIR}
 166     [[ ${simulateInTemporaryDir} -eq 1 ]] && simrunpath=$(mktemp -d -t cpass0MC.XXXXXX)
 167     mkdir -p ${outputDirMC}
 168     mkdir -p ${simrunpath}
 169     if cd ${simrunpath}; then
 170
 171       filesMC=(
 172       "${batchWorkingDirectory}/sim.C"
 173       "${batchWorkingDirectory}/rec.C"
 174       "${batchWorkingDirectory}/Config.C"
 175       "${batchWorkingDirectory}/OCDB_*.root"
 176       )
 177       for file in ${filesMC[*]}; do
 178         [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
 179       done
 180
 181       generateMC ${runNumber} ${ocdbPath} ${nEvents}
 182
 183       [[ ! "${simrunpath}" =~ "${outputDirMC}" ]] && mv * ${outputDirMC} #TODO check if it works
 184       cd ${olddir}
 185
 186       ln -s ${outputDirMC}/* ${runpath}/
 187
 188       inputList=${outputDirMC}/galice.root #TODO not valid outside shell !!!
 189       infile=""
 190     fi
 191   fi
 192   ######
 193
 194   if [[ ! -f ${inputList} && -z ${pretend} ]]; then
 195     touch ${doneFileTmp}
 196     echo "input file ${inputList} not found, exiting..." >> ${doneFileTmp}
 197     cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
 198     [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
 199     return 1
 200   fi
 201
 202   logOutputDir=${runpath}
 203   [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir}
 204   [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/stdout
 205   #[[ -z ${dontRedirectStdOutToLog} ]] && exec 2> ${logOutputDir}/stderr
 206   echo "${0} $*"
 207
 208   echo "#####################"
 209   echo CPass0:
 210   echo JOB setup
 211   echo nEvents               ${nEvents}
 212   echo runNumber             ${runNumber}
 213   echo ocdbPath              ${ocdbPath}
 214   echo infile                ${infile}
 215   echo chunkName             ${chunkName}
 216   echo jobindex              ${jobindex}
 217   echo recoTriggerOptions    ${recoTriggerOptions}
 218   echo targetDirectory       ${targetDirectory}
 219   echo commonOutputPath      ${commonOutputPath}
 220   echo doneFile              ${doneFile}
 221   echo batchWorkingDirectory ${batchWorkingDirectory}
 222   echo runpath               ${runpath}
 223   echo outputDir             ${outputDir}
 224   echo PWD                   ${PWD}
 225   echo ALICE_ROOT            ${ALICE_ROOT}
 226   echo "########## ###########"
 227
 228   alirootInfo > ALICE_ROOT.log
 229
 230   filesCPass0=(
 231                "${batchWorkingDirectory}/runCPass0.sh"
 232                "${batchWorkingDirectory}/recCPass0.C"
 233                "${batchWorkingDirectory}/runCalibTrain.C"
 234                "${batchWorkingDirectory}/localOCDBaccessConfig.C"
 235                "${batchWorkingDirectory}/OCDB.root"
 236                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/runCPass0.sh"
 237                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/recCPass0.C"
 238                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/runCalibTrain.C"
 239   )
 240
 241   for file in ${filesCPass0[*]}; do
 242     [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
 243     [[ ${file##*/} =~ .*\.sh ]] && chmod +x ${file##*/}
 244   done
 245
 246   echo "this directory (${PWD}) contents:"
 247   /bin/ls
 248   echo
 249   chmod u+x runCPass0.sh
 250
 251   sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh
 252
 253   if [[ -n ${postSetUpActionCPass0} ]]; then
 254     echo "running ${postSetUpActionCPass0}"
 255     eval ${postSetUpActionCPass0}
 256   fi
 257
 258   #run CPass0
 259   echo "${runpath}/runCPass0.sh ${infile} ${nEvents} ${runNumber} ${ocdbPath} ${recoTriggerOptions}"
 260   if [[ -n ${pretend} ]]; then
 261     sleep ${pretendDelay}
 262     touch AliESDs.root
 263     touch AliESDfriends.root
 264     touch AliESDfriends_v1.root
 265     touch rec.log
 266     touch calib.log
 267   else
 268     echo ./runCPass0.sh "${infile}" "${nEvents}" "${runNumber}" "${ocdbPath}" "${recoTriggerOptions}"
 269     ./runCPass0.sh "${infile}" "${nEvents}" "${runNumber}" "${ocdbPath}" "${recoTriggerOptions}"
 270   fi
 271
 272   #move stuff to final destination
 273   echo "this directory (${PWD}) contents:"
 274   /bin/ls
 275   echo
 276
 277   # [dberzano] OK this is fine!
 278   echo rm -f ./${chunkName}
 279   rm -f ./${chunkName}
 280   echo "paranoidCp ${runpath}/* ${outputDir}"
 281   paranoidCp ${runpath}/* ${outputDir}
 282   echo
 283
 284   #validate CPass0
 285   cd ${outputDir}
 286   if summarizeLogs >> ${doneFileTmp}; then
 287     [[ -f ${outputDirMC}/galice.root ]] && echo "sim ${outputDirMC}/galice.root" >> ${doneFileTmp}
 288     [[ -f AliESDfriends_v1.root ]] && echo "calibfile ${outputDir}/AliESDfriends_v1.root" >> ${doneFileTmp}
 289     [[ -f AliESDs.root ]] && echo "esd ${outputDir}/AliESDs.root" >> ${doneFileTmp}
 290   fi
 291
 292   [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath} && echo "removing ${runpath}"
 293   cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
 294   [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
 295   return 0
 296 )
 297
 298 goCPass1()
 299 (
 300   umask 0002
 301
 302   targetDirectory=${1}
 303   inputList=${2}
 304   nEvents=${3}
 305   ocdbPath=${4}
 306   configFile=${5}
 307   runNumber=${6}
 308   jobindex=${7}
 309   shift 7
 310   extraOpts=("$@")
 311   if ! parseConfig ${configFile} "$@"; then return 1; fi
 312
 313   #record the working directory provided by the batch system
 314   batchWorkingDirectory=${PWD}
 315
 316   #use the jobindex only if set and non-negative
 317   if [[ -z ${jobindex} || ${jobindex} -lt 0 ]]; then
 318     [[ -n "${LSB_JOBINDEX}" ]] && jobindex=${LSB_JOBINDEX}
 319     [[ -n "${SGE_TASK_ID}" ]] && jobindex=${SGE_TASK_ID}
 320     if [[ -z ${jobindex} ]]; then
 321       echo "no jobindex!"
 322       return 1
 323     fi
 324   fi
 325
 326   [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
 327
 328   # This file signals that/if everything went fine
 329   doneFileBase="cpass1.job${jobindex}.run${runNumber}.done"
 330   [[ -n ${useProfilingCommand} ]] && doneFileBase="profiling.cpass0.job${jobindex}.run${runNumber}.done"
 331
 332   # We will have two copies of the file
 333   mkdir -p "${commonOutputPath}/meta" || return 1
 334   doneFileTmp="${batchWorkingDirectory}/${doneFileBase}"
 335   doneFile="${commonOutputPath}/meta/${doneFileBase}"
 336
 337   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
 338
 339   if [[ -n ${ALIROOT_FORCE_COREDUMP} ]]; then
 340     ulimit -c unlimited
 341     export ALIROOT_FORCE_COREDUMP
 342   fi
 343
 344   #the contents of this is stored in the tree and used later (e.g. AliAnalysisTaskPIDResponse)!
 345   #at the QA stage the pass number is guessed from the path stored here.
 346   #The Format is:
 347   #Packages= ;OutputDir= ;LPMPass= ;TriggerAlias= ;LPMRunNumber= ;LPMProductionType= ;LPMInteractionType= ;LPMProductionTag= ;LPMAnchorRun= ;LPMAnchorProduction= ;LPMAnchorYear=
 348   export PRODUCTION_METADATA="OutputDir=cpass1"
 349
 350   if [[ ! -f ${inputList} && -z ${pretend} ]]; then
 351     touch ${doneFileTmp}
 352     echo "input file ${inputList} not found, exiting..." >> ${doneFileTmp}
 353     cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
 354     [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
 355     return 1
 356   fi
 357   if [[ "${inputList}" =~ \.root$ ]]; then
 358     infile=${inputList}
 359   else
 360     infile=$(sed -ne "${jobindex}p" ${inputList} | egrep '\s*\w*/\w*')
 361   fi
 362   chunkName=${infile##*/}
 363
 364   outputDir=${targetDirectory}/${jobindex}_${chunkName%.*}
 365   mkdir -p ${outputDir}
 366   if [[ ! -d ${outputDir} ]];then
 367     touch ${doneFileTmp}
 368     echo "cannot make ${outputDir}" >> ${doneFileTmp}
 369     cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
 370     [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
 371     return 1
 372   fi
 373
 374   runpath=${outputDir}
 375   [[ ${reconstructInTemporaryDir} -eq 1 ]] && runpath=$(mktemp -d -t cpass1.XXXXXX)
 376   [[ ${reconstructInTemporaryDir} -eq 2 ]] && runpath=${PWD}/rundir_cpass1_${runNumber}_${jobindex}
 377
 378   #MC
 379   if [[ "${infile}" =~ galice\.root ]]; then
 380     ln -s ${inputList%/*}/* ${runpath}
 381     infile=""
 382   fi
 383
 384   #init the running path
 385   mkdir -p ${runpath}
 386   if [[ ! -d ${runpath} ]]; then
 387    touch ${doneFileTmp}
 388    echo "cannot make runpath ${runpath}" >> ${doneFileTmp}
 389    cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
 390     [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
 391    return 1
 392  fi
 393   if ! cd ${runpath}; then
 394     touch ${doneFileTmp}
 395     echo "PWD=$PWD is not the runpath=${runpath}" >> ${doneFileTmp}
 396     cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
 397     [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
 398     return 1
 399   fi
 400
 401   #this is needed for runCPass1.sh
 402   ln -s ${infile} ${runpath}/${chunkName}
 403
 404   logOutputDir=${runpath}
 405   [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir}
 406   [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/stdout
 407   #[[ -z ${dontRedirectStdOutToLog} ]] && exec 2> ${logOutputDir}/stderr
 408   echo "${0} $*"
 409
 410   echo "#####################"
 411   echo CPass1:
 412   echo JOB setup
 413   echo nEvents               ${nEvents}
 414   echo runNumber             ${runNumber}
 415   echo ocdbPath              ${ocdbPath}
 416   echo infile                ${infile}
 417   echo chunkName             ${chunkName}
 418   echo jobindex              ${jobindex}
 419   echo recoTriggerOptions    ${recoTriggerOptions}
 420   echo targetDirectory       ${targetDirectory}
 421   echo commonOutputPath      ${commonOutputPath}
 422   echo doneFile              ${doneFile}
 423   echo runpath               ${runpath}
 424   echo outputDir             ${outputDir}
 425   echo batchWorkingDirectory ${batchWorkingDirectory}
 426   echo ALICE_ROOT            ${ALICE_ROOT}
 427   echo PWD                   ${PWD}
 428   echo "#####################"
 429
 430   alirootInfo > ALICE_ROOT.log
 431
 432   filesCPass1=(
 433                "${batchWorkingDirectory}/runCPass1.sh"
 434                "${batchWorkingDirectory}/recCPass1.C"
 435                "${batchWorkingDirectory}/recCPass1_OuterDet.C"
 436                "${batchWorkingDirectory}/runCalibTrain.C"
 437                "${batchWorkingDirectory}/QAtrain_duo.C"
 438                "${batchWorkingDirectory}/localOCDBaccessConfig.C"
 439                "${batchWorkingDirectory}/${configFile}"
 440                "${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz"
 441                "${batchWorkingDirectory}/OCDB.root"
 442                "${trustedQAtrainMacro}"
 443                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/runCPass1.sh"
 444                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/recCPass1.C"
 445                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/recCPass1_OuterDet.C"
 446                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/runCalibTrain.C"
 447                "${ALICE_ROOT}/ANALYSIS/macros/QAtrain_duo.C"
 448   )
 449
 450   for file in "${filesCPass1[@]}"; do
 451     [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
 452     [[ ${file##*/} =~ .*\.sh ]] && echo "making ${file##*/} executable" && chmod +x ${file##*/}
 453   done
 454
 455   echo "this directory (${PWD}) contents:"
 456   /bin/ls
 457   echo
 458
 459   #remove spaces around commas from calls to root
 460   sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh
 461
 462   if [[ -n ${postSetUpActionCPass1} ]]; then
 463     echo "running ${postSetUpActionCPass1}"
 464     eval ${postSetUpActionCPass1}
 465     echo
 466   fi
 467
 468   #configure local OCDB storage from CPass0 (creates the localOCDBaccessConfig.C script)
 469   if [[ -f cpass0.localOCDB.${runNumber}.tgz ]]; then
 470     echo goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz"
 471     goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz"
 472   else
 473     echo "WARNING: file cpass0.localOCDB.${runNumber}.tgz not found!"
 474   fi
 475
 476   if [[ ! $(/bin/ls -1 OCDB/*/*/*/*.root 2>/dev/null) ]]; then
 477     touch ${doneFileTmp}
 478     echo "cpass0 produced no calibration! exiting..." >> ${doneFileTmp}
 479     cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
 480     [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
 481     return 1
 482   fi
 483
 484   #create the Barrel and OuterDet directories for CPass1 and link the local OCDB directory
 485   #there to make the localOCDBaccessConfig.C file work, since it may point to the OCDB
 486   #entries using a relative path, e.g. local://./OCDB
 487   echo "linking the OCDB/ for Barrel and OuterDet directories"
 488   mkdir Barrel OuterDet
 489   ls -l
 490   ln -s ../OCDB Barrel/OCDB
 491   ln -s ../OCDB OuterDet/OCDB
 492
 493   #setup the filtering
 494   #the following option enables the filtering task inside the QAtrain_duo.C
 495   [[ -n $runESDfiltering ]] && export QA_TaskFilteredTree=1
 496   #set the downscaling factors during the filtering fro expert QA (overrides the previous values)
 497   if [[ -n ${filteringFactorHighPt} ]]; then
 498     export AliAnalysisTaskFilteredTree_fLowPtTrackDownscaligF=${filteringFactorHighPt}
 499   fi
 500   if [[ -n ${filteringFactorV0s} ]]; then
 501     export AliAnalysisTaskFilteredTree_fLowPtV0DownscaligF=${filteringFactorV0s}
 502   fi
 503
 504   #run CPass1
 505   chmod u+x runCPass1.sh
 506   echo "${runpath}/runCPass1.sh ${infile} ${nEvents} ${runNumber} ${ocdbPath} ${recoTriggerOptions}"
 507   if [[ -n ${pretend} ]]; then
 508     sleep ${pretendDelay}
 509     touch AliESDs_Barrel.root
 510     touch AliESDfriends_Barrel.root
 511     touch AliESDfriends_v1.root
 512     touch QAresults_barrel.root
 513     touch EventStat_temp_barrel.root
 514     touch AODtpITS.root
 515     touch AliESDs_Outer.root
 516     touch AliESDfriends_Outer.root
 517     touch QAresults_outer.root
 518     touch EventStat_temp_outer.root
 519     touch rec.log
 520     touch calib.log
 521     touch qa.log
 522     touch filtering.log FilterEvents_Trees.root
 523   else
 524     ./runCPass1.sh "${infile}" "${nEvents}" "${runNumber}" "${ocdbPath}" "${recoTriggerOptions}"
 525
 526     [[ ! -f AliESDs_Barrel.root && -f Barrel/AliESDs.root ]] && mv Barrel/AliESDs.root AliESDs_Barrel.root
 527     [[ ! -f AliESDfriends_Barrel.root && -f Barrel/AliESDfriends.root ]] && mv Barrel/AliESDfriends.root AliESDfriends_Barrel.root
 528     [[ ! -f AliESDfriends_v1.root && -f Barrel/AliESDfriends_v1.root ]] && mv Barrel/AliESDfriends_v1.root .
 529     [[ ! -f QAresults_barrel.root && -f Barrel/QAresults_barrel.root ]] && mv Barrel/QAresults_barrel.root .
 530     [[ ! -f AliESDs_Outer.root && -f OuterDet/AliESDs.root ]] && mv OuterDet/AliESDs.root AliESDs_Outer.root
 531     [[ ! -f AliESDfriends_Outer.root && -f OuterDet/AliESDfriends.root ]] && mv OuterDet/AliESDfriends.root AliESDfriends_Outer.root
 532     [[ ! -f QAresults_outer.root && -f OuterDet/QAresults_outer.root ]] && mv OuterDet/QAresults_outer.root .
 533     [[ ! -f FilterEvents_Trees.root && -f Barrel/FilterEvents_Trees.root ]] && mv Barrel/FilterEvents_Trees.root .
 534
 535     #make the filtered tree (if requested and not already produced by QA
 536     [[ -f AliESDs_Barrel.root ]] && echo "AliESDs_Barrel.root" > filtered.list
 537     if [[ -n ${runESDfiltering} && ! -f FilterEvents_Trees.root && -f filtered.list ]]; then
 538       goMakeFilteredTrees ${PWD} ${runNumber} "${PWD}/filtered.list" ${filteringFactorHighPt} ${filteringFactorV0s} ${ocdbPath} 1000000 0 10000000 0 ${configFile} AliESDs_Barrel.root "${extraOpts[@]}" >filtering.log
 539     else
 540       echo ""
 541     fi
 542
 543   fi
 544
 545   ##handle possible crashes in QA (happens often in trunk)
 546   ##rerun QA with a trusted aliroot version
 547   #if [[ $(validateLog qa_barrel.log) ]]; then
 548   #  echo "qa_barrel.log not validated!"
 549   #fi
 550   #if [[ ! -f QAresults_barrel.root && -f ${setupTrustedAliROOTenvInCurrentShell} || $(validateLog qa_barrel.log) ]]; then
 551   #  echo "WARNING: using trusted QA aliroot ${ALICE_ROOT}"
 552   #  source ${setupTrustedAliROOTenvInCurrentShell}
 553   #  cd Barrel
 554   #  rm QAresults_barrel.root
 555   #  rm EventStat_temp_barrel.root
 556   #  rm AODtpITS.root
 557   #  [[ ! -f AliESDs.root ]] && ln -s ../AliESDs_Barrel.root AliESDs.root
 558   #  [[ ! -f AliESDfriends.root ]] && ln -s ../AliESDfriends_Barrel.root AliESDfriends.root
 559   #  if [[ -n ${trustedQAtrainMacro} ]]; then
 560   #    eval "cp ${trustedQAtrainMacro} QAtrain_duo_trusted.C"
 561   #  fi
 562   #  echo executing aliroot -b -q "QAtrain_duo_trusted.C(\"_barrel\",${runNumber},\"wn.xml\",0,\"${ocdbPath}\")"
 563   #  time aliroot -b -q "QAtrain_duo.C(\"_barrel\",${runNumber},\"wn.xml\",0,\"${ocdbPath}\")" &> ../qa_barrel_trusted.log
 564   #  cd ../
 565   #fi
 566
 567   #move stuff to final destination
 568   echo "this directory (${PWD}) contents:"
 569   /bin/ls
 570   echo rm -f ./${chunkName}
 571   rm -f ./${chunkName}
 572   echo "paranoidCp ${runpath}/* ${outputDir}"
 573   paranoidCp ${runpath}/* ${outputDir}
 574   echo
 575
 576   #validate CPass1
 577   cd ${outputDir}
 578   if summarizeLogs >> ${doneFileTmp}; then
 579     [[ -f AliESDs_Barrel.root ]] && echo "esd ${outputDir}/AliESDs_Barrel.root" >> ${doneFileTmp}
 580     [[ -f AliESDfriends_v1.root ]] && echo "calibfile ${outputDir}/AliESDfriends_v1.root" >> ${doneFileTmp}
 581     [[ -f QAresults_Barrel.root ]] && echo "qafile ${outputDir}/QAresults_Barrel.root" >> ${doneFileTmp}
 582     [[ -f QAresults_Outer.root ]] && echo "qafile ${outputDir}/QAresults_Outer.root" >> ${doneFileTmp}
 583     [[ -f QAresults_barrel.root ]] && echo "qafile ${outputDir}/QAresults_barrel.root" >> ${doneFileTmp}
 584     [[ -f QAresults_outer.root ]] && echo "qafile ${outputDir}/QAresults_outer.root" >> ${doneFileTmp}
 585     [[ -f FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFileTmp}
 586   else
 587     if grep "qa_outer.log.*OK" ${doneFileTmp} > /dev/null; then
 588       [[ -f QAresults_Outer.root ]] && echo "qafile ${outputDir}/QAresults_Outer.root" >> ${doneFileTmp}
 589       [[ -f QAresults_outer.root ]] && echo "qafile ${outputDir}/QAresults_outer.root" >> ${doneFileTmp}
 590     fi
 591     if grep "qa_barrel.log.*OK" ${doneFileTmp} > /dev/null; then
 592       [[ -f QAresults_Barrel.root ]] && echo "qafile ${outputDir}/QAresults_Barrel.root" >> ${doneFileTmp}
 593       [[ -f QAresults_barrel.root ]] && echo "qafile ${outputDir}/QAresults_barrel.root" >> ${doneFileTmp}
 594     fi
 595     if grep "filtering.log.*OK" ${doneFileTmp} > /dev/null; then
 596       [[ -f FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFileTmp}
 597     fi
 598   fi
 599
 600   [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
 601   cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
 602   [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
 603   return 0
 604 )
 605
 606
 607 goMergeCPass0()
 608 (
 609   #
 610   # find the output files and merge them
 611   #
 612
 613   outputDir=${1}
 614   ocdbStorage=${2}
 615   configFile=${3}
 616   runNumber=${4}
 617   calibrationFilesToMerge=${5}  #can be a non-existent file, will then be produced on the fly
 618   shift 5
 619   if ! parseConfig ${configFile} "$@"; then return 1; fi
 620
 621   #record the working directory provided by the batch system
 622   batchWorkingDirectory=${PWD}
 623
 624   [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
 625
 626   # This file signals that everything went fine
 627   doneFileBase="merge.cpass0.run${runNumber}.done"
 628
 629   # We will have two copies of the file
 630   mkdir -p "${commonOutputPath}/meta" || return 1
 631   doneFileTmp="${batchWorkingDirectory}/${doneFileBase}"
 632   doneFile="${commonOutputPath}/meta/${doneFileBase}"
 633
 634   umask 0002
 635   ulimit -c unlimited
 636
 637   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
 638
 639   runpath=${outputDir}
 640   [[ ${reconstructInTemporaryDir} -eq 1 ]] && runpath=$(mktemp -d -t mergeCPass0.XXXXXX)
 641   [[ ${reconstructInTemporaryDir} -eq 2 ]] && runpath=${PWD}/rundir_mergeCPass0_${runNumber}
 642
 643   mkdir -p ${runpath}
 644   if [[ ! -d ${runpath} ]]; then
 645     touch ${doneFileTmp}
 646     echo "not able to make the runpath ${runpath}" >> ${doneFileTmp}
 647     cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
 648     [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
 649     return 1
 650   fi
 651   if ! cd ${runpath}; then
 652     touch ${doneFileTmp}
 653     echo "PWD=$PWD is not the runpath=${runpath}" >> ${doneFileTmp}
 654     cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
 655     [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
 656     return 1
 657   fi
 658
 659   logOutputDir=${runpath}
 660   [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir}
 661   [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/stdout
 662   echo "${0} $*"
 663
 664   mergingScript="mergeMakeOCDB.byComponent.sh"
 665
 666   echo goMergeCPass0 SETUP:
 667   echo runNumber=${runNumber}
 668   echo outputDir=${outputDir}
 669   echo ocdbStorage=${ocdbStorage}
 670   echo calibrationFilesToMerge=${calibrationFilesToMerge}
 671   echo mergingScript=${mergingScript}
 672   echo commonOutputPath=${commonOutputPath}
 673   echo runpath=${runpath}
 674
 675   # copy files in case they are not already there
 676   filesMergeCPass0=(
 677                     "${batchWorkingDirectory}/${calibrationFilesToMerge}"
 678                     "${batchWorkingDirectory}/OCDB.root"
 679                     "${batchWorkingDirectory}/localOCDBaccessConfig.C"
 680                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/mergeMakeOCDB.byComponent.sh"
 681                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/mergeByComponent.C"
 682                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/makeOCDB.C"
 683                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/merge.C"
 684                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/mergeMakeOCDB.sh"
 685   )
 686   for file in ${filesMergeCPass0[*]}; do
 687     [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
 688     [[ ${file##*/} =~ .*\.sh ]] && chmod +x ${file##*/}
 689   done
 690
 691   sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh
 692
 693   alirootInfo > ALICE_ROOT.log
 694
 695   #
 696   echo "PWD"
 697   /bin/ls
 698   echo "PWD/.."
 699   /bin/ls ../
 700
 701
 702   #merge calibration
 703   chmod u+x ${mergingScript}
 704   mkdir -p ./OCDB
 705   if [[ ! -f ${calibrationFilesToMerge} ]]; then
 706     echo "/bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root > ${calibrationFilesToMerge}"
 707     /bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root 2>/dev/null > ${calibrationFilesToMerge}
 708   fi
 709
 710   echo "${mergingScript} ${calibrationFilesToMerge} ${runNumber} local://./OCDB ${ocdbStorage}"
 711   if [[ -n ${pretend} ]]; then
 712     sleep ${pretendDelay}
 713     touch CalibObjects.root
 714     touch ocdb.log
 715     touch merge.log
 716     touch dcsTime.root
 717     mkdir -p ./OCDB/TPC/Calib/TimeGain/
 718     mkdir -p ./OCDB/TPC/Calib/TimeDrift/
 719     echo "some calibration" >> ./OCDB/TPC/Calib/TimeGain/someCalibObject_0-999999_cpass0.root
 720     echo "some calibration" >> ./OCDB/TPC/Calib/TimeDrift/otherCalibObject_0-999999_cpass0.root
 721   else
 722     ./${mergingScript} ${calibrationFilesToMerge} ${runNumber} "local://./OCDB" ${ocdbStorage} >> "mergeMakeOCDB.log"
 723
 724     #produce the calib trees for expert QA (dcsTime.root)
 725     goMakeLocalOCDBaccessConfig ./OCDB
 726     echo aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")"
 727     aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")"
 728   fi
 729
 730   ### produce the output
 731   #tar the produced OCDB for reuse
 732   #tar czf ${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz ./OCDB
 733
 734   # Create tarball with OCDB, store on the shared directory, create signal file on batch directory
 735   mkdir -p ${commonOutputPath}/meta
 736   baseTar="cpass0.localOCDB.${runNumber}.tgz"
 737   tar czf ${batchWorkingDirectory}/${baseTar} ./OCDB && \
 738     mv ${batchWorkingDirectory}/${baseTar} ${commonOutputPath}/meta/${baseTar} && \
 739     touch ${batchWorkingDirectory}/${baseTar}.done
 740
 741   /bin/ls
 742
 743   #copy all to output dir
 744   echo "paranoidCp ${runpath}/* ${outputDir}"
 745   paranoidCp ${runpath}/* ${outputDir}
 746
 747   if [[ -n ${generateMC} ]]; then
 748     goPrintValues sim ${commonOutputPath}/meta/sim.run${runNumber}.list ${commonOutputPath}/meta/cpass0.job*.run${runNumber}.done
 749   fi
 750
 751   #validate merging cpass0
 752   cd ${outputDir}
 753   if summarizeLogs >> ${doneFileTmp}; then
 754     [[ -f CalibObjects.root ]] && echo "calibfile ${outputDir}/CalibObjects.root" >> ${doneFileTmp}
 755     [[ -f dcsTime.root ]] && echo "dcsTree ${outputDir}/dcsTime.root" >> ${doneFileTmp}
 756   fi
 757
 758   [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
 759   cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
 760   [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
 761   return 0
 762 )
 763
 764 goMergeCPass1()
 765 (
 766   #
 767   # find the output files and merge them
 768   #
 769
 770   outputDir=${1}
 771   ocdbStorage=${2}
 772   configFile=${3}
 773   runNumber=${4}
 774   calibrationFilesToMerge=${5}
 775   qaFilesToMerge=${6}
 776   filteredFilesToMerge=${7}
 777   shift 7
 778   if ! parseConfig ${configFile} "$@"; then return 1; fi
 779
 780   #record the working directory provided by the batch system
 781   batchWorkingDirectory=${PWD}
 782
 783   [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
 784
 785   # This file signals that everything went fine
 786   doneFileBase="merge.cpass1.run${runNumber}.done"
 787
 788   # We will have two copies of the file
 789   mkdir -p "${commonOutputPath}/meta" || return 1
 790   doneFileTmp="${batchWorkingDirectory}/${doneFileBase}"
 791   doneFile="${commonOutputPath}/meta/${doneFileBase}"
 792
 793   umask 0002
 794   ulimit -c unlimited
 795
 796   #clean up first:
 797   rm -f ${outputDir}/*.log
 798   rm -f ${outputDir}/*.root
 799   rm -f ${outputDir}/*done
 800
 801   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
 802
 803   runpath=${outputDir}
 804   [[ ${reconstructInTemporaryDir} -eq 1 ]] && runpath=$(mktemp -d -t mergeCPass1.XXXXXX)
 805   [[ ${reconstructInTemporaryDir} -eq 2 ]] && runpath=${PWD}/rundir_mergeCPass1_${runNumber}
 806
 807   mkdir -p ${runpath}
 808   if [[ ! -d ${runpath} ]]; then
 809     touch ${doneFileTmp}
 810     echo "not able to make the runpath ${runpath}" >> ${doneFileTmp}
 811     cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
 812     [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
 813     return 1
 814   fi
 815   if ! cd ${runpath}; then
 816     touch ${doneFileTmp}
 817     echo "PWD=$PWD is not the runpath=${runpath}" >> ${doneFileTmp}
 818     cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
 819     [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
 820     return 1
 821   fi
 822
 823   logOutputDir=${runpath}
 824   [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir}
 825   [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/mergeMakeOCDB.log
 826   echo "${0} $*"
 827
 828   calibrationOutputFileName='AliESDfriends_v1.root'
 829   qaOutputFileName='QAresults*.root'
 830   mergingScript="mergeMakeOCDB.byComponent.sh"
 831   #important to have the string "Stage.txt" in the filename to trigger the merging
 832   #it has to be a list of directories containing the files
 833   qaMergedOutputFileName="QAresults_merged.root"
 834
 835   echo goMergeCPass1 SETUP:
 836   echo runNumber=${runNumber}
 837   echo outputDir=${outputDir}
 838   echo ocdbStorage=${ocdbStorage}
 839   echo calibrationFilesToMerge=$calibrationFilesToMerge
 840   echo qaFilesToMerge=$qaFilesToMerge
 841   echo calibrationOutputFileName=${calibrationOutputFileName}
 842   echo mergingScript=${mergingScript}
 843
 844   # copy files in case they are not already there
 845   filesMergeCPass1=(
 846                     "${batchWorkingDirectory}/${calibrationFilesToMerge}"
 847                     "${batchWorkingDirectory}/${qaFilesToMerge}"
 848                     "${batchWorkingDirectory}/OCDB.root"
 849                     "${batchWorkingDirectory}/localOCDBaccessConfig.C"
 850                     "${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz"
 851                     "${batchWorkingDirectory}/QAtrain_duo.C"
 852                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/mergeMakeOCDB.byComponent.sh"
 853                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/mergeByComponent.C"
 854                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/makeOCDB.C"
 855                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/merge.C"
 856                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/mergeMakeOCDB.sh"
 857                     "${trustedQAtrainMacro}"
 858                     "${ALICE_ROOT}/ANALYSIS/macros/QAtrain_duo.C"
 859   )
 860   for file in ${filesMergeCPass1[*]}; do
 861     [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
 862     [[ ${file##*/} =~ .*\.sh ]] && chmod +x ${file##*/}
 863   done
 864
 865   sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh
 866
 867   #configure local OCDB storage from CPass0 (creates the localOCDBaccessConfig.C script)
 868   if [[ -f cpass0.localOCDB.${runNumber}.tgz ]]; then
 869     echo goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz"
 870     goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz"
 871   else
 872     echo "WARNING: file cpass0.localOCDB.${runNumber}.tgz not found!"
 873   fi
 874
 875   alirootInfo > ALICE_ROOT.log
 876
 877   #
 878   /bin/ls
 879
 880   #merge calibration
 881   chmod u+x ${mergingScript}
 882   mkdir -p OCDB
 883
 884   #if not provided, create the lists of files to merge
 885   if [[ ! -f ${filteredFilesToMerge} ]]; then
 886     echo "/bin/ls -1 ${outputDir}/*/FilterEvents_Trees.root > ${filteredFilesToMerge}"
 887     /bin/ls -1 ${outputDir}/*/FilterEvents_Trees.root 2>/dev/null > ${filteredFilesToMerge}
 888   fi
 889   if [[ ! -f ${calibrationFilesToMerge} ]]; then
 890     echo "/bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root > ${calibrationFilesToMerge}"
 891     /bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root 2>/dev/null > ${calibrationFilesToMerge}
 892   fi
 893   if [[ ! -f ${qaFilesToMerge} ]]; then
 894     #find the files, but only store the directories (QAtrain_duo.C requires this)
 895     echo "/bin/ls -1 ${outputDir}/*/QAresults*.root | while read x; do echo ${x%/*}; done | sort | uniq > ${qaFilesToMerge}"
 896     /bin/ls -1 ${outputDir}/*/QAresults*.root | while read x; do echo ${x%/*}; done | sort | uniq > ${qaFilesToMerge}
 897   fi
 898
 899   echo "${mergingScript} ${calibrationFilesToMerge} ${runNumber} local://./OCDB ${ocdbStorage}"
 900   if [[ -n ${pretend} ]]; then
 901     sleep ${pretendDelay}
 902     touch ocdb.log
 903     touch cpass1.localOCDB.${runNumber}.tgz
 904     touch ${qaMergedOutputFileName}
 905     touch merge.log
 906     touch trending.root
 907     touch FilterEvents_Trees.root
 908     touch CalibObjects.root
 909     touch dcsTime.root
 910     touch ${qaMergedOutputFileName}
 911     mkdir -p OCDB
 912   else
 913     ./${mergingScript} ${calibrationFilesToMerge} ${runNumber} "local://./OCDB" ${ocdbStorage}
 914
 915     #merge QA (and filtered trees)
 916     [[ -n ${AliAnalysisTaskFilteredTree_fLowPtTrackDownscaligF} ]] && export AliAnalysisTaskFilteredTree_fLowPtTrackDownscaligF
 917     [[ -n ${AliAnalysisTaskFilteredTree_fLowPtV0DownscaligF} ]] && export AliAnalysisTaskFilteredTree_fLowPtV0DownscaligF
 918
 919     #echo aliroot -l -b -q "merge.C(\"${qaFilesToMerge}\",\"\",kFALSE,\"${qaMergedOutputFileName}\")"
 920     echo aliroot -b -q "QAtrain_duo.C(\"_barrel\",${runNumber},\"${qaFilesToMerge}\",1,\"${ocdbStorage}\")"
 921     #aliroot -l -b -q "merge.C(\"${qaFilesToMerge}\",\"\",kFALSE,\"${qaMergedOutputFileName}\")"
 922     aliroot -b -q "QAtrain_duo.C(\"_barrel\",${runNumber},\"${qaFilesToMerge}\",1,\"${ocdbStorage}\")" > mergeQA.log
 923     mv QAresults_barrel.root ${qaMergedOutputFileName}
 924     mv trending_barrel.root trending.root
 925
 926     #merge filtered trees
 927     echo aliroot -l -b -q "merge.C(\"${qaFilesToMerge}\",\"\",kFALSE,\"${qaMergedOutputFileName}\")"
 928     aliroot -l -b -q "merge.C(\"${filteredFilesToMerge}\",\"\",kFALSE,\"FilterEvents_Trees.root\")" > mergeFilteredTrees.log
 929
 930     #produce the calib trees for expert QA
 931     echo aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")"
 932     aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")" > calibTree.log
 933   fi
 934
 935   # Create tarball with OCDB, store on the shared directory, create signal file on batch directory
 936   mkdir -p ${commonOutputPath}/meta
 937   baseTar="cpass1.localOCDB.${runNumber}.tgz"
 938   tar czf ${batchWorkingDirectory}/${baseTar} ./OCDB && \
 939     mv ${batchWorkingDirectory}/${baseTar} ${commonOutputPath}/meta/${baseTar} && \
 940     touch ${batchWorkingDirectory}/${baseTar}.done
 941
 942   /bin/ls
 943
 944   #copy all to output dir
 945   echo "paranoidCp ${runpath}/* ${outputDir}"
 946   paranoidCp ${runpath}/* ${outputDir}
 947
 948   #validate merge cpass1
 949   cd ${outputDir}
 950   if summarizeLogs >>  ${doneFileTmp}; then
 951     [[ -f CalibObjects.root ]] && echo "calibfile ${outputDir}/CalibObjects.root" >> ${doneFileTmp}
 952     [[ -f ${qaMergedOutputFileName} ]] && echo "qafile ${outputDir}/${qaMergedOutputFileName}" >> ${doneFileTmp}
 953     [[ -f trending.root ]] && echo "trendingfile ${outputDir}/trending.root" >> ${doneFileTmp}
 954     [[ -f dcsTime.root ]] && echo "dcsTree ${outputDir}/dcsTime.root" >> ${doneFileTmp}
 955     [[ -f FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFileTmp}
 956   else
 957     if grep "mergeQA.log.*OK" ${doneFileTmp} > /dev/null; then
 958       [[ -f ${qaMergedOutputFileName} ]] && echo "qafile ${outputDir}/${qaMergedOutputFileName}" >> ${doneFileTmp}
 959     fi
 960     if grep "mergeFilteredTrees.log.*OK" ${doneFileTmp} > /dev/null; then
 961       [[ -f FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFileTmp}
 962     fi
 963   fi
 964
 965   [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
 966   cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
 967   [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
 968   return 0
 969 )
 970
 971 goMerge()
 972 (
 973   #generic root merge using CPass1 merge.C script
 974   inputList=${1}
 975   outputFile=${2}
 976   configFile=${3-"benchmark.config"}
 977   shift 3
 978   if ! parseConfig ${configFile} "$@"; then return 1; fi
 979
 980   #record the working directory provided by the batch system
 981   batchWorkingDirectory=${PWD}
 982
 983   [[ ! -f ${inputList} ]] && echo "inputList ${inputList} does not exist!" && return 1
 984   [[ ! -f ${configFile} ]] && echo "configFile ${configFile} does not exist!" && return 1
 985   umask 0002
 986   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
 987   rm -f ${outputFile}
 988   aliroot -b -q "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/merge.C(\"${inputList}\",\"\",kFALSE,\"${outputFile}\")" > merge_${inputList}.log
 989   return 0
 990 )
 991
 992 goSubmitMakeflow()
 993 {
 994   #run
 995   productionID=${1}
 996   inputFileList=${2}
 997   configFile=${3}
 998   shift 3
 999   extraOpts=("$@")
1000   if ! parseConfig ${configFile} "${extraOpts[@]}"; then return 1; fi
1001
1002   #record the working directory provided by the batch system
1003   batchWorkingDirectory=${PWD}
1004
1005   [[ -z ${configFile} ]] && configFile="benchmark.config"
1006   [[ ! -f ${configFile} ]] && echo "no config file found (${configFile})" && return 1
1007
1008   if [[ ! $(which makeflow &>/dev/null) && -n ${makeflowPath} ]]; then
1009     echo "setting the makflow path from the config: "
1010     echo "  export PATH=${makeflowPath}:${PATH}"
1011     export PATH=${makeflowPath}:${PATH}
1012   fi
1013
1014   #create the common output dir and the meta dir
1015   commonOutputPath=${baseOutputDirectory}/${productionID}
1016   if [[ -d ${commonOutputPath} ]]; then
1017     echo "output dir ${commonOutputPath} exists!"
1018     #return 1
1019   else
1020     mkdir -p ${commonOutputPath}
1021   fi
1022   mkdir -p ${commonOutputPath}/meta
1023
1024   self=${0}
1025   #if which greadlink; then self=$(greadlink -f "${0}"); fi
1026
1027   #for reference copy the setup to the output dir
1028   cp ${self} ${commonOutputPath}
1029   cp ${configFile} ${commonOutputPath}
1030   cp ${inputFileList} ${commonOutputPath}
1031
1032   #submit - use makeflow if available, fall back to old stuff when makeflow not there
1033   if which makeflow; then
1034     goGenerateMakeflow ${productionID} ${inputFileList} ${configFile} "${extraOpts[@]}" commonOutputPath=${commonOutputPath} > benchmark.makeflow
1035     cp benchmark.makeflow ${commonOutputPath}
1036     makeflow ${makeflowOptions} benchmark.makeflow
1037   else
1038     echo "no makeflow!"
1039   fi
1040
1041   #summarize the run based on the makeflow log
1042   #and add it to the end of summary log
1043   awk '/STARTED/   {startTime=$3}
1044        /COMPLETED/ {endTime=$3}
1045        END         {print "makeflow running time: "(endTime-startTime)/1000000/3600" hours"}' \
1046       benchmark.makeflow.makeflowlog | tee -a summary.log
1047   paranoidCp summary.log ${commonOutputPath}
1048
1049   return 0
1050 }
1051
1052 goGenerateMakeflow()
1053 (
1054   #generate the makeflow file
1055   [[ $# -lt 3 ]] && echo "args: id inputFileList configFile" && return 1
1056   productionID=${1}
1057   inputFileList=${2}
1058   configFile=${3}
1059   shift 3
1060   extraOpts=("$@")
1061
1062   #batch systems/makeflow sometimes handle spaces in arguments poorly, so encode them
1063   for (( i=0;i<${#extraOpts[@]};i++ )); do
1064     extraOpts[i]=$(encSpaces "${extraOpts[i]}")
1065   done
1066   extraOpts+=("encodedSpaces=1")
1067
1068   if ! parseConfig ${configFile} "${extraOpts[@]}" &>/dev/null; then return 1; fi
1069
1070   #extra safety
1071   if [[ -z ${commonOutputPath} ]]; then
1072     commonOutputPath=${baseOutputDirectory}/${productionID}
1073     extraOpts=( "${extraOpts[@]}" "commonOutputPath=${commonOutputPath}" )
1074   fi
1075
1076   #record the working directory provided by the batch system
1077   batchWorkingDirectory=${PWD}
1078
1079   [[ -z ${configFile} ]] && configFile="benchmark.config"
1080   [[ ! -f ${configFile} ]] && echo "no config file found (${configFile})" && return 1
1081
1082   #these files will be made a dependency - will be copied to the working dir of the jobs
1083   declare -a copyFiles
1084   inputFiles=(
1085               "OCDB.root"
1086               "localOCDBaccessConfig.C"
1087               "QAtrain_duo.C"
1088               "runCPass1.sh"
1089               "recCPass1.C"
1090               "recCPass1_OuterDet.C"
1091               "runCalibTrain.C"
1092               "runCPass0.sh"
1093               "recCPass0.C"
1094               "runQA.sh"
1095   )
1096   for file in ${inputFiles[*]}; do
1097     [[ -f ${file} ]] && copyFiles+=("${file}")
1098   done
1099
1100   #create the makeflow file
1101   [[ -n ${batchFlags} ]] && echo "BATCH_OPTIONS = ${batchFlags}"
1102   declare -A arr_cpass0_merged arr_cpass1_merged
1103   declare -A arr_cpass0_calib_list arr_cpass1_calib_list
1104   declare -A arr_cpass1_QA_list arr_cpass1_ESD_list arr_cpass1_filtered_list
1105   declare -A arr_cpass0_profiled_outputs
1106   declare -A listOfRuns
1107   [[ -n ${runNumber} ]] && listOfRuns[${runNumber}]=1
1108   while read x; do tmpRun=$(guessRunNumber ${x}); [[ -n ${tmpRun} ]] && listOfRuns[${tmpRun}]=1; done < ${inputFileList}
1109   for runNumber in "${!listOfRuns[@]}"; do
1110     [[ -z ${runNumber} ]] && continue
1111     [[ ! ${runNumber} =~ ^[0-9]*[0-9]$ ]] && continue
1112
1113     unset arr_cpass0_outputs
1114     unset arr_cpass1_outputs
1115     declare -a arr_cpass0_outputs
1116     declare -a arr_cpass1_outputs
1117
1118     #Header
1119     echo "### Automatically generated on $(LANG=C date) ###"
1120     echo ; echo
1121
1122     jobindex=0
1123     inputFile=""
1124     while read inputFile; do
1125       currentDefaultOCDB=${defaultOCDB}
1126       [[ -z ${autoOCDB} ]] && autoOCDB=1
1127       if [[ ${autoOCDB} -ne 0 ]]; then
1128         currentDefaultOCDB=$(setYear ${inputFile} ${defaultOCDB})
1129       fi
1130       guessRunData ${inputFile}
1131
1132       #Set variables
1133       echo "### Variables ###"
1134       echo "OUTPATH=\"${commonOutputPath}/${year}/${period}\""
1135       echo ; echo
1136
1137       #CPass0
1138       #arr_cpass0_outputs[${jobindex}]="${commonOutputPath}/meta/cpass0.job${jobindex}.run${runNumber}.done"
1139       arr_cpass0_outputs[${jobindex}]="cpass0.job${jobindex}.run${runNumber}.done"
1140       echo "### CPass0 ###"
1141       echo "${arr_cpass0_outputs[${jobindex}]}: benchmark.sh ${configFile} ${copyFiles[@]}"
1142       echo "    ${alirootEnv} ./benchmark.sh CPass0 \$OUTPATH/000${runNumber}/cpass0 ${inputFile} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} ${jobindex} ${extraOpts[@]}"" "
1143       echo ; echo
1144
1145       #CPass1
1146       #arr_cpass1_outputs[${jobindex}]="${commonOutputPath}/meta/cpass1.job${jobindex}.run${runNumber}.done"
1147       arr_cpass1_outputs[${jobindex}]="cpass1.job${jobindex}.run${runNumber}.done"
1148       echo "### CPass1 ###"
1149       echo "${arr_cpass1_outputs[${jobindex}]}: benchmark.sh ${configFile} cpass0.localOCDB.${runNumber}.tgz.done ${copyFiles[@]}"
1150       echo "    ${alirootEnv} ./benchmark.sh CPass1 \$OUTPATH/000${runNumber}/cpass1 ${inputFile} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} ${jobindex} ${extraOpts[@]}"" "
1151       echo ; echo
1152       ((jobindex++))
1153
1154     done< <(grep "/000${runNumber}/" ${inputFileList})
1155
1156     #CPass0 list of Calib files to merge
1157     #arr_cpass0_calib_list[${runNumber}]="${commonOutputPath}/meta/cpass0.calib.run${runNumber}.list"
1158     arr_cpass0_calib_list[${runNumber}]="cpass0.calib.run${runNumber}.list"
1159     echo "### Produces the list of CPass0 files to merge (executes locally) ###"
1160     echo "${arr_cpass0_calib_list[${runNumber}]}: benchmark.sh ${arr_cpass0_outputs[*]}"
1161     echo "    LOCAL ./benchmark.sh PrintValues calibfile ${arr_cpass0_calib_list[${runNumber}]} ${arr_cpass0_outputs[*]} && mkdir -p \$OUTPATH/meta && cp ${arr_cpass0_calib_list[${runNumber}]} \$OUTPATH/meta/${arr_cpass0_calib_list[${runNumber}]}"
1162     echo ; echo
1163
1164     #CPass0 merging
1165     echo "### Merges CPass0 files ###"
1166     #arr_cpass0_merged[${runNumber}]="${commonOutputPath}/meta/merge.cpass0.run${runNumber}.done"
1167     arr_cpass0_merged[${runNumber}]="merge.cpass0.run${runNumber}.done"
1168     echo "cpass0.localOCDB.${runNumber}.tgz.done ${arr_cpass0_merged[${runNumber}]}: benchmark.sh ${configFile} ${arr_cpass0_calib_list[${runNumber}]} ${copyFiles[@]}"
1169     echo "    ${alirootEnv} ./benchmark.sh MergeCPass0 \$OUTPATH/000${runNumber}/cpass0 ${currentDefaultOCDB} ${configFile} ${runNumber} ${arr_cpass0_calib_list[${runNumber}]} ${extraOpts[@]}"" "
1170     echo ; echo
1171
1172     #CPass1 list of Calib/QA/ESD/filtered files
1173     # the trick with QA is to have the string "Stage.txt" in the file name of the list of directories with QA output to trigger
1174     # the production of the QA trending tree (only then the task->Finish() will be called in QAtrain_duo.C, on the grid
1175     # this corresponds to the last merging stage)
1176     #arr_cpass1_QA_list[${runNumber}]="${commonOutputPath}/meta/cpass1.QA.run${runNumber}.lastMergingStage.txt.list"
1177     arr_cpass1_QA_list[${runNumber}]="cpass1.QA.run${runNumber}.lastMergingStage.txt.list"
1178     echo "### Lists CPass1 QA ###"
1179     echo "${arr_cpass1_QA_list[${runNumber}]}: benchmark.sh ${arr_cpass1_outputs[*]}"
1180     echo "    LOCAL ./benchmark.sh PrintValues dir ${arr_cpass1_QA_list[${runNumber}]} ${arr_cpass1_outputs[*]} && mkdir -p \$OUTPATH/meta && cp ${arr_cpass1_QA_list[${runNumber}]} \$OUTPATH/meta/${arr_cpass1_QA_list[${runNumber}]}"
1181     echo ; echo
1182
1183     #arr_cpass1_calib_list[${runNumber}]="${commonOutputPath}/meta/cpass1.calib.run${runNumber}.list"
1184     arr_cpass1_calib_list[${runNumber}]="cpass1.calib.run${runNumber}.list"
1185     echo "### Lists CPass1 Calib ###"
1186     echo "${arr_cpass1_calib_list[${runNumber}]}: benchmark.sh ${arr_cpass1_outputs[*]}"
1187     echo "    LOCAL ./benchmark.sh PrintValues calibfile ${arr_cpass1_calib_list[${runNumber}]} ${arr_cpass1_outputs[*]} && mkdir -p \$OUTPATH/meta && cp ${arr_cpass1_calib_list[${runNumber}]} \$OUTPATH/meta/${arr_cpass1_calib_list[${runNumber}]}"
1188     echo ; echo
1189
1190     #arr_cpass1_ESD_list[${runNumber}]="${commonOutputPath}/meta/cpass1.ESD.run${runNumber}.list"
1191     arr_cpass1_ESD_list[${runNumber}]="cpass1.ESD.run${runNumber}.list"
1192     echo "### Lists CPass1 ESDs ###"
1193     echo "${arr_cpass1_ESD_list[${runNumber}]}: benchmark.sh ${arr_cpass1_outputs[*]}"
1194     echo "    LOCAL ./benchmark.sh PrintValues esd ${arr_cpass1_ESD_list[${runNumber}]} ${arr_cpass1_outputs[*]} && mkdir -p \$OUTPATH/meta && cp ${arr_cpass1_ESD_list[${runNumber}]} \$OUTPATH/meta/${arr_cpass1_ESD_list[${runNumber}]}"
1195     echo ; echo
1196
1197     #arr_cpass1_filtered_list[${runNumber}]="${commonOutputPath}/meta/cpass1.filtered.run${runNumber}.list"
1198     arr_cpass1_filtered_list[${runNumber}]="cpass1.filtered.run${runNumber}.list"
1199     echo "### Lists CPass1 filtered ###"
1200     echo "${arr_cpass1_filtered_list[${runNumber}]}: benchmark.sh ${arr_cpass1_outputs[*]}"
1201     echo "    LOCAL ./benchmark.sh PrintValues filteredTree ${arr_cpass1_filtered_list[${runNumber}]} ${arr_cpass1_outputs[*]} && mkdir -p \$OUTPATH/meta && cp ${arr_cpass1_filtered_list[${runNumber}]} \$OUTPATH/meta/${arr_cpass1_filtered_list[${runNumber}]}"
1202     echo ; echo
1203
1204     #CPass1 merging
1205     #arr_cpass1_merged[${runNumber}]="${commonOutputPath}/meta/merge.cpass1.run${runNumber}.done"
1206     arr_cpass1_merged[${runNumber}]="merge.cpass1.run${runNumber}.done"
1207     echo "### Merges CPass1 files ###"
1208     echo "cpass1.localOCDB.${runNumber}.tgz.done ${arr_cpass1_merged[${runNumber}]}: benchmark.sh ${configFile} ${arr_cpass1_calib_list[${runNumber}]} ${arr_cpass1_QA_list[${runNumber}]} ${copyFiles[@]}"
1209     echo "    ${alirootEnv} ./benchmark.sh MergeCPass1 \$OUTPATH/000${runNumber}/cpass1 ${currentDefaultOCDB} ${configFile} ${runNumber} ${arr_cpass1_calib_list[${runNumber}]} ${arr_cpass1_QA_list[${runNumber}]} ${arr_cpass1_filtered_list[${runNumber}]} ${extraOpts[@]}"" "
1210     echo ; echo
1211
1212     #CPass0 wrapped in a profiling tool (valgrind,....)
1213     if [[ -n ${profilingCommand} ]]; then
1214       inputFile=$(grep -m1 "${runNumber}/" ${inputFileList})
1215       [[ -z ${nEventsProfiling} ]] && nEventsProfiling=2
1216       currentDefaultOCDB=$(setYear ${inputFile} ${defaultOCDB})
1217       jobindex="profiling"
1218
1219       #arr_cpass0_profiled_outputs[${runNumber}]="${commonOutputPath}/meta/profiling.cpass0.job${jobindex}.run${runNumber}.done"
1220       arr_cpass0_profiled_outputs[${runNumber}]="profiling.cpass0.job${jobindex}.run${runNumber}.done"
1221       echo "### CPass0 in a profiler ###"
1222       echo "${arr_cpass0_profiled_outputs[${runNumber}]}: benchmark.sh ${configFile} ${copyFiles[@]}"
1223       profilingCommand=$(encSpaces "${profilingCommand}")
1224       echo "    ${alirootEnv} ./benchmark.sh CPass0 \$OUTPATH/000${runNumber}/${jobindex} ${inputFile} ${nEventsProfiling} ${currentDefaultOCDB} ${configFile} ${runNumber} ${jobindex} ${extraOpts[@]} useProfilingCommand=${profilingCommand}"
1225       echo ; echo
1226     fi
1227
1228   done #runs
1229
1230   #Summary
1231   echo "### Summary ###"
1232   echo "summary.log: benchmark.sh ${configFile} ${arr_cpass1_merged[*]}"
1233   echo "     ${alirootEnv} ./benchmark.sh MakeSummary ${configFile} ${extraOpts[@]}"
1234   echo ; echo
1235
1236   return 0
1237 )
1238
1239 goPrintValues()
1240 (
1241   #print the values given the key from any number of files (store in output file on second argument)
1242   if [[ $# -lt 3 ]]; then
1243     echo "goPrintValues key outputFile inputFiles"
1244     echo "if outputFile is \"-\" don't redirect to a file"
1245     return
1246   fi
1247   key=${1}
1248   outputFile=${2}
1249   [[ ${outputFile} =~ "-" ]] && outputFile=""
1250   shift 2 #remove 2 first arguments from arg list to only pass the input files to awk
1251   awk -v key=${key} '$0 ~ key" " {print $2}' "$@" | tee ${outputFile}
1252   return 0
1253 )
1254
1255 goCreateQAplots()
1256 (
1257   umask 0002
1258   mergedQAfileList=${1}
1259   productionID=${2}
1260   outputDir=${3}
1261   configFile=${4}
1262   shift 4
1263   if ! parseConfig ${configFile} "$@"; then return 1; fi
1264
1265   #record the working directory provided by the batch system
1266   batchWorkingDirectory=${PWD}
1267
1268   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
1269
1270   [[ -z ${logOutputDir} ]] && logOutputDir=${PWD}
1271   [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/makeQAplots.log
1272   echo "${0} $*"
1273
1274   olddir=${PWD}
1275   mkdir -p ${outputDir}
1276   cd ${outputDir}
1277   [[ ! "${PWD}" =~ "${outputDir}" ]] && echo "PWD is not equal to outputDir=${outputDir}" && cd ${olddir} && return 1
1278
1279   inputFiles=(
1280               "${batchWorkingDirectory}/runQA.sh"
1281               "${ALICE_ROOT}/PWGPP/QA/scripts/runQA.sh"
1282   )
1283   for file in ${inputFiles[*]}; do
1284     [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
1285   done
1286
1287   echo "running QA with command:"
1288   echo ./runQA.sh inputList="${mergedQAfileList}" inputListHighPtTrees="${filteringList}" ocdbStorage="${defaultOCDB}"
1289   ./runQA.sh inputList="${mergedQAfileList}" inputListHighPtTrees="${filteringList}" ocdbStorage="${defaultOCDB}"
1290   cd ${olddir}
1291   return 0
1292 )
1293
1294 goTest()
1295 (
1296   echo AA
1297 )
1298
1299 alirootInfo()
1300 (
1301   umask 0002
1302   # save aliroot repository info
1303   [[ -z "${ALICE_ROOT}" ]] && return 1
1304
1305   echo "\${ALICE_ROOT}=${ALICE_ROOT}"
1306   echo "\${ROOTSYS}=${ROOTSYS}"
1307   echo "\${PATH}=${PATH}"
1308   echo "\${LD_LIBRARY_PATH}=${LD_LIBRARY_PATH}"
1309   echo
1310
1311   pushd ${PWD}
1312   cd ${ALICE_ROOT}
1313
1314   currentBranch=$(git rev-parse --abbrev-ref HEAD)
1315   git status
1316   echo ""
1317   echo ""
1318   git diff ${currentBranch}
1319   popd
1320   return 0
1321 )
1322
1323 setYear()
1324 (
1325   #set the year
1326   #  ${1} - year to be set
1327   #  ${2} - where to set the year
1328   year1=$(guessYear ${1})
1329   year2=$(guessYear ${2})
1330   local path=${2}
1331   [[ ${year1} -ne ${year2} && -n ${year2} && -n ${year1} ]] && path=${2/\/${year2}\//\/${year1}\/}
1332   echo ${path}
1333   return 0
1334 )
1335
1336 guessPeriod()
1337 (
1338   #guess the period from the path, pick the rightmost one
1339   local IFS="/"
1340   declare -a path=( ${1} )
1341   local dirDepth=${#path[*]}
1342   for ((x=${dirDepth}-1;x>=0;x--)); do
1343     local field=${path[${x}]}
1344     [[ ${field} =~ ^LHC[0-9][0-9][a-z]$ ]] && period=${field} && break
1345   done
1346   echo ${period}
1347   return 0
1348 )
1349
1350 guessYear()
1351 (
1352   #guess the year from the path, pick the rightmost one
1353   local IFS="/"
1354   declare -a path=( ${1} )
1355   local dirDepth=${#path[*]}
1356   for ((x=${dirDepth}-1;x>=0;x--)); do
1357     local field=${path[${x}]}
1358     [[ ${field} =~ ^20[0-9][0-9]$ ]] && year=${field} && break
1359   done
1360   echo ${year}
1361   return 0
1362 )
1363
1364 guessRunNumber()
1365 (
1366   #guess the run number from the path, pick the rightmost one
1367   #works for /path/foo/000123456/bar/...
1368   #and       /path/foo.run123456.bar
1369   local IFS="/."
1370   declare -a path=( ${1} )
1371   local dirDepth=${#path[*]}
1372   for ((x=${dirDepth}-1;x>=0;x--)); do
1373     local field=${path[${x}]}
1374     field=${field/run/000}
1375     [[ ${field} =~ [0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && runNumber=${field#000} && break
1376   done
1377   echo ${runNumber}
1378   return 0
1379 )
1380
1381 validateLog()
1382 (
1383   log=${1}
1384   errorConditions=(
1385                     'There was a crash'
1386                     'floating'
1387                     'error while loading shared libraries'
1388                     'std::bad_alloc'
1389                     's_err_syswatch_'
1390                     'Thread [0-9]* (Thread'
1391                     'AliFatal'
1392                     'core dumped'
1393                     '\.C.*error:.*\.h: No such file'
1394                     'line.*Aborted'
1395   )
1396
1397   warningConditions=(
1398                      'This is serious !'
1399                      'rocVoltage out of range:'
1400   )
1401
1402   local logstatus=0
1403   local errorSummary=""
1404   local warningSummary=""
1405
1406   for ((i=0; i<${#errorConditions[@]};i++)); do
1407     local tmp=$(grep -m1 -e "${errorConditions[${i}]}" ${log})
1408     [[ -n ${tmp} ]] && tmp+=" : "
1409     errorSummary+=${tmp}
1410   done
1411
1412   for ((i=0; i<${#warningConditions[@]};i++)); do
1413     local tmp=$(grep -m1 -e "${warningConditions[${i}]}" ${log})
1414     [[ -n ${tmp} ]] && tmp+=" : "
1415     warningSummary+=${tmp}
1416   done
1417
1418   if [[ -n ${errorSummary} ]]; then
1419     echo "${errorSummary}"
1420     return 1
1421   fi
1422
1423   if [[ -n ${warningSummary} ]]; then
1424     echo "${warningSummary}"
1425     return 2
1426   fi
1427
1428   return 0
1429 )
1430
1431 summarizeLogs()
1432 (
1433   #print a summary of logs
1434   logFiles=(
1435             "*.log"
1436             "stdout"
1437             "stderr"
1438   )
1439
1440   #put dir information in the output
1441   echo "dir $PWD"
1442
1443   #check logs
1444   local logstatus=0
1445   for log in ${logFiles[*]}; do
1446     finallog=${PWD%/}/${log}
1447     [[ ! -f ${log} ]] && continue
1448     errorSummary=$(validateLog ${log})
1449     validationStatus=$?
1450     if [[ ${validationStatus} -eq 0 ]]; then
1451       #in pretend mode randomly report an error in rec.log some cases
1452       if [[ -n ${pretend} && "${log}" == "rec.log" ]]; then
1453         #[[ $(( ${RANDOM}%2 )) -ge 1 ]] && echo "${finallog} BAD random error" || echo "${finallog} OK"
1454         echo "${finallog} OK"
1455       else
1456         echo "${finallog} OK"
1457       fi
1458     elif [[ ${validationStatus} -eq 1 ]]; then
1459       echo "${finallog} BAD ${errorSummary}"
1460       logstatus=1
1461     elif [[ ${validationStatus} -eq 2 ]]; then
1462       echo "${finallog} OK MWAH ${errorSummary}"
1463     fi
1464   done
1465
1466   #report core files
1467   while read x; do
1468     echo ${x}
1469     chmod 644 ${x}
1470     gdb --batch --quiet -ex "bt" -ex "quit" aliroot ${x} > stacktrace_${x//\//_}.log
1471   done < <(/bin/ls ${PWD}/*/core 2>/dev/null; /bin/ls ${PWD}/core 2>/dev/null)
1472
1473   return ${logstatus}
1474 )
1475
1476 spitOutLocalOCDBaccessConfig()
1477 {
1478   umask 0002
1479   #find ${1} -name "*root" | \
1480   /bin/ls -1 ${1}/*/*/*/*.root 2>/dev/null | \
1481   while read line
1482   do
1483     local tmp=${line#${1}}
1484     echo ${tmp%/*} | \
1485     awk -v ocdb=${1} '{print "  man->SetSpecificStorage(\""$1"\",\"local://"ocdb"\");"}'
1486   done
1487   return 0
1488 }
1489
1490 goMakeLocalOCDBaccessConfig()
1491 {
1492   umask 0002
1493   # make a script that sets the specific storages form all the root files produced by CPass0
1494   local localOCDBpathCPass0=${1}
1495   local OCDBpathPrefix=${2}
1496   [[ -z ${OCDBpathPrefix} ]] && OCDBpathPrefix="."
1497
1498   if [[ -f ${localOCDBpathCPass0} && ${localOCDBpathCPass0} =~ \.tgz$ ]]; then
1499     tar xzf ${localOCDBpathCPass0}
1500     local localOCDBpathCPass0="${OCDBpathPrefix}/OCDB"
1501   fi
1502
1503   echo
1504   echo creating the specific storage script
1505   echo   localOCDBaccessConfig.C
1506   echo   based on OCDB: ${localOCDBaccessConfig}
1507   echo
1508
1509   local tempLocalOCDB=""
1510   if [[ -f localOCDBaccessConfig.C ]]; then
1511     tempLocalOCDB=$(mktemp -t tempLocalOCDB.XXXXXX)
1512     echo "egrep "SetSpecificStorage" localOCDBaccessConfig.C > ${tempLocalOCDB}"
1513     egrep "SetSpecificStorage" localOCDBaccessConfig.C > ${tempLocalOCDB}
1514   fi
1515
1516   echo "localOCDBaccessConfig()"                               >  localOCDBaccessConfig.C
1517   echo "{"                                                     >> localOCDBaccessConfig.C
1518   echo "  AliCDBManager* man = AliCDBManager::Instance();"     >> localOCDBaccessConfig.C
1519   spitOutLocalOCDBaccessConfig ${localOCDBpathCPass0}|sort|uniq  >> localOCDBaccessConfig.C
1520   [[ -f "${tempLocalOCDB}" ]] && cat ${tempLocalOCDB}              >> localOCDBaccessConfig.C
1521   echo "}"                                                     >> localOCDBaccessConfig.C
1522
1523   [[ -f "${tempLocalOCDB}" ]] && rm -f ${tempLocalOCDB}
1524
1525   if ! grep SetSpecificStorage localOCDBaccessConfig.C; then
1526     echo
1527     echo "!!!!!!! CPass0 produced no OCDB entries"
1528     return 1
1529   fi
1530   return 0
1531 }
1532
1533 goMakeFilteredTrees()
1534 (
1535   outputDir=${1}
1536   runNumber=${2}
1537   #get path to input list
1538   inputListfiles=${3}
1539   #get scale number for tracks
1540   filterT=${4}
1541   #get scale number for V0s
1542   filterV=${5}
1543   #get OCDB path (optional)
1544   OCDBpath=${6}
1545   #get max number of files
1546   maxFiles=${7-"1000000"}
1547   #get offset of first file
1548   offsetFile=${8-"0"}
1549   #get max number of events
1550   maxEvents=${9-"30000000"}
1551   #get offset of first event
1552   offsetEvent=${10-"0"}
1553   configFile=${11-"benchmark.config"}
1554   esdFileName=${12-"AliESDs_Barrel.root"}
1555   shift 12
1556   if ! parseConfig ${configFile} "$@"; then return 1; fi
1557
1558   #record the working directory provided by the batch system
1559   batchWorkingDirectory=${PWD}
1560
1561   [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
1562   doneFileBase=filtering.cpass1.run${runNumber}.done
1563   doneFileTmp=${batchWorkingDirectory}/${doneFileBase}
1564   doneFile=${commonOutputPath}/meta/${doneFileBase}
1565
1566   cat > filtering.log << EOF
1567   goMakeFilteredTrees config:
1568   runpath=${runpath}
1569   outputDir=${outputDir}
1570   commonOutputPath=${commonOutputPath}
1571   ALICE_ROOT=${ALICE_ROOT}
1572   PATH=${PATH}
1573   offsetEvent=$offsetEvent
1574   configFile=$configFile
1575   esdFileName=$esdFileName
1576   inputListfiles=$inputListfiles
1577   doneFile=$doneFile
1578 EOF
1579
1580   #runpath=${outputDir}
1581   #[[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
1582   #[[ ${reconstructInTemporaryDir} -eq 1 ]] && runpath=$(mktemp -d -t goMakeFilteredTrees.XXXXXX)
1583   #mkdir -p ${outputDir}
1584   #mkdir -p ${runpath}
1585   #if ! cd ${runpath}; then
1586   #  echo "PWD=$PWD is not the runpath=${runpath}"
1587   #  touch ${doneFile}
1588   #  return 1
1589   #fi
1590
1591   if [[ -z ${pretend} ]];then
1592     aliroot -l -b -q "${ALICE_ROOT}/PWGPP/macros/runFilteringTask.C(\"${inputListfiles}\",${filterT},${filterV},\"${OCDBpath}\",${maxFiles},${offsetFile},${maxEvents},${offsetEvent},\"${esdFileName}\")" &>> filtering.log
1593   else
1594     sleep ${pretendDelay}
1595     touch filtering.log FilterEvents_Trees.root
1596   fi
1597   pwd
1598   /bin/ls
1599   summarizeLogs >>  ${doneFile}
1600
1601   #echo mv -f * ${outputDir}
1602   #mv -f * ${outputDir}
1603   #[[ -f ${outputDir}/FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFile}
1604   #cd ${commonOutputPath}
1605   #[[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
1606
1607   cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
1608   [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
1609
1610   return 0
1611 )
1612
1613 submit()
1614 {
1615   umask 0002
1616   [[ $# -lt 5 ]] && echo "at least 5 args needed, you supplied $#" && return 1
1617   JobID=${1}
1618   startID=${2}
1619   endID=${3}
1620   waitForJOBID=${4}
1621   command=${5}
1622   shift 5
1623   local commandArgs=("$@")
1624
1625   #add quote strings around the extra arguments
1626   for ((i=0; i<${#commandArgs[@]}; i++)); do
1627     commandArgs[i]=\"${commandArgs[i]}\"
1628   done
1629
1630   [[ -z ${waitForJOBID} ]] && waitForJOBID=0
1631
1632   newFarm=$(which qsub|grep "^/usr/bin/qsub")
1633
1634   batchSystem="SGE"
1635
1636   if [[ -z "${newFarm}" ]]
1637   then
1638     #old LSF
1639     # submit it (as job array)
1640     nFiles=$(( ${endID}-${startID}+1 ))
1641     while [ ${startID} -le ${nFiles}  ] ; do
1642       if [ $(expr ${nFiles} - ${startID}) -gt 999 ] ; then
1643         endID=$(expr ${startID} + 999)
1644       else
1645         endID=${nFiles}
1646       fi
1647     if [[ ${waitForJOBID} -eq 0 ]]; then
1648         echo ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}"
1649         ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}"
1650       else
1651         echo ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -w "ended(${waitForJOBID})" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}"
1652         ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -w "ended(${waitForJOBID})" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}"
1653       fi
1654       startID=$(expr ${endID} + 1)
1655     done
1656   else
1657     #new SGE farm
1658     if [[ ${waitForJOBID} =~ "000" ]]; then
1659       echo ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}"
1660       ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}"
1661     else
1662       echo ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -hold_jid "${waitForJOBID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}"
1663       ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -hold_jid "${waitForJOBID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}"
1664     fi
1665   fi
1666   return 0
1667 }
1668
1669 goSubmitBatch()
1670 {
1671   if [[ $# -lt 3 ]]; then
1672     echo "minimal use:"
1673     echo " ${0} submit fileList productionID configFile"
1674     return 0
1675   fi
1676
1677   productionID=${1}
1678   inputList=${2}
1679   configFile=${3:-"benchmark.config"}
1680   #if which greadlink; then configFile=$(greadlink -f ${configFile}); fi
1681   shift 3
1682   extraOpts=("$@")
1683   if ! parseConfig ${configFile} "${extraOpts[@]}"; then return 1; fi
1684
1685   #batch systems/makeflow sometimes handle spaces in arguments poorly, so encode them
1686   for (( i=0;i<${#extraOpts[@]};i++ )); do
1687     extraOpts[i]=$(encSpaces "${extraOpts[i]}")
1688   done
1689   extraOpts+=("encodedSpaces=1")
1690   #this removes the copy of the done file used by makeflow (in the running dir)
1691   extraOpts+=("removeTMPdoneFile=1")
1692
1693   #record the working directory provided by the batch system
1694   batchWorkingDirectory=${PWD}
1695
1696   #redirect all output to submit.log
1697   echo "redirecting all output to ${PWD}/submit_${productionID//"/"/_}.log"
1698   exec 7>&1
1699   exec 1>submit_${productionID//"/"/_}.log 2>&1
1700
1701   umask 0002
1702   echo ${0}" submit $*"
1703   if [[ -z "${inputList}" || -z "${productionID}" ]]
1704   then
1705     echo
1706     echo " Usage: ${0} submit inputList productionID [configFile=benchmark.config]"
1707     echo
1708     return
1709   fi
1710
1711   # check if config file is there
1712   if [ ! -f ${configFile} ]; then
1713     echo "ERROR! Config File '${configFile}' not found" >&2
1714     return
1715   else
1716     echo "Using Config File: '${configFile}'"
1717   fi
1718
1719   [[ ! -f ${alirootEnv} ]] && echo "alirootEnv script ${alirootEnv} not found!..." && return 1
1720
1721   #move the script, config and some other stuff to ${commonOutputPath} first, then use them from there
1722   self=${0}
1723   #if which greadlink; then self=$(greadlink -f "${0}"); fi
1724   configPath=$(dirname ${configFile})
1725   export commonOutputPath=${baseOutputDirectory}/${productionID}
1726
1727   mkdir -p ${commonOutputPath}
1728   mkdir -p ${commonOutputPath}/logs
1729   mkdir -p ${commonOutputPath}/meta
1730
1731   cp ${self} ${commonOutputPath}
1732   cp ${configFile} ${commonOutputPath}
1733   cp ${inputList} ${commonOutputPath}
1734   self=${commonOutputPath}/${self##*/}
1735   chmod u+x ${self}
1736   configFile=${commonOutputPath}/${configFile##*/}
1737   inputList=${commonOutputPath}/${inputList##*/}
1738
1739   #convert to absolut pathnames
1740   #if which greadlink; then inputList=$(greadlink -f "${inputList}"); fi
1741   #make list of runs
1742   if [[ -z ${runNumber} ]]; then
1743     listOfRuns=($(while read x; do guessRunNumber ${x}; done < ${inputList} | sort | uniq))
1744   else
1745     listOfRuns=${runNumber}
1746   fi
1747
1748   #if which greadlink; then alirootSource=$(greadlink -f "${alirootSource}"); fi
1749
1750   echo ""
1751   echo "### BEGIN CONFIGURATION ###"
1752   echo ""
1753   echo "GENERAL:"
1754   echo ""
1755   echo "    productionID:    ${productionID}"
1756   echo "    batchCommand:    ${batchCommand}"
1757   echo "    batchFlags:      ${batchFlags}"
1758   echo "    alirootEnv:   ${alirootEnv}"
1759   ${alirootEnv} echo '    ALICE_ROOT:      ${ALICE_ROOT}'
1760   ${alirootEnv} echo '    ALIROOT_RELEASE: ${ALICE_RELEASE}'
1761   echo "    inputList:       ${inputList}"
1762   echo "    configPath:      ${configPath}"
1763   echo "    commonOutputPath:      ${commonOutputPath}"
1764   echo "    defaultOCDB:     ${defaultOCDB}"
1765   echo "      autoOCDB: ${autoOCDB}"
1766   echo "    recoTriggerOptions:   ${recoTriggerOptions}"
1767   echo "    runs:"
1768   echo "      ${listOfRuns[*]}"
1769   echo ""
1770   echo "THE TRAIN WILL RUN:"
1771
1772   if [ ${runCPass0reco} -eq 1 ]; then
1773     echo "    Pass0 - Recontruction"
1774   fi
1775
1776   if [ ${runCPass0MergeMakeOCDB} -eq 1 ]; then
1777     echo "    Pass0 - merging and OCDB export"
1778   fi
1779
1780   if [ ${runCPass1reco} -eq 1 ]; then
1781     echo "    Pass1 - Recontruction"
1782   fi
1783   if [ ${runCPass1MergeMakeOCDB} -eq 1 ]; then
1784     echo "    Pass1 - merging and OCDB export"
1785   fi
1786
1787   echo ""
1788   echo "LIMITS:"
1789   echo "    max. Events/Chunk:   ${nEvents}"
1790   echo "    max. Number of Chunks per Run:     ${nMaxChunks}"
1791   echo ""
1792   echo "### END CONFIGURATION ###"
1793   echo ""
1794
1795
1796   # check if input file is there
1797   if [ ! -f ${inputList} ]; then
1798     echo "ERROR! Input List '${inputList}' not found" >&2
1799     return
1800   fi
1801
1802   # define jobid (for dependent jobs)
1803   date=$(date +%Y_%m_%d_%H%M%S)
1804   #for each run we submit one jobarray:
1805   for runNumber in ${listOfRuns[*]}; do
1806
1807     [[ -z ${runNumber} ]] && continue
1808     [[ ! ${runNumber} =~ ^[0-9]*[0-9]$ ]] && continue
1809
1810     JOBpostfix="${productionID//"/"/_}_${runNumber}_${date}"
1811     JOBID1="p0_${JOBpostfix}"
1812     JOBID1wait="w0_${JOBpostfix}"
1813     JOBID2="m0_${JOBpostfix}"
1814     JOBID2wait="wm0_${JOBpostfix}"
1815     JOBID3="op0_${JOBpostfix}"
1816     JOBID3wait="wop0_${JOBpostfix}"
1817     JOBID4="p1_${JOBpostfix}"
1818     JOBID4wait="w1_${JOBpostfix}"
1819     JOBID5="m1_${JOBpostfix}"
1820     JOBID5wait="wm1_${JOBpostfix}"
1821     JOBID6="s1_${JOBpostfix}"
1822     JOBID6wait="ws1_${JOBpostfix}"
1823     JOBID7="QA_${JOBpostfix}"
1824     JOBmakeESDlistCPass1="lp1_${JOBpostfix}"
1825     JOBfilterESDcpass1="fp1_${JOBpostfix}"
1826     LASTJOB="000"
1827
1828     oneInputFile=$(egrep -m1 "${runNumber}/" ${inputList})
1829
1830     currentDefaultOCDB=${defaultOCDB}
1831     [[ -z ${autoOCDB} ]] && autoOCDB=1
1832     if [[ ${autoOCDB} -ne 0 ]]; then
1833       currentDefaultOCDB=$(setYear ${oneInputFile} ${defaultOCDB})
1834     fi
1835     period=$(guessPeriod ${oneInputFile})
1836     year=$(guessYear ${oneInputFile})
1837
1838     echo "submitting run ${runNumber} with OCDB ${currentDefaultOCDB}"
1839
1840     ###############################################################################
1841     #run one chunk with valgrind:
1842     if [[ -n ${profilingCommand} ]]; then
1843       [[ -z ${nEventsProfiling} ]] && nEventsProfiling=2
1844       [[ -z ${profilingCommand} ]] && profilingCommand="/usr/bin/valgrind --tool=callgrind --num-callers=40 -v --trace-children=yes"
1845       submit "profile-${JOBpostfix}" 1 1 000 "${alirootEnv} ${self}" CPass0 ${commonOutputPath}/${year}/${period}/000${runNumber}/${jobindex} ${oneInputFile} ${nEventsProfiling} ${currentDefaultOCDB} ${configFile} ${runNumber} ${jobindex} useProfilingCommand=$(encSpaces "${profilingCommand}") "${extraOpts[@]}"
1846     fi
1847
1848     ################################################################################
1849     ################################################################################
1850     # run the CPass0 if requested
1851
1852     if [ ${runCPass0reco} -eq 1 ]; then
1853
1854       echo
1855       echo "starting CPass0... for run ${runNumber}"
1856       echo
1857
1858       # create directory and copy all files that are needed
1859       targetDirectory="${commonOutputPath}/${year}/${period}/000${runNumber}/cpass0"
1860       mkdir -p ${targetDirectory}
1861
1862       filesCPass0=(
1863                     "${configPath}/runCPass0.sh"
1864                     "${configPath}/recCPass0.C"
1865                     "${configPath}/runCalibTrain.C"
1866                     "${configPath}/localOCDBaccessConfig.C"
1867                     "${configPath}/OCDB*.root"
1868                     "${configPath}/sim.C"
1869                     "${configPath}/rec.C"
1870                     "${configPath}/Config.C"
1871       )
1872       for file in ${filesCPass0[*]}; do
1873         [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
1874       done
1875
1876       localInputList=${targetDirectory}/${inputList##*/}
1877       [[ ! -f ${localInputList} ]] && egrep "\/000${runNumber}\/" ${inputList} > ${localInputList}
1878       # limit nFiles to nMaxChunks
1879       nFiles=$(wc -l < ${localInputList})
1880       [[ ${nFiles} -eq 0 ]] && echo "list contains ZERO files! exiting..." && return 1
1881       echo "raw files in list:    ${nFiles}"
1882       if [[ ${nMaxChunks} -gt 0 && ${nMaxChunks} -le ${nFiles} ]]; then
1883         nFiles=${nMaxChunks}
1884       fi
1885       echo "raw files to process: ${nFiles}"
1886       [[ -z "${percentProcessedFilesToContinue}" ]] && percentProcessedFilesToContinue=100
1887       if [[ ${percentProcessedFilesToContinue} -eq 100 ]]; then
1888         nFilesToWaitFor=${nFiles}
1889       else
1890         nFilesToWaitFor=$(( ${nFiles}-${nFiles}/(100/(100-${percentProcessedFilesToContinue})) ))
1891       fi
1892       echo "requested success rate is ${percentProcessedFilesToContinue}%"
1893       echo "merging will start after ${nFilesToWaitFor} jobs are done"
1894
1895       submit ${JOBID1} 1 ${nFiles} 000 "${alirootEnv} ${self}" CPass0 ${targetDirectory} ${localInputList} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} -1 "${extraOpts[@]}"
1896
1897       ## submit a monitoring job that will run until a certain number of jobs are done with reconstruction
1898       submit "${JOBID1wait}" 1 1 000 "${alirootEnv} ${self}" WaitForOutput ${commonOutputPath} "meta/cpass0.job*.run${runNumber}.done" ${nFilesToWaitFor} ${maxSecondsToWait}
1899       LASTJOB=${JOBID1wait}
1900
1901     fi #end running CPass0
1902     ################################################################################
1903
1904
1905     ################################################################################
1906     # submit merging of CPass0, depends on the reconstruction
1907
1908     if [ ${runCPass0MergeMakeOCDB} -eq 1 ]; then
1909
1910       echo
1911       echo "submit CPass0 merging for run ${runNumber}"
1912       echo
1913
1914       targetDirectory="${commonOutputPath}/${year}/${period}/000${runNumber}/cpass0"
1915       mkdir -p ${targetDirectory}
1916
1917       #copy the scripts
1918       filesMergeCPass0=(
1919                         "${configPath}/OCDB.root"
1920                         "${configPath}/mergeMakeOCDB.byComponent.sh"
1921                         "${configPath}/mergeMakeOCDB.sh"
1922                         "${configPath}/localOCDBaccessConfig.C"
1923                         "${configPath}/mergeByComponent.C"
1924                         "${configPath}/makeOCDB.C"
1925                         "${configPath}/merge.C"
1926       )
1927       for file in ${filesMergeCPass0[*]}; do
1928         [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
1929       done
1930
1931       submit ${JOBID2} 1 1 "${LASTJOB}" "${alirootEnv} ${self}" MergeCPass0 ${targetDirectory} ${currentDefaultOCDB} ${configFile} ${runNumber} cpass0.calib.run${runNumber}.list "${extraOpts[@]}"
1932       LASTJOB=${JOBID2}
1933
1934       if [[ -n ${generateMC} ]]; then
1935         submit "mrl${JOBpostfix}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" PrintValues sim ${commonOutputPath}/meta/sim.run${runNumber}.list ${commonOutputPath}/meta/cpass0.job*.run${runNumber}.done
1936         LASTJOB="mrl${JOBpostfix}"
1937       fi
1938
1939       echo
1940     fi
1941     # end of merging CPass0
1942     ################################################################################
1943
1944     ################################################################################
1945     ################################################################################
1946     # run the CPass1 if requested
1947
1948     if [ ${runCPass1reco} -eq 1 ]; then
1949
1950       targetDirectory="${commonOutputPath}/${year}/${period}/000${runNumber}/cpass1"
1951       rm -f ${commonOutputPath}/meta/cpass1.job*.run${runNumber}.done
1952
1953       # safety feature: if we are re-running for any reason we want to delete the previous output first.
1954       [[ -d ${targetDirectory} ]] && rm -rf ${targetDirectory}/* && echo "removed old output at ${targetDirectory}/*"
1955
1956       echo
1957       echo "starting CPass1... for run ${runNumber}"
1958       echo
1959
1960       # create directory and copy all files that are needed
1961       mkdir -p ${targetDirectory}
1962
1963       filesCPass1=(
1964                     "${configPath}/runCPass1.sh"
1965                     "${configPath}/recCPass1.C"
1966                     "${configPath}/recCPass1_OuterDet.C"
1967                     "${configPath}/runCalibTrain.C"
1968                     "${configPath}/QAtrain_duo.C"
1969                     "${configPath}/localOCDBaccessConfig.C"
1970                     "${configPath}/OCDB.root"
1971       )
1972       for file in ${filesCPass1[*]}; do
1973         [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
1974       done
1975
1976       if [[ -n ${generateMC} ]]; then
1977         localInputList=${commonOutputPath}/meta/sim.run${runNumber}.list
1978       else
1979         localInputList=${targetDirectory}/${inputList##*/}
1980         [[ ! -f ${localInputList} ]] && egrep "\/000${runNumber}\/" ${inputList} > ${localInputList}
1981       fi
1982       # limit nFiles to nMaxChunks
1983       nFiles=$(wc -l < ${localInputList})
1984       [[ ${nFiles} -eq 0 ]] && echo "list contains ZERO files! continuing..." && continue
1985       echo "raw files in list:    ${nFiles}"
1986       if [[ ${nMaxChunks} -gt 0 && ${nMaxChunks} -le ${nFiles} ]]; then
1987         nFiles=${nMaxChunks}
1988       fi
1989       echo "raw files to process: ${nFiles}"
1990       [[ -z "${percentProcessedFilesToContinue}" ]] && percentProcessedFilesToContinue=100
1991       if [[ ${percentProcessedFilesToContinue} -eq 100 ]]; then
1992         nFilesToWaitFor=${nFiles}
1993       else
1994         nFilesToWaitFor=$(( ${nFiles}-${nFiles}/(100/(100-${percentProcessedFilesToContinue})) ))
1995       fi
1996       echo "requested success rate is ${percentProcessedFilesToContinue}%"
1997       echo "merging will start after ${nFilesToWaitFor} jobs are done"
1998
1999       submit ${JOBID4} 1 ${nFiles} "${LASTJOB}" "${alirootEnv} ${self}" CPass1 ${targetDirectory} ${localInputList} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} -1 "${extraOpts[@]}"
2000
2001       ################################################################################
2002       ## submit a monitoring job that will run until a certain number of jobs are done with reconstruction
2003       submit "${JOBID4wait}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" WaitForOutput ${commonOutputPath} "meta/cpass1.job*.run${runNumber}.done" ${nFilesToWaitFor} ${maxSecondsToWait}
2004       LASTJOB=${JOBID4wait}
2005       ################################################################################
2006
2007       echo
2008     fi #end running CPass1
2009
2010     ################################################################################
2011     # submit merging of CPass1, depends on the reconstruction
2012     if [ ${runCPass1MergeMakeOCDB} -eq 1 ]; then
2013
2014       echo
2015       echo "submit CPass1 merging for run ${runNumber}"
2016       echo
2017
2018       targetDirectory="${commonOutputPath}/${year}/${period}/000${runNumber}/cpass1"
2019       rm -f ${commonOutputPath}/meta/merge.cpass1.run${runNumber}.done
2020       mkdir -p ${targetDirectory}
2021
2022       # copy files
2023       filesMergeCPass1=(
2024                         "${configPath}/OCDB.root"
2025                         "${configPath}/localOCDBaccessConfig.C"
2026                         "${configPath}/mergeMakeOCDB.byComponent.sh"
2027                         "${configPath}/mergeByComponent.C"
2028                         "${configPath}/makeOCDB.C"
2029                         "${configPath}/merge.C"
2030                         "${configPath}/mergeMakeOCDB.sh"
2031                         "${configPath}/QAtrain_duo.C"
2032       )
2033       for file in ${filesMergeCPass1[*]}; do
2034         [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
2035       done
2036
2037       submit "${JOBID5}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" MergeCPass1 ${targetDirectory} ${currentDefaultOCDB} ${configFile} ${runNumber} cpass1.calib.run${runNumber}.list cpass1.QA.run${runNumber}.lastMergingStage.txt.list cpass1.filtered.run${runNumber}.list "${extraOpts[@]}"
2038       LASTJOB=${JOBID5}
2039       echo
2040     fi
2041
2042     ###############################
2043     #if [ ${runESDfiltering} -eq 1 ]; then
2044     #  rm -f ${commonOutputPath}/cpass1.ESD.run${runNumber}.list
2045     #  rm -f ${commonOutputPath}/meta/filtering.cpass1.run*.done
2046     #  echo
2047     #  echo submitting filtering for run ${runNumber}
2048     #  echo
2049     #  submit "${JOBmakeESDlistCPass1}" 1 1 "${LASTJOB}" "${self}" PrintValues esd ${commonOutputPath}/meta/cpass1.ESD.run${runNumber}.list ${commonOutputPath}/meta/cpass1.job*.run${runNumber}.done
2050     #  submit "${JOBfilterESDcpass1}" 1 1 "${JOBmakeESDlistCPass1}" "${alirootEnv} ${self}" MakeFilteredTrees ${commonOutputPath}/${year}/${period}/000${runNumber}/cpass1 ${runNumber} ${commonOutputPath}/meta/cpass1.ESD.run${runNumber}.list ${filteringFactorHighPt} ${filteringFactorV0s} ${currentDefaultOCDB} 1000000 0 10000000 0 ${configFile} AliESDs_Barrel.root "${extraOpts[@]}"
2051     #  LASTJOB=${JOBfilterESDcpass1}
2052     #fi
2053
2054   done
2055
2056   #################################################################################
2057   #################################################################################
2058   #if [ ${runESDfiltering} -eq 1 ]; then
2059   #  submit "${JOBID5wait}" 1 1 "${LASTJOB}" "${self}" WaitForOutput ${commonOutputPath} "meta/filtering.cpass1.run*.done" "${#listOfRuns[@]}" ${maxSecondsToWait}
2060   #else
2061     submit "${JOBID5wait}" 1 1 "${LASTJOB}" "${self}" WaitForOutput ${commonOutputPath} "meta/merge.cpass1.run*.done" ${#listOfRuns[@]} ${maxSecondsToWait}
2062   #fi
2063   LASTJOB=${JOBID5wait}
2064
2065   #################################################################################
2066   echo
2067   echo "submit make a summary"
2068   echo
2069
2070   [[ -z ${alirootEnvQA} ]] && alirootEnvQA=$(encSpaces "${alirootEnv}")
2071   submit "${JOBID6}" 1 1 "${LASTJOB}" "${alirootEnvQA} ${self}" MakeSummary ${configFile} "commonOutputPath=${commonOutputPath}"
2072   LASTJOB=${JOBID6}
2073   #################################################################################
2074
2075   #restore stdout
2076   exec 1>&7 7>&-
2077   echo "jobs submitted."
2078   return 0
2079 }
2080
2081 goWaitForOutput()
2082 (
2083   umask 0002
2084   [[ $# -lt 3 ]] && echo "goWaitForOutput() wrong number of arguments, exiting.." && return 1
2085   echo searchPath=${1}
2086   echo fileName=${2}
2087   echo numberOfFiles=${3}
2088   echo maxSecondsToWait=${4}
2089   searchPath=${1}
2090   fileName=${2}
2091   numberOfFiles=${3}
2092   maxSecondsToWait=${4}
2093   echo "command to be executed: /bin/ls -1 ${searchPath}/${fileName}"
2094   [[ -z "${maxSecondsToWait}" ]] && maxSecondsToWait=$(( 3600*12 ))
2095   while true; do
2096     n=$(/bin/ls -1 ${searchPath}/${fileName} 2>/dev/null | wc -l)
2097     [[ ${n} -gt 0 ]] && echo "found ${n} X ${fileName}"
2098     [[ ${n} -ge ${numberOfFiles} ]] && break
2099     [[ ${SECONDS} -gt ${maxSecondsToWait} ]] && echo "timeout of ${maxSecondsToWait}s!" && break
2100     sleep 60
2101   done
2102   echo "DONE! exiting..."
2103   return 0
2104 )
2105
2106 mergeSysLogs()
2107 (
2108   outputFile=${1}
2109   shift
2110   inputFiles="$@"
2111   i=0
2112   if ! ls -1 ${inputFiles} &>/dev/null; then echo "the files dont exist!: ${inputFiles}"; return 1; fi
2113   while read x; do
2114     runNumber=$(guessRunNumber ${x})
2115     [[ -z ${runNumber} ]] && echo "run number cannot be guessed for ${x}" && continue
2116     awk -v run=${runNumber} -v i=${i} 'NR > 1 {print run" "$0} NR==1 && i==0 {print "run/I:"$0}' ${x}
2117     (( i++ ))
2118   done < <(ls -1 ${inputFiles}) > ${outputFile}
2119   return 0
2120 )
2121
2122 goMakeMergedSummaryTree()
2123 (
2124   # create list of calibration entries
2125   # takes no arguments, just run it in the base output
2126   # directory with the following files in the working directory
2127   #
2128   #  Calibration file lists:
2129   #       cpass0.dcsTree.list, cpass1.dcsTree.list
2130   #  QA trending root file:
2131   #       trending.root
2132   #
2133   #  Production infoo ascii files:
2134   #       summary_pass0.tree
2135   #       summary_pass1.tree
2136   #
2137
2138   [[ ! -f cpass0.dcsTree.list ]] && echo "no cpass0.dcsTree.list" && return 1
2139   [[ ! -f cpass1.dcsTree.list ]] && echo "no cpass1.dcsTree.list" && return 1
2140   [[ ! -f trending.root ]] && echo "no trending.root" && return 1
2141   [[ ! -f summary_pass0.tree ]] && echo "no summary_pass0.tree" && return 1
2142   [[ ! -f summary_pass1.tree ]] && echo "no summary_pass1.tree" && return 1
2143
2144   #first, dump the C macro to file
2145   cat << EOF > mergeTree.C
2146   //
2147   // Merge summary information
2148   // Following files are expected to be in the working directory
2149   //
2150   // Calibration file lists:
2151   //      cpass0.dcsTree.list, cpass1.dcsTree.list
2152   // QA trending root file:
2153   //      trending.root
2154   //
2155   // Production infoo ascii files:
2156   //      summary_pass0.tree
2157   //      summary_pass1.tree
2158   //
2159   void mergeTree(){
2160     //
2161     //
2162     //
2163     // Calibration values dump
2164     //
2165     //Printf("MakeTreeFromList cpass0.dcsTree.list");
2166     AliXRDPROOFtoolkit::MakeTreeFromList("Calib.TPC.CPass0.root", "dcs","dcs","cpass0.dcsTree.list",1);
2167     //Printf("MakeTreeFromList cpass1.dcsTree.list");
2168     AliXRDPROOFtoolkit::MakeTreeFromList("Calib.TPC.CPass1.root", "dcs","dcs","cpass1.dcsTree.list",1);
2169     //
2170     // Calibration status dump
2171     //
2172     TFile *fprod = TFile::Open("fproduction.root","recreate");
2173     TTree  tree0, tree1;
2174     //Printf("reading summary_pass0.tree");
2175     tree0.ReadFile("summary_pass0.tree");
2176     //Printf("reading summary_pass1.tree");
2177     tree1.ReadFile("summary_pass1.tree");
2178     tree0.Write("CPass0");
2179     tree1.Write("CPass1");
2180     fprod->Close();
2181     //
2182     //
2183     //
2184     TString stringSetup="";
2185     stringSetup+="1#QA.TPC#run#SummaryTPCQA/tpcQA#trending.root+";  //
2186     stringSetup+="1#Calib.TPC.CPass0#run#dcs#Calib.TPC.CPass0.root+";  //
2187     stringSetup+="1#Calib.TPC.CPass1#run#dcs#Calib.TPC.CPass1.root+";  //
2188     //
2189     stringSetup+="1#CPass0#runnumber#CPass0#fproduction.root+";  //
2190     stringSetup+="1#CPass1#runnumber#CPass1#fproduction.root+";  //
2191     //
2192     //Printf("stringSetup: %s", stringSetup.Data());
2193     AliXRDPROOFtoolkit::JoinTreesIndex("outAll.root","joinAll","run",stringSetup.Data(), 1);
2194   }
2195 EOF
2196
2197   aliroot -b -q "mergeTree.C" > mergeTrees.log
2198   return $?
2199 )
2200
2201 stackTraceTree()
2202 (
2203   if [[ $# -lt 1 ]]; then
2204     echo 'make stacktrace processing  in case of standard root crash log'
2205     echo 'input is a (list of) text files with the stack trace (either gdb aoutput'
2206     echo 'produced with e.g. gdb --batch --quiet -ex "bt" -ex "quit" aliroot core,'
2207     echo 'or the root crash log), output is a TTree formatted table.'
2208     echo 'example usage:'
2209     echo 'benchmark.sh stackTraceTree /foo/*/rec.log'
2210     echo 'benchmark.sh stackTraceTree $(cat file.list)'
2211     echo 'benchmark.sh stackTraceTree `cat file.list`'
2212     return 0
2213   fi
2214   gawk '
2215        BEGIN {
2216                print "frame/I:method/C:line/C:cpass/I:aliroot/I:file/C";
2217                RS="#[0-9]*";
2218                aliroot=0;
2219                read=1;
2220              }
2221       /There was a crash/ {read=1;}
2222       /The lines below might hint at the cause of the crash/ {read=0;}
2223       read==1 {
2224                if ($3 ~ /Ali*/) aliroot=1; else aliroot=0;
2225                gsub("#","",RT);
2226                if ($NF!="" && RT!="" && $3!="") print RT" "$3" "$NF" "0" "aliroot" "FILENAME
2227              }
2228       ' "$@" 2>/dev/null
2229 )
2230
2231 goMakeSummary()
2232 (
2233   #all the final stuff goes in here for ease of use:
2234   # summary logs
2235   # qa plot making
2236   # final file lists
2237   # runs in current dir - in makeflow mode it can run LOCAL, then the QA plots and summaries
2238   # will appear in the submission dir.
2239   #some defaults:
2240   log="summary.log"
2241   productionID="qa"
2242
2243   configFile=${1}
2244   shift 1
2245   extraOpts=("$@")
2246   if ! parseConfig ${configFile} "${extraOpts[@]}"; then return 1; fi
2247
2248   #if which greadlink; then configFile=$(greadlink -f ${configFile}); fi
2249
2250   #record the working directory provided by the batch system
2251   batchWorkingDirectory=${PWD}
2252
2253   logTmp=${batchWorkingDirectory}/${log}
2254   logDest=${commonOutputPath}/${log}
2255
2256   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
2257
2258   [[ ! -f ${configFile} ]] && echo "no config file ${configFile}!" && return
2259
2260   [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
2261
2262   #copy some useful stuff
2263   #and go to the commonOutputPath
2264   cp ${configFile} ${commonOutputPath}
2265
2266   exec &> >(tee ${logTmp})
2267
2268   #summarize the global stuff
2269   echo "env script: ${alirootSource} ${alirootEnv}"
2270   echo "\$ALICE_ROOT=${ALICE_ROOT}"
2271   echo "commonOutputPath=${commonOutputPath}"
2272
2273   #summarize the stacktraces
2274   stackTraceTree ${commonOutputPath}/*/*/000*/cpass0/*/stacktrace* > stacktrace_cpass0.tree
2275   stackTraceTree ${commonOutputPath}/*/*/000*/cpass1/*/stacktrace* > stacktrace_cpass1.tree
2276
2277   echo total numbers for the production:
2278   echo
2279   awk 'BEGIN {nFiles=0;nCore=0;}
2280   /^calibfile/ {nFiles++;}
2281   /core dumped/ {nCore++i;}
2282   END {print     "cpass0 produced "nFiles" calib files, "nCore" core files";}' ${commonOutputPath}/meta/cpass0.job*done 2>/dev/null
2283   awk 'BEGIN {nOK=0; nBAD=0; }
2284   /\/rec.log OK/ {nOK++;}
2285   /\/rec.log BAD/ {nBAD++;}
2286   /stderr BAD/ {if ($0 ~ /rec.log/){nBAD++;}}
2287   END {print     "cpass0 reco:  OK: "nOK"\tBAD: "nBAD;}' ${commonOutputPath}/meta/cpass0.job*done 2>/dev/null
2288   awk 'BEGIN {nOK=0; nBAD=0; }
2289   /\/calib.log OK/ {nOK++;}
2290   /\/calib.log BAD/ {nBAD++;}
2291   END {print "cpass0 calib: OK: "nOK"\tBAD: "nBAD;}' ${commonOutputPath}/meta/cpass0.job*done 2>/dev/null
2292
2293   awk 'BEGIN {nOK=0; nBAD=0; }
2294   /merge.log OK/ {nOK++;}
2295   /merge.log BAD/ {nBAD++;}
2296   END {print "cpass0 merge: OK: "nOK"\tBAD: "nBAD;}' ${commonOutputPath}/meta/merge.cpass0*done 2>/dev/null
2297   awk 'BEGIN {nOK=0; nBAD=0; }
2298   /ocdb.log OK/ {nOK++;}
2299   /ocdb.log BAD/ {nBAD++;}
2300   END {print   "cpass0 OCDB:  OK: "nOK"\tBAD: "nBAD;}' ${commonOutputPath}/meta/merge.cpass0*done 2>/dev/null
2301
2302   echo
2303   awk 'BEGIN {nFiles=0;nCore=0;}
2304   /^calibfile/ {nFiles++;}
2305   /core dumped/ {nCore++;}
2306   END {print     "cpass1 produced "nFiles" calib files, "nCore" core files";}' ${commonOutputPath}/meta/cpass1.job*done 2>/dev/null
2307   awk 'BEGIN {nOK=0; nBAD=0; }
2308   /\/rec.log OK/ {nOK++;}
2309   /\/rec.log BAD/ {nBAD++;}
2310   /stderr BAD/ {if ($0 ~ /rec.log/){nBAD++;}}
2311   END {print     "cpass1 reco:  OK: "nOK"\tBAD: "nBAD;}' ${commonOutputPath}/meta/cpass1.job*done 2>/dev/null
2312   awk 'BEGIN {nOK=0; nBAD=0; }
2313   /\/calib.log OK/ {nOK++;}
2314   /\/calib.log BAD/ {nBAD++;}
2315   END {print "cpass1 calib: OK: "nOK"\tBAD: "nBAD;}' ${commonOutputPath}/meta/cpass1.job*done 2>/dev/null
2316
2317   awk 'BEGIN {nOK=0; nBAD=0; }
2318   /merge.log OK/ {nOK++;}
2319   /merge.log BAD/ {nBAD++;}
2320   END {print "cpass1 merge: OK: "nOK"\tBAD: "nBAD;}' ${commonOutputPath}/meta/merge.cpass1*done 2>/dev/null
2321   awk 'BEGIN {nOK=0; nBAD=0; }
2322   /ocdb.log OK/ {nOK++;}
2323   /ocdb.log BAD/ {nBAD++;}
2324   END {print   "cpass1 OCDB:  OK: "nOK"\tBAD: "nBAD;}' ${commonOutputPath}/meta/merge.cpass1*done 2>/dev/null
2325
2326   echo
2327   echo per run stats:
2328   /bin/ls -1 ${commonOutputPath}/meta/merge.cpass0.run*.done | while read x
2329 do
2330   dir=$(goPrintValues dir - ${x})
2331   runNumber=$(guessRunNumber ${dir})
2332   [[ -z ${runNumber} ]] && continue
2333
2334   if $(/bin/ls ${commonOutputPath}/meta/cpass0.job*.run${runNumber}.done &> /dev/null); then
2335     statusCPass0=( $(
2336     awk 'BEGIN {nOKrec=0;nBADrec=0;nOKcalib=0;nBADcalib=0;nOKstderr=0;nBADstderr=0;}
2337     /\/rec.log OK/ {nOKrec++;}
2338     /\/rec.log BAD/ {nBADrec++;}
2339     /stderr BAD/ {if ($0 ~ /rec.log/) {nBADrec++;} nBADstderr++;}
2340     /stderr OK/ {nOKstderr++;}
2341     /\/calib.log OK/ {nOKcalib++;}
2342     /\/calib.log BAD/ {nBADcalib++}
2343     END {print ""nOKrec" "nBADrec" "nOKstderr" "nBADstderr" "nOKcalib" "nBADcalib;}' ${commonOutputPath}/meta/cpass0.job*.run${runNumber}.done 2>/dev/null
2344     ) )
2345   fi
2346
2347   if $(/bin/ls ${commonOutputPath}/meta/cpass1.job*.run${runNumber}.done &>/dev/null); then
2348     statusCPass1=( $(
2349     awk 'BEGIN {nOKrec=0;nBADrec=0;nOKcalib=0;nBADcalib=0;nOKstderr=0;nBADstderr=0;nQAbarrelOK=0;nQAbarrelBAD=0;nQAouterOK=0;nQAouterBAD=0;}
2350     /\/rec.log OK/ {nOKrec++;}
2351     /\/rec.log BAD/ {nBADrec++;}
2352     /stderr BAD/ {if ($0 ~ /rec.log/) nBADrec++;nBADstderr++;}
2353     /stderr OK/ {nOKstderr++;}
2354     /\/calib.log OK/ {nOKcalib++;}
2355     /\/calib.log BAD/ {nBADcalib++}
2356     /\/qa_barrel.log OK/ {nQAbarrelOK++;}
2357     /\/qa_barrel.log BAD/ {nQAbarrelBAD++;}
2358     /\/qa_outer.log OK/ {nQAouterOK++;}
2359     /\/qa_outer.log BAD/ {nQAouterBAD++;}
2360     END {print ""nOKrec" "nBADrec" "nOKstderr" "nBADstderr" "nOKcalib" "nBADcalib" "nQAbarrelOK" "nQAbarrelBAD" "nQAouterOK" "nQAouterBAD;}' ${commonOutputPath}/meta/cpass1.job*.run${runNumber}.done 2>/dev/null
2361     ) )
2362   fi
2363
2364   statusOCDBcpass0=$(awk '/ocdb.log/ {print $2} ' ${x} 2>/dev/null)
2365   statusOCDBcpass1=$(awk '/ocdb.log/ {print $2}' ${x/cpass0/cpass1} 2>/dev/null)
2366   statusQA=$(awk '/mergeMakeOCDB.log/ {print $2}' ${x/cpass0/cpass1} 2>/dev/null)
2367
2368   printf "%s\t ocdb.log cpass0: %s\t ocdb.log cpass1: %s\tqa.log:%s\t| cpass0: rec:%s/%s stderr:%s/%s calib:%s/%s cpass1: rec:%s/%s stderr:%s/%s calib:%s/%s QAbarrel:%s/%s QAouter:%s/%s\n" ${runNumber} ${statusOCDBcpass0} ${statusOCDBcpass1} ${statusQA} ${statusCPass0[0]} ${statusCPass0[1]} ${statusCPass0[2]} ${statusCPass0[3]} ${statusCPass0[4]} ${statusCPass0[5]} ${statusCPass1[0]} ${statusCPass1[1]} ${statusCPass1[2]} ${statusCPass1[3]} ${statusCPass1[4]} ${statusCPass1[5]} ${statusCPass1[6]} ${statusCPass1[7]} ${statusCPass1[8]} ${statusCPass1[9]}
2369 done
2370
2371   #make lists with output files - QA, trending, filtering and calibration
2372   ### wait for the merging of all runs to be over ###
2373   rm -f qa.list
2374   goPrintValues qafile qa.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2375   rm -f calib.list
2376   goPrintValues calibfile calib.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2377   rm -f trending.list
2378   goPrintValues trendingfile trending.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2379   rm -f filtering.list
2380   goPrintValues filteredTree filtering.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2381   rm -f cpass0.dcsTree.list
2382   goPrintValues dcsTree cpass0.dcsTree.list ${commonOutputPath}/meta/merge.cpass0.run*.done &>/dev/null
2383   rm -f cpass1.dcsTree.list
2384   goPrintValues dcsTree cpass1.dcsTree.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2385
2386   #merge trending
2387   rm -f trending.root
2388   goMerge trending.list trending.root ${configFile} "${extraOpts[@]}" &> mergeTrending.log
2389
2390   goMakeSummaryTree ${commonOutputPath} 0
2391   goMakeSummaryTree ${commonOutputPath} 1
2392
2393   goCreateQAplots "${PWD}/qa.list" "${productionID}" "QAplots" "${configFile}" "${extraOpts[@]}" filteringList="${PWD}/filtering.list" &>createQAplots.log
2394
2395   #make a merged summary tree out of the QA trending, dcs trees and log summary trees
2396   goMakeMergedSummaryTree
2397
2398   #if set, email the summary
2399   [[ -n ${MAILTO} ]] && cat ${logTmp} | mail -s "benchmark ${productionID} done" ${MAILTO}
2400
2401   # Copy log to destination (delete all on failure to signal error)
2402   cp "$logTmp" "$logDest" || rm -f "$logTmp" "$logDest"
2403
2404   #copy output files
2405   exec &> >(tee fileCopy.log)
2406   paranoidCp QAplots ${commonOutputPath}
2407   paranoidCp *.list ${commonOutputPath}
2408   paranoidCp *.root ${commonOutputPath}
2409   paranoidCp *.log ${commonOutputPath}
2410   paranoidCp fileCopy.log ${commonOutputPath}
2411
2412   return 0
2413 )
2414
2415 goMakeSummaryTree()
2416 (
2417   if [[ $# -lt 1 ]] ; then
2418     return
2419   fi
2420   #1. define vars/arrays
2421   DIR=${1} #use input or exec in current dir
2422   pass=${2-"0"} #pass from input
2423   outfile="summary_pass${pass}.tree"
2424   Ncolumns=0
2425   test -f ${outfile} && : >${outfile}
2426   errfile=${outfile/tree/err}
2427   test -f ${errfile} && : >${errfile}
2428
2429   declare -a counterString=(TOFevents TOFtracks TPCevents TPCtracks TRDevents TRDtracks T0events SDDevents SDDtracks MeanVertexevents)
2430   Ncounter=${#counterString[@]}
2431
2432   declare -a statusString=(TRDStatus TOFStatus TPCStatus T0Status MeanVertexStatus)
2433   Nstatus=${#statusString[@]}
2434
2435
2436   declare -a ratesString=(rec stderr calib qa_barrel qa_outer)
2437   Nrates=${#ratesString[@]}
2438
2439   runs=( $(ls -1 ${DIR}/meta/merge.cpass0* | while read x; do guessRunNumber $x; done) )
2440   Nruns=${#runs[@]}
2441
2442   echo -n runnumber/I >>${outfile}
2443   echo -n :cpass${pass}status/I >>${outfile}
2444   echo -n :cpass${pass}QAstatus/I >>${outfile}
2445   for i in ${ratesString[@]}; do
2446     echo -n :${i}OK/I >>${outfile}
2447     echo -n :${i}BAD/I >>${outfile}
2448
2449   done
2450   for i in ${counterString[@]} ${statusString[@]} ; do
2451     echo -n :${i}/I >>${outfile}
2452   done
2453   Ncolumns=$((2 + 2*Nrates + Ncounter + Nstatus))
2454   echo >> ${outfile}
2455
2456   #2. loop runs
2457
2458   for runnumber in ${runs[@]} ; do
2459
2460
2461
2462     filejob="${DIR}/meta/cpass${pass}.job*.run${runnumber}.done"
2463     filemerge="${DIR}/meta/merge.cpass${pass}.run${runnumber}.done"
2464     fileOCDB=$(grep /ocdb.log ${filemerge} | awk '{print $1}')
2465     if ! $(/bin/ls ${filemerge} &>/dev/null) ; then
2466       echo "${filemerge} does not exist!" >>${errfile}
2467       continue
2468     elif ! $(/bin/ls ${filejob} &>/dev/null) ; then
2469       echo "${filejob} does not exist!" >>${errfile}
2470       echo -n ${runnumber} >> ${outfile}
2471       for i in $(seq ${Ncolumns}) ; do
2472         echo -n "-1" >> ${outfile}
2473       done
2474       echo >> ${outfile}
2475       continue
2476     fi
2477     echo -n ${runnumber} >> ${outfile}
2478     #pass0status= grep '/ocdb.log' ${filemerge} |  cut -d' ' -f2 | tr OK x1 | tr BAD xx0 | tr -d 'x'
2479     passStatus=$(grep '/ocdb.log' ${filemerge} | grep OK | wc -l)
2480     echo -n " ${passStatus}" >> ${outfile}
2481     qaStatus=$(grep '/mergeMakeOCDB.log' ${filemerge} | grep OK | wc -l)
2482     echo -n " ${qaStatus}" >> ${outfile}
2483
2484
2485     #fill OK/BAD rates
2486     for i in $(seq 0 $((${Nrates}-1))) ; do
2487       var1=$(grep "/${ratesString[${i}]}.log" ${filejob} | grep OK | wc -l)
2488       var2=$(grep "/${ratesString[${i}]}.log" ${filejob} | grep BAD | wc -l)
2489
2490       if [[ ${ratesString[${i}]} == "stderr" ]] ; then
2491         var1=$(grep "stderr" ${filejob} | grep OK | wc -l)
2492         var2=$(grep "stderr" ${filejob} | grep "rec.log" | grep BAD | wc -l)
2493       fi
2494       echo -n " ${var1}" >> ${outfile}
2495       echo -n " ${var2}" >> ${outfile}
2496     done
2497
2498     if [[ -f ${fileOCDB} ]] ; then
2499       #fill counter
2500       for i in $(seq 0 $((${Ncounter}-1))) ; do
2501         var1=$(grep Monalisa ${fileOCDB} | grep ${counterString[${i}]} | cut -f2)
2502         echo -n " ${var1:-"-1"}" >> ${outfile}
2503       done
2504
2505       #fill status
2506       for i in $(seq 0 $((${Nstatus}-1))) ; do
2507         var1=$(grep "calibration status=" ${fileOCDB} | grep ${statusString[${i}]/Status/} | cut -d'=' -f2)
2508         echo -n " ${var1:-"-1"}" >> ${outfile}
2509       done
2510     fi
2511     echo >> ${outfile}
2512   done
2513
2514   return 0
2515 )
2516
2517 parseConfig()
2518 {
2519   configFile=${1}
2520   shift
2521   args=("$@")
2522
2523
2524   #some defaults
2525   #autoOCDB=0
2526   defaultOCDB="raw://"
2527   #runNumber=167123
2528   #makeflowPath="/hera/alice/aux/cctools/bin"
2529   #makeflowOptions="-T wq -N alice -d all -C ali-copilot.cern.ch:9097"
2530   #makeflowOptions="-T wq -N alice -C ali-copilot.cern.ch:9097"
2531   makeflowOptions=""
2532   #batchCommand="/usr/bin/qsub"
2533   batchFlags=""
2534   baseOutputDirectory="$PWD/output"
2535   #alirootEnv="/cvmfs/alice.cern.ch/bin/alienv setenv AliRoot/v5-04-34-AN -c"
2536   #alirootEnv="/home/mkrzewic/alisoft/balice_master.sh"
2537   #trustedQAtrainMacro='/hera/alice/mkrzewic/gsisvn/Calibration/QAtrain_duo.C'
2538   reconstructInTemporaryDir=0
2539   recoTriggerOptions="\"\""
2540   percentProcessedFilesToContinue=100
2541   maxSecondsToWait=$(( 3600*24 ))
2542   nEvents=-1
2543   nMaxChunks=0
2544   postSetUpActionCPass0=""
2545   postSetUpActionCPass1=""
2546   runCPass0reco=1
2547   runCPass0MergeMakeOCDB=1
2548   runCPass1reco=1
2549   runCPass1MergeMakeOCDB=1
2550   runESDfiltering=1
2551   filteringFactorHighPt=1e2
2552   filteringFactorV0s=1e1
2553   MAILTO=""
2554   #pretend=1
2555   #dontRedirectStdOutToLog=1
2556   logToFinalDestination=1
2557   ALIROOT_FORCE_COREDUMP=1
2558   pretendDelay=0
2559
2560   #first, source the config file
2561   if [ -f ${configFile} ]; then
2562     source ${configFile}
2563   else
2564     echo "config file ${configFile} not found!"
2565     return 1
2566   fi
2567
2568   unset encodedSpaces
2569   for opt in "${args[@]}"; do
2570     [[ "${opt}" =~ encodedSpaces=.* ]] && encodedSpaces=1 && echo "encodedSpaces!" && break
2571   done
2572
2573   #then, parse the options as they override the options from file
2574   for opt in "${args[@]}"; do
2575     [[ -z ${opt} ]] && continue
2576     [[ -n ${encodedSpaces} ]] && opt="$(decSpaces ${opt})"
2577     [[ "${opt}" =~ ^[[:space:]]*$ ]] && continue
2578     if [[ ! "${opt}" =~ .*=.* ]]; then
2579       echo "badly formatted option \"${opt}\" should be: option=value, stopping..."
2580       return 1
2581     fi
2582     local var="${opt%%=*}"
2583     local value="${opt#*=}"
2584     echo "${var}=${value}"
2585     export ${var}="${value}"
2586   done
2587
2588   #do some checking
2589   [[ -z ${alirootEnv} ]] && echo "alirootEnv not defined!" && return 1
2590
2591   #export the aliroot function if defined to override normal behaviour
2592   [[ $(type -t aliroot) =~ "function" ]] && export -f aliroot && echo "exporting aliroot() function..."
2593
2594   return 0
2595 }
2596
2597 aliroot()
2598 {
2599   args=("$@")
2600   if [[ -n ${useProfilingCommand} ]]; then
2601     profilerLogFile="cpu.txt"
2602     [[ "${args[@]}" =~ rec ]] && profilerLogFile="cpu_rec.txt"
2603     [[ "${args[@]}" =~ Calib ]] && profilerLogFile="cpu_calib.txt"
2604     echo running "${useProfilingCommand} aliroot ${args[@]} &> ${profilerLogFile}"
2605     ${useProfilingCommand} aliroot "${args[@]}" &> ${profilerLogFile}
2606   else
2607     #to prevent an infinite recursion use "command aliroot" to disable
2608     #aliases and functions
2609     echo running command aliroot "${args[@]}"
2610     command aliroot "${args[@]}"
2611   fi
2612   return 0
2613 }
2614
2615 paranoidCp()
2616 (
2617   #recursively copy files and directories
2618   #to avoid using find and the like as they kill
2619   #the performance on some cluster file systems
2620   #does not copy links to avoid problems
2621   sourceFiles=("${@}")
2622   destination="${sourceFiles[@]:(-1)}" #last element
2623   unset sourceFiles[${#sourceFiles[@]}-1] #remove last element (dst)
2624   for src in "${sourceFiles[@]}"; do
2625     if [[ -f "${src}" && ! -h  "${src}" ]]; then
2626       paranoidCopyFile "${src}" "${destination}"
2627     elif [[ -d "${src}" && ! -h "${src}" ]]; then
2628       src="${src%/}"
2629       dst="${destination}/${src##*/}"
2630       mkdir -p "${dst}"
2631       paranoidCp "${src}"/* "${dst}"
2632     fi
2633   done
2634 )
2635
2636 paranoidCopyFile()
2637 (
2638   #copy a single file to a target in an existing dir
2639   #repeat a few times if copy fails
2640   src="${1}"
2641   dst="${2}"
2642   [[ -d "${dst}" ]] && dst="${dst}/${src##*/}"
2643   [[ -z "${maxCopyTries}" ]] && maxCopyTries=5
2644   #echo "maxCopyTries=${maxCopyTries}"
2645   echo "cp ${src} ${dst}"
2646   cp "${src}" "${dst}"
2647   i=0
2648   until cmp -s "${src}" "${dst}"; do
2649     echo "try: ${i}"
2650     [[ -f "${dst}" ]] && rm "${dst}"
2651     cp "${src}" "${dst}"
2652     [[ ${i} -gt ${maxCopyTries} ]] && ret=1 && return 1
2653     (( i++ ))
2654   done
2655   return 0
2656 )
2657
2658 guessRunData()
2659 {
2660   #guess the period from the path, pick the rightmost one
2661   period=""
2662   runNumber=""
2663   year=""
2664   pass=""
2665   legoTrainRunNumber=""
2666   dataType=""
2667
2668   local shortRunNumber=""
2669   local IFS="/"
2670   declare -a path=( $1 )
2671   local dirDepth=$(( ${#path[*]}-1 ))
2672   i=0
2673   #for ((x=${dirDepth};x>=0;x--)); do
2674   for ((x=0;x<=${dirDepth};x++)); do
2675
2676     [[ $((x-1)) -ge 0 ]] && local fieldPrev=${path[$((x-1))]}
2677     local field=${path[${x}]}
2678     local fieldNext=${path[$((x+1))]}
2679
2680     [[ ${field} =~ ^[0-9]*$ && ${fieldNext} =~ (.*\.zip$|.*\.root$) ]] && legoTrainRunNumber=${field}
2681     [[ -n ${legoTrainRunNumber} && -z ${pass} ]] && pass=${fieldPrev}
2682     [[ ${field} =~ ^LHC[0-9][0-9][a-z].*$ ]] && period=${field%_*}
2683     [[ ${field} =~ ^000[0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && runNumber=${field#000}
2684     [[ ${field} =~ ^[0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && shortRunNumber=${field}
2685     [[ ${field} =~ ^20[0-9][0-9]$ ]] && year=${field}
2686     [[ ${field} =~ ^(^sim$|^data$) ]] && dataType=${field}
2687     (( i++ ))
2688   done
2689   [[ -z ${legoTrainRunNumber} ]] && pass=${path[$((dirDepth-1))]}
2690   [[ "${dataType}" =~ ^sim$ ]] && pass="passMC" && runNumber=${shortRunNumber}
2691
2692   #if [[ -z ${dataType} || -z ${year} || -z ${period} || -z ${runNumber}} || -z ${pass} ]];
2693   if [[ -z ${runNumber}} ]];
2694   then
2695     #error condition
2696     return 1
2697   else
2698     #ALL OK
2699     return 0
2700   fi
2701   return 0
2702 }
2703
2704 #these functions encode strings to and from a space-less form
2705 #use when spaces are not well handled (e.g. in arguments to
2706 #commands in makeflow files, etc.
2707 encSpaces()(echo "${1// /@@@@}")
2708 decSpaces()(echo "${1//@@@@/ }")
2709
2710 main "$@"