this contains the fixes by Dario to workaround some of the quirks of EOS
[u/mrichter/AliRoot.git] / PWGPP / benchmark / benchmark.sh
1 #!/bin/bash
2 #include benchmark.config
3
4 # this script runs the CPass0/CPass1 train
5 # produced OCDB updates are local
6
7 main()
8 {
9   #run in proper mode depending on the selection
10   if [[ $# -lt 1 ]]; then
11     if [[ ! "${0}" =~ "bash" ]]; then
12       echo "uses makeflow:"
13       echo " ${0} \"run\" productionID inputList configFile [extraOpts]"
14       echo "uses a batch system (SGE):"
15       echo " ${0} \"submit\" productionID inputList configFile [extraOpts]"
16       echo "extraOpts if non-empty override the config file, e.g.:"
17       echo " ${0} submit test1 benchmark.list benchmark.config runNumber=169123 nEvents=10"
18     fi
19     return
20   fi
21
22   #define some aliases - default is to call one of the functions directly
23   runMode=${1}
24   umask 0002
25   shift
26   case ${runMode} in
27     "CPass0") goCPass0 "$@";;
28     "CPass1") goCPass1 "$@";;
29     "MakeLocalOCDBaccessConfig") goMakeLocalOCDBaccessConfig "$@";;
30     "MergeCPass0") goMergeCPass0 "$@";;
31     "MergeCPass1") goMergeCPass1 "$@";;
32     "MakeFilteredTrees") goMakeFilteredTrees "$@";;
33     "MakeSummary") goMakeSummary "$@";;
34     "run") goSubmitMakeflow "$@";;
35     "submit") goSubmitBatch "$@";;
36     "test") goTest "$@";;
37     "GenerateMakeflow") goGenerateMakeflow "$@";;
38     "PrintValues") goPrintValues "$@";;
39     "CreateQAplots") goCreateQAplots "$@";;
40     "WaitForOutput") goWaitForOutput "$@";;
41     "Merge") goMerge "$@";;
42     *) 
43       ${runMode} "$@"
44     ;;
45   esac
46   return 0
47 }
48
49 generateMC()
50 {
51   #generate one raw chunk in current directory
52   SEED=${JOB_ID}${SGE_TASK_ID}
53   export CONFIG_SEED=${SEED}
54   runNumber=${1}
55   OCDBpath=${2}
56   nEventsim=${3}
57   if [[ -n ${pretend} ]]; then
58     sleep ${pretendDelay}
59     touch galice.root
60   else
61     if [[ -f sim.C && -f Config.C ]] ; then
62         time aliroot -b -q -x sim.C\(${runNumber},\"${OCDBpath}\",${nEventsim}\) >sim.log 2>&1
63         mv syswatch.log simwatch.log
64     fi
65   fi
66 }
67
68 goCPass0()
69 (
70   umask 0002
71   
72   targetDirectory=${1}
73   inputList=${2}
74   nEvents=${3}
75   ocdbPath=${4}
76   configFile=${5}
77   runNumber=${6}
78   jobindex=${7}
79   shift 7
80   if ! parseConfig ${configFile} "$@"; then return 1; fi
81
82   #record the working directory provided by the batch system
83   batchWorkingDirectory=${PWD}
84
85   #use the jobindex only if set and non-negative
86   if [[ -z ${jobindex} || ${jobindex} -lt 0 ]]; then
87     [[ -n "${LSB_JOBINDEX}" ]] && jobindex=${LSB_JOBINDEX}
88     [[ -n "${SGE_TASK_ID}" ]] && jobindex=${SGE_TASK_ID}
89     if [[ -z ${jobindex} ]]; then 
90       echo "no jobindex!"
91       return 1
92     fi
93   fi
94
95   [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
96
97   # This file signals that/if everything went fine
98   doneFileBase="cpass0.job${jobindex}.run${runNumber}.done"
99   [[ -n ${useProfilingCommand} ]] && doneFileBase="profiling.cpass0.job${jobindex}.run${runNumber}.done"
100
101   # We will have two copies of the file
102   mkdir -p "${commonOutputPath}/meta" || return 1
103   doneFileTmp="${batchWorkingDirectory}/${doneFileBase}"
104   doneFile="${commonOutputPath}/meta/${doneFileBase}"
105
106   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
107   
108   if [[ -n ${ALIROOT_FORCE_COREDUMP} ]]; then
109     ulimit -c unlimited 
110     export ALIROOT_FORCE_COREDUMP
111   fi
112
113   #the contents of this is stored in the tree and used later (e.g. AliAnalysisTaskPIDResponse)!
114   #at the QA stage the pass number is guessed from the path stored here.
115   #The Format is:
116   #Packages= ;OutputDir= ;LPMPass= ;TriggerAlias= ;LPMRunNumber= ;LPMProductionType= ;LPMInteractionType= ;LPMProductionTag= ;LPMAnchorRun= ;LPMAnchorProduction= ;LPMAnchorYear= 
117   export PRODUCTION_METADATA="OutputDir=cpass0"
118
119   if [[ "${inputList}" =~ \.root$ ]]; then
120     infile=${inputList}
121   else
122     infile=$(sed -ne "${jobindex}p" ${inputList} | egrep '\s*\w*/\w*')
123   fi
124   chunkName=${infile##*/}
125
126   outputDir=${targetDirectory}/${jobindex}_${chunkName%.*}
127   mkdir -p ${outputDir}
128   if [[ ! -d ${outputDir} ]]; then 
129     touch ${doneFileTmp}
130     echo "cannot make ${outputDir}" >> ${doneFileTmp}
131     cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
132     [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
133     return 1  
134   fi
135   
136   #runpath=${PWD}/rundir_cpass0_${runNumber}_${jobindex}
137   runpath=${outputDir}
138   #[[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
139   [[ ${reconstructInTemporaryDir} -eq 1 ]] && runpath=$(mktemp -d -t cpass0.XXXXXX)
140   mkdir -p ${runpath}
141   if [[ ! -d ${runpath} ]]; then
142     touch ${doneFileTmp} 
143     echo "cannot make runpath ${runpath}" >> ${doneFileTmp}
144     cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
145     [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
146     return 1
147   fi
148   if ! cd ${runpath}; then
149     touch ${doneFileTmp}
150     echo "PWD=$PWD is not the runpath=${runpath}" >> ${doneFileTmp}
151     cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
152     [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
153     return 1
154   fi
155
156   #runCPassX/C expects the raw chunk to be linked in the run dir
157   #despite it being accessed by the full path
158   ln -s ${infile} ${runpath}/${chunkName}
159
160   #####MC
161   if [[ -n ${generateMC} ]]; then
162     olddir=${PWD}
163     outputDirMC=${commonOutputPath}/000${runNumber}/sim/${jobindex}
164     simrunpath=${outputDirMC}
165     #[[ ${simulateInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && simrunpath=${TMPDIR}
166     [[ ${simulateInTemporaryDir} -eq 1 ]] && simrunpath=$(mktemp -d -t cpass0MC.XXXXXX)
167     mkdir -p ${outputDirMC}
168     mkdir -p ${simrunpath}
169     if cd ${simrunpath}; then
170
171       filesMC=( 
172       "${batchWorkingDirectory}/sim.C"
173       "${batchWorkingDirectory}/rec.C"
174       "${batchWorkingDirectory}/Config.C"
175       "${batchWorkingDirectory}/OCDB_*.root"
176       )
177       for file in ${filesMC[*]}; do
178         [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
179       done
180
181       generateMC ${runNumber} ${ocdbPath} ${nEvents}
182
183       [[ ! "${simrunpath}" =~ "${outputDirMC}" ]] && mv * ${outputDirMC} #TODO check if it works
184       cd ${olddir}
185
186       ln -s ${outputDirMC}/* ${runpath}/ 
187
188       inputList=${outputDirMC}/galice.root #TODO not valid outside shell !!!
189       infile=""
190     fi
191   fi
192   ######
193   
194   if [[ ! -f ${inputList} && -z ${pretend} ]]; then
195     touch ${doneFileTmp}
196     echo "input file ${inputList} not found, exiting..." >> ${doneFileTmp}
197     cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
198     [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
199     return 1
200   fi
201
202   logOutputDir=${runpath}
203   [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir}
204   [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/stdout
205   #[[ -z ${dontRedirectStdOutToLog} ]] && exec 2> ${logOutputDir}/stderr
206   echo "${0} $*"
207
208   echo "#####################"
209   echo CPass0:
210   echo JOB setup
211   echo nEvents               ${nEvents}
212   echo runNumber             ${runNumber}
213   echo ocdbPath              ${ocdbPath}
214   echo infile                ${infile}
215   echo chunkName             ${chunkName}
216   echo jobindex              ${jobindex}
217   echo recoTriggerOptions    ${recoTriggerOptions}
218   echo targetDirectory       ${targetDirectory}
219   echo commonOutputPath      ${commonOutputPath}
220   echo doneFile              ${doneFile}
221   echo batchWorkingDirectory ${batchWorkingDirectory}
222   echo runpath               ${runpath}  
223   echo outputDir             ${outputDir}
224   echo PWD                   ${PWD}
225   echo ALICE_ROOT            ${ALICE_ROOT}
226   echo "########## ###########"
227
228   alirootInfo > ALICE_ROOT.log
229
230   filesCPass0=( 
231                "${batchWorkingDirectory}/runCPass0.sh"
232                "${batchWorkingDirectory}/recCPass0.C"
233                "${batchWorkingDirectory}/runCalibTrain.C"
234                "${batchWorkingDirectory}/localOCDBaccessConfig.C"
235                "${batchWorkingDirectory}/OCDB.root"
236                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/runCPass0.sh"
237                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/recCPass0.C" 
238                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/runCalibTrain.C"
239   )
240
241   for file in ${filesCPass0[*]}; do
242     [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
243     [[ ${file##*/} =~ .*\.sh ]] && chmod +x ${file##*/}
244   done
245
246   echo "this directory (${PWD}) contents:"
247   /bin/ls
248   echo
249   chmod u+x runCPass0.sh
250
251   sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh
252
253   if [[ -n ${postSetUpActionCPass0} ]]; then
254     echo "running ${postSetUpActionCPass0}"
255     eval ${postSetUpActionCPass0}
256   fi
257
258   #run CPass0
259   echo "${runpath}/runCPass0.sh ${infile} ${nEvents} ${runNumber} ${ocdbPath} ${recoTriggerOptions}"
260   if [[ -n ${pretend} ]]; then
261     sleep ${pretendDelay}
262     touch AliESDs.root
263     touch AliESDfriends.root
264     touch AliESDfriends_v1.root
265     touch rec.log
266     touch calib.log
267   else
268     echo ./runCPass0.sh "${infile}" "${nEvents}" "${runNumber}" "${ocdbPath}" "${recoTriggerOptions}"
269     ./runCPass0.sh "${infile}" "${nEvents}" "${runNumber}" "${ocdbPath}" "${recoTriggerOptions}"
270   fi
271   
272   #move stuff to final destination
273   echo "this directory (${PWD}) contents:"
274   /bin/ls
275   echo
276
277   # [dberzano] OK this is fine!
278   echo rm -f ./${chunkName}
279   rm -f ./${chunkName}
280   echo "cp -R ${runpath}/* ${outputDir}"
281   cp -p -R ${runpath}/* ${outputDir}
282   echo
283   
284   #validate CPass0
285   cd ${outputDir}
286   touch ${doneFileTmp}
287   echo "dir ${outputDir}" >> ${doneFileTmp}
288   if summarizeLogs >> ${doneFileTmp}; then
289     [[ -f ${outputDirMC}/galice.root ]] && echo "sim ${outputDirMC}/galice.root" >> ${doneFileTmp}
290     [[ -f AliESDfriends_v1.root ]] && echo "calibfile ${outputDir}/AliESDfriends_v1.root" >> ${doneFileTmp}
291     [[ -f AliESDs.root ]] && echo "esd ${outputDir}/AliESDs.root" >> ${doneFileTmp}
292   fi
293
294   [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath} && echo "removing ${runpath}"
295   cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
296   [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
297   return 0
298 )
299
300 goCPass1()
301 (
302   umask 0002
303   
304   targetDirectory=${1}
305   inputList=${2}
306   nEvents=${3}
307   ocdbPath=${4}
308   configFile=${5}
309   runNumber=${6}
310   jobindex=${7}
311   shift 7
312   extraOpts=("$@")
313   if ! parseConfig ${configFile} "$@"; then return 1; fi
314
315   #record the working directory provided by the batch system
316   batchWorkingDirectory=${PWD}
317
318   #use the jobindex only if set and non-negative
319   if [[ -z ${jobindex} || ${jobindex} -lt 0 ]]; then
320     [[ -n "${LSB_JOBINDEX}" ]] && jobindex=${LSB_JOBINDEX}
321     [[ -n "${SGE_TASK_ID}" ]] && jobindex=${SGE_TASK_ID}
322     if [[ -z ${jobindex} ]]; then 
323       echo "no jobindex!"
324       return 1
325     fi
326   fi
327
328   [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
329
330   # This file signals that/if everything went fine
331   doneFileBase="cpass1.job${jobindex}.run${runNumber}.done"
332   [[ -n ${useProfilingCommand} ]] && doneFileBase="profiling.cpass0.job${jobindex}.run${runNumber}.done"
333
334   # We will have two copies of the file
335   mkdir -p "${commonOutputPath}/meta" || return 1
336   doneFileTmp="${batchWorkingDirectory}/${doneFileBase}"
337   doneFile="${commonOutputPath}/meta/${doneFileBase}"
338
339   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
340   
341   if [[ -n ${ALIROOT_FORCE_COREDUMP} ]]; then
342     ulimit -c unlimited 
343     export ALIROOT_FORCE_COREDUMP
344   fi
345
346   #the contents of this is stored in the tree and used later (e.g. AliAnalysisTaskPIDResponse)!
347   #at the QA stage the pass number is guessed from the path stored here.
348   #The Format is:
349   #Packages= ;OutputDir= ;LPMPass= ;TriggerAlias= ;LPMRunNumber= ;LPMProductionType= ;LPMInteractionType= ;LPMProductionTag= ;LPMAnchorRun= ;LPMAnchorProduction= ;LPMAnchorYear= 
350   export PRODUCTION_METADATA="OutputDir=cpass1"
351
352   if [[ ! -f ${inputList} && -z ${pretend} ]]; then
353     touch ${doneFileTmp}
354     echo "input file ${inputList} not found, exiting..." >> ${doneFileTmp}
355     cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
356     [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
357     return 1
358   fi
359   if [[ "${inputList}" =~ \.root$ ]]; then
360     infile=${inputList}
361   else
362     infile=$(sed -ne "${jobindex}p" ${inputList} | egrep '\s*\w*/\w*')
363   fi
364   chunkName=${infile##*/}
365
366   outputDir=${targetDirectory}/${jobindex}_${chunkName%.*}
367   mkdir -p ${outputDir}
368   if [[ ! -d ${outputDir} ]];then
369     touch ${doneFileTmp}
370     echo "cannot make ${outputDir}" >> ${doneFileTmp}
371     cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
372     [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
373     return 1
374   fi
375   
376   #runpath=${PWD}/rundir_cpass1_${runNumber}_${jobindex}
377   runpath=${outputDir}
378   #[[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
379   [[ ${reconstructInTemporaryDir} -eq 1 ]] && runpath=$(mktemp -d -t cpass1.XXXXXX)
380
381   #MC
382   if [[ "${infile}" =~ galice\.root ]]; then
383     ln -s ${inputList%/*}/* ${runpath}
384     infile=""
385   fi
386
387   #init the running path
388   mkdir -p ${runpath}
389   if [[ ! -d ${runpath} ]]; then
390    touch ${doneFileTmp}
391    echo "cannot make runpath ${runpath}" >> ${doneFileTmp}
392    cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
393     [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
394    return 1
395  fi
396   if ! cd ${runpath}; then
397     touch ${doneFileTmp}
398     echo "PWD=$PWD is not the runpath=${runpath}" >> ${doneFileTmp}
399     cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
400     [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
401     return 1
402   fi
403
404   #this is needed for runCPass1.sh
405   ln -s ${infile} ${runpath}/${chunkName}
406
407   logOutputDir=${runpath}
408   [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir}
409   [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/stdout
410   #[[ -z ${dontRedirectStdOutToLog} ]] && exec 2> ${logOutputDir}/stderr
411   echo "${0} $*"
412
413   echo "#####################"
414   echo CPass1:
415   echo JOB setup
416   echo nEvents               ${nEvents}
417   echo runNumber             ${runNumber}
418   echo ocdbPath              ${ocdbPath}
419   echo infile                ${infile}
420   echo chunkName             ${chunkName}
421   echo jobindex              ${jobindex}
422   echo recoTriggerOptions    ${recoTriggerOptions}
423   echo targetDirectory       ${targetDirectory}
424   echo commonOutputPath      ${commonOutputPath}
425   echo doneFile              ${doneFile}
426   echo runpath               ${runpath}  
427   echo outputDir             ${outputDir}
428   echo batchWorkingDirectory ${batchWorkingDirectory}
429   echo ALICE_ROOT            ${ALICE_ROOT}
430   echo PWD                   ${PWD}
431   echo "#####################"
432
433   alirootInfo > ALICE_ROOT.log
434
435   filesCPass1=( 
436                "${batchWorkingDirectory}/runCPass1.sh"
437                "${batchWorkingDirectory}/recCPass1.C"
438                "${batchWorkingDirectory}/recCPass1_OuterDet.C"
439                "${batchWorkingDirectory}/runCalibTrain.C"
440                "${batchWorkingDirectory}/QAtrain_duo.C"
441                "${batchWorkingDirectory}/localOCDBaccessConfig.C"
442                "${batchWorkingDirectory}/${configFile}"
443                "${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz"
444                "${batchWorkingDirectory}/OCDB.root"
445                "${trustedQAtrainMacro}"
446                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/runCPass1.sh"
447                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/recCPass1.C" 
448                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/recCPass1_OuterDet.C" 
449                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/runCalibTrain.C"
450                "${ALICE_ROOT}/ANALYSIS/macros/QAtrain_duo.C"
451   )
452
453   for file in "${filesCPass1[@]}"; do
454     [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
455     [[ ${file##*/} =~ .*\.sh ]] && echo "making ${file##*/} executable" && chmod +x ${file##*/}
456   done
457
458   echo "this directory (${PWD}) contents:"
459   /bin/ls
460   echo
461
462   #remove spaces around commas from calls to root
463   sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh
464
465   if [[ -n ${postSetUpActionCPass1} ]]; then
466     echo "running ${postSetUpActionCPass1}"
467     eval ${postSetUpActionCPass1}
468     echo
469   fi
470
471   #configure local OCDB storage from CPass0 (creates the localOCDBaccessConfig.C script)
472   if [[ -f cpass0.localOCDB.${runNumber}.tgz ]]; then
473     echo goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz"
474     goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz"
475   else
476     echo "WARNING: file cpass0.localOCDB.${runNumber}.tgz not found!"
477   fi
478
479   if [[ ! $(/bin/ls -1 OCDB/*/*/*/*.root 2>/dev/null) ]]; then
480     touch ${doneFileTmp}
481     echo "cpass0 produced no calibration! exiting..." >> ${doneFileTmp}
482     cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
483     [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
484     return 1
485   fi
486
487   #create the Barrel and OuterDet directories for CPass1 and link the local OCDB directory
488   #there to make the localOCDBaccessConfig.C file work, since it may point to the OCDB
489   #entries using a relative path, e.g. local://./OCDB
490   echo "linking the OCDB/ for Barrel and OuterDet directories"
491   mkdir Barrel OuterDet
492   ls -l
493   ln -s ../OCDB Barrel/OCDB
494   ln -s ../OCDB OuterDet/OCDB
495
496   #setup the filtering
497   #the following option enables the filtering task inside the QAtrain_duo.C
498   [[ -n $runESDfiltering ]] && export QA_TaskFilteredTree=1
499   #set the downscaling factors during the filtering fro expert QA (overrides the previous values)
500   if [[ -n ${filteringFactorHighPt} ]]; then
501     export AliAnalysisTaskFilteredTree_fLowPtTrackDownscaligF=${filteringFactorHighPt}
502   fi
503   if [[ -n ${filteringFactorV0s} ]]; then
504     export AliAnalysisTaskFilteredTree_fLowPtV0DownscaligF=${filteringFactorV0s} 
505   fi
506
507   #run CPass1
508   chmod u+x runCPass1.sh
509   echo "${runpath}/runCPass1.sh ${infile} ${nEvents} ${runNumber} ${ocdbPath} ${recoTriggerOptions}"
510   if [[ -n ${pretend} ]]; then
511     sleep ${pretendDelay}
512     touch AliESDs_Barrel.root
513     touch AliESDfriends_Barrel.root
514     touch AliESDfriends_v1.root
515     touch QAresults_barrel.root
516     touch EventStat_temp_barrel.root
517     touch AODtpITS.root
518     touch AliESDs_Outer.root
519     touch AliESDfriends_Outer.root
520     touch QAresults_outer.root
521     touch EventStat_temp_outer.root
522     touch rec.log
523     touch calib.log
524     touch qa.log
525     touch filtering.log FilterEvents_Trees.root
526   else
527     ./runCPass1.sh "${infile}" "${nEvents}" "${runNumber}" "${ocdbPath}" "${recoTriggerOptions}"
528     
529     [[ ! -f AliESDs_Barrel.root && -f Barrel/AliESDs.root ]] && mv Barrel/AliESDs.root AliESDs_Barrel.root
530     [[ ! -f AliESDfriends_Barrel.root && -f Barrel/AliESDfriends.root ]] && mv Barrel/AliESDfriends.root AliESDfriends_Barrel.root
531     [[ ! -f AliESDfriends_v1.root && -f Barrel/AliESDfriends_v1.root ]] && mv Barrel/AliESDfriends_v1.root .
532     [[ ! -f QAresults_barrel.root && -f Barrel/QAresults_barrel.root ]] && mv Barrel/QAresults_barrel.root .
533     [[ ! -f AliESDs_Outer.root && -f OuterDet/AliESDs.root ]] && mv OuterDet/AliESDs.root AliESDs_Outer.root
534     [[ ! -f AliESDfriends_Outer.root && -f OuterDet/AliESDfriends.root ]] && mv OuterDet/AliESDfriends.root AliESDfriends_Outer.root
535     [[ ! -f QAresults_outer.root && -f OuterDet/QAresults_outer.root ]] && mv OuterDet/QAresults_outer.root .
536     [[ ! -f FilterEvents_Trees.root && -f Barrel/FilterEvents_Trees.root ]] && mv Barrel/FilterEvents_Trees.root .
537
538     #make the filtered tree (if requested and not already produced by QA
539     [[ -f AliESDs_Barrel.root ]] && echo "AliESDs_Barrel.root" > filtered.list
540     if [[ -n ${runESDfiltering} && ! -f FilterEvents_Trees.root && -f filtered.list ]]; then 
541       goMakeFilteredTrees ${PWD} ${runNumber} "${PWD}/filtered.list" ${filteringFactorHighPt} ${filteringFactorV0s} ${ocdbPath} 1000000 0 10000000 0 ${configFile} AliESDs_Barrel.root "${extraOpts[@]}" >filtering.log
542     else
543       echo ""
544     fi
545
546   fi
547
548   ##handle possible crashes in QA (happens often in trunk)
549   ##rerun QA with a trusted aliroot version
550   #if [[ $(validateLog qa_barrel.log) ]]; then
551   #  echo "qa_barrel.log not validated!"
552   #fi
553   #if [[ ! -f QAresults_barrel.root && -f ${setupTrustedAliROOTenvInCurrentShell} || $(validateLog qa_barrel.log) ]]; then
554   #  echo "WARNING: using trusted QA aliroot ${ALICE_ROOT}"
555   #  source ${setupTrustedAliROOTenvInCurrentShell}
556   #  cd Barrel
557   #  rm QAresults_barrel.root
558   #  rm EventStat_temp_barrel.root
559   #  rm AODtpITS.root
560   #  [[ ! -f AliESDs.root ]] && ln -s ../AliESDs_Barrel.root AliESDs.root
561   #  [[ ! -f AliESDfriends.root ]] && ln -s ../AliESDfriends_Barrel.root AliESDfriends.root
562   #  if [[ -n ${trustedQAtrainMacro} ]]; then
563   #    eval "cp ${trustedQAtrainMacro} QAtrain_duo_trusted.C"
564   #  fi
565   #  echo executing aliroot -b -q "QAtrain_duo_trusted.C(\"_barrel\",${runNumber},\"wn.xml\",0,\"${ocdbPath}\")"
566   #  time aliroot -b -q "QAtrain_duo.C(\"_barrel\",${runNumber},\"wn.xml\",0,\"${ocdbPath}\")" &> ../qa_barrel_trusted.log
567   #  cd ../
568   #fi
569
570   #move stuff to final destination
571   echo "this directory (${PWD}) contents:"
572   /bin/ls
573   echo rm -f ./${chunkName}
574   rm -f ./${chunkName}
575   echo "cp -R ${runpath}/* ${outputDir}"
576   cp -pf -R ${runpath}/* ${outputDir}
577   echo
578
579   #validate CPass1
580   cd ${outputDir}
581   touch ${doneFileTmp}
582   echo "dir ${outputDir}" >> ${doneFileTmp}
583   if summarizeLogs >> ${doneFileTmp}; then
584     [[ -f AliESDs_Barrel.root ]] && echo "esd ${outputDir}/AliESDs_Barrel.root" >> ${doneFileTmp}
585     [[ -f AliESDfriends_v1.root ]] && echo "calibfile ${outputDir}/AliESDfriends_v1.root" >> ${doneFileTmp}
586     [[ -f QAresults_Barrel.root ]] && echo "qafile ${outputDir}/QAresults_Barrel.root" >> ${doneFileTmp}
587     [[ -f QAresults_Outer.root ]] && echo "qafile ${outputDir}/QAresults_Outer.root" >> ${doneFileTmp}
588     [[ -f QAresults_barrel.root ]] && echo "qafile ${outputDir}/QAresults_barrel.root" >> ${doneFileTmp}
589     [[ -f QAresults_outer.root ]] && echo "qafile ${outputDir}/QAresults_outer.root" >> ${doneFileTmp}
590     [[ -f FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFileTmp}
591   else
592     if grep "qa_outer.log.*OK" ${doneFileTmp} > /dev/null; then
593       [[ -f QAresults_Outer.root ]] && echo "qafile ${outputDir}/QAresults_Outer.root" >> ${doneFileTmp}
594       [[ -f QAresults_outer.root ]] && echo "qafile ${outputDir}/QAresults_outer.root" >> ${doneFileTmp}
595     fi
596     if grep "qa_barrel.log.*OK" ${doneFileTmp} > /dev/null; then
597       [[ -f QAresults_Barrel.root ]] && echo "qafile ${outputDir}/QAresults_Barrel.root" >> ${doneFileTmp}
598       [[ -f QAresults_barrel.root ]] && echo "qafile ${outputDir}/QAresults_barrel.root" >> ${doneFileTmp}
599     fi
600     if grep "filtering.log.*OK" ${doneFileTmp} > /dev/null; then
601       [[ -f FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFileTmp}
602     fi
603   fi
604
605   [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
606   cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
607   [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
608   return 0
609 )
610
611
612 goMergeCPass0()
613 (
614   #
615   # find the output files and merge them
616   #
617
618   outputDir=${1}
619   ocdbStorage=${2}
620   configFile=${3}
621   runNumber=${4}
622   calibrationFilesToMerge=${5}  #can be a non-existent file, will then be produced on the fly
623   shift 5
624   if ! parseConfig ${configFile} "$@"; then return 1; fi
625
626   #record the working directory provided by the batch system
627   batchWorkingDirectory=${PWD}
628
629   [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
630
631   # This file signals that everything went fine
632   doneFileBase="merge.cpass0.run${runNumber}.done"
633
634   # We will have two copies of the file
635   mkdir -p "${commonOutputPath}/meta" || return 1
636   doneFileTmp="${batchWorkingDirectory}/${doneFileBase}"
637   doneFile="${commonOutputPath}/meta/${doneFileBase}"
638
639   umask 0002
640   ulimit -c unlimited 
641
642   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
643
644   #runpath=${PWD}/rundir_cpass0_Merge_${runNumber}
645   runpath=${outputDir}
646   #[[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
647   [[ ${reconstructInTemporaryDir} -eq 1 ]] && runpath=$(mktemp -d -t mergeCPass0.XXXXXX)
648
649   mkdir -p ${runpath}
650   if [[ ! -d ${runpath} ]]; then
651     touch ${doneFileTmp}
652     echo "not able to make the runpath ${runpath}" >> ${doneFileTmp}
653     cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
654     [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
655     return 1
656   fi
657   if ! cd ${runpath}; then 
658     touch ${doneFileTmp}
659     echo "PWD=$PWD is not the runpath=${runpath}" >> ${doneFileTmp}
660     cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
661     [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
662     return 1
663   fi
664
665   logOutputDir=${runpath}
666   [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir}
667   [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/stdout
668   echo "${0} $*"
669
670   mergingScript="mergeMakeOCDB.byComponent.sh"
671
672   echo goMergeCPass0 SETUP:
673   echo runNumber=${runNumber}
674   echo outputDir=${outputDir}
675   echo ocdbStorage=${ocdbStorage}
676   echo calibrationFilesToMerge=${calibrationFilesToMerge}
677   echo mergingScript=${mergingScript}
678   echo commonOutputPath=${commonOutputPath}
679   echo runpath=${runpath}
680   
681   # copy files in case they are not already there
682   filesMergeCPass0=(
683                     "${batchWorkingDirectory}/${calibrationFilesToMerge}"
684                     "${batchWorkingDirectory}/OCDB.root"
685                     "${batchWorkingDirectory}/localOCDBaccessConfig.C"
686                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/mergeMakeOCDB.byComponent.sh"
687                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/mergeByComponent.C"
688                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/makeOCDB.C"
689                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/merge.C"
690                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/mergeMakeOCDB.sh"
691   )
692   for file in ${filesMergeCPass0[*]}; do
693     [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
694     [[ ${file##*/} =~ .*\.sh ]] && chmod +x ${file##*/}
695   done
696   
697   sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh
698
699   alirootInfo > ALICE_ROOT.log
700
701   #
702   echo "PWD"
703   /bin/ls
704   echo "PWD/.."
705   /bin/ls ../
706
707
708   #merge calibration
709   chmod u+x ${mergingScript}  
710   mkdir -p ./OCDB
711   if [[ ! -f ${calibrationFilesToMerge} ]]; then
712     echo "/bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root > ${calibrationFilesToMerge}"
713     /bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root 2>/dev/null > ${calibrationFilesToMerge}
714   fi
715   
716   echo "${mergingScript} ${calibrationFilesToMerge} ${runNumber} local://./OCDB ${ocdbStorage}"
717   if [[ -n ${pretend} ]]; then
718     sleep ${pretendDelay}
719     touch CalibObjects.root
720     touch ocdb.log
721     touch merge.log
722     touch dcsTime.root
723     mkdir -p ./OCDB/TPC/Calib/TimeGain/
724     mkdir -p ./OCDB/TPC/Calib/TimeDrift/
725     echo "some calibration" >> ./OCDB/TPC/Calib/TimeGain/someCalibObject_0-999999_cpass0.root
726     echo "some calibration" >> ./OCDB/TPC/Calib/TimeDrift/otherCalibObject_0-999999_cpass0.root
727   else
728     ./${mergingScript} ${calibrationFilesToMerge} ${runNumber} "local://./OCDB" ${ocdbStorage} >> "mergeMakeOCDB.log"
729
730     #produce the calib trees for expert QA (dcsTime.root)
731     goMakeLocalOCDBaccessConfig ./OCDB
732     echo aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")"
733     aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")"
734   fi
735   
736   ### produce the output
737   #tar the produced OCDB for reuse
738   #tar czf ${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz ./OCDB
739
740   # Create tarball with OCDB, store on the shared directory, create signal file on batch directory
741   mkdir -p ${commonOutputPath}/meta
742   baseTar="cpass0.localOCDB.${runNumber}.tgz"
743   tar czf ${batchWorkingDirectory}/${baseTar} ./OCDB && \
744     mv ${batchWorkingDirectory}/${baseTar} ${commonOutputPath}/meta/${baseTar} && \
745     touch ${batchWorkingDirectory}/${baseTar}.done
746
747   /bin/ls
748
749   #copy all to output dir
750   cp -pf -R ${runpath}/* ${outputDir}
751
752   if [[ -n ${generateMC} ]]; then
753     goPrintValues sim ${commonOutputPath}/meta/sim.run${runNumber}.list ${commonOutputPath}/meta/cpass0.job*.run${runNumber}.done
754   fi
755
756   #validate merging cpass0
757   cd ${outputDir}
758   touch ${doneFileTmp}
759   echo "dir ${outputDir}" >> ${doneFileTmp}
760   if summarizeLogs >> ${doneFileTmp}; then
761     [[ -f CalibObjects.root ]] && echo "calibfile ${outputDir}/CalibObjects.root" >> ${doneFileTmp}
762     [[ -f dcsTime.root ]] && echo "dcsTree ${outputDir}/dcsTime.root" >> ${doneFileTmp}
763   fi
764
765   [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
766   cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
767   [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
768   return 0
769 )
770
771 goMergeCPass1()
772 (
773   #
774   # find the output files and merge them
775   #
776
777   outputDir=${1}
778   ocdbStorage=${2}
779   configFile=${3}
780   runNumber=${4}
781   calibrationFilesToMerge=${5}
782   qaFilesToMerge=${6}
783   filteredFilesToMerge=${7}
784   shift 7
785   if ! parseConfig ${configFile} "$@"; then return 1; fi
786
787   #record the working directory provided by the batch system
788   batchWorkingDirectory=${PWD}
789
790   [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
791
792   # This file signals that everything went fine
793   doneFileBase="merge.cpass1.run${runNumber}.done"
794
795   # We will have two copies of the file
796   mkdir -p "${commonOutputPath}/meta" || return 1
797   doneFileTmp="${batchWorkingDirectory}/${doneFileBase}"
798   doneFile="${commonOutputPath}/meta/${doneFileBase}"
799
800   umask 0002
801   ulimit -c unlimited 
802
803   #clean up first:
804   rm -f ${outputDir}/*.log
805   rm -f ${outputDir}/*.root
806   rm -f ${outputDir}/*done
807
808   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
809
810   #runpath=${PWD}/rundir_cpass1_Merge_${runNumber}
811   runpath=${outputDir}
812   #[[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
813   [[ ${reconstructInTemporaryDir} -eq 1 ]] && runpath=$(mktemp -d -t mergeCPass1.XXXXXX)
814
815   mkdir -p ${runpath}
816   if [[ ! -d ${runpath} ]]; then
817     touch ${doneFileTmp}
818     echo "not able to make the runpath ${runpath}" >> ${doneFileTmp}
819     cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
820     [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
821     return 1
822   fi
823   if ! cd ${runpath}; then 
824     touch ${doneFileTmp}
825     echo "PWD=$PWD is not the runpath=${runpath}" >> ${doneFileTmp}
826     cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
827     [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
828     return 1
829   fi
830
831   logOutputDir=${runpath}
832   [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir}
833   [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/mergeMakeOCDB.log
834   echo "${0} $*"
835
836   calibrationOutputFileName='AliESDfriends_v1.root'
837   qaOutputFileName='QAresults*.root'
838   mergingScript="mergeMakeOCDB.byComponent.sh"
839   #important to have the string "Stage.txt" in the filename to trigger the merging
840   #it has to be a list of directories containing the files
841   qaMergedOutputFileName="QAresults_merged.root"
842
843   echo goMergeCPass1 SETUP:
844   echo runNumber=${runNumber}
845   echo outputDir=${outputDir}
846   echo ocdbStorage=${ocdbStorage}
847   echo calibrationFilesToMerge=$calibrationFilesToMerge
848   echo qaFilesToMerge=$qaFilesToMerge
849   echo calibrationOutputFileName=${calibrationOutputFileName}
850   echo mergingScript=${mergingScript}
851   
852   # copy files in case they are not already there
853   filesMergeCPass1=(
854                     "${batchWorkingDirectory}/${calibrationFilesToMerge}"
855                     "${batchWorkingDirectory}/${qaFilesToMerge}"
856                     "${batchWorkingDirectory}/OCDB.root"
857                     "${batchWorkingDirectory}/localOCDBaccessConfig.C"
858                     "${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz"
859                     "${batchWorkingDirectory}/QAtrain_duo.C"
860                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/mergeMakeOCDB.byComponent.sh"
861                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/mergeByComponent.C"
862                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/makeOCDB.C"
863                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/merge.C"
864                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/mergeMakeOCDB.sh"
865                     "${trustedQAtrainMacro}"
866                     "${ALICE_ROOT}/ANALYSIS/macros/QAtrain_duo.C"
867   )
868   for file in ${filesMergeCPass1[*]}; do
869     [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
870     [[ ${file##*/} =~ .*\.sh ]] && chmod +x ${file##*/}
871   done
872
873   sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh
874
875   #configure local OCDB storage from CPass0 (creates the localOCDBaccessConfig.C script)
876   if [[ -f cpass0.localOCDB.${runNumber}.tgz ]]; then
877     echo goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz"
878     goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz"
879   else
880     echo "WARNING: file cpass0.localOCDB.${runNumber}.tgz not found!"
881   fi
882
883   alirootInfo > ALICE_ROOT.log
884
885   #
886   /bin/ls
887
888   #merge calibration
889   chmod u+x ${mergingScript}  
890   mkdir -p OCDB
891   
892   #if not provided, create the lists of files to merge
893   if [[ ! -f ${filteredFilesToMerge} ]]; then
894     echo "/bin/ls -1 ${outputDir}/*/FilterEvents_Trees.root > ${filteredFilesToMerge}"
895     /bin/ls -1 ${outputDir}/*/FilterEvents_Trees.root 2>/dev/null > ${filteredFilesToMerge}
896   fi
897   if [[ ! -f ${calibrationFilesToMerge} ]]; then
898     echo "/bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root > ${calibrationFilesToMerge}"
899     /bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root 2>/dev/null > ${calibrationFilesToMerge}
900   fi
901   if [[ ! -f ${qaFilesToMerge} ]]; then
902     #find the files, but only store the directories (QAtrain_duo.C requires this)
903     echo "/bin/ls -1 ${outputDir}/*/QAresults*.root | while read x; do echo ${x%/*}; done | sort | uniq > ${qaFilesToMerge}"
904     /bin/ls -1 ${outputDir}/*/QAresults*.root | while read x; do echo ${x%/*}; done | sort | uniq > ${qaFilesToMerge}
905   fi
906   
907   echo "${mergingScript} ${calibrationFilesToMerge} ${runNumber} local://./OCDB ${ocdbStorage}"
908   if [[ -n ${pretend} ]]; then
909     sleep ${pretendDelay}
910     touch ocdb.log
911     touch cpass1.localOCDB.${runNumber}.tgz
912     touch ${qaMergedOutputFileName}
913     touch merge.log
914     touch trending.root
915     touch FilterEvents_Trees.root
916     touch CalibObjects.root
917     touch dcsTime.root
918     touch ${qaMergedOutputFileName}
919     mkdir -p OCDB
920   else
921     ./${mergingScript} ${calibrationFilesToMerge} ${runNumber} "local://./OCDB" ${ocdbStorage}
922
923     #merge QA (and filtered trees)
924     [[ -n ${AliAnalysisTaskFilteredTree_fLowPtTrackDownscaligF} ]] && export AliAnalysisTaskFilteredTree_fLowPtTrackDownscaligF
925     [[ -n ${AliAnalysisTaskFilteredTree_fLowPtV0DownscaligF} ]] && export AliAnalysisTaskFilteredTree_fLowPtV0DownscaligF
926
927     #echo aliroot -l -b -q "merge.C(\"${qaFilesToMerge}\",\"\",kFALSE,\"${qaMergedOutputFileName}\")"
928     echo aliroot -b -q "QAtrain_duo.C(\"_barrel\",${runNumber},\"${qaFilesToMerge}\",1,\"${ocdbStorage}\")"
929     #aliroot -l -b -q "merge.C(\"${qaFilesToMerge}\",\"\",kFALSE,\"${qaMergedOutputFileName}\")"
930     aliroot -b -q "QAtrain_duo.C(\"_barrel\",${runNumber},\"${qaFilesToMerge}\",1,\"${ocdbStorage}\")" > mergeQA.log
931     mv QAresults_barrel.root ${qaMergedOutputFileName}
932     mv trending_barrel.root trending.root
933
934     #merge filtered trees
935     echo aliroot -l -b -q "merge.C(\"${qaFilesToMerge}\",\"\",kFALSE,\"${qaMergedOutputFileName}\")"
936     aliroot -l -b -q "merge.C(\"${filteredFilesToMerge}\",\"\",kFALSE,\"FilterEvents_Trees.root\")" > mergeFilteredTrees.log
937
938     #produce the calib trees for expert QA
939     echo aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")"
940     aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")" > calibTree.log
941   fi
942
943   # Create tarball with OCDB, store on the shared directory, create signal file on batch directory
944   mkdir -p ${commonOutputPath}/meta
945   baseTar="cpass1.localOCDB.${runNumber}.tgz"
946   tar czf ${batchWorkingDirectory}/${baseTar} ./OCDB && \
947     mv ${batchWorkingDirectory}/${baseTar} ${commonOutputPath}/meta/${baseTar} && \
948     touch ${batchWorkingDirectory}/${baseTar}.done
949
950   /bin/ls
951
952   #copy all to output dir
953   cp -pf -R ${runpath}/* ${outputDir}
954   
955   #validate merge cpass1
956   cd ${outputDir}
957   touch ${doneFileTmp}
958   echo "dir ${outputDir}" >> ${doneFileTmp}
959   if summarizeLogs >>  ${doneFileTmp}; then
960     [[ -f CalibObjects.root ]] && echo "calibfile ${outputDir}/CalibObjects.root" >> ${doneFileTmp}
961     [[ -f ${qaMergedOutputFileName} ]] && echo "qafile ${outputDir}/${qaMergedOutputFileName}" >> ${doneFileTmp}
962     [[ -f trending.root ]] && echo "trendingfile ${outputDir}/trending.root" >> ${doneFileTmp}
963     [[ -f dcsTime.root ]] && echo "dcsTree ${outputDir}/dcsTime.root" >> ${doneFileTmp}
964     [[ -f FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFileTmp}
965   else
966     if grep "mergeQA.log.*OK" ${doneFileTmp} > /dev/null; then
967       [[ -f ${qaMergedOutputFileName} ]] && echo "qafile ${outputDir}/${qaMergedOutputFileName}" >> ${doneFileTmp}
968     fi
969     if grep "mergeFilteredTrees.log.*OK" ${doneFileTmp} > /dev/null; then
970       [[ -f FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFileTmp}
971     fi
972   fi
973       
974   [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
975   cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
976   [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
977   return 0
978 )
979
980 goMerge()
981 (
982   #generic root merge using CPass1 merge.C script
983   inputList=${1}
984   outputFile=${2}  
985   configFile=${3-"benchmark.config"}
986   shift 3
987   if ! parseConfig ${configFile} "$@"; then return 1; fi
988   
989   #record the working directory provided by the batch system
990   batchWorkingDirectory=${PWD}
991
992   [[ ! -f ${inputList} ]] && echo "inputList ${inputList} does not exist!" && return 1
993   [[ ! -f ${configFile} ]] && echo "configFile ${configFile} does not exist!" && return 1
994   umask 0002
995   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
996   rm -f ${outputFile}
997   aliroot -b -q "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/merge.C(\"${inputList}\",\"\",kFALSE,\"${outputFile}\")" > merge_${inputList}.log
998   return 0
999 )
1000
1001 goSubmitMakeflow()
1002 {
1003   #run
1004   productionID=${1}
1005   inputFileList=${2}
1006   configFile=${3}
1007   shift 3
1008   extraOpts=("$@")
1009   if ! parseConfig ${configFile} "${extraOpts[@]}"; then return 1; fi
1010  
1011   #record the working directory provided by the batch system
1012   batchWorkingDirectory=${PWD}
1013
1014   [[ -z ${configFile} ]] && configFile="benchmark.config"
1015   [[ ! -f ${configFile} ]] && echo "no config file found (${configFile})" && return 1
1016
1017   if [[ ! $(which makeflow &>/dev/null) && -n ${makeflowPath} ]]; then
1018     echo "setting the makflow path from the config: "
1019     echo "  export PATH=${makeflowPath}:${PATH}"
1020     export PATH=${makeflowPath}:${PATH}
1021   fi
1022
1023   #create the common output dir and the meta dir
1024   commonOutputPath=${baseOutputDirectory}/${productionID}
1025   if [[ -d ${commonOutputPath} ]]; then
1026     echo "output dir ${commonOutputPath} exists!"
1027     #return 1
1028   else
1029     mkdir -p ${commonOutputPath}
1030   fi
1031   mkdir -p ${commonOutputPath}/meta
1032   
1033   self=${0}
1034   #if which greadlink; then self=$(greadlink -f "${0}"); fi
1035   
1036   #for reference copy the setup to the output dir
1037   cp ${self} ${commonOutputPath}
1038   cp ${configFile} ${commonOutputPath}
1039   cp ${inputFileList} ${commonOutputPath}
1040
1041   #submit - use makeflow if available, fall back to old stuff when makeflow not there
1042   if which makeflow; then
1043     goGenerateMakeflow ${productionID} ${inputFileList} ${configFile} "${extraOpts[@]}" commonOutputPath=${commonOutputPath} > benchmark.makeflow
1044     cp benchmark.makeflow ${commonOutputPath}
1045     makeflow ${makeflowOptions} benchmark.makeflow
1046   else 
1047     echo "no makeflow!"
1048   fi
1049   return 0
1050 }
1051
1052 goGenerateMakeflow()
1053 (
1054   #generate the makeflow file
1055   [[ $# -lt 3 ]] && echo "args: id inputFileList configFile" && return 1
1056   productionID=${1}
1057   inputFileList=${2}
1058   configFile=${3}
1059   shift 3
1060   extraOpts=("$@")
1061   
1062   #batch systems/makeflow sometimes handle spaces in arguments poorly, so encode them
1063   for (( i=0;i<${#extraOpts[@]};i++ )); do 
1064     extraOpts[i]=$(encSpaces "${extraOpts[i]}")
1065   done
1066   extraOpts+=("encodedSpaces=1")
1067
1068   if ! parseConfig ${configFile} "${extraOpts[@]}" &>/dev/null; then return 1; fi
1069  
1070   #extra safety
1071   if [[ -z ${commonOutputPath} ]]; then
1072     commonOutputPath=${baseOutputDirectory}/${productionID}
1073     extraOpts=( "${extraOpts[@]}" "commonOutputPath=${commonOutputPath}" )
1074   fi
1075
1076   #record the working directory provided by the batch system
1077   batchWorkingDirectory=${PWD}
1078
1079   [[ -z ${configFile} ]] && configFile="benchmark.config"
1080   [[ ! -f ${configFile} ]] && echo "no config file found (${configFile})" && return 1
1081
1082   #these files will be made a dependency - will be copied to the working dir of the jobs
1083   declare -a copyFiles
1084   inputFiles=(
1085               "OCDB.root"
1086               "localOCDBaccessConfig.C"
1087               "QAtrain_duo.C"
1088               "runCPass1.sh"
1089               "recCPass1.C"
1090               "recCPass1_OuterDet.C"
1091               "runCalibTrain.C"
1092               "runCPass0.sh"
1093               "recCPass0.C"
1094               "runQA.sh"
1095   )
1096   for file in ${inputFiles[*]}; do
1097     [[ -f ${file} ]] && copyFiles+=("${file}")
1098   done
1099
1100   #create the makeflow file
1101   [[ -n ${batchFlags} ]] && echo "BATCH_OPTIONS = ${batchFlags}"
1102   declare -A arr_cpass0_merged arr_cpass1_merged
1103   declare -A arr_cpass0_calib_list arr_cpass1_calib_list 
1104   declare -A arr_cpass1_QA_list arr_cpass1_ESD_list arr_cpass1_filtered_list
1105   declare -A arr_cpass0_profiled_outputs
1106   declare -A listOfRuns
1107   [[ -n ${runNumber} ]] && listOfRuns[${runNumber}]=1
1108   while read x; do tmpRun=$(guessRunNumber ${x}); [[ -n ${tmpRun} ]] && listOfRuns[${tmpRun}]=1; done < ${inputFileList}
1109   for runNumber in "${!listOfRuns[@]}"; do
1110     [[ -z ${runNumber} ]] && continue
1111     [[ ! ${runNumber} =~ ^[0-9]*[0-9]$ ]] && continue
1112
1113     unset arr_cpass0_outputs
1114     unset arr_cpass1_outputs
1115     declare -a arr_cpass0_outputs
1116     declare -a arr_cpass1_outputs
1117
1118     #Header
1119     echo "### Automatically generated on $(LANG=C date) ###"
1120     echo ; echo
1121
1122     jobindex=0
1123     inputFile=""
1124     while read inputFile; do
1125       currentDefaultOCDB=${defaultOCDB}
1126       [[ -z ${autoOCDB} ]] && autoOCDB=1
1127       if [[ ${autoOCDB} -ne 0 ]]; then
1128         currentDefaultOCDB=$(setYear ${inputFile} ${defaultOCDB})
1129       fi
1130       guessRunData ${inputFile}
1131
1132       #Set variables
1133       echo "### Variables ###"
1134       echo "OUTPATH=\"${commonOutputPath}/${year}/${period}\""
1135       echo ; echo
1136
1137       #CPass0
1138       #arr_cpass0_outputs[${jobindex}]="${commonOutputPath}/meta/cpass0.job${jobindex}.run${runNumber}.done"
1139       arr_cpass0_outputs[${jobindex}]="cpass0.job${jobindex}.run${runNumber}.done"
1140       echo "### CPass0 ###"
1141       echo "${arr_cpass0_outputs[${jobindex}]}: benchmark.sh ${configFile} ${copyFiles[@]}"
1142       echo "    ${alirootEnv} ./benchmark.sh CPass0 \$OUTPATH/000${runNumber}/cpass0 ${inputFile} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} ${jobindex} ${extraOpts[@]}"" "
1143       echo ; echo
1144
1145       #CPass1
1146       #arr_cpass1_outputs[${jobindex}]="${commonOutputPath}/meta/cpass1.job${jobindex}.run${runNumber}.done"
1147       arr_cpass1_outputs[${jobindex}]="cpass1.job${jobindex}.run${runNumber}.done"
1148       echo "### CPass1 ###"
1149       echo "${arr_cpass1_outputs[${jobindex}]}: benchmark.sh ${configFile} cpass0.localOCDB.${runNumber}.tgz.done ${copyFiles[@]}"
1150       echo "    ${alirootEnv} ./benchmark.sh CPass1 \$OUTPATH/000${runNumber}/cpass1 ${inputFile} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} ${jobindex} ${extraOpts[@]}"" "
1151       echo ; echo
1152       ((jobindex++))
1153
1154     done< <(grep "/000${runNumber}/" ${inputFileList})
1155     
1156     #CPass0 list of Calib files to merge
1157     #arr_cpass0_calib_list[${runNumber}]="${commonOutputPath}/meta/cpass0.calib.run${runNumber}.list"
1158     arr_cpass0_calib_list[${runNumber}]="cpass0.calib.run${runNumber}.list"
1159     echo "### Produces the list of CPass0 files to merge (executes locally) ###"
1160     echo "${arr_cpass0_calib_list[${runNumber}]}: benchmark.sh ${arr_cpass0_outputs[*]}"
1161     echo "    LOCAL ./benchmark.sh PrintValues calibfile ${arr_cpass0_calib_list[${runNumber}]} ${arr_cpass0_outputs[*]} && mkdir -p \$OUTPATH/meta && cp ${arr_cpass0_calib_list[${runNumber}]} \$OUTPATH/meta/${arr_cpass0_calib_list[${runNumber}]}"
1162     echo ; echo
1163
1164     #CPass0 merging
1165     echo "### Merges CPass0 files ###"
1166     #arr_cpass0_merged[${runNumber}]="${commonOutputPath}/meta/merge.cpass0.run${runNumber}.done"
1167     arr_cpass0_merged[${runNumber}]="merge.cpass0.run${runNumber}.done"
1168     echo "cpass0.localOCDB.${runNumber}.tgz.done ${arr_cpass0_merged[${runNumber}]}: benchmark.sh ${configFile} ${arr_cpass0_calib_list[${runNumber}]} ${copyFiles[@]}"
1169     echo "    ${alirootEnv} ./benchmark.sh MergeCPass0 \$OUTPATH/000${runNumber}/cpass0 ${currentDefaultOCDB} ${configFile} ${runNumber} ${arr_cpass0_calib_list[${runNumber}]} ${extraOpts[@]}"" "
1170     echo ; echo
1171
1172     #CPass1 list of Calib/QA/ESD/filtered files
1173     # the trick with QA is to have the string "Stage.txt" in the file name of the list of directories with QA output to trigger
1174     # the production of the QA trending tree (only then the task->Finish() will be called in QAtrain_duo.C, on the grid
1175     # this corresponds to the last merging stage)
1176     #arr_cpass1_QA_list[${runNumber}]="${commonOutputPath}/meta/cpass1.QA.run${runNumber}.lastMergingStage.txt.list"
1177     arr_cpass1_QA_list[${runNumber}]="cpass1.QA.run${runNumber}.lastMergingStage.txt.list"
1178     echo "### Lists CPass1 QA ###"
1179     echo "${arr_cpass1_QA_list[${runNumber}]}: benchmark.sh ${arr_cpass1_outputs[*]}"
1180     echo "    LOCAL ./benchmark.sh PrintValues dir ${arr_cpass1_QA_list[${runNumber}]} ${arr_cpass1_outputs[*]} && mkdir -p \$OUTPATH/meta && cp ${arr_cpass1_QA_list[${runNumber}]} \$OUTPATH/meta/${arr_cpass1_QA_list[${runNumber}]}"
1181     echo ; echo
1182
1183     #arr_cpass1_calib_list[${runNumber}]="${commonOutputPath}/meta/cpass1.calib.run${runNumber}.list"
1184     arr_cpass1_calib_list[${runNumber}]="cpass1.calib.run${runNumber}.list"
1185     echo "### Lists CPass1 Calib ###"
1186     echo "${arr_cpass1_calib_list[${runNumber}]}: benchmark.sh ${arr_cpass1_outputs[*]}"
1187     echo "    LOCAL ./benchmark.sh PrintValues calibfile ${arr_cpass1_calib_list[${runNumber}]} ${arr_cpass1_outputs[*]} && mkdir -p \$OUTPATH/meta && cp ${arr_cpass1_calib_list[${runNumber}]} \$OUTPATH/meta/${arr_cpass1_calib_list[${runNumber}]}"
1188     echo ; echo
1189
1190     #arr_cpass1_ESD_list[${runNumber}]="${commonOutputPath}/meta/cpass1.ESD.run${runNumber}.list"
1191     arr_cpass1_ESD_list[${runNumber}]="cpass1.ESD.run${runNumber}.list"
1192     echo "### Lists CPass1 ESDs ###"
1193     echo "${arr_cpass1_ESD_list[${runNumber}]}: benchmark.sh ${arr_cpass1_outputs[*]}"
1194     echo "    LOCAL ./benchmark.sh PrintValues esd ${arr_cpass1_ESD_list[${runNumber}]} ${arr_cpass1_outputs[*]} && mkdir -p \$OUTPATH/meta && cp ${arr_cpass1_ESD_list[${runNumber}]} \$OUTPATH/meta/${arr_cpass1_ESD_list[${runNumber}]}"
1195     echo ; echo
1196
1197     #arr_cpass1_filtered_list[${runNumber}]="${commonOutputPath}/meta/cpass1.filtered.run${runNumber}.list"
1198     arr_cpass1_filtered_list[${runNumber}]="cpass1.filtered.run${runNumber}.list"
1199     echo "### Lists CPass1 filtered ###"
1200     echo "${arr_cpass1_filtered_list[${runNumber}]}: benchmark.sh ${arr_cpass1_outputs[*]}"
1201     echo "    LOCAL ./benchmark.sh PrintValues filteredTree ${arr_cpass1_filtered_list[${runNumber}]} ${arr_cpass1_outputs[*]} && mkdir -p \$OUTPATH/meta && cp ${arr_cpass1_filtered_list[${runNumber}]} \$OUTPATH/meta/${arr_cpass1_filtered_list[${runNumber}]}"
1202     echo ; echo
1203   
1204     #CPass1 merging
1205     #arr_cpass1_merged[${runNumber}]="${commonOutputPath}/meta/merge.cpass1.run${runNumber}.done"
1206     arr_cpass1_merged[${runNumber}]="merge.cpass1.run${runNumber}.done"
1207     echo "### Merges CPass1 files ###"
1208     echo "cpass1.localOCDB.${runNumber}.tgz.done ${arr_cpass1_merged[${runNumber}]}: benchmark.sh ${configFile} ${arr_cpass1_calib_list[${runNumber}]} ${arr_cpass1_QA_list[${runNumber}]} ${copyFiles[@]}"
1209     echo "    ${alirootEnv} ./benchmark.sh MergeCPass1 \$OUTPATH/000${runNumber}/cpass1 ${currentDefaultOCDB} ${configFile} ${runNumber} ${arr_cpass1_calib_list[${runNumber}]} ${arr_cpass1_QA_list[${runNumber}]} ${arr_cpass1_filtered_list[${runNumber}]} ${extraOpts[@]}"" "
1210     echo ; echo
1211
1212     #CPass0 wrapped in a profiling tool (valgrind,....)
1213     if [[ -n ${profilingCommand} ]]; then
1214       inputFile=$(grep -m1 "${runNumber}/" ${inputFileList})
1215       [[ -z ${nEventsProfiling} ]] && nEventsProfiling=2
1216       currentDefaultOCDB=$(setYear ${inputFile} ${defaultOCDB})
1217       jobindex="profiling"
1218
1219       #arr_cpass0_profiled_outputs[${runNumber}]="${commonOutputPath}/meta/profiling.cpass0.job${jobindex}.run${runNumber}.done"
1220       arr_cpass0_profiled_outputs[${runNumber}]="profiling.cpass0.job${jobindex}.run${runNumber}.done"
1221       echo "### CPass0 in a profiler ###"
1222       echo "${arr_cpass0_profiled_outputs[${runNumber}]}: benchmark.sh ${configFile} ${copyFiles[@]}"
1223       profilingCommand=$(encSpaces "${profilingCommand}")
1224       echo "    ${alirootEnv} ./benchmark.sh CPass0 \$OUTPATH/000${runNumber}/${jobindex} ${inputFile} ${nEventsProfiling} ${currentDefaultOCDB} ${configFile} ${runNumber} ${jobindex} ${extraOpts[@]} useProfilingCommand=${profilingCommand}"
1225       echo ; echo
1226     fi
1227
1228   done #runs
1229
1230   #Summary
1231   echo "### Summary ###"
1232   echo "summary.log: benchmark.sh ${configFile} ${arr_cpass1_merged[*]}"
1233   echo "     ${alirootEnv} ./benchmark.sh MakeSummary ${configFile} ${extraOpts[@]}"
1234   echo ; echo
1235
1236   return 0
1237 )
1238
1239 goPrintValues()
1240 (
1241   #print the values given the key from any number of files (store in output file on second argument)
1242   if [[ $# -lt 3 ]]; then
1243     echo "goPrintValues key outputFile inputFiles"
1244     echo "if outputFile is \"-\" don't redirect to a file"
1245     return
1246   fi
1247   key=${1}
1248   outputFile=${2}
1249   [[ ${outputFile} =~ "-" ]] && outputFile=""
1250   shift 2 #remove 2 first arguments from arg list to only pass the input files to awk
1251   awk -v key=${key} '$0 ~ key" " {print $2}' "$@" | tee ${outputFile}
1252   return 0
1253 )
1254
1255 goCreateQAplots()
1256 (
1257   umask 0002
1258   mergedQAfileList=${1}
1259   productionID=${2}
1260   outputDir=${3}
1261   configFile=${4}
1262   shift 4
1263   if ! parseConfig ${configFile} "$@"; then return 1; fi
1264   
1265   #record the working directory provided by the batch system
1266   batchWorkingDirectory=${PWD}
1267
1268   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
1269
1270   [[ -z ${logOutputDir} ]] && logOutputDir=${PWD}
1271   [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/makeQAplots.log
1272   echo "${0} $*"
1273
1274   olddir=${PWD}
1275   mkdir -p ${outputDir}
1276   cd ${outputDir}
1277   [[ ! "${PWD}" =~ "${outputDir}" ]] && echo "PWD is not equal to outputDir=${outputDir}" && cd ${olddir} && return 1
1278
1279   inputFiles=(
1280               "${batchWorkingDirectory}/runQA.sh"
1281               "${ALICE_ROOT}/PWGPP/QA/scripts/runQA.sh"
1282   )
1283   for file in ${inputFiles[*]}; do
1284     [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
1285   done
1286
1287   echo "running QA with command:"
1288   echo ./runQA.sh inputList="${mergedQAfileList}" inputListHighPtTrees="${filteringList}" ocdbStorage="${defaultOCDB}"
1289   ./runQA.sh inputList="${mergedQAfileList}" inputListHighPtTrees="${filteringList}" ocdbStorage="${defaultOCDB}"
1290   cd ${olddir}
1291   return 0
1292 )
1293
1294 goTest()
1295 (
1296   echo AA
1297 )
1298
1299 alirootInfo()
1300 (
1301   umask 0002
1302   # save aliroot repository info
1303   [[ -z "${ALICE_ROOT}" ]] && return 1
1304   
1305   echo "\${ALICE_ROOT}=${ALICE_ROOT}"
1306   echo "\${ROOTSYS}=${ROOTSYS}"
1307   echo "\${PATH}=${PATH}"
1308   echo "\${LD_LIBRARY_PATH}=${LD_LIBRARY_PATH}"
1309   echo
1310   
1311   pushd ${PWD}
1312   cd ${ALICE_ROOT}
1313
1314   currentBranch=$(git rev-parse --abbrev-ref HEAD)
1315   git status
1316   echo ""
1317   echo ""
1318   git diff ${currentBranch}
1319   popd
1320   return 0
1321 )
1322
1323 setYear()
1324 (
1325   #set the year
1326   #  ${1} - year to be set
1327   #  ${2} - where to set the year
1328   year1=$(guessYear ${1})
1329   year2=$(guessYear ${2})
1330   local path=${2}
1331   [[ ${year1} -ne ${year2} && -n ${year2} && -n ${year1} ]] && path=${2/\/${year2}\//\/${year1}\/}
1332   echo ${path}
1333   return 0
1334 )
1335
1336 guessPeriod()
1337 (
1338   #guess the period from the path, pick the rightmost one
1339   local IFS="/"
1340   declare -a path=( ${1} )
1341   local dirDepth=${#path[*]}
1342   for ((x=${dirDepth}-1;x>=0;x--)); do
1343     local field=${path[${x}]}
1344     [[ ${field} =~ ^LHC[0-9][0-9][a-z]$ ]] && period=${field} && break
1345   done
1346   echo ${period}
1347   return 0
1348 )
1349
1350 guessYear()
1351 (
1352   #guess the year from the path, pick the rightmost one
1353   local IFS="/"
1354   declare -a path=( ${1} )
1355   local dirDepth=${#path[*]}
1356   for ((x=${dirDepth}-1;x>=0;x--)); do
1357     local field=${path[${x}]}
1358     [[ ${field} =~ ^20[0-9][0-9]$ ]] && year=${field} && break
1359   done
1360   echo ${year}
1361   return 0
1362 )
1363
1364 guessRunNumber()
1365 (
1366   #guess the run number from the path, pick the rightmost one
1367   #works for /path/foo/000123456/bar/...
1368   #and       /path/foo.run123456.bar
1369   local IFS="/."
1370   declare -a path=( ${1} )
1371   local dirDepth=${#path[*]}
1372   for ((x=${dirDepth}-1;x>=0;x--)); do
1373     local field=${path[${x}]}
1374     field=${field/run/000}
1375     [[ ${field} =~ [0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && runNumber=${field#000} && break
1376   done
1377   echo ${runNumber}
1378   return 0
1379 )
1380
1381 validateLog()
1382 (
1383   log=${1}
1384   errorConditions=(
1385                     'There was a crash'
1386                     'floating'
1387                     'error while loading shared libraries'
1388                     'std::bad_alloc'
1389                     's_err_syswatch_'
1390                     'Thread [0-9]* (Thread'
1391                     'AliFatal'
1392                     'core dumped'
1393                     '\.C.*error:.*\.h: No such file'
1394                     'line.*Aborted'
1395   )
1396
1397   warningConditions=(
1398                      'This is serious !'
1399                      'rocVoltage out of range:'
1400   )
1401   
1402   local logstatus=0
1403   local errorSummary=""
1404   local warningSummary=""
1405
1406   for ((i=0; i<${#errorConditions[@]};i++)); do
1407     local tmp=$(grep -m1 -e "${errorConditions[${i}]}" ${log})
1408     [[ -n ${tmp} ]] && tmp+=" : "
1409     errorSummary+=${tmp}
1410   done
1411
1412   for ((i=0; i<${#warningConditions[@]};i++)); do
1413     local tmp=$(grep -m1 -e "${warningConditions[${i}]}" ${log})
1414     [[ -n ${tmp} ]] && tmp+=" : "
1415     warningSummary+=${tmp}
1416   done
1417
1418   if [[ -n ${errorSummary} ]]; then 
1419     echo "${errorSummary}"
1420     return 1
1421   fi
1422   
1423   if [[ -n ${warningSummary} ]]; then
1424     echo "${warningSummary}"
1425     return 2
1426   fi
1427
1428   return 0
1429 )
1430
1431 summarizeLogs()
1432 (
1433   #print a summary of logs
1434   logFiles=(
1435             "*.log"
1436             "stdout"
1437             "stderr"
1438   )
1439
1440   #check logs
1441   local logstatus=0
1442   for log in ${logFiles[*]}; do
1443     finallog=${PWD%/}/${log}
1444     [[ ! -f ${log} ]] && continue
1445     errorSummary=$(validateLog ${log})
1446     validationStatus=$?
1447     if [[ ${validationStatus} -eq 0 ]]; then 
1448       #in pretend mode randomly report an error in rec.log some cases
1449       if [[ -n ${pretend} && "${log}" == "rec.log" ]]; then
1450         #[[ $(( ${RANDOM}%2 )) -ge 1 ]] && echo "${finallog} BAD random error" || echo "${finallog} OK"
1451         echo "${finallog} OK"
1452       else
1453         echo "${finallog} OK"
1454       fi
1455     elif [[ ${validationStatus} -eq 1 ]]; then
1456       echo "${finallog} BAD ${errorSummary}"
1457       logstatus=1
1458     elif [[ ${validationStatus} -eq 2 ]]; then
1459       echo "${finallog} OK MWAH ${errorSummary}"
1460     fi
1461   done
1462   
1463   #report core files
1464   while read x; do
1465     echo ${x}
1466     chmod 644 ${x}
1467     gdb --batch --quiet -ex "bt" -ex "quit" aliroot ${x} > stacktrace_${x//\//_}.log
1468   done < <(/bin/ls ${PWD}/*/core 2>/dev/null; /bin/ls ${PWD}/core 2>/dev/null)
1469   
1470   return ${logstatus}
1471 )
1472
1473 spitOutLocalOCDBaccessConfig()
1474 {
1475   umask 0002
1476   #find ${1} -name "*root" | \
1477   /bin/ls -1 ${1}/*/*/*/*.root 2>/dev/null | \
1478   while read line
1479   do 
1480     local tmp=${line#${1}}
1481     echo ${tmp%/*} | \
1482     awk -v ocdb=${1} '{print "  man->SetSpecificStorage(\""$1"\",\"local://"ocdb"\");"}'
1483   done
1484   return 0
1485 }
1486
1487 goMakeLocalOCDBaccessConfig()
1488 {
1489   umask 0002
1490   # make a script that sets the specific storages form all the root files produced by CPass0
1491   local localOCDBpathCPass0=${1}
1492   local OCDBpathPrefix=${2}
1493   [[ -z ${OCDBpathPrefix} ]] && OCDBpathPrefix="."
1494
1495   if [[ -f ${localOCDBpathCPass0} && ${localOCDBpathCPass0} =~ \.tgz$ ]]; then
1496     tar xzf ${localOCDBpathCPass0}
1497     local localOCDBpathCPass0="${OCDBpathPrefix}/OCDB"
1498   fi
1499
1500   echo
1501   echo creating the specific storage script
1502   echo   localOCDBaccessConfig.C
1503   echo   based on OCDB: ${localOCDBaccessConfig}
1504   echo
1505
1506   local tempLocalOCDB=""
1507   if [[ -f localOCDBaccessConfig.C ]]; then
1508     tempLocalOCDB=$(mktemp -t tempLocalOCDB.XXXXXX)
1509     echo "egrep "SetSpecificStorage" localOCDBaccessConfig.C > ${tempLocalOCDB}"
1510     egrep "SetSpecificStorage" localOCDBaccessConfig.C > ${tempLocalOCDB}
1511   fi
1512
1513   echo "localOCDBaccessConfig()"                               >  localOCDBaccessConfig.C
1514   echo "{"                                                     >> localOCDBaccessConfig.C
1515   echo "  AliCDBManager* man = AliCDBManager::Instance();"     >> localOCDBaccessConfig.C
1516   spitOutLocalOCDBaccessConfig ${localOCDBpathCPass0}|sort|uniq  >> localOCDBaccessConfig.C
1517   [[ -f "${tempLocalOCDB}" ]] && cat ${tempLocalOCDB}              >> localOCDBaccessConfig.C
1518   echo "}"                                                     >> localOCDBaccessConfig.C
1519
1520   [[ -f "${tempLocalOCDB}" ]] && rm -f ${tempLocalOCDB}
1521
1522   if ! grep SetSpecificStorage localOCDBaccessConfig.C; then 
1523     echo
1524     echo "!!!!!!! CPass0 produced no OCDB entries"
1525     return 1
1526   fi
1527   return 0
1528 }
1529
1530 goMakeFilteredTrees()
1531 (
1532   outputDir=${1}
1533   runNumber=${2}
1534   #get path to input list
1535   inputListfiles=${3}
1536   #get scale number for tracks
1537   filterT=${4}
1538   #get scale number for V0s
1539   filterV=${5}
1540   #get OCDB path (optional)
1541   OCDBpath=${6}
1542   #get max number of files 
1543   maxFiles=${7-"1000000"}
1544   #get offset of first file
1545   offsetFile=${8-"0"}
1546   #get max number of events
1547   maxEvents=${9-"30000000"}
1548   #get offset of first event
1549   offsetEvent=${10-"0"}
1550   configFile=${11-"benchmark.config"}
1551   esdFileName=${12-"AliESDs_Barrel.root"}
1552   shift 12
1553   if ! parseConfig ${configFile} "$@"; then return 1; fi
1554   
1555   #record the working directory provided by the batch system
1556   batchWorkingDirectory=${PWD}
1557
1558   [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
1559   doneFileBase=filtering.cpass1.run${runNumber}.done
1560   doneFileTmp=${batchWorkingDirectory}/${doneFileBase}
1561   doneFile=${commonOutputPath}/meta/${doneFileBase}
1562
1563   cat > filtering.log << EOF
1564   goMakeFilteredTrees config:
1565   runpath=${runpath}
1566   outputDir=${outputDir}
1567   commonOutputPath=${commonOutputPath}
1568   ALICE_ROOT=${ALICE_ROOT}
1569   PATH=${PATH}
1570   offsetEvent=$offsetEvent
1571   configFile=$configFile
1572   esdFileName=$esdFileName
1573   inputListfiles=$inputListfiles
1574   doneFile=$doneFile
1575 EOF
1576
1577   #runpath=${outputDir}
1578   #[[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
1579   #[[ ${reconstructInTemporaryDir} -eq 1 ]] && runpath=$(mktemp -d -t goMakeFilteredTrees.XXXXXX)
1580   #mkdir -p ${outputDir}
1581   #mkdir -p ${runpath}
1582   #if ! cd ${runpath}; then 
1583   #  echo "PWD=$PWD is not the runpath=${runpath}"
1584   #  touch ${doneFile}
1585   #  return 1
1586   #fi
1587   
1588   if [[ -z ${pretend} ]];then
1589     aliroot -l -b -q "${ALICE_ROOT}/PWGPP/macros/runFilteringTask.C(\"${inputListfiles}\",${filterT},${filterV},\"${OCDBpath}\",${maxFiles},${offsetFile},${maxEvents},${offsetEvent},\"${esdFileName}\")" &>> filtering.log
1590   else
1591     sleep ${pretendDelay}
1592     touch filtering.log FilterEvents_Trees.root
1593   fi
1594   pwd
1595   /bin/ls
1596   touch ${doneFile}
1597   summarizeLogs >>  ${doneFile}
1598   
1599   #echo mv -f * ${outputDir}
1600   #mv -f * ${outputDir}
1601   #[[ -f ${outputDir}/FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFile}
1602   #cd ${commonOutputPath}
1603   #[[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
1604  
1605   cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
1606   [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
1607
1608   return 0
1609 )
1610
1611 submit()
1612 {
1613   umask 0002
1614   [[ $# -lt 5 ]] && echo "at least 5 args needed, you supplied $#" && return 1
1615   JobID=${1}
1616   startID=${2}
1617   endID=${3}
1618   waitForJOBID=${4}
1619   command=${5}
1620   shift 5
1621   local commandArgs=("$@")
1622
1623   #add quote strings around the extra arguments
1624   for ((i=0; i<${#commandArgs[@]}; i++)); do 
1625     commandArgs[i]=\"${commandArgs[i]}\"
1626   done
1627
1628   [[ -z ${waitForJOBID} ]] && waitForJOBID=0
1629
1630   newFarm=$(which qsub|grep "^/usr/bin/qsub")
1631   
1632   batchSystem="SGE"
1633
1634   if [[ -z "${newFarm}" ]]
1635   then
1636     #old LSF
1637     # submit it (as job array)
1638     nFiles=$(( ${endID}-${startID}+1 ))
1639     while [ ${startID} -le ${nFiles}  ] ; do
1640       if [ $(expr ${nFiles} - ${startID}) -gt 999 ] ; then 
1641         endID=$(expr ${startID} + 999)
1642       else
1643         endID=${nFiles}
1644       fi      
1645     if [[ ${waitForJOBID} -eq 0 ]]; then
1646         echo ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}"     
1647         ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}"     
1648       else
1649         echo ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -w "ended(${waitForJOBID})" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}"     
1650         ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -w "ended(${waitForJOBID})" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}"     
1651       fi
1652       startID=$(expr ${endID} + 1)
1653     done
1654   else 
1655     #new SGE farm
1656     if [[ ${waitForJOBID} =~ "000" ]]; then
1657       echo ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}"
1658       ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}"
1659     else
1660       echo ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -hold_jid "${waitForJOBID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}"
1661       ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -hold_jid "${waitForJOBID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}"
1662     fi
1663   fi
1664   return 0
1665 }
1666
1667 goSubmitBatch()
1668 {
1669   if [[ $# -lt 3 ]]; then
1670     echo "minimal use:"
1671     echo " ${0} submit fileList productionID configFile"
1672     return 0
1673   fi
1674
1675   productionID=${1}
1676   inputList=${2}
1677   configFile=${3:-"benchmark.config"}
1678   #if which greadlink; then configFile=$(greadlink -f ${configFile}); fi
1679   shift 3
1680   extraOpts=("$@")
1681   if ! parseConfig ${configFile} "${extraOpts[@]}"; then return 1; fi
1682   
1683   #batch systems/makeflow sometimes handle spaces in arguments poorly, so encode them
1684   for (( i=0;i<${#extraOpts[@]};i++ )); do 
1685     extraOpts[i]=$(encSpaces "${extraOpts[i]}")
1686   done
1687   extraOpts+=("encodedSpaces=1")
1688   #this removes the copy of the done file used by makeflow (in the running dir)
1689   extraOpts+=("removeTMPdoneFile=1")
1690
1691   #record the working directory provided by the batch system
1692   batchWorkingDirectory=${PWD}
1693
1694   #redirect all output to submit.log
1695   echo "redirecting all output to ${PWD}/submit_${productionID//"/"/_}.log"
1696   exec 7>&1
1697   exec 1>submit_${productionID//"/"/_}.log 2>&1
1698
1699   umask 0002
1700   echo ${0}" submit $*"
1701   if [[ -z "${inputList}" || -z "${productionID}" ]]
1702   then
1703     echo
1704     echo " Usage: ${0} submit inputList productionID [configFile=benchmark.config]"
1705     echo
1706     return
1707   fi
1708
1709   # check if config file is there
1710   if [ ! -f ${configFile} ]; then
1711     echo "ERROR! Config File '${configFile}' not found" >&2
1712     return
1713   else
1714     echo "Using Config File: '${configFile}'"
1715   fi
1716
1717   [[ ! -f ${alirootEnv} ]] && echo "alirootEnv script ${alirootEnv} not found!..." && return 1
1718
1719   #move the script, config and some other stuff to ${commonOutputPath} first, then use them from there
1720   self=${0}
1721   #if which greadlink; then self=$(greadlink -f "${0}"); fi
1722   configPath=$(dirname ${configFile})
1723   export commonOutputPath=${baseOutputDirectory}/${productionID}
1724   
1725   mkdir -p ${commonOutputPath}
1726   mkdir -p ${commonOutputPath}/logs
1727   mkdir -p ${commonOutputPath}/meta
1728
1729   cp ${self} ${commonOutputPath}
1730   cp ${configFile} ${commonOutputPath}
1731   cp ${inputList} ${commonOutputPath}
1732   self=${commonOutputPath}/${self##*/}
1733   chmod u+x ${self}
1734   configFile=${commonOutputPath}/${configFile##*/}
1735   inputList=${commonOutputPath}/${inputList##*/}
1736
1737   #convert to absolut pathnames
1738   #if which greadlink; then inputList=$(greadlink -f "${inputList}"); fi
1739   #make list of runs
1740   if [[ -z ${runNumber} ]]; then
1741     listOfRuns=($(while read x; do guessRunNumber ${x}; done < ${inputList} | sort | uniq))
1742   else
1743     listOfRuns=${runNumber}
1744   fi
1745
1746   #if which greadlink; then alirootSource=$(greadlink -f "${alirootSource}"); fi
1747
1748   echo ""
1749   echo "### BEGIN CONFIGURATION ###"
1750   echo ""
1751   echo "GENERAL:"
1752   echo ""
1753   echo "    productionID:    ${productionID}"
1754   echo "    batchCommand:    ${batchCommand}"
1755   echo "    batchFlags:      ${batchFlags}"
1756   echo "    alirootEnv:   ${alirootEnv}"
1757   ${alirootEnv} echo '    ALICE_ROOT:      ${ALICE_ROOT}'
1758   ${alirootEnv} echo '    ALIROOT_RELEASE: ${ALICE_RELEASE}'
1759   echo "    inputList:       ${inputList}"
1760   echo "    configPath:      ${configPath}"
1761   echo "    commonOutputPath:      ${commonOutputPath}"
1762   echo "    defaultOCDB:     ${defaultOCDB}"
1763   echo "      autoOCDB: ${autoOCDB}"
1764   echo "    recoTriggerOptions:   ${recoTriggerOptions}"
1765   echo "    runs:"
1766   echo "      ${listOfRuns[*]}"
1767   echo ""
1768   echo "THE TRAIN WILL RUN:"
1769
1770   if [ ${runCPass0reco} -eq 1 ]; then
1771     echo "    Pass0 - Recontruction"
1772   fi
1773
1774   if [ ${runCPass0MergeMakeOCDB} -eq 1 ]; then
1775     echo "    Pass0 - merging and OCDB export"
1776   fi
1777
1778   if [ ${runCPass1reco} -eq 1 ]; then
1779     echo "    Pass1 - Recontruction"
1780   fi
1781   if [ ${runCPass1MergeMakeOCDB} -eq 1 ]; then
1782     echo "    Pass1 - merging and OCDB export"
1783   fi
1784
1785   echo ""
1786   echo "LIMITS:"
1787   echo "    max. Events/Chunk:   ${nEvents}"
1788   echo "    max. Number of Chunks per Run:     ${nMaxChunks}"
1789   echo ""
1790   echo "### END CONFIGURATION ###"
1791   echo ""
1792
1793
1794   # check if input file is there
1795   if [ ! -f ${inputList} ]; then
1796     echo "ERROR! Input List '${inputList}' not found" >&2
1797     return
1798   fi
1799
1800   # define jobid (for dependent jobs)
1801   date=$(date +%Y_%m_%d_%H%M%S)
1802   #for each run we submit one jobarray:
1803   for runNumber in ${listOfRuns[*]}; do
1804     
1805     [[ -z ${runNumber} ]] && continue
1806     [[ ! ${runNumber} =~ ^[0-9]*[0-9]$ ]] && continue
1807
1808     JOBpostfix="${productionID//"/"/_}_${runNumber}_${date}"
1809     JOBID1="p0_${JOBpostfix}"
1810     JOBID1wait="w0_${JOBpostfix}"
1811     JOBID2="m0_${JOBpostfix}"
1812     JOBID2wait="wm0_${JOBpostfix}"
1813     JOBID3="op0_${JOBpostfix}"
1814     JOBID3wait="wop0_${JOBpostfix}"
1815     JOBID4="p1_${JOBpostfix}"
1816     JOBID4wait="w1_${JOBpostfix}"
1817     JOBID5="m1_${JOBpostfix}"
1818     JOBID5wait="wm1_${JOBpostfix}"
1819     JOBID6="s1_${JOBpostfix}"
1820     JOBID6wait="ws1_${JOBpostfix}"
1821     JOBID7="QA_${JOBpostfix}"
1822     JOBmakeESDlistCPass1="lp1_${JOBpostfix}"
1823     JOBfilterESDcpass1="fp1_${JOBpostfix}"
1824     LASTJOB="000"
1825
1826     oneInputFile=$(egrep -m1 "${runNumber}/" ${inputList})
1827
1828     currentDefaultOCDB=${defaultOCDB}
1829     [[ -z ${autoOCDB} ]] && autoOCDB=1
1830     if [[ ${autoOCDB} -ne 0 ]]; then
1831       currentDefaultOCDB=$(setYear ${oneInputFile} ${defaultOCDB})
1832     fi
1833     period=$(guessPeriod ${oneInputFile})
1834     year=$(guessYear ${oneInputFile})
1835
1836     echo "submitting run ${runNumber} with OCDB ${currentDefaultOCDB}"
1837
1838     ###############################################################################
1839     #run one chunk with valgrind:
1840     if [[ -n ${profilingCommand} ]]; then
1841       [[ -z ${nEventsProfiling} ]] && nEventsProfiling=2
1842       [[ -z ${profilingCommand} ]] && profilingCommand="/usr/bin/valgrind --tool=callgrind --num-callers=40 -v --trace-children=yes"
1843       submit "profile-${JOBpostfix}" 1 1 000 "${alirootEnv} ${self}" CPass0 ${commonOutputPath}/${year}/${period}/000${runNumber}/${jobindex} ${oneInputFile} ${nEventsProfiling} ${currentDefaultOCDB} ${configFile} ${runNumber} ${jobindex} useProfilingCommand=$(encSpaces "${profilingCommand}") "${extraOpts[@]}"
1844     fi 
1845
1846     ################################################################################
1847     ################################################################################
1848     # run the CPass0 if requested
1849
1850     if [ ${runCPass0reco} -eq 1 ]; then
1851
1852       echo
1853       echo "starting CPass0... for run ${runNumber}"
1854       echo
1855
1856       # create directory and copy all files that are needed
1857       targetDirectory="${commonOutputPath}/${year}/${period}/000${runNumber}/cpass0"
1858       mkdir -p ${targetDirectory}
1859
1860       filesCPass0=( 
1861                     "${configPath}/runCPass0.sh"
1862                     "${configPath}/recCPass0.C"
1863                     "${configPath}/runCalibTrain.C"
1864                     "${configPath}/localOCDBaccessConfig.C"
1865                     "${configPath}/OCDB*.root"
1866                     "${configPath}/sim.C"
1867                     "${configPath}/rec.C"
1868                     "${configPath}/Config.C"
1869       )
1870       for file in ${filesCPass0[*]}; do
1871         [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
1872       done
1873
1874       localInputList=${targetDirectory}/${inputList##*/}
1875       [[ ! -f ${localInputList} ]] && egrep "\/000${runNumber}\/" ${inputList} > ${localInputList}
1876       # limit nFiles to nMaxChunks
1877       nFiles=$(wc -l < ${localInputList})
1878       [[ ${nFiles} -eq 0 ]] && echo "list contains ZERO files! exiting..." && return 1
1879       echo "raw files in list:    ${nFiles}"
1880       if [[ ${nMaxChunks} -gt 0 && ${nMaxChunks} -le ${nFiles} ]]; then
1881         nFiles=${nMaxChunks}
1882       fi
1883       echo "raw files to process: ${nFiles}"
1884       [[ -z "${percentProcessedFilesToContinue}" ]] && percentProcessedFilesToContinue=100
1885       if [[ ${percentProcessedFilesToContinue} -eq 100 ]]; then
1886         nFilesToWaitFor=${nFiles}
1887       else
1888         nFilesToWaitFor=$(( ${nFiles}-${nFiles}/(100/(100-${percentProcessedFilesToContinue})) ))
1889       fi
1890       echo "requested success rate is ${percentProcessedFilesToContinue}%"
1891       echo "merging will start after ${nFilesToWaitFor} jobs are done"
1892
1893       submit ${JOBID1} 1 ${nFiles} 000 "${alirootEnv} ${self}" CPass0 ${targetDirectory} ${localInputList} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} -1 "${extraOpts[@]}"
1894
1895       ## submit a monitoring job that will run until a certain number of jobs are done with reconstruction
1896       submit "${JOBID1wait}" 1 1 000 "${alirootEnv} ${self}" WaitForOutput ${commonOutputPath} "meta/cpass0.job*.run${runNumber}.done" ${nFilesToWaitFor} ${maxSecondsToWait}
1897       LASTJOB=${JOBID1wait}
1898
1899     fi #end running CPass0
1900     ################################################################################
1901
1902
1903     ################################################################################
1904     # submit merging of CPass0, depends on the reconstruction
1905
1906     if [ ${runCPass0MergeMakeOCDB} -eq 1 ]; then
1907
1908       echo
1909       echo "submit CPass0 merging for run ${runNumber}"
1910       echo
1911
1912       targetDirectory="${commonOutputPath}/${year}/${period}/000${runNumber}/cpass0"
1913       mkdir -p ${targetDirectory}
1914
1915       #copy the scripts
1916       filesMergeCPass0=(
1917                         "${configPath}/OCDB.root"
1918                         "${configPath}/mergeMakeOCDB.byComponent.sh"
1919                         "${configPath}/mergeMakeOCDB.sh"
1920                         "${configPath}/localOCDBaccessConfig.C"
1921                         "${configPath}/mergeByComponent.C"
1922                         "${configPath}/makeOCDB.C"
1923                         "${configPath}/merge.C"
1924       )
1925       for file in ${filesMergeCPass0[*]}; do
1926         [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
1927       done
1928   
1929       submit ${JOBID2} 1 1 "${LASTJOB}" "${alirootEnv} ${self}" MergeCPass0 ${targetDirectory} ${currentDefaultOCDB} ${configFile} ${runNumber} cpass0.calib.run${runNumber}.list "${extraOpts[@]}"
1930       LASTJOB=${JOBID2}
1931
1932       if [[ -n ${generateMC} ]]; then
1933         submit "mrl${JOBpostfix}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" PrintValues sim ${commonOutputPath}/meta/sim.run${runNumber}.list ${commonOutputPath}/meta/cpass0.job*.run${runNumber}.done
1934         LASTJOB="mrl${JOBpostfix}"
1935       fi
1936
1937       echo
1938     fi
1939     # end of merging CPass0
1940     ################################################################################
1941
1942     ################################################################################
1943     ################################################################################
1944     # run the CPass1 if requested
1945
1946     if [ ${runCPass1reco} -eq 1 ]; then
1947
1948       targetDirectory="${commonOutputPath}/${year}/${period}/000${runNumber}/cpass1"
1949       rm -f ${commonOutputPath}/meta/cpass1.job*.run${runNumber}.done
1950
1951       # safety feature: if we are re-running for any reason we want to delete the previous output first.
1952       [[ -d ${targetDirectory} ]] && rm -rf ${targetDirectory}/* && echo "removed old output at ${targetDirectory}/*"
1953
1954       echo
1955       echo "starting CPass1... for run ${runNumber}"
1956       echo
1957
1958       # create directory and copy all files that are needed
1959       mkdir -p ${targetDirectory}
1960       
1961       filesCPass1=( 
1962                     "${configPath}/runCPass1.sh"
1963                     "${configPath}/recCPass1.C"
1964                     "${configPath}/recCPass1_OuterDet.C"
1965                     "${configPath}/runCalibTrain.C"
1966                     "${configPath}/QAtrain_duo.C"
1967                     "${configPath}/localOCDBaccessConfig.C"
1968                     "${configPath}/OCDB.root"
1969       )
1970       for file in ${filesCPass1[*]}; do
1971         [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
1972       done
1973
1974       if [[ -n ${generateMC} ]]; then
1975         localInputList=${commonOutputPath}/meta/sim.run${runNumber}.list
1976       else
1977         localInputList=${targetDirectory}/${inputList##*/}
1978         [[ ! -f ${localInputList} ]] && egrep "\/000${runNumber}\/" ${inputList} > ${localInputList}
1979       fi
1980       # limit nFiles to nMaxChunks
1981       nFiles=$(wc -l < ${localInputList})
1982       [[ ${nFiles} -eq 0 ]] && echo "list contains ZERO files! continuing..." && continue
1983       echo "raw files in list:    ${nFiles}"
1984       if [[ ${nMaxChunks} -gt 0 && ${nMaxChunks} -le ${nFiles} ]]; then
1985         nFiles=${nMaxChunks}
1986       fi
1987       echo "raw files to process: ${nFiles}"
1988       [[ -z "${percentProcessedFilesToContinue}" ]] && percentProcessedFilesToContinue=100
1989       if [[ ${percentProcessedFilesToContinue} -eq 100 ]]; then
1990         nFilesToWaitFor=${nFiles}
1991       else
1992         nFilesToWaitFor=$(( ${nFiles}-${nFiles}/(100/(100-${percentProcessedFilesToContinue})) ))
1993       fi
1994       echo "requested success rate is ${percentProcessedFilesToContinue}%"
1995       echo "merging will start after ${nFilesToWaitFor} jobs are done"
1996
1997       submit ${JOBID4} 1 ${nFiles} "${LASTJOB}" "${alirootEnv} ${self}" CPass1 ${targetDirectory} ${localInputList} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} -1 "${extraOpts[@]}"
1998
1999       ################################################################################
2000       ## submit a monitoring job that will run until a certain number of jobs are done with reconstruction
2001       submit "${JOBID4wait}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" WaitForOutput ${commonOutputPath} "meta/cpass1.job*.run${runNumber}.done" ${nFilesToWaitFor} ${maxSecondsToWait}
2002       LASTJOB=${JOBID4wait}
2003       ################################################################################
2004
2005       echo
2006     fi #end running CPass1
2007
2008     ################################################################################
2009     # submit merging of CPass1, depends on the reconstruction
2010     if [ ${runCPass1MergeMakeOCDB} -eq 1 ]; then
2011
2012       echo
2013       echo "submit CPass1 merging for run ${runNumber}"
2014       echo
2015
2016       targetDirectory="${commonOutputPath}/${year}/${period}/000${runNumber}/cpass1"
2017       rm -f ${commonOutputPath}/meta/merge.cpass1.run${runNumber}.done
2018       mkdir -p ${targetDirectory}
2019
2020       # copy files 
2021       filesMergeCPass1=(
2022                         "${configPath}/OCDB.root"
2023                         "${configPath}/localOCDBaccessConfig.C"
2024                         "${configPath}/mergeMakeOCDB.byComponent.sh"
2025                         "${configPath}/mergeByComponent.C"
2026                         "${configPath}/makeOCDB.C"
2027                         "${configPath}/merge.C"
2028                         "${configPath}/mergeMakeOCDB.sh"
2029                         "${configPath}/QAtrain_duo.C"
2030       )
2031       for file in ${filesMergeCPass1[*]}; do
2032         [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
2033       done
2034
2035       submit "${JOBID5}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" MergeCPass1 ${targetDirectory} ${currentDefaultOCDB} ${configFile} ${runNumber} cpass1.calib.run${runNumber}.list cpass1.QA.run${runNumber}.lastMergingStage.txt.list cpass1.filtered.run${runNumber}.list "${extraOpts[@]}"
2036       LASTJOB=${JOBID5}
2037       echo
2038     fi
2039
2040     ###############################
2041     #if [ ${runESDfiltering} -eq 1 ]; then
2042     #  rm -f ${commonOutputPath}/cpass1.ESD.run${runNumber}.list
2043     #  rm -f ${commonOutputPath}/meta/filtering.cpass1.run*.done
2044     #  echo
2045     #  echo submitting filtering for run ${runNumber}
2046     #  echo
2047     #  submit "${JOBmakeESDlistCPass1}" 1 1 "${LASTJOB}" "${self}" PrintValues esd ${commonOutputPath}/meta/cpass1.ESD.run${runNumber}.list ${commonOutputPath}/meta/cpass1.job*.run${runNumber}.done 
2048     #  submit "${JOBfilterESDcpass1}" 1 1 "${JOBmakeESDlistCPass1}" "${alirootEnv} ${self}" MakeFilteredTrees ${commonOutputPath}/${year}/${period}/000${runNumber}/cpass1 ${runNumber} ${commonOutputPath}/meta/cpass1.ESD.run${runNumber}.list ${filteringFactorHighPt} ${filteringFactorV0s} ${currentDefaultOCDB} 1000000 0 10000000 0 ${configFile} AliESDs_Barrel.root "${extraOpts[@]}"
2049     #  LASTJOB=${JOBfilterESDcpass1}
2050     #fi
2051
2052   done
2053
2054   #################################################################################
2055   #################################################################################
2056   #if [ ${runESDfiltering} -eq 1 ]; then
2057   #  submit "${JOBID5wait}" 1 1 "${LASTJOB}" "${self}" WaitForOutput ${commonOutputPath} "meta/filtering.cpass1.run*.done" "${#listOfRuns[@]}" ${maxSecondsToWait}
2058   #else
2059     submit "${JOBID5wait}" 1 1 "${LASTJOB}" "${self}" WaitForOutput ${commonOutputPath} "meta/merge.cpass1.run*.done" ${#listOfRuns[@]} ${maxSecondsToWait}
2060   #fi
2061   LASTJOB=${JOBID5wait}
2062
2063   #################################################################################
2064   echo
2065   echo "submit make a summary"
2066   echo
2067
2068   submit "${JOBID6}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" MakeSummary ${configFile}
2069   LASTJOB=${JOBID6}
2070   #################################################################################
2071   
2072   #restore stdout
2073   exec 1>&7 7>&-
2074   echo "jobs submitted."
2075   return 0
2076 }
2077
2078 goWaitForOutput()
2079 (
2080   umask 0002
2081   [[ $# -lt 3 ]] && echo "goWaitForOutput() wrong number of arguments, exiting.." && return 1
2082   echo searchPath=${1}
2083   echo fileName=${2}
2084   echo numberOfFiles=${3}
2085   echo maxSecondsToWait=${4}
2086   searchPath=${1}
2087   fileName=${2}
2088   numberOfFiles=${3}
2089   maxSecondsToWait=${4}
2090   echo "command to be executed: /bin/ls -1 ${searchPath}/${fileName}"
2091   [[ -z "${maxSecondsToWait}" ]] && maxSecondsToWait=$(( 3600*12 ))
2092   while true; do
2093     n=$(/bin/ls -1 ${searchPath}/${fileName} 2>/dev/null | wc -l)
2094     [[ ${n} -gt 0 ]] && echo "found ${n} X ${fileName}"
2095     [[ ${n} -ge ${numberOfFiles} ]] && break
2096     [[ ${SECONDS} -gt ${maxSecondsToWait} ]] && echo "timeout of ${maxSecondsToWait}s!" && break
2097     sleep 60
2098   done
2099   echo "DONE! exiting..."
2100   return 0
2101 )
2102
2103 mergeSysLogs()
2104 (
2105   outputFile=${1}
2106   shift
2107   inputFiles="$@"
2108   i=0
2109   if ! ls -1 ${inputFiles} &>/dev/null; then echo "the files dont exist!: ${inputFiles}"; return 1; fi
2110   while read x; do 
2111     runNumber=$(guessRunNumber ${x})
2112     [[ -z ${runNumber} ]] && echo "run number cannot be guessed for ${x}" && continue
2113     awk -v run=${runNumber} -v i=${i} 'NR > 1 {print run" "$0} NR==1 && i==0 {print "run/I:"$0}' ${x}
2114     (( i++ ))
2115   done < <(ls -1 ${inputFiles}) > ${outputFile}
2116   return 0
2117 )
2118
2119 goMakeMergedSummaryTree()
2120 (
2121   # create list of calibration entries
2122   # takes no arguments, just run it in the base output
2123   # directory with the following files in the working directory
2124   # 
2125   #  Calibration file lists:
2126   #       cpass0.dcsTree.list, cpass1.dcsTree.list
2127   #  QA trending root file:
2128   #       trending.root
2129   # 
2130   #  Production infoo ascii files:
2131   #       summary_pass0.tree
2132   #       summary_pass1.tree
2133   #    
2134
2135   [[ ! -f cpass0.dcsTree.list ]] && echo "no cpass0.dcsTree.list" && return 1
2136   [[ ! -f cpass1.dcsTree.list ]] && echo "no cpass1.dcsTree.list" && return 1
2137   [[ ! -f trending.root ]] && echo "no trending.root" && return 1
2138   [[ ! -f summary_pass0.tree ]] && echo "no summary_pass0.tree" && return 1
2139   [[ ! -f summary_pass1.tree ]] && echo "no summary_pass1.tree" && return 1
2140
2141   #first, dump the C macro to file
2142   cat << EOF > mergeTree.C
2143   //
2144   // Merge summary information
2145   // Following files are expected to be in the working directory
2146   //
2147   // Calibration file lists:
2148   //      cpass0.dcsTree.list, cpass1.dcsTree.list
2149   // QA trending root file:
2150   //      trending.root
2151   //
2152   // Production infoo ascii files:
2153   //      summary_pass0.tree
2154   //      summary_pass1.tree
2155   //   
2156   void mergeTree(){
2157     //
2158     //
2159     //
2160     // Calibration values dump
2161     //
2162     //Printf("MakeTreeFromList cpass0.dcsTree.list");
2163     AliXRDPROOFtoolkit::MakeTreeFromList("Calib.TPC.CPass0.root", "dcs","dcs","cpass0.dcsTree.list",1);
2164     //Printf("MakeTreeFromList cpass1.dcsTree.list");
2165     AliXRDPROOFtoolkit::MakeTreeFromList("Calib.TPC.CPass1.root", "dcs","dcs","cpass1.dcsTree.list",1);
2166     //
2167     // Calibration status dump
2168     //
2169     TFile *fprod = TFile::Open("fproduction.root","recreate");
2170     TTree  tree0, tree1;
2171     //Printf("reading summary_pass0.tree");
2172     tree0.ReadFile("summary_pass0.tree");
2173     //Printf("reading summary_pass1.tree");
2174     tree1.ReadFile("summary_pass1.tree");
2175     tree0.Write("CPass0");
2176     tree1.Write("CPass1");
2177     fprod->Close();
2178     //
2179     //
2180     //
2181     TString stringSetup="";
2182     stringSetup+="1#QA.TPC#run#SummaryTPCQA/tpcQA#trending.root+";  // 
2183     stringSetup+="1#Calib.TPC.CPass0#run#dcs#Calib.TPC.CPass0.root+";  // 
2184     stringSetup+="1#Calib.TPC.CPass1#run#dcs#Calib.TPC.CPass1.root+";  // 
2185     //
2186     stringSetup+="1#CPass0#runnumber#CPass0#fproduction.root+";  // 
2187     stringSetup+="1#CPass1#runnumber#CPass1#fproduction.root+";  // 
2188     //
2189     //Printf("stringSetup: %s", stringSetup.Data());
2190     AliXRDPROOFtoolkit::JoinTreesIndex("outAll.root","joinAll","run",stringSetup.Data(), 1);
2191   }
2192 EOF
2193
2194   aliroot -b -q "mergeTree.C" > mergeTrees.log
2195   return $?
2196 )
2197
2198 stackTraceTree()
2199 (
2200   # make stacktrace processing  in case of standard root crash log
2201   # input is a (list of) text files with the stack trace (either gdb aoutput
2202   # produced with e.g. gdb --batch --quiet -ex "bt" -ex "quit" aliroot core, or the root crash log), output is a TTree formatted table.
2203 # example usage:
2204 # benchmark.sh stackTraceTree /foo/*/rec.log
2205   gawk '
2206        BEGIN { 
2207                print "frame/I:method/C:line/C:cpass/I:aliroot/I";
2208                RS="#[0-9]*";
2209                aliroot=0;
2210                read=1;
2211              } 
2212       /There was a crash/ {read=1;}
2213       /The lines below might hint at the cause of the crash/ {read=0;}
2214       read==1 { 
2215                if ($3 ~ /Ali*/) aliroot=1; else aliroot=0;
2216                gsub("#","",RT); 
2217                if ($NF!="" && RT!="" && $3!="") print RT" "$3" "$NF" "0" "aliroot
2218              }
2219       ' "$@" 2>/dev/null
2220 )
2221
2222 goMakeSummary()
2223 (
2224   #all the final stuff goes in here for ease of use:
2225   # summary logs
2226   # qa plot making
2227   # final file lists
2228   # runs in current dir - in makeflow mode it can run LOCAL, then the QA plots and summaries
2229   # will appear in the submission dir.
2230   #some defaults:
2231   log="summary.log"
2232   productionID="qa"
2233
2234   configFile=${1}
2235   shift 1
2236   extraOpts=("$@")
2237   if ! parseConfig ${configFile} "${extraOpts[@]}"; then return 1; fi
2238   
2239   #if which greadlink; then configFile=$(greadlink -f ${configFile}); fi
2240   
2241   #record the working directory provided by the batch system
2242   batchWorkingDirectory=${PWD}
2243
2244   logTmp=${batchWorkingDirectory}/${log}
2245   logDest=${commonOutputPath}/${log}
2246   
2247   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
2248
2249   [[ ! -f ${configFile} ]] && echo "no config file ${configFile}!" && return
2250
2251   [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
2252
2253   #copy some useful stuff
2254   #and go to the commonOutputPath
2255   cp ${configFile} ${commonOutputPath}
2256
2257   exec &> >(tee ${logTmp})
2258
2259   #summarize the global stuff
2260   echo "env script: ${alirootSource} ${alirootEnv}"
2261   echo "\$ALICE_ROOT=${ALICE_ROOT}"
2262   echo "commonOutputPath=${commonOutputPath}"
2263
2264   #summarize the stacktraces
2265   stackTraceTree ${commonOutputPath}/*/*/000*/cpass0/*/stacktrace* > stacktrace_cpass0.tree
2266   stackTraceTree ${commonOutputPath}/*/*/000*/cpass1/*/stacktrace* > stacktrace_cpass1.tree
2267
2268   echo total numbers for the production:
2269   echo
2270   awk 'BEGIN {nFiles=0;nCore=0;} 
2271   /^calibfile/ {nFiles++;} 
2272   /core dumped/ {nCore++i;}
2273   END {print     "cpass0 produced "nFiles" calib files, "nCore" core files";}' ${commonOutputPath}/meta/cpass0.job*done 2>/dev/null
2274   awk 'BEGIN {nOK=0; nBAD=0; } 
2275   /\/rec.log OK/ {nOK++;} 
2276   /\/rec.log BAD/ {nBAD++;} 
2277   /stderr BAD/ {if ($0 ~ /rec.log/){nBAD++;}}
2278   END {print     "cpass0 reco:  OK: "nOK"\tBAD: "nBAD;}' ${commonOutputPath}/meta/cpass0.job*done 2>/dev/null
2279   awk 'BEGIN {nOK=0; nBAD=0; } 
2280   /\/calib.log OK/ {nOK++;} 
2281   /\/calib.log BAD/ {nBAD++;} 
2282   END {print "cpass0 calib: OK: "nOK"\tBAD: "nBAD;}' ${commonOutputPath}/meta/cpass0.job*done 2>/dev/null
2283
2284   awk 'BEGIN {nOK=0; nBAD=0; } 
2285   /merge.log OK/ {nOK++;} 
2286   /merge.log BAD/ {nBAD++;} 
2287   END {print "cpass0 merge: OK: "nOK"\tBAD: "nBAD;}' ${commonOutputPath}/meta/merge.cpass0*done 2>/dev/null
2288   awk 'BEGIN {nOK=0; nBAD=0; } 
2289   /ocdb.log OK/ {nOK++;} 
2290   /ocdb.log BAD/ {nBAD++;} 
2291   END {print   "cpass0 OCDB:  OK: "nOK"\tBAD: "nBAD;}' ${commonOutputPath}/meta/merge.cpass0*done 2>/dev/null
2292
2293   echo
2294   awk 'BEGIN {nFiles=0;nCore=0;} 
2295   /^calibfile/ {nFiles++;} 
2296   /core dumped/ {nCore++;}
2297   END {print     "cpass1 produced "nFiles" calib files, "nCore" core files";}' ${commonOutputPath}/meta/cpass1.job*done 2>/dev/null
2298   awk 'BEGIN {nOK=0; nBAD=0; } 
2299   /\/rec.log OK/ {nOK++;} 
2300   /\/rec.log BAD/ {nBAD++;} 
2301   /stderr BAD/ {if ($0 ~ /rec.log/){nBAD++;}}
2302   END {print     "cpass1 reco:  OK: "nOK"\tBAD: "nBAD;}' ${commonOutputPath}/meta/cpass1.job*done 2>/dev/null
2303   awk 'BEGIN {nOK=0; nBAD=0; } 
2304   /\/calib.log OK/ {nOK++;} 
2305   /\/calib.log BAD/ {nBAD++;} 
2306   END {print "cpass1 calib: OK: "nOK"\tBAD: "nBAD;}' ${commonOutputPath}/meta/cpass1.job*done 2>/dev/null
2307
2308   awk 'BEGIN {nOK=0; nBAD=0; } 
2309   /merge.log OK/ {nOK++;} 
2310   /merge.log BAD/ {nBAD++;} 
2311   END {print "cpass1 merge: OK: "nOK"\tBAD: "nBAD;}' ${commonOutputPath}/meta/merge.cpass1*done 2>/dev/null
2312   awk 'BEGIN {nOK=0; nBAD=0; } 
2313   /ocdb.log OK/ {nOK++;} 
2314   /ocdb.log BAD/ {nBAD++;} 
2315   END {print   "cpass1 OCDB:  OK: "nOK"\tBAD: "nBAD;}' ${commonOutputPath}/meta/merge.cpass1*done 2>/dev/null
2316
2317   echo
2318   echo per run stats:
2319   /bin/ls -1 ${commonOutputPath}/meta/merge.cpass0.run*.done | while read x 
2320 do
2321   dir=$(goPrintValues calibfile - ${x})
2322   runNumber=$(guessRunNumber ${dir})
2323   [[ -z ${runNumber} ]] && continue
2324
2325   if $(/bin/ls ${commonOutputPath}/meta/cpass0.job*.run${runNumber}.done &> /dev/null); then
2326     statusCPass0=( $(
2327     awk 'BEGIN {nOKrec=0;nBADrec=0;nOKcalib=0;nBADcalib=0;nOKstderr=0;nBADstderr=0;}
2328     /\/rec.log OK/ {nOKrec++;} 
2329     /\/rec.log BAD/ {nBADrec++;}
2330     /stderr BAD/ {if ($0 ~ /rec.log/) {nBADrec++;} nBADstderr++;}
2331     /stderr OK/ {nOKstderr++;}
2332     /\/calib.log OK/ {nOKcalib++;}
2333     /\/calib.log BAD/ {nBADcalib++}
2334     END {print ""nOKrec" "nBADrec" "nOKstderr" "nBADstderr" "nOKcalib" "nBADcalib;}' ${commonOutputPath}/meta/cpass0.job*.run${runNumber}.done 2>/dev/null
2335     ) ) 
2336   fi
2337
2338   if $(/bin/ls ${commonOutputPath}/meta/cpass1.job*.run${runNumber}.done &>/dev/null); then
2339     statusCPass1=( $(
2340     awk 'BEGIN {nOKrec=0;nBADrec=0;nOKcalib=0;nBADcalib=0;nOKstderr=0;nBADstderr=0;nQAbarrelOK=0;nQAbarrelBAD=0;nQAouterOK=0;nQAouterBAD=0;}
2341     /\/rec.log OK/ {nOKrec++;} 
2342     /\/rec.log BAD/ {nBADrec++;}
2343     /stderr BAD/ {if ($0 ~ /rec.log/) nBADrec++;nBADstderr++;}
2344     /stderr OK/ {nOKstderr++;}
2345     /\/calib.log OK/ {nOKcalib++;}
2346     /\/calib.log BAD/ {nBADcalib++}
2347     /\/qa_barrel.log OK/ {nQAbarrelOK++;}
2348     /\/qa_barrel.log BAD/ {nQAbarrelBAD++;}
2349     /\/qa_outer.log OK/ {nQAouterOK++;}
2350     /\/qa_outer.log BAD/ {nQAouterBAD++;}
2351     END {print ""nOKrec" "nBADrec" "nOKstderr" "nBADstderr" "nOKcalib" "nBADcalib" "nQAbarrelOK" "nQAbarrelBAD" "nQAouterOK" "nQAouterBAD;}' ${commonOutputPath}/meta/cpass1.job*.run${runNumber}.done 2>/dev/null
2352     ) ) 
2353   fi
2354
2355   statusOCDBcpass0=$(awk '/ocdb.log/ {print $2} ' ${x} 2>/dev/null)
2356   statusOCDBcpass1=$(awk '/ocdb.log/ {print $2}' ${x/cpass0/cpass1} 2>/dev/null)
2357   statusQA=$(awk '/mergeMakeOCDB.log/ {print $2}' ${x/cpass0/cpass1} 2>/dev/null)
2358
2359   printf "%s\t ocdb.log cpass0: %s\t ocdb.log cpass1: %s\tqa.log:%s\t| cpass0: rec:%s/%s stderr:%s/%s calib:%s/%s cpass1: rec:%s/%s stderr:%s/%s calib:%s/%s QAbarrel:%s/%s QAouter:%s/%s\n" ${runNumber} ${statusOCDBcpass0} ${statusOCDBcpass1} ${statusQA} ${statusCPass0[0]} ${statusCPass0[1]} ${statusCPass0[2]} ${statusCPass0[3]} ${statusCPass0[4]} ${statusCPass0[5]} ${statusCPass1[0]} ${statusCPass1[1]} ${statusCPass1[2]} ${statusCPass1[3]} ${statusCPass1[4]} ${statusCPass1[5]} ${statusCPass1[6]} ${statusCPass1[7]} ${statusCPass1[8]} ${statusCPass1[9]}
2360 done
2361
2362   #make lists with output files - QA, trending, filtering and calibration
2363   ### wait for the merging of all runs to be over ###
2364   rm -f qa.list
2365   goPrintValues qafile qa.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2366   rm -f calib.list
2367   goPrintValues calibfile calib.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2368   rm -f trending.list
2369   goPrintValues trendingfile trending.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2370   rm -f filtering.list
2371   goPrintValues filteredTree filtering.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2372   rm -f cpass0.dcsTree.list
2373   goPrintValues dcsTree cpass0.dcsTree.list ${commonOutputPath}/meta/merge.cpass0.run*.done &>/dev/null
2374   rm -f cpass1.dcsTree.list
2375   goPrintValues dcsTree cpass1.dcsTree.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2376  
2377   #merge trending
2378   rm -f ${commonOutputPath}/trending_merged.root
2379   goMerge trending.list ${commonOutputPath}/trending.root ${configFile} "${extraOpts[@]}" &> mergeTrending.log
2380
2381   goMakeSummaryTree ${commonOutputPath} 0
2382   goMakeSummaryTree ${commonOutputPath} 1
2383
2384   goCreateQAplots "${PWD}/qa.list" "${productionID}" "QAplots" "${configFile}" "${extraOpts[@]}" filteringList="${PWD}/filtering.list" &>createQAplots.log
2385
2386   #make a merged summary tree out of the QA trending, dcs trees and log summary trees
2387   goMakeMergedSummaryTree
2388
2389   #if set, email the summary
2390   [[ -n ${MAILTO} ]] && cat ${logTmp} | mail -s "benchmark ${productionID} done" ${MAILTO}
2391
2392   # Copy log to destination (delete all on failure to signal error)
2393   cp "$logTmp" "$logDest" || rm -f "$logTmp" "$logDest"
2394   cp -r QAplots ${commonOutputPath}
2395   cp *.list ${commonOutputPath}
2396   cp *.root ${commonOutputPath}
2397   cp *.log ${commonOutputPath}
2398
2399   return 0
2400 )
2401
2402 goMakeSummaryTree()
2403 (
2404   if [[ $# -lt 1 ]] ; then
2405     return
2406   fi
2407   #1. define vars/arrays
2408   DIR=${1} #use input or exec in current dir
2409   pass=${2-"0"} #pass from input
2410   outfile="summary_pass${pass}.tree"
2411   Ncolumns=0
2412   test -f ${outfile} && : >${outfile}
2413   errfile=${outfile/tree/err}
2414   test -f ${errfile} && : >${errfile}
2415
2416   declare -a counterString=(TOFevents TOFtracks TPCevents TPCtracks TRDevents TRDtracks T0events SDDevents SDDtracks MeanVertexevents)
2417   Ncounter=${#counterString[@]}
2418
2419   declare -a statusString=(TRDStatus TOFStatus TPCStatus T0Status MeanVertexStatus)
2420   Nstatus=${#statusString[@]}
2421
2422
2423   declare -a ratesString=(rec stderr calib qa_barrel qa_outer)
2424   Nrates=${#ratesString[@]}
2425
2426   runs=( $(ls -1 ${DIR}/meta/merge.cpass0* | while read x; do guessRunNumber $x; done) )
2427   Nruns=${#runs[@]}
2428
2429   echo -n runnumber/I >>${outfile}
2430   echo -n :cpass${pass}status/I >>${outfile}
2431   echo -n :cpass${pass}QAstatus/I >>${outfile}
2432   for i in ${ratesString[@]}; do
2433     echo -n :${i}OK/I >>${outfile}
2434     echo -n :${i}BAD/I >>${outfile}
2435
2436   done
2437   for i in ${counterString[@]} ${statusString[@]} ; do
2438     echo -n :${i}/I >>${outfile}
2439   done
2440   Ncolumns=$((2 + 2*Nrates + Ncounter + Nstatus))
2441   echo >> ${outfile}
2442
2443   #2. loop runs 
2444
2445   for runnumber in ${runs[@]} ; do 
2446
2447
2448
2449     filejob="${DIR}/meta/cpass${pass}.job*.run${runnumber}.done"
2450     filemerge="${DIR}/meta/merge.cpass${pass}.run${runnumber}.done"
2451     fileOCDB=$(grep /ocdb.log ${filemerge} | awk '{print $1}')
2452     if ! $(/bin/ls ${filemerge} &>/dev/null) ; then
2453       echo "${filemerge} does not exist!" >>${errfile}
2454       continue
2455     elif ! $(/bin/ls ${filejob} &>/dev/null) ; then
2456       echo "${filejob} does not exist!" >>${errfile}
2457       echo -n ${runnumber} >> ${outfile}
2458       for i in $(seq ${Ncolumns}) ; do 
2459         echo -n "-1" >> ${outfile}
2460       done
2461       echo >> ${outfile}
2462       continue
2463     fi
2464     echo -n ${runnumber} >> ${outfile}
2465     #pass0status= grep '/ocdb.log' ${filemerge} |  cut -d' ' -f2 | tr OK x1 | tr BAD xx0 | tr -d 'x'
2466     passStatus=$(grep '/ocdb.log' ${filemerge} | grep OK | wc -l)
2467     echo -n " ${passStatus}" >> ${outfile}
2468     qaStatus=$(grep '/mergeMakeOCDB.log' ${filemerge} | grep OK | wc -l)
2469     echo -n " ${qaStatus}" >> ${outfile}
2470
2471
2472     #fill OK/BAD rates
2473     for i in $(seq 0 $((${Nrates}-1))) ; do
2474       var1=$(grep "/${ratesString[${i}]}.log" ${filejob} | grep OK | wc -l)
2475       var2=$(grep "/${ratesString[${i}]}.log" ${filejob} | grep BAD | wc -l)
2476
2477       if [[ ${ratesString[${i}]} == "stderr" ]] ; then
2478         var1=$(grep "stderr" ${filejob} | grep OK | wc -l)
2479         var2=$(grep "stderr" ${filejob} | grep "rec.log" | grep BAD | wc -l)
2480       fi
2481       echo -n " ${var1}" >> ${outfile}
2482       echo -n " ${var2}" >> ${outfile}
2483     done
2484
2485     if [[ -f ${fileOCDB} ]] ; then
2486       #fill counter
2487       for i in $(seq 0 $((${Ncounter}-1))) ; do
2488         var1=$(grep Monalisa ${fileOCDB} | grep ${counterString[${i}]} | cut -f2)
2489         echo -n " ${var1:-"-1"}" >> ${outfile}
2490       done
2491
2492       #fill status
2493       for i in $(seq 0 $((${Nstatus}-1))) ; do
2494         var1=$(grep "calibration status=" ${fileOCDB} | grep ${statusString[${i}]/Status/} | cut -d'=' -f2)
2495         echo -n " ${var1:-"-1"}" >> ${outfile}
2496       done
2497     fi
2498     echo >> ${outfile}
2499   done
2500
2501   return 0
2502 )
2503
2504 parseConfig()
2505 {
2506   configFile=${1}
2507   shift
2508   args=("$@")
2509
2510
2511   #some defaults
2512   #autoOCDB=0
2513   defaultOCDB="raw://"
2514   #runNumber=167123
2515   #makeflowPath="/hera/alice/aux/cctools/bin"
2516   #makeflowOptions="-T wq -N alice -d all -C ali-copilot.cern.ch:9097"
2517   #makeflowOptions="-T wq -N alice -C ali-copilot.cern.ch:9097"
2518   makeflowOptions=""
2519   #batchCommand="/usr/bin/qsub"
2520   batchFlags=""
2521   baseOutputDirectory="$PWD/output"
2522   #alirootEnv="/cvmfs/alice.cern.ch/bin/alienv setenv AliRoot/v5-04-34-AN -c"
2523   #alirootEnv="/home/mkrzewic/alisoft/balice_master.sh"
2524   #trustedQAtrainMacro='/hera/alice/mkrzewic/gsisvn/Calibration/QAtrain_duo.C'
2525   reconstructInTemporaryDir=0
2526   recoTriggerOptions="\"\""
2527   percentProcessedFilesToContinue=100
2528   maxSecondsToWait=$(( 3600*24 ))
2529   nEvents=-1
2530   nMaxChunks=0
2531   postSetUpActionCPass0=""
2532   postSetUpActionCPass1=""
2533   runCPass0reco=1
2534   runCPass0MergeMakeOCDB=1
2535   runCPass1reco=1
2536   runCPass1MergeMakeOCDB=1
2537   runESDfiltering=1
2538   filteringFactorHighPt=1e2
2539   filteringFactorV0s=1e1
2540   MAILTO=""
2541   #pretend=1
2542   #dontRedirectStdOutToLog=1
2543   logToFinalDestination=1
2544   ALIROOT_FORCE_COREDUMP=1
2545   pretendDelay=0
2546
2547   #first, source the config file
2548   if [ -f ${configFile} ]; then
2549     source ${configFile}
2550   else
2551     echo "config file ${configFile} not found!"
2552     return 1
2553   fi
2554
2555   unset encodedSpaces
2556   for opt in "${args[@]}"; do
2557     [[ "${opt}" =~ encodedSpaces=.* ]] && encodedSpaces=1 && echo "encodedSpaces!" && break
2558   done
2559
2560   #then, parse the options as they override the options from file
2561   for opt in "${args[@]}"; do
2562     [[ -z ${opt} ]] && continue
2563     [[ -n ${encodedSpaces} ]] && opt="$(decSpaces ${opt})"
2564     [[ "${opt}" =~ ^[[:space:]]*$ ]] && continue
2565     if [[ ! "${opt}" =~ .*=.* ]]; then
2566       echo "badly formatted option \"${opt}\" should be: option=value, stopping..."
2567       return 1
2568     fi
2569     local var="${opt%%=*}"
2570     local value="${opt#*=}"
2571     echo "${var}=${value}"
2572     export ${var}="${value}"
2573   done
2574
2575   #do some checking
2576   [[ -z ${alirootEnv} ]] && echo "alirootEnv not defined!" && return 1
2577
2578   #export the aliroot function if defined to override normal behaviour
2579   [[ $(type -t aliroot) =~ "function" ]] && export -f aliroot
2580
2581   return 0
2582 }
2583
2584 aliroot()
2585 {
2586   args=("$@")
2587   if [[ -n ${useProfilingCommand} ]]; then
2588     profilerLogFile="cpu.txt"
2589     [[ "${args}" =~ rec ]] && profilerLogFile="cpu_rec.txt"
2590     [[ "${args}}" =~ Calib ]] && profilerLogFile="cpu_calib.txt"
2591     echo running "${useProfilingCommand} aliroot ${args} &> ${profilerLogFile}"
2592     ${useProfilingCommand} aliroot "${args[@]}" &> ${profilerLogFile}
2593   else
2594     #to prevent an infinite recursion use "command aliroot" to disable
2595     #aliases and functions
2596     echo running command aliroot "${args[@]}"
2597     command aliroot "${args[@]}"
2598   fi
2599   return 0
2600 }
2601
2602 guessRunData()
2603 {
2604   #guess the period from the path, pick the rightmost one
2605   period=""
2606   runNumber=""
2607   year=""
2608   pass=""
2609   legoTrainRunNumber=""
2610   dataType=""
2611
2612   local shortRunNumber=""
2613   local IFS="/"
2614   declare -a path=( $1 )
2615   local dirDepth=$(( ${#path[*]}-1 ))
2616   i=0
2617   #for ((x=${dirDepth};x>=0;x--)); do
2618   for ((x=0;x<=${dirDepth};x++)); do
2619
2620     [[ $((x-1)) -ge 0 ]] && local fieldPrev=${path[$((x-1))]}
2621     local field=${path[${x}]}
2622     local fieldNext=${path[$((x+1))]}
2623
2624     [[ ${field} =~ ^[0-9]*$ && ${fieldNext} =~ (.*\.zip$|.*\.root$) ]] && legoTrainRunNumber=${field}
2625     [[ -n ${legoTrainRunNumber} && -z ${pass} ]] && pass=${fieldPrev}
2626     [[ ${field} =~ ^LHC[0-9][0-9][a-z].*$ ]] && period=${field%_*}
2627     [[ ${field} =~ ^000[0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && runNumber=${field#000}
2628     [[ ${field} =~ ^[0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && shortRunNumber=${field}
2629     [[ ${field} =~ ^20[0-9][0-9]$ ]] && year=${field}
2630     [[ ${field} =~ ^(^sim$|^data$) ]] && dataType=${field}
2631     (( i++ ))
2632   done
2633   [[ -z ${legoTrainRunNumber} ]] && pass=${path[$((dirDepth-1))]}
2634   [[ "${dataType}" =~ ^sim$ ]] && pass="passMC" && runNumber=${shortRunNumber}
2635   
2636   #if [[ -z ${dataType} || -z ${year} || -z ${period} || -z ${runNumber}} || -z ${pass} ]];
2637   if [[ -z ${runNumber}} ]];
2638   then
2639     #error condition
2640     return 1
2641   else
2642     #ALL OK
2643     return 0
2644   fi
2645   return 0
2646 }
2647
2648 #these functions encode strings to and from a space-less form
2649 #use when spaces are not well handled (e.g. in arguments to 
2650 #commands in makeflow files, etc.
2651 encSpaces()(echo "${1// /@@@@}")
2652 decSpaces()(echo "${1//@@@@/ }")
2653
2654 main "$@"