RelVal: makeflow's exitcode may not be meaningful
[u/mrichter/AliRoot.git] / PWGPP / benchmark / benchmark.sh
1 #!/bin/bash
2 #include benchmark.config
3
4 # blame: Mikolaj Krzewicki, mkrzewic@cern.ch
5 # this script runs the CPass0/CPass1 train
6 # produced OCDB updates are local
7
8 main()
9 {
10   #run in proper mode depending on the selection
11   if [[ $# -lt 1 ]]; then
12     if [[ ! "${0}" =~ "bash" ]]; then
13       echo "uses makeflow:"
14       echo " ${0} \"run\" productionID inputList configFile [extraOpts]"
15       echo "uses a batch system (SGE):"
16       echo " ${0} \"submit\" productionID inputList configFile [extraOpts]"
17       echo "extraOpts if non-empty override the config file, e.g.:"
18       echo " ${0} submit test1 benchmark.list benchmark.config runNumber=169123 nEvents=10"
19     fi
20     return
21   fi
22
23   #define some aliases - default is to call one of the functions directly
24   runMode=${1}
25   umask 0002
26   shift
27   case ${runMode} in
28     "CPass0") goCPass0 "$@";;
29     "CPass1") goCPass1 "$@";;
30     "MakeLocalOCDBaccessConfig") goMakeLocalOCDBaccessConfig "$@";;
31     "MergeCPass0") goMergeCPass0 "$@";;
32     "MergeCPass1") goMergeCPass1 "$@";;
33     "MakeFilteredTrees") goMakeFilteredTrees "$@";;
34     "MakeSummary") goMakeSummary "$@";;
35     "run") goSubmitMakeflow "$@";;
36     "submit") goSubmitBatch "$@";;
37     "test") goTest "$@";;
38     "GenerateMakeflow") goGenerateMakeflow "$@";;
39     "PrintValues") goPrintValues "$@";;
40     "CreateQAplots") goCreateQAplots "$@";;
41     "WaitForOutput") goWaitForOutput "$@";;
42     "Merge") goMerge "$@";;
43     *) 
44       ${runMode} "$@"
45     ;;
46   esac
47   return 0
48 }
49
50 generateMC()
51 {
52   #generate one raw chunk in current directory
53   SEED=${JOB_ID}${SGE_TASK_ID}
54   export CONFIG_SEED=${SEED}
55   runNumber=${1}
56   OCDBpath=${2}
57   nEventsim=${3}
58   if [[ -n ${pretend} ]]; then
59     sleep ${pretendDelay}
60     touch galice.root
61   else
62     if [[ -f sim.C && -f Config.C ]] ; then
63         time aliroot -b -q -x sim.C\(${runNumber},\"${OCDBpath}\",${nEventsim}\) >sim.log 2>&1
64         mv syswatch.log simwatch.log
65     fi
66   fi
67 }
68
69 goCPass0()
70 (
71   umask 0002
72   
73   targetDirectory=${1}
74   inputList=${2}
75   nEvents=${3}
76   ocdbPath=${4}
77   configFile=${5}
78   runNumber=${6}
79   jobindex=${7}
80   shift 7
81   if ! parseConfig ${configFile} "$@"; then return 1; fi
82
83   #record the working directory provided by the batch system
84   batchWorkingDirectory=${PWD}
85
86   #use the jobindex only if set and non-negative
87   if [[ -z ${jobindex} || ${jobindex} -lt 0 ]]; then
88     [[ -n "${LSB_JOBINDEX}" ]] && jobindex=${LSB_JOBINDEX}
89     [[ -n "${SGE_TASK_ID}" ]] && jobindex=${SGE_TASK_ID}
90     if [[ -z ${jobindex} ]]; then 
91       echo "no jobindex!"
92       return 1
93     fi
94   fi
95
96   [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
97
98   # This file signals that/if everything went fine
99   doneFileBase="cpass0.job${jobindex}.run${runNumber}.done"
100   [[ -n ${useProfilingCommand} ]] && doneFileBase="profiling.cpass0.job${jobindex}.run${runNumber}.done"
101
102   # We will have two copies of the file
103   mkdir -p "${commonOutputPath}/meta" || return 1
104   doneFileTmp="${batchWorkingDirectory}/${doneFileBase}"
105   doneFile="${commonOutputPath}/meta/${doneFileBase}"
106
107   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
108   
109   if [[ -n ${ALIROOT_FORCE_COREDUMP} ]]; then
110     ulimit -c unlimited 
111     export ALIROOT_FORCE_COREDUMP
112   fi
113
114   #the contents of this is stored in the tree and used later (e.g. AliAnalysisTaskPIDResponse)!
115   #at the QA stage the pass number is guessed from the path stored here.
116   #The Format is:
117   #Packages= ;OutputDir= ;LPMPass= ;TriggerAlias= ;LPMRunNumber= ;LPMProductionType= ;LPMInteractionType= ;LPMProductionTag= ;LPMAnchorRun= ;LPMAnchorProduction= ;LPMAnchorYear= 
118   export PRODUCTION_METADATA="OutputDir=cpass0"
119
120   if [[ "${inputList}" =~ \.root$ ]]; then
121     infile=${inputList}
122   else
123     infile=$(sed -ne "${jobindex}p" ${inputList} | egrep '\s*\w*/\w*')
124   fi
125   chunkName=${infile##*/}
126
127   outputDir=${targetDirectory}/${jobindex}_${chunkName%.*}
128   mkdir -p ${outputDir}
129   if [[ ! -d ${outputDir} ]]; then 
130     touch ${doneFileTmp}
131     echo "cannot make ${outputDir}" >> ${doneFileTmp}
132     cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
133     [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
134     return 1  
135   fi
136   
137   runpath=${outputDir}
138   [[ ${reconstructInTemporaryDir} -eq 1 ]] && runpath=$(mktemp -d -t cpass0.XXXXXX)
139   [[ ${reconstructInTemporaryDir} -eq 2 ]] && runpath=${PWD}/rundir_cpass0_${runNumber}_${jobindex}
140   mkdir -p ${runpath}
141   if [[ ! -d ${runpath} ]]; then
142     touch ${doneFileTmp} 
143     echo "cannot make runpath ${runpath}" >> ${doneFileTmp}
144     cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
145     [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
146     return 1
147   fi
148   if ! cd ${runpath}; then
149     touch ${doneFileTmp}
150     echo "PWD=$PWD is not the runpath=${runpath}" >> ${doneFileTmp}
151     cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
152     [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
153     return 1
154   fi
155
156   #runCPassX/C expects the raw chunk to be linked in the run dir
157   #despite it being accessed by the full path
158   ln -s ${infile} ${runpath}/${chunkName}
159
160   #####MC
161   if [[ -n ${generateMC} ]]; then
162     olddir=${PWD}
163     outputDirMC=${commonOutputPath}/000${runNumber}/sim/${jobindex}
164     simrunpath=${outputDirMC}
165     #[[ ${simulateInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && simrunpath=${TMPDIR}
166     [[ ${simulateInTemporaryDir} -eq 1 ]] && simrunpath=$(mktemp -d -t cpass0MC.XXXXXX)
167     mkdir -p ${outputDirMC}
168     mkdir -p ${simrunpath}
169     if cd ${simrunpath}; then
170
171       filesMC=( 
172       "${batchWorkingDirectory}/sim.C"
173       "${batchWorkingDirectory}/rec.C"
174       "${batchWorkingDirectory}/Config.C"
175       "${batchWorkingDirectory}/OCDB_*.root"
176       )
177       for file in ${filesMC[*]}; do
178         [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
179       done
180
181       generateMC ${runNumber} ${ocdbPath} ${nEvents}
182
183       [[ ! "${simrunpath}" =~ "${outputDirMC}" ]] && mv * ${outputDirMC} #TODO check if it works
184       cd ${olddir}
185
186       ln -s ${outputDirMC}/* ${runpath}/ 
187
188       inputList=${outputDirMC}/galice.root #TODO not valid outside shell !!!
189       infile=""
190     fi
191   fi
192   ######
193   
194   if [[ ! -f ${inputList} && -z ${pretend} ]]; then
195     touch ${doneFileTmp}
196     echo "input file ${inputList} not found, exiting..." >> ${doneFileTmp}
197     cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
198     [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
199     return 1
200   fi
201
202   logOutputDir=${runpath}
203   [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir}
204   [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/stdout
205   #[[ -z ${dontRedirectStdOutToLog} ]] && exec 2> ${logOutputDir}/stderr
206   echo "${0} $*"
207
208   echo "#####################"
209   echo CPass0:
210   echo JOB setup
211   echo nEvents               ${nEvents}
212   echo runNumber             ${runNumber}
213   echo ocdbPath              ${ocdbPath}
214   echo infile                ${infile}
215   echo chunkName             ${chunkName}
216   echo jobindex              ${jobindex}
217   echo recoTriggerOptions    ${recoTriggerOptions}
218   echo targetDirectory       ${targetDirectory}
219   echo commonOutputPath      ${commonOutputPath}
220   echo doneFile              ${doneFile}
221   echo batchWorkingDirectory ${batchWorkingDirectory}
222   echo runpath               ${runpath}  
223   echo outputDir             ${outputDir}
224   echo PWD                   ${PWD}
225   echo ALICE_ROOT            ${ALICE_ROOT}
226   echo "########## ###########"
227
228   alirootInfo > ALICE_ROOT.log
229
230   filesCPass0=( 
231                "${batchWorkingDirectory}/runCPass0.sh"
232                "${batchWorkingDirectory}/recCPass0.C"
233                "${batchWorkingDirectory}/runCalibTrain.C"
234                "${batchWorkingDirectory}/localOCDBaccessConfig.C"
235                "${batchWorkingDirectory}/OCDB.root"
236                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/runCPass0.sh"
237                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/recCPass0.C" 
238                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/runCalibTrain.C"
239   )
240
241   for file in ${filesCPass0[*]}; do
242     [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
243     [[ ${file##*/} =~ .*\.sh ]] && chmod +x ${file##*/}
244   done
245
246   echo "this directory (${PWD}) contents:"
247   /bin/ls
248   echo
249   chmod u+x runCPass0.sh
250
251   sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh
252
253   if [[ -n ${postSetUpActionCPass0} ]]; then
254     echo "running ${postSetUpActionCPass0}"
255     eval ${postSetUpActionCPass0}
256   fi
257
258   #run CPass0
259   echo "${runpath}/runCPass0.sh ${infile} ${nEvents} ${runNumber} ${ocdbPath} ${recoTriggerOptions}"
260   if [[ -n ${pretend} ]]; then
261     sleep ${pretendDelay}
262     touch AliESDs.root
263     touch AliESDfriends.root
264     touch AliESDfriends_v1.root
265     touch rec.log
266     touch calib.log
267   else
268     echo ./runCPass0.sh "${infile}" "${nEvents}" "${runNumber}" "${ocdbPath}" "${recoTriggerOptions}"
269     ./runCPass0.sh "${infile}" "${nEvents}" "${runNumber}" "${ocdbPath}" "${recoTriggerOptions}"
270   fi
271   
272   #move stuff to final destination
273   echo "this directory (${PWD}) contents:"
274   /bin/ls
275   echo
276
277   # [dberzano] OK this is fine!
278   echo rm -f ./${chunkName}
279   rm -f ./${chunkName}
280   echo "paranoidCp ${runpath}/* ${outputDir}"
281   paranoidCp ${runpath}/* ${outputDir}
282   echo
283   
284   #validate CPass0
285   cd ${outputDir}
286   if summarizeLogs >> ${doneFileTmp}; then
287     [[ -f ${outputDirMC}/galice.root ]] && echo "sim ${outputDirMC}/galice.root" >> ${doneFileTmp}
288     [[ -f AliESDfriends_v1.root ]] && echo "calibfile ${outputDir}/AliESDfriends_v1.root" >> ${doneFileTmp}
289     [[ -f AliESDs.root ]] && echo "esd ${outputDir}/AliESDs.root" >> ${doneFileTmp}
290   fi
291
292   [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath} && echo "removing ${runpath}"
293   cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
294   [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
295   return 0
296 )
297
298 goCPass1()
299 (
300   umask 0002
301   
302   targetDirectory=${1}
303   inputList=${2}
304   nEvents=${3}
305   ocdbPath=${4}
306   configFile=${5}
307   runNumber=${6}
308   jobindex=${7}
309   shift 7
310   extraOpts=("$@")
311   if ! parseConfig ${configFile} "$@"; then return 1; fi
312
313   #record the working directory provided by the batch system
314   batchWorkingDirectory=${PWD}
315
316   #use the jobindex only if set and non-negative
317   if [[ -z ${jobindex} || ${jobindex} -lt 0 ]]; then
318     [[ -n "${LSB_JOBINDEX}" ]] && jobindex=${LSB_JOBINDEX}
319     [[ -n "${SGE_TASK_ID}" ]] && jobindex=${SGE_TASK_ID}
320     if [[ -z ${jobindex} ]]; then 
321       echo "no jobindex!"
322       return 1
323     fi
324   fi
325
326   [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
327
328   # This file signals that/if everything went fine
329   doneFileBase="cpass1.job${jobindex}.run${runNumber}.done"
330   [[ -n ${useProfilingCommand} ]] && doneFileBase="profiling.cpass0.job${jobindex}.run${runNumber}.done"
331
332   # We will have two copies of the file
333   mkdir -p "${commonOutputPath}/meta" || return 1
334   doneFileTmp="${batchWorkingDirectory}/${doneFileBase}"
335   doneFile="${commonOutputPath}/meta/${doneFileBase}"
336
337   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
338   
339   if [[ -n ${ALIROOT_FORCE_COREDUMP} ]]; then
340     ulimit -c unlimited 
341     export ALIROOT_FORCE_COREDUMP
342   fi
343
344   #the contents of this is stored in the tree and used later (e.g. AliAnalysisTaskPIDResponse)!
345   #at the QA stage the pass number is guessed from the path stored here.
346   #The Format is:
347   #Packages= ;OutputDir= ;LPMPass= ;TriggerAlias= ;LPMRunNumber= ;LPMProductionType= ;LPMInteractionType= ;LPMProductionTag= ;LPMAnchorRun= ;LPMAnchorProduction= ;LPMAnchorYear= 
348   export PRODUCTION_METADATA="OutputDir=cpass1"
349
350   if [[ ! -f ${inputList} && -z ${pretend} ]]; then
351     touch ${doneFileTmp}
352     echo "input file ${inputList} not found, exiting..." >> ${doneFileTmp}
353     cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
354     [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
355     return 1
356   fi
357   if [[ "${inputList}" =~ \.root$ ]]; then
358     infile=${inputList}
359   else
360     infile=$(sed -ne "${jobindex}p" ${inputList} | egrep '\s*\w*/\w*')
361   fi
362   chunkName=${infile##*/}
363
364   outputDir=${targetDirectory}/${jobindex}_${chunkName%.*}
365   mkdir -p ${outputDir}
366   if [[ ! -d ${outputDir} ]];then
367     touch ${doneFileTmp}
368     echo "cannot make ${outputDir}" >> ${doneFileTmp}
369     cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
370     [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
371     return 1
372   fi
373   
374   runpath=${outputDir}
375   [[ ${reconstructInTemporaryDir} -eq 1 ]] && runpath=$(mktemp -d -t cpass1.XXXXXX)
376   [[ ${reconstructInTemporaryDir} -eq 2 ]] && runpath=${PWD}/rundir_cpass1_${runNumber}_${jobindex}
377
378   #MC
379   if [[ "${infile}" =~ galice\.root ]]; then
380     ln -s ${inputList%/*}/* ${runpath}
381     infile=""
382   fi
383
384   #init the running path
385   mkdir -p ${runpath}
386   if [[ ! -d ${runpath} ]]; then
387    touch ${doneFileTmp}
388    echo "cannot make runpath ${runpath}" >> ${doneFileTmp}
389    cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
390     [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
391    return 1
392  fi
393   if ! cd ${runpath}; then
394     touch ${doneFileTmp}
395     echo "PWD=$PWD is not the runpath=${runpath}" >> ${doneFileTmp}
396     cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
397     [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
398     return 1
399   fi
400
401   #this is needed for runCPass1.sh
402   ln -s ${infile} ${runpath}/${chunkName}
403
404   logOutputDir=${runpath}
405   [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir}
406   [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/stdout
407   #[[ -z ${dontRedirectStdOutToLog} ]] && exec 2> ${logOutputDir}/stderr
408   echo "${0} $*"
409
410   echo "#####################"
411   echo CPass1:
412   echo JOB setup
413   echo nEvents               ${nEvents}
414   echo runNumber             ${runNumber}
415   echo ocdbPath              ${ocdbPath}
416   echo infile                ${infile}
417   echo chunkName             ${chunkName}
418   echo jobindex              ${jobindex}
419   echo recoTriggerOptions    ${recoTriggerOptions}
420   echo targetDirectory       ${targetDirectory}
421   echo commonOutputPath      ${commonOutputPath}
422   echo doneFile              ${doneFile}
423   echo runpath               ${runpath}  
424   echo outputDir             ${outputDir}
425   echo batchWorkingDirectory ${batchWorkingDirectory}
426   echo ALICE_ROOT            ${ALICE_ROOT}
427   echo PWD                   ${PWD}
428   echo "#####################"
429
430   alirootInfo > ALICE_ROOT.log
431
432   filesCPass1=( 
433                "${batchWorkingDirectory}/runCPass1.sh"
434                "${batchWorkingDirectory}/recCPass1.C"
435                "${batchWorkingDirectory}/recCPass1_OuterDet.C"
436                "${batchWorkingDirectory}/runCalibTrain.C"
437                "${batchWorkingDirectory}/QAtrain_duo.C"
438                "${batchWorkingDirectory}/localOCDBaccessConfig.C"
439                "${batchWorkingDirectory}/${configFile}"
440                "${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz"
441                "${batchWorkingDirectory}/OCDB.root"
442                "${trustedQAtrainMacro}"
443                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/runCPass1.sh"
444                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/recCPass1.C" 
445                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/recCPass1_OuterDet.C" 
446                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/runCalibTrain.C"
447                "${ALICE_ROOT}/ANALYSIS/macros/QAtrain_duo.C"
448   )
449
450   for file in "${filesCPass1[@]}"; do
451     [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
452     [[ ${file##*/} =~ .*\.sh ]] && echo "making ${file##*/} executable" && chmod +x ${file##*/}
453   done
454
455   echo "this directory (${PWD}) contents:"
456   /bin/ls
457   echo
458
459   #remove spaces around commas from calls to root
460   sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh
461
462   if [[ -n ${postSetUpActionCPass1} ]]; then
463     echo "running ${postSetUpActionCPass1}"
464     eval ${postSetUpActionCPass1}
465     echo
466   fi
467
468   #configure local OCDB storage from CPass0 (creates the localOCDBaccessConfig.C script)
469   if [[ -f cpass0.localOCDB.${runNumber}.tgz ]]; then
470     echo goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz"
471     goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz"
472   else
473     echo "WARNING: file cpass0.localOCDB.${runNumber}.tgz not found!"
474   fi
475
476   if [[ ! $(/bin/ls -1 OCDB/*/*/*/*.root 2>/dev/null) ]]; then
477     touch ${doneFileTmp}
478     echo "cpass0 produced no calibration! exiting..." >> ${doneFileTmp}
479     cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
480     [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
481     return 1
482   fi
483
484   #create the Barrel and OuterDet directories for CPass1 and link the local OCDB directory
485   #there to make the localOCDBaccessConfig.C file work, since it may point to the OCDB
486   #entries using a relative path, e.g. local://./OCDB
487   echo "linking the OCDB/ for Barrel and OuterDet directories"
488   mkdir Barrel OuterDet
489   ls -l
490   ln -s ../OCDB Barrel/OCDB
491   ln -s ../OCDB OuterDet/OCDB
492
493   #setup the filtering
494   #the following option enables the filtering task inside the QAtrain_duo.C
495   [[ -n $runESDfiltering ]] && export QA_TaskFilteredTree=1
496   #set the downscaling factors during the filtering fro expert QA (overrides the previous values)
497   if [[ -n ${filteringFactorHighPt} ]]; then
498     export AliAnalysisTaskFilteredTree_fLowPtTrackDownscaligF=${filteringFactorHighPt}
499   fi
500   if [[ -n ${filteringFactorV0s} ]]; then
501     export AliAnalysisTaskFilteredTree_fLowPtV0DownscaligF=${filteringFactorV0s} 
502   fi
503
504   #run CPass1
505   chmod u+x runCPass1.sh
506   echo "${runpath}/runCPass1.sh ${infile} ${nEvents} ${runNumber} ${ocdbPath} ${recoTriggerOptions}"
507   if [[ -n ${pretend} ]]; then
508     sleep ${pretendDelay}
509     touch AliESDs_Barrel.root
510     touch AliESDfriends_Barrel.root
511     touch AliESDfriends_v1.root
512     touch QAresults_barrel.root
513     touch EventStat_temp_barrel.root
514     touch AODtpITS.root
515     touch AliESDs_Outer.root
516     touch AliESDfriends_Outer.root
517     touch QAresults_outer.root
518     touch EventStat_temp_outer.root
519     touch rec.log
520     touch calib.log
521     touch qa.log
522     touch filtering.log FilterEvents_Trees.root
523   else
524     ./runCPass1.sh "${infile}" "${nEvents}" "${runNumber}" "${ocdbPath}" "${recoTriggerOptions}"
525     
526     [[ ! -f AliESDs_Barrel.root && -f Barrel/AliESDs.root ]] && mv Barrel/AliESDs.root AliESDs_Barrel.root
527     [[ ! -f AliESDfriends_Barrel.root && -f Barrel/AliESDfriends.root ]] && mv Barrel/AliESDfriends.root AliESDfriends_Barrel.root
528     [[ ! -f AliESDfriends_v1.root && -f Barrel/AliESDfriends_v1.root ]] && mv Barrel/AliESDfriends_v1.root .
529     [[ ! -f QAresults_barrel.root && -f Barrel/QAresults_barrel.root ]] && mv Barrel/QAresults_barrel.root .
530     [[ ! -f AliESDs_Outer.root && -f OuterDet/AliESDs.root ]] && mv OuterDet/AliESDs.root AliESDs_Outer.root
531     [[ ! -f AliESDfriends_Outer.root && -f OuterDet/AliESDfriends.root ]] && mv OuterDet/AliESDfriends.root AliESDfriends_Outer.root
532     [[ ! -f QAresults_outer.root && -f OuterDet/QAresults_outer.root ]] && mv OuterDet/QAresults_outer.root .
533     [[ ! -f FilterEvents_Trees.root && -f Barrel/FilterEvents_Trees.root ]] && mv Barrel/FilterEvents_Trees.root .
534
535     #make the filtered tree (if requested and not already produced by QA
536     [[ -f AliESDs_Barrel.root ]] && echo "AliESDs_Barrel.root" > filtered.list
537     if [[ -n ${runESDfiltering} && ! -f FilterEvents_Trees.root && -f filtered.list ]]; then 
538       goMakeFilteredTrees ${PWD} ${runNumber} "${PWD}/filtered.list" ${filteringFactorHighPt} ${filteringFactorV0s} ${ocdbPath} 1000000 0 10000000 0 ${configFile} AliESDs_Barrel.root "${extraOpts[@]}" >filtering.log
539     else
540       echo ""
541     fi
542
543   fi
544
545   ##handle possible crashes in QA (happens often in trunk)
546   ##rerun QA with a trusted aliroot version
547   #if [[ $(validateLog qa_barrel.log) ]]; then
548   #  echo "qa_barrel.log not validated!"
549   #fi
550   #if [[ ! -f QAresults_barrel.root && -f ${setupTrustedAliROOTenvInCurrentShell} || $(validateLog qa_barrel.log) ]]; then
551   #  echo "WARNING: using trusted QA aliroot ${ALICE_ROOT}"
552   #  source ${setupTrustedAliROOTenvInCurrentShell}
553   #  cd Barrel
554   #  rm QAresults_barrel.root
555   #  rm EventStat_temp_barrel.root
556   #  rm AODtpITS.root
557   #  [[ ! -f AliESDs.root ]] && ln -s ../AliESDs_Barrel.root AliESDs.root
558   #  [[ ! -f AliESDfriends.root ]] && ln -s ../AliESDfriends_Barrel.root AliESDfriends.root
559   #  if [[ -n ${trustedQAtrainMacro} ]]; then
560   #    eval "cp ${trustedQAtrainMacro} QAtrain_duo_trusted.C"
561   #  fi
562   #  echo executing aliroot -b -q "QAtrain_duo_trusted.C(\"_barrel\",${runNumber},\"wn.xml\",0,\"${ocdbPath}\")"
563   #  time aliroot -b -q "QAtrain_duo.C(\"_barrel\",${runNumber},\"wn.xml\",0,\"${ocdbPath}\")" &> ../qa_barrel_trusted.log
564   #  cd ../
565   #fi
566
567   #move stuff to final destination
568   echo "this directory (${PWD}) contents:"
569   /bin/ls
570   echo rm -f ./${chunkName}
571   rm -f ./${chunkName}
572   echo "paranoidCp ${runpath}/* ${outputDir}"
573   paranoidCp ${runpath}/* ${outputDir}
574   echo
575
576   #validate CPass1
577   cd ${outputDir}
578   if summarizeLogs >> ${doneFileTmp}; then
579     [[ -f AliESDs_Barrel.root ]] && echo "esd ${outputDir}/AliESDs_Barrel.root" >> ${doneFileTmp}
580     [[ -f AliESDfriends_v1.root ]] && echo "calibfile ${outputDir}/AliESDfriends_v1.root" >> ${doneFileTmp}
581     [[ -f QAresults_Barrel.root ]] && echo "qafile ${outputDir}/QAresults_Barrel.root" >> ${doneFileTmp}
582     [[ -f QAresults_Outer.root ]] && echo "qafile ${outputDir}/QAresults_Outer.root" >> ${doneFileTmp}
583     [[ -f QAresults_barrel.root ]] && echo "qafile ${outputDir}/QAresults_barrel.root" >> ${doneFileTmp}
584     [[ -f QAresults_outer.root ]] && echo "qafile ${outputDir}/QAresults_outer.root" >> ${doneFileTmp}
585     [[ -f FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFileTmp}
586   else
587     if grep "qa_outer.log.*OK" ${doneFileTmp} > /dev/null; then
588       [[ -f QAresults_Outer.root ]] && echo "qafile ${outputDir}/QAresults_Outer.root" >> ${doneFileTmp}
589       [[ -f QAresults_outer.root ]] && echo "qafile ${outputDir}/QAresults_outer.root" >> ${doneFileTmp}
590     fi
591     if grep "qa_barrel.log.*OK" ${doneFileTmp} > /dev/null; then
592       [[ -f QAresults_Barrel.root ]] && echo "qafile ${outputDir}/QAresults_Barrel.root" >> ${doneFileTmp}
593       [[ -f QAresults_barrel.root ]] && echo "qafile ${outputDir}/QAresults_barrel.root" >> ${doneFileTmp}
594     fi
595     if grep "filtering.log.*OK" ${doneFileTmp} > /dev/null; then
596       [[ -f FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFileTmp}
597     fi
598   fi
599
600   [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
601   cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
602   [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
603   return 0
604 )
605
606
607 goMergeCPass0()
608 (
609   #
610   # find the output files and merge them
611   #
612
613   outputDir=${1}
614   ocdbStorage=${2}
615   configFile=${3}
616   runNumber=${4}
617   calibrationFilesToMerge=${5}  #can be a non-existent file, will then be produced on the fly
618   shift 5
619   if ! parseConfig ${configFile} "$@"; then return 1; fi
620
621   #record the working directory provided by the batch system
622   batchWorkingDirectory=${PWD}
623
624   [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
625
626   # This file signals that everything went fine
627   doneFileBase="merge.cpass0.run${runNumber}.done"
628
629   # We will have two copies of the file
630   mkdir -p "${commonOutputPath}/meta" || return 1
631   doneFileTmp="${batchWorkingDirectory}/${doneFileBase}"
632   doneFile="${commonOutputPath}/meta/${doneFileBase}"
633
634   umask 0002
635   ulimit -c unlimited 
636
637   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
638
639   runpath=${outputDir}
640   [[ ${reconstructInTemporaryDir} -eq 1 ]] && runpath=$(mktemp -d -t mergeCPass0.XXXXXX)
641   [[ ${reconstructInTemporaryDir} -eq 2 ]] && runpath=${PWD}/rundir_mergeCPass0_${runNumber}
642
643   mkdir -p ${runpath}
644   if [[ ! -d ${runpath} ]]; then
645     touch ${doneFileTmp}
646     echo "not able to make the runpath ${runpath}" >> ${doneFileTmp}
647     cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
648     [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
649     return 1
650   fi
651   if ! cd ${runpath}; then 
652     touch ${doneFileTmp}
653     echo "PWD=$PWD is not the runpath=${runpath}" >> ${doneFileTmp}
654     cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
655     [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
656     return 1
657   fi
658
659   logOutputDir=${runpath}
660   [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir}
661   [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/stdout
662   echo "${0} $*"
663
664   mergingScript="mergeMakeOCDB.byComponent.sh"
665
666   echo goMergeCPass0 SETUP:
667   echo runNumber=${runNumber}
668   echo outputDir=${outputDir}
669   echo ocdbStorage=${ocdbStorage}
670   echo calibrationFilesToMerge=${calibrationFilesToMerge}
671   echo mergingScript=${mergingScript}
672   echo commonOutputPath=${commonOutputPath}
673   echo runpath=${runpath}
674   
675   # copy files in case they are not already there
676   filesMergeCPass0=(
677                     "${batchWorkingDirectory}/${calibrationFilesToMerge}"
678                     "${batchWorkingDirectory}/OCDB.root"
679                     "${batchWorkingDirectory}/localOCDBaccessConfig.C"
680                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/mergeMakeOCDB.byComponent.sh"
681                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/mergeByComponent.C"
682                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/makeOCDB.C"
683                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/merge.C"
684                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/mergeMakeOCDB.sh"
685   )
686   for file in ${filesMergeCPass0[*]}; do
687     [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
688     [[ ${file##*/} =~ .*\.sh ]] && chmod +x ${file##*/}
689   done
690   
691   sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh
692
693   alirootInfo > ALICE_ROOT.log
694
695   #
696   echo "PWD"
697   /bin/ls
698   echo "PWD/.."
699   /bin/ls ../
700
701
702   #merge calibration
703   chmod u+x ${mergingScript}  
704   mkdir -p ./OCDB
705   if [[ ! -f ${calibrationFilesToMerge} ]]; then
706     echo "/bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root > ${calibrationFilesToMerge}"
707     /bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root 2>/dev/null > ${calibrationFilesToMerge}
708   fi
709   
710   echo "${mergingScript} ${calibrationFilesToMerge} ${runNumber} local://./OCDB ${ocdbStorage}"
711   if [[ -n ${pretend} ]]; then
712     sleep ${pretendDelay}
713     touch CalibObjects.root
714     touch ocdb.log
715     touch merge.log
716     touch dcsTime.root
717     mkdir -p ./OCDB/TPC/Calib/TimeGain/
718     mkdir -p ./OCDB/TPC/Calib/TimeDrift/
719     echo "some calibration" >> ./OCDB/TPC/Calib/TimeGain/someCalibObject_0-999999_cpass0.root
720     echo "some calibration" >> ./OCDB/TPC/Calib/TimeDrift/otherCalibObject_0-999999_cpass0.root
721   else
722     ./${mergingScript} ${calibrationFilesToMerge} ${runNumber} "local://./OCDB" ${ocdbStorage} >> "mergeMakeOCDB.log"
723
724     #produce the calib trees for expert QA (dcsTime.root)
725     goMakeLocalOCDBaccessConfig ./OCDB
726     echo aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")"
727     aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")"
728   fi
729   
730   ### produce the output
731   #tar the produced OCDB for reuse
732   #tar czf ${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz ./OCDB
733
734   # Create tarball with OCDB, store on the shared directory, create signal file on batch directory
735   mkdir -p ${commonOutputPath}/meta
736   baseTar="cpass0.localOCDB.${runNumber}.tgz"
737   tar czf ${batchWorkingDirectory}/${baseTar} ./OCDB && \
738     mv ${batchWorkingDirectory}/${baseTar} ${commonOutputPath}/meta/${baseTar} && \
739     touch ${batchWorkingDirectory}/${baseTar}.done
740
741   /bin/ls
742
743   #copy all to output dir
744   echo "paranoidCp ${runpath}/* ${outputDir}"
745   paranoidCp ${runpath}/* ${outputDir}
746
747   if [[ -n ${generateMC} ]]; then
748     goPrintValues sim ${commonOutputPath}/meta/sim.run${runNumber}.list ${commonOutputPath}/meta/cpass0.job*.run${runNumber}.done
749   fi
750
751   #validate merging cpass0
752   cd ${outputDir}
753   if summarizeLogs >> ${doneFileTmp}; then
754     [[ -f CalibObjects.root ]] && echo "calibfile ${outputDir}/CalibObjects.root" >> ${doneFileTmp}
755     [[ -f dcsTime.root ]] && echo "dcsTree ${outputDir}/dcsTime.root" >> ${doneFileTmp}
756   fi
757
758   [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
759   cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
760   [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
761   return 0
762 )
763
764 goMergeCPass1()
765 (
766   #
767   # find the output files and merge them
768   #
769
770   outputDir=${1}
771   ocdbStorage=${2}
772   configFile=${3}
773   runNumber=${4}
774   calibrationFilesToMerge=${5}
775   qaFilesToMerge=${6}
776   filteredFilesToMerge=${7}
777   shift 7
778   if ! parseConfig ${configFile} "$@"; then return 1; fi
779
780   #record the working directory provided by the batch system
781   batchWorkingDirectory=${PWD}
782
783   [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
784
785   # This file signals that everything went fine
786   doneFileBase="merge.cpass1.run${runNumber}.done"
787
788   # We will have two copies of the file
789   mkdir -p "${commonOutputPath}/meta" || return 1
790   doneFileTmp="${batchWorkingDirectory}/${doneFileBase}"
791   doneFile="${commonOutputPath}/meta/${doneFileBase}"
792
793   umask 0002
794   ulimit -c unlimited 
795
796   #clean up first:
797   rm -f ${outputDir}/*.log
798   rm -f ${outputDir}/*.root
799   rm -f ${outputDir}/*done
800
801   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
802
803   runpath=${outputDir}
804   [[ ${reconstructInTemporaryDir} -eq 1 ]] && runpath=$(mktemp -d -t mergeCPass1.XXXXXX)
805   [[ ${reconstructInTemporaryDir} -eq 2 ]] && runpath=${PWD}/rundir_mergeCPass1_${runNumber}
806
807   mkdir -p ${runpath}
808   if [[ ! -d ${runpath} ]]; then
809     touch ${doneFileTmp}
810     echo "not able to make the runpath ${runpath}" >> ${doneFileTmp}
811     cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
812     [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
813     return 1
814   fi
815   if ! cd ${runpath}; then 
816     touch ${doneFileTmp}
817     echo "PWD=$PWD is not the runpath=${runpath}" >> ${doneFileTmp}
818     cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
819     [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
820     return 1
821   fi
822
823   logOutputDir=${runpath}
824   [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir}
825   [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/mergeMakeOCDB.log
826   echo "${0} $*"
827
828   calibrationOutputFileName='AliESDfriends_v1.root'
829   qaOutputFileName='QAresults*.root'
830   mergingScript="mergeMakeOCDB.byComponent.sh"
831   #important to have the string "Stage.txt" in the filename to trigger the merging
832   #it has to be a list of directories containing the files
833   qaMergedOutputFileName="QAresults_merged.root"
834
835   echo goMergeCPass1 SETUP:
836   echo runNumber=${runNumber}
837   echo outputDir=${outputDir}
838   echo ocdbStorage=${ocdbStorage}
839   echo calibrationFilesToMerge=$calibrationFilesToMerge
840   echo qaFilesToMerge=$qaFilesToMerge
841   echo calibrationOutputFileName=${calibrationOutputFileName}
842   echo mergingScript=${mergingScript}
843   
844   # copy files in case they are not already there
845   filesMergeCPass1=(
846                     "${batchWorkingDirectory}/${calibrationFilesToMerge}"
847                     "${batchWorkingDirectory}/${qaFilesToMerge}"
848                     "${batchWorkingDirectory}/OCDB.root"
849                     "${batchWorkingDirectory}/localOCDBaccessConfig.C"
850                     "${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz"
851                     "${batchWorkingDirectory}/QAtrain_duo.C"
852                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/mergeMakeOCDB.byComponent.sh"
853                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/mergeByComponent.C"
854                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/makeOCDB.C"
855                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/merge.C"
856                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/mergeMakeOCDB.sh"
857                     "${trustedQAtrainMacro}"
858                     "${ALICE_ROOT}/ANALYSIS/macros/QAtrain_duo.C"
859   )
860   for file in ${filesMergeCPass1[*]}; do
861     [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
862     [[ ${file##*/} =~ .*\.sh ]] && chmod +x ${file##*/}
863   done
864
865   sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh
866
867   #configure local OCDB storage from CPass0 (creates the localOCDBaccessConfig.C script)
868   if [[ -f cpass0.localOCDB.${runNumber}.tgz ]]; then
869     echo goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz"
870     goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz"
871   else
872     echo "WARNING: file cpass0.localOCDB.${runNumber}.tgz not found!"
873   fi
874
875   alirootInfo > ALICE_ROOT.log
876
877   #
878   /bin/ls
879
880   #merge calibration
881   chmod u+x ${mergingScript}  
882   mkdir -p OCDB
883   
884   #if not provided, create the lists of files to merge
885   if [[ ! -f ${filteredFilesToMerge} ]]; then
886     echo "/bin/ls -1 ${outputDir}/*/FilterEvents_Trees.root > ${filteredFilesToMerge}"
887     /bin/ls -1 ${outputDir}/*/FilterEvents_Trees.root 2>/dev/null > ${filteredFilesToMerge}
888   fi
889   if [[ ! -f ${calibrationFilesToMerge} ]]; then
890     echo "/bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root > ${calibrationFilesToMerge}"
891     /bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root 2>/dev/null > ${calibrationFilesToMerge}
892   fi
893   if [[ ! -f ${qaFilesToMerge} ]]; then
894     #find the files, but only store the directories (QAtrain_duo.C requires this)
895     echo "/bin/ls -1 ${outputDir}/*/QAresults*.root | while read x; do echo ${x%/*}; done | sort | uniq > ${qaFilesToMerge}"
896     /bin/ls -1 ${outputDir}/*/QAresults*.root | while read x; do echo ${x%/*}; done | sort | uniq > ${qaFilesToMerge}
897   fi
898   
899   echo "${mergingScript} ${calibrationFilesToMerge} ${runNumber} local://./OCDB ${ocdbStorage}"
900   if [[ -n ${pretend} ]]; then
901     sleep ${pretendDelay}
902     touch ocdb.log
903     touch cpass1.localOCDB.${runNumber}.tgz
904     touch ${qaMergedOutputFileName}
905     touch merge.log
906     touch trending.root
907     touch FilterEvents_Trees.root
908     touch CalibObjects.root
909     touch dcsTime.root
910     touch ${qaMergedOutputFileName}
911     mkdir -p OCDB
912   else
913     ./${mergingScript} ${calibrationFilesToMerge} ${runNumber} "local://./OCDB" ${ocdbStorage}
914
915     #merge QA (and filtered trees)
916     [[ -n ${AliAnalysisTaskFilteredTree_fLowPtTrackDownscaligF} ]] && export AliAnalysisTaskFilteredTree_fLowPtTrackDownscaligF
917     [[ -n ${AliAnalysisTaskFilteredTree_fLowPtV0DownscaligF} ]] && export AliAnalysisTaskFilteredTree_fLowPtV0DownscaligF
918
919     #echo aliroot -l -b -q "merge.C(\"${qaFilesToMerge}\",\"\",kFALSE,\"${qaMergedOutputFileName}\")"
920     echo aliroot -b -q "QAtrain_duo.C(\"_barrel\",${runNumber},\"${qaFilesToMerge}\",1,\"${ocdbStorage}\")"
921     #aliroot -l -b -q "merge.C(\"${qaFilesToMerge}\",\"\",kFALSE,\"${qaMergedOutputFileName}\")"
922     aliroot -b -q "QAtrain_duo.C(\"_barrel\",${runNumber},\"${qaFilesToMerge}\",1,\"${ocdbStorage}\")" > mergeQA.log
923     mv QAresults_barrel.root ${qaMergedOutputFileName}
924     mv trending_barrel.root trending.root
925
926     #merge filtered trees
927     echo aliroot -l -b -q "merge.C(\"${qaFilesToMerge}\",\"\",kFALSE,\"${qaMergedOutputFileName}\")"
928     aliroot -l -b -q "merge.C(\"${filteredFilesToMerge}\",\"\",kFALSE,\"FilterEvents_Trees.root\")" > mergeFilteredTrees.log
929
930     #produce the calib trees for expert QA
931     echo aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")"
932     aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")" > calibTree.log
933   fi
934
935   # Create tarball with OCDB, store on the shared directory, create signal file on batch directory
936   mkdir -p ${commonOutputPath}/meta
937   baseTar="cpass1.localOCDB.${runNumber}.tgz"
938   tar czf ${batchWorkingDirectory}/${baseTar} ./OCDB && \
939     mv ${batchWorkingDirectory}/${baseTar} ${commonOutputPath}/meta/${baseTar} && \
940     touch ${batchWorkingDirectory}/${baseTar}.done
941
942   /bin/ls
943
944   #copy all to output dir
945   echo "paranoidCp ${runpath}/* ${outputDir}"
946   paranoidCp ${runpath}/* ${outputDir}
947   
948   #validate merge cpass1
949   cd ${outputDir}
950   if summarizeLogs >>  ${doneFileTmp}; then
951     [[ -f CalibObjects.root ]] && echo "calibfile ${outputDir}/CalibObjects.root" >> ${doneFileTmp}
952     [[ -f ${qaMergedOutputFileName} ]] && echo "qafile ${outputDir}/${qaMergedOutputFileName}" >> ${doneFileTmp}
953     [[ -f trending.root ]] && echo "trendingfile ${outputDir}/trending.root" >> ${doneFileTmp}
954     [[ -f dcsTime.root ]] && echo "dcsTree ${outputDir}/dcsTime.root" >> ${doneFileTmp}
955     [[ -f FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFileTmp}
956   else
957     if grep "mergeQA.log.*OK" ${doneFileTmp} > /dev/null; then
958       [[ -f ${qaMergedOutputFileName} ]] && echo "qafile ${outputDir}/${qaMergedOutputFileName}" >> ${doneFileTmp}
959     fi
960     if grep "mergeFilteredTrees.log.*OK" ${doneFileTmp} > /dev/null; then
961       [[ -f FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFileTmp}
962     fi
963   fi
964       
965   [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
966   cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
967   [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
968   return 0
969 )
970
971 goMerge()
972 (
973   #generic root merge using CPass1 merge.C script
974   inputList=${1}
975   outputFile=${2}  
976   configFile=${3-"benchmark.config"}
977   shift 3
978   if ! parseConfig ${configFile} "$@"; then return 1; fi
979   
980   #record the working directory provided by the batch system
981   batchWorkingDirectory=${PWD}
982
983   [[ ! -f ${inputList} ]] && echo "inputList ${inputList} does not exist!" && return 1
984   [[ ! -f ${configFile} ]] && echo "configFile ${configFile} does not exist!" && return 1
985   umask 0002
986   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
987   rm -f ${outputFile}
988   aliroot -b -q "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/merge.C(\"${inputList}\",\"\",kFALSE,\"${outputFile}\")" > merge_${inputList}.log
989   return 0
990 )
991
992 goSubmitMakeflow()
993 {
994   #run
995   productionID=${1}
996   inputFileList=${2}
997   configFile=${3}
998   shift 3
999   extraOpts=("$@")
1000   if ! parseConfig ${configFile} "${extraOpts[@]}"; then return 1; fi
1001  
1002   #record the working directory provided by the batch system
1003   batchWorkingDirectory=${PWD}
1004
1005   [[ -z ${configFile} ]] && configFile="benchmark.config"
1006   [[ ! -f ${configFile} ]] && echo "no config file found (${configFile})" && return 1
1007
1008   if [[ ! $(which makeflow &>/dev/null) && -n ${makeflowPath} ]]; then
1009     echo "setting the makflow path from the config: "
1010     echo "  export PATH=${makeflowPath}:${PATH}"
1011     export PATH=${makeflowPath}:${PATH}
1012   fi
1013
1014   #create the common output dir and the meta dir
1015   commonOutputPath=${baseOutputDirectory}/${productionID}
1016   if [[ -d ${commonOutputPath} ]]; then
1017     echo "output dir ${commonOutputPath} exists!"
1018     #return 1
1019   else
1020     mkdir -p ${commonOutputPath}
1021   fi
1022   mkdir -p ${commonOutputPath}/meta
1023   
1024   self=${0}
1025   #if which greadlink; then self=$(greadlink -f "${0}"); fi
1026   
1027   #for reference copy the setup to the output dir
1028   cp ${self} ${commonOutputPath}
1029   cp ${configFile} ${commonOutputPath}
1030   cp ${inputFileList} ${commonOutputPath}
1031
1032   #submit - use makeflow if available, fall back to old stuff when makeflow not there
1033   if which makeflow; then
1034     goGenerateMakeflow ${productionID} ${inputFileList} ${configFile} "${extraOpts[@]}" commonOutputPath=${commonOutputPath} > benchmark.makeflow
1035     cp benchmark.makeflow ${commonOutputPath}
1036     makeflow ${makeflowOptions} benchmark.makeflow
1037   else 
1038     echo "no makeflow!"
1039   fi
1040   
1041   #summarize the run based on the makeflow log
1042   #and add it to the end of summary log
1043   awk '/STARTED/   {startTime=$3} 
1044        /COMPLETED/ {endTime=$3} 
1045        END         {print "makeflow running time: "(endTime-startTime)/1000000/3600" hours"}' \
1046       benchmark.makeflow.makeflowlog | tee -a summary.log
1047   paranoidCp summary.log ${commonOutputPath}
1048
1049   return 0
1050 }
1051
1052 goGenerateMakeflow()
1053 (
1054   #generate the makeflow file
1055   [[ $# -lt 3 ]] && echo "args: id inputFileList configFile" && return 1
1056   productionID=${1}
1057   inputFileList=${2}
1058   configFile=${3}
1059   shift 3
1060   extraOpts=("$@")
1061   
1062   #batch systems/makeflow sometimes handle spaces in arguments poorly, so encode them
1063   for (( i=0;i<${#extraOpts[@]};i++ )); do 
1064     extraOpts[i]=$(encSpaces "${extraOpts[i]}")
1065   done
1066   extraOpts+=("encodedSpaces=1")
1067
1068   if ! parseConfig ${configFile} "${extraOpts[@]}" &>/dev/null; then return 1; fi
1069  
1070   #extra safety
1071   if [[ -z ${commonOutputPath} ]]; then
1072     commonOutputPath=${baseOutputDirectory}/${productionID}
1073     extraOpts=( "${extraOpts[@]}" "commonOutputPath=${commonOutputPath}" )
1074   fi
1075
1076   #record the working directory provided by the batch system
1077   batchWorkingDirectory=${PWD}
1078
1079   [[ -z ${configFile} ]] && configFile="benchmark.config"
1080   [[ ! -f ${configFile} ]] && echo "no config file found (${configFile})" && return 1
1081
1082   #these files will be made a dependency - will be copied to the working dir of the jobs
1083   declare -a copyFiles
1084   inputFiles=(
1085               "OCDB.root"
1086               "localOCDBaccessConfig.C"
1087               "QAtrain_duo.C"
1088               "runCPass1.sh"
1089               "recCPass1.C"
1090               "recCPass1_OuterDet.C"
1091               "runCalibTrain.C"
1092               "runCPass0.sh"
1093               "recCPass0.C"
1094               "runQA.sh"
1095   )
1096   for file in ${inputFiles[*]}; do
1097     [[ -f ${file} ]] && copyFiles+=("${file}")
1098   done
1099
1100   #create the makeflow file
1101   [[ -n ${batchFlags} ]] && echo "BATCH_OPTIONS = ${batchFlags}"
1102   declare -A arr_cpass0_merged arr_cpass1_merged
1103   declare -A arr_cpass0_calib_list arr_cpass1_calib_list 
1104   declare -A arr_cpass1_QA_list arr_cpass1_ESD_list arr_cpass1_filtered_list
1105   declare -A arr_cpass0_profiled_outputs
1106   declare -A listOfRuns
1107   [[ -n ${runNumber} ]] && listOfRuns[${runNumber}]=1
1108   while read x; do tmpRun=$(guessRunNumber ${x}); [[ -n ${tmpRun} ]] && listOfRuns[${tmpRun}]=1; done < ${inputFileList}
1109   for runNumber in "${!listOfRuns[@]}"; do
1110     [[ -z ${runNumber} ]] && continue
1111     [[ ! ${runNumber} =~ ^[0-9]*[0-9]$ ]] && continue
1112
1113     unset arr_cpass0_outputs
1114     unset arr_cpass1_outputs
1115     declare -a arr_cpass0_outputs
1116     declare -a arr_cpass1_outputs
1117
1118     #Header
1119     echo "### Automatically generated on $(LANG=C date) ###"
1120     echo ; echo
1121
1122     jobindex=0
1123     inputFile=""
1124     while read inputFile; do
1125       currentDefaultOCDB=${defaultOCDB}
1126       [[ -z ${autoOCDB} ]] && autoOCDB=1
1127       if [[ ${autoOCDB} -ne 0 ]]; then
1128         currentDefaultOCDB=$(setYear ${inputFile} ${defaultOCDB})
1129       fi
1130       guessRunData ${inputFile}
1131
1132       #Set variables
1133       echo "### Variables ###"
1134       echo "OUTPATH=\"${commonOutputPath}/${year}/${period}\""
1135       echo ; echo
1136
1137       #CPass0
1138       #arr_cpass0_outputs[${jobindex}]="${commonOutputPath}/meta/cpass0.job${jobindex}.run${runNumber}.done"
1139       arr_cpass0_outputs[${jobindex}]="cpass0.job${jobindex}.run${runNumber}.done"
1140       echo "### CPass0 ###"
1141       echo "${arr_cpass0_outputs[${jobindex}]}: benchmark.sh ${configFile} ${copyFiles[@]}"
1142       echo "    ${alirootEnv} ./benchmark.sh CPass0 \$OUTPATH/000${runNumber}/cpass0 ${inputFile} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} ${jobindex} ${extraOpts[@]}"" "
1143       echo ; echo
1144
1145       #CPass1
1146       #arr_cpass1_outputs[${jobindex}]="${commonOutputPath}/meta/cpass1.job${jobindex}.run${runNumber}.done"
1147       arr_cpass1_outputs[${jobindex}]="cpass1.job${jobindex}.run${runNumber}.done"
1148       echo "### CPass1 ###"
1149       echo "${arr_cpass1_outputs[${jobindex}]}: benchmark.sh ${configFile} cpass0.localOCDB.${runNumber}.tgz.done ${copyFiles[@]}"
1150       echo "    ${alirootEnv} ./benchmark.sh CPass1 \$OUTPATH/000${runNumber}/cpass1 ${inputFile} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} ${jobindex} ${extraOpts[@]}"" "
1151       echo ; echo
1152       ((jobindex++))
1153
1154     done< <(grep "/000${runNumber}/" ${inputFileList})
1155     
1156     #CPass0 list of Calib files to merge
1157     #arr_cpass0_calib_list[${runNumber}]="${commonOutputPath}/meta/cpass0.calib.run${runNumber}.list"
1158     arr_cpass0_calib_list[${runNumber}]="cpass0.calib.run${runNumber}.list"
1159     echo "### Produces the list of CPass0 files to merge (executes locally) ###"
1160     echo "${arr_cpass0_calib_list[${runNumber}]}: benchmark.sh ${arr_cpass0_outputs[*]}"
1161     echo "    LOCAL ./benchmark.sh PrintValues calibfile ${arr_cpass0_calib_list[${runNumber}]} ${arr_cpass0_outputs[*]} && mkdir -p \$OUTPATH/meta && cp ${arr_cpass0_calib_list[${runNumber}]} \$OUTPATH/meta/${arr_cpass0_calib_list[${runNumber}]}"
1162     echo ; echo
1163
1164     #CPass0 merging
1165     echo "### Merges CPass0 files ###"
1166     #arr_cpass0_merged[${runNumber}]="${commonOutputPath}/meta/merge.cpass0.run${runNumber}.done"
1167     arr_cpass0_merged[${runNumber}]="merge.cpass0.run${runNumber}.done"
1168     echo "cpass0.localOCDB.${runNumber}.tgz.done ${arr_cpass0_merged[${runNumber}]}: benchmark.sh ${configFile} ${arr_cpass0_calib_list[${runNumber}]} ${copyFiles[@]}"
1169     echo "    ${alirootEnv} ./benchmark.sh MergeCPass0 \$OUTPATH/000${runNumber}/cpass0 ${currentDefaultOCDB} ${configFile} ${runNumber} ${arr_cpass0_calib_list[${runNumber}]} ${extraOpts[@]}"" "
1170     echo ; echo
1171
1172     #CPass1 list of Calib/QA/ESD/filtered files
1173     # the trick with QA is to have the string "Stage.txt" in the file name of the list of directories with QA output to trigger
1174     # the production of the QA trending tree (only then the task->Finish() will be called in QAtrain_duo.C, on the grid
1175     # this corresponds to the last merging stage)
1176     #arr_cpass1_QA_list[${runNumber}]="${commonOutputPath}/meta/cpass1.QA.run${runNumber}.lastMergingStage.txt.list"
1177     arr_cpass1_QA_list[${runNumber}]="cpass1.QA.run${runNumber}.lastMergingStage.txt.list"
1178     echo "### Lists CPass1 QA ###"
1179     echo "${arr_cpass1_QA_list[${runNumber}]}: benchmark.sh ${arr_cpass1_outputs[*]}"
1180     echo "    LOCAL ./benchmark.sh PrintValues dir ${arr_cpass1_QA_list[${runNumber}]} ${arr_cpass1_outputs[*]} && mkdir -p \$OUTPATH/meta && cp ${arr_cpass1_QA_list[${runNumber}]} \$OUTPATH/meta/${arr_cpass1_QA_list[${runNumber}]}"
1181     echo ; echo
1182
1183     #arr_cpass1_calib_list[${runNumber}]="${commonOutputPath}/meta/cpass1.calib.run${runNumber}.list"
1184     arr_cpass1_calib_list[${runNumber}]="cpass1.calib.run${runNumber}.list"
1185     echo "### Lists CPass1 Calib ###"
1186     echo "${arr_cpass1_calib_list[${runNumber}]}: benchmark.sh ${arr_cpass1_outputs[*]}"
1187     echo "    LOCAL ./benchmark.sh PrintValues calibfile ${arr_cpass1_calib_list[${runNumber}]} ${arr_cpass1_outputs[*]} && mkdir -p \$OUTPATH/meta && cp ${arr_cpass1_calib_list[${runNumber}]} \$OUTPATH/meta/${arr_cpass1_calib_list[${runNumber}]}"
1188     echo ; echo
1189
1190     #arr_cpass1_ESD_list[${runNumber}]="${commonOutputPath}/meta/cpass1.ESD.run${runNumber}.list"
1191     arr_cpass1_ESD_list[${runNumber}]="cpass1.ESD.run${runNumber}.list"
1192     echo "### Lists CPass1 ESDs ###"
1193     echo "${arr_cpass1_ESD_list[${runNumber}]}: benchmark.sh ${arr_cpass1_outputs[*]}"
1194     echo "    LOCAL ./benchmark.sh PrintValues esd ${arr_cpass1_ESD_list[${runNumber}]} ${arr_cpass1_outputs[*]} && mkdir -p \$OUTPATH/meta && cp ${arr_cpass1_ESD_list[${runNumber}]} \$OUTPATH/meta/${arr_cpass1_ESD_list[${runNumber}]}"
1195     echo ; echo
1196
1197     #arr_cpass1_filtered_list[${runNumber}]="${commonOutputPath}/meta/cpass1.filtered.run${runNumber}.list"
1198     arr_cpass1_filtered_list[${runNumber}]="cpass1.filtered.run${runNumber}.list"
1199     echo "### Lists CPass1 filtered ###"
1200     echo "${arr_cpass1_filtered_list[${runNumber}]}: benchmark.sh ${arr_cpass1_outputs[*]}"
1201     echo "    LOCAL ./benchmark.sh PrintValues filteredTree ${arr_cpass1_filtered_list[${runNumber}]} ${arr_cpass1_outputs[*]} && mkdir -p \$OUTPATH/meta && cp ${arr_cpass1_filtered_list[${runNumber}]} \$OUTPATH/meta/${arr_cpass1_filtered_list[${runNumber}]}"
1202     echo ; echo
1203   
1204     #CPass1 merging
1205     #arr_cpass1_merged[${runNumber}]="${commonOutputPath}/meta/merge.cpass1.run${runNumber}.done"
1206     arr_cpass1_merged[${runNumber}]="merge.cpass1.run${runNumber}.done"
1207     echo "### Merges CPass1 files ###"
1208     echo "cpass1.localOCDB.${runNumber}.tgz.done ${arr_cpass1_merged[${runNumber}]}: benchmark.sh ${configFile} ${arr_cpass1_calib_list[${runNumber}]} ${arr_cpass1_QA_list[${runNumber}]} ${copyFiles[@]}"
1209     echo "    ${alirootEnv} ./benchmark.sh MergeCPass1 \$OUTPATH/000${runNumber}/cpass1 ${currentDefaultOCDB} ${configFile} ${runNumber} ${arr_cpass1_calib_list[${runNumber}]} ${arr_cpass1_QA_list[${runNumber}]} ${arr_cpass1_filtered_list[${runNumber}]} ${extraOpts[@]}"" "
1210     echo ; echo
1211
1212     #CPass0 wrapped in a profiling tool (valgrind,....)
1213     if [[ -n ${profilingCommand} ]]; then
1214       inputFile=$(grep -m1 "${runNumber}/" ${inputFileList})
1215       [[ -z ${nEventsProfiling} ]] && nEventsProfiling=2
1216       currentDefaultOCDB=$(setYear ${inputFile} ${defaultOCDB})
1217       jobindex="profiling"
1218
1219       #arr_cpass0_profiled_outputs[${runNumber}]="${commonOutputPath}/meta/profiling.cpass0.job${jobindex}.run${runNumber}.done"
1220       arr_cpass0_profiled_outputs[${runNumber}]="profiling.cpass0.job${jobindex}.run${runNumber}.done"
1221       echo "### CPass0 in a profiler ###"
1222       echo "${arr_cpass0_profiled_outputs[${runNumber}]}: benchmark.sh ${configFile} ${copyFiles[@]}"
1223       profilingCommand=$(encSpaces "${profilingCommand}")
1224       echo "    ${alirootEnv} ./benchmark.sh CPass0 \$OUTPATH/000${runNumber}/${jobindex} ${inputFile} ${nEventsProfiling} ${currentDefaultOCDB} ${configFile} ${runNumber} ${jobindex} ${extraOpts[@]} useProfilingCommand=${profilingCommand}"
1225       echo ; echo
1226     fi
1227
1228   done #runs
1229
1230   #Summary
1231   echo "### Summary ###"
1232   echo "summary.log: benchmark.sh ${configFile} ${arr_cpass1_merged[*]}"
1233   echo "     ${alirootEnv} ./benchmark.sh MakeSummary ${configFile} ${extraOpts[@]}"
1234   echo ; echo
1235
1236   return 0
1237 )
1238
1239 goPrintValues()
1240 (
1241   #print the values given the key from any number of files (store in output file on second argument)
1242   if [[ $# -lt 3 ]]; then
1243     echo "goPrintValues key outputFile inputFiles"
1244     echo "if outputFile is \"-\" don't redirect to a file"
1245     return
1246   fi
1247   key=${1}
1248   outputFile=${2}
1249   [[ ${outputFile} =~ "-" ]] && outputFile=""
1250   shift 2 #remove 2 first arguments from arg list to only pass the input files to awk
1251   awk -v key=${key} '$0 ~ key" " {print $2}' "$@" | tee ${outputFile}
1252   return 0
1253 )
1254
1255 goCreateQAplots()
1256 (
1257   umask 0002
1258   mergedQAfileList=${1}
1259   productionID=${2}
1260   outputDir=${3}
1261   configFile=${4}
1262   shift 4
1263   if ! parseConfig ${configFile} "$@"; then return 1; fi
1264   
1265   #record the working directory provided by the batch system
1266   batchWorkingDirectory=${PWD}
1267
1268   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
1269
1270   [[ -z ${logOutputDir} ]] && logOutputDir=${PWD}
1271   [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/makeQAplots.log
1272   echo "${0} $*"
1273
1274   olddir=${PWD}
1275   mkdir -p ${outputDir}
1276   cd ${outputDir}
1277   [[ ! "${PWD}" =~ "${outputDir}" ]] && echo "PWD is not equal to outputDir=${outputDir}" && cd ${olddir} && return 1
1278
1279   inputFiles=(
1280               "${batchWorkingDirectory}/runQA.sh"
1281               "${ALICE_ROOT}/PWGPP/QA/scripts/runQA.sh"
1282   )
1283   for file in ${inputFiles[*]}; do
1284     [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
1285   done
1286
1287   echo "running QA with command:"
1288   echo ./runQA.sh inputList="${mergedQAfileList}" inputListHighPtTrees="${filteringList}" ocdbStorage="${defaultOCDB}"
1289   ./runQA.sh inputList="${mergedQAfileList}" inputListHighPtTrees="${filteringList}" ocdbStorage="${defaultOCDB}"
1290   cd ${olddir}
1291   return 0
1292 )
1293
1294 goTest()
1295 (
1296   echo AA
1297 )
1298
1299 alirootInfo()
1300 (
1301   umask 0002
1302   # save aliroot repository info
1303   [[ -z "${ALICE_ROOT}" ]] && return 1
1304   
1305   echo "\${ALICE_ROOT}=${ALICE_ROOT}"
1306   echo "\${ROOTSYS}=${ROOTSYS}"
1307   echo "\${PATH}=${PATH}"
1308   echo "\${LD_LIBRARY_PATH}=${LD_LIBRARY_PATH}"
1309   echo
1310   
1311   pushd ${PWD}
1312   cd ${ALICE_ROOT}
1313
1314   currentBranch=$(git rev-parse --abbrev-ref HEAD)
1315   git status
1316   echo ""
1317   echo ""
1318   git diff ${currentBranch}
1319   popd
1320   return 0
1321 )
1322
1323 setYear()
1324 (
1325   #set the year
1326   #  ${1} - year to be set
1327   #  ${2} - where to set the year
1328   year1=$(guessYear ${1})
1329   year2=$(guessYear ${2})
1330   local path=${2}
1331   [[ ${year1} -ne ${year2} && -n ${year2} && -n ${year1} ]] && path=${2/\/${year2}\//\/${year1}\/}
1332   echo ${path}
1333   return 0
1334 )
1335
1336 guessPeriod()
1337 (
1338   #guess the period from the path, pick the rightmost one
1339   local IFS="/"
1340   declare -a path=( ${1} )
1341   local dirDepth=${#path[*]}
1342   for ((x=${dirDepth}-1;x>=0;x--)); do
1343     local field=${path[${x}]}
1344     [[ ${field} =~ ^LHC[0-9][0-9][a-z]$ ]] && period=${field} && break
1345   done
1346   echo ${period}
1347   return 0
1348 )
1349
1350 guessYear()
1351 (
1352   #guess the year from the path, pick the rightmost one
1353   local IFS="/"
1354   declare -a path=( ${1} )
1355   local dirDepth=${#path[*]}
1356   for ((x=${dirDepth}-1;x>=0;x--)); do
1357     local field=${path[${x}]}
1358     [[ ${field} =~ ^20[0-9][0-9]$ ]] && year=${field} && break
1359   done
1360   echo ${year}
1361   return 0
1362 )
1363
1364 guessRunNumber()
1365 (
1366   #guess the run number from the path, pick the rightmost one
1367   #works for /path/foo/000123456/bar/...
1368   #and       /path/foo.run123456.bar
1369   local IFS="/."
1370   declare -a path=( ${1} )
1371   local dirDepth=${#path[*]}
1372   for ((x=${dirDepth}-1;x>=0;x--)); do
1373     local field=${path[${x}]}
1374     field=${field/run/000}
1375     [[ ${field} =~ [0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && runNumber=${field#000} && break
1376   done
1377   echo ${runNumber}
1378   return 0
1379 )
1380
1381 validateLog()
1382 (
1383   log=${1}
1384   errorConditions=(
1385                     'There was a crash'
1386                     'floating'
1387                     'error while loading shared libraries'
1388                     'std::bad_alloc'
1389                     's_err_syswatch_'
1390                     'Thread [0-9]* (Thread'
1391                     'AliFatal'
1392                     'core dumped'
1393                     '\.C.*error:.*\.h: No such file'
1394                     'line.*Aborted'
1395   )
1396
1397   warningConditions=(
1398                      'This is serious !'
1399                      'rocVoltage out of range:'
1400   )
1401   
1402   local logstatus=0
1403   local errorSummary=""
1404   local warningSummary=""
1405
1406   for ((i=0; i<${#errorConditions[@]};i++)); do
1407     local tmp=$(grep -m1 -e "${errorConditions[${i}]}" ${log})
1408     [[ -n ${tmp} ]] && tmp+=" : "
1409     errorSummary+=${tmp}
1410   done
1411
1412   for ((i=0; i<${#warningConditions[@]};i++)); do
1413     local tmp=$(grep -m1 -e "${warningConditions[${i}]}" ${log})
1414     [[ -n ${tmp} ]] && tmp+=" : "
1415     warningSummary+=${tmp}
1416   done
1417
1418   if [[ -n ${errorSummary} ]]; then 
1419     echo "${errorSummary}"
1420     return 1
1421   fi
1422   
1423   if [[ -n ${warningSummary} ]]; then
1424     echo "${warningSummary}"
1425     return 2
1426   fi
1427
1428   return 0
1429 )
1430
1431 summarizeLogs()
1432 (
1433   #print a summary of logs
1434   logFiles=(
1435             "*.log"
1436             "stdout"
1437             "stderr"
1438   )
1439
1440   #put dir information in the output
1441   echo "dir $PWD"
1442
1443   #check logs
1444   local logstatus=0
1445   for log in ${logFiles[*]}; do
1446     finallog=${PWD%/}/${log}
1447     [[ ! -f ${log} ]] && continue
1448     errorSummary=$(validateLog ${log})
1449     validationStatus=$?
1450     if [[ ${validationStatus} -eq 0 ]]; then 
1451       #in pretend mode randomly report an error in rec.log some cases
1452       if [[ -n ${pretend} && "${log}" == "rec.log" ]]; then
1453         #[[ $(( ${RANDOM}%2 )) -ge 1 ]] && echo "${finallog} BAD random error" || echo "${finallog} OK"
1454         echo "${finallog} OK"
1455       else
1456         echo "${finallog} OK"
1457       fi
1458     elif [[ ${validationStatus} -eq 1 ]]; then
1459       echo "${finallog} BAD ${errorSummary}"
1460       logstatus=1
1461     elif [[ ${validationStatus} -eq 2 ]]; then
1462       echo "${finallog} OK MWAH ${errorSummary}"
1463     fi
1464   done
1465   
1466   #report core files
1467   while read x; do
1468     echo ${x}
1469     chmod 644 ${x}
1470     gdb --batch --quiet -ex "bt" -ex "quit" aliroot ${x} > stacktrace_${x//\//_}.log
1471   done < <(/bin/ls ${PWD}/*/core 2>/dev/null; /bin/ls ${PWD}/core 2>/dev/null)
1472   
1473   return ${logstatus}
1474 )
1475
1476 spitOutLocalOCDBaccessConfig()
1477 {
1478   umask 0002
1479   #find ${1} -name "*root" | \
1480   /bin/ls -1 ${1}/*/*/*/*.root 2>/dev/null | \
1481   while read line
1482   do 
1483     local tmp=${line#${1}}
1484     echo ${tmp%/*} | \
1485     awk -v ocdb=${1} '{print "  man->SetSpecificStorage(\""$1"\",\"local://"ocdb"\");"}'
1486   done
1487   return 0
1488 }
1489
1490 goMakeLocalOCDBaccessConfig()
1491 {
1492   umask 0002
1493   # make a script that sets the specific storages form all the root files produced by CPass0
1494   local localOCDBpathCPass0=${1}
1495   local OCDBpathPrefix=${2}
1496   [[ -z ${OCDBpathPrefix} ]] && OCDBpathPrefix="."
1497
1498   if [[ -f ${localOCDBpathCPass0} && ${localOCDBpathCPass0} =~ \.tgz$ ]]; then
1499     tar xzf ${localOCDBpathCPass0}
1500     local localOCDBpathCPass0="${OCDBpathPrefix}/OCDB"
1501   fi
1502
1503   echo
1504   echo creating the specific storage script
1505   echo   localOCDBaccessConfig.C
1506   echo   based on OCDB: ${localOCDBaccessConfig}
1507   echo
1508
1509   local tempLocalOCDB=""
1510   if [[ -f localOCDBaccessConfig.C ]]; then
1511     tempLocalOCDB=$(mktemp -t tempLocalOCDB.XXXXXX)
1512     echo "egrep "SetSpecificStorage" localOCDBaccessConfig.C > ${tempLocalOCDB}"
1513     egrep "SetSpecificStorage" localOCDBaccessConfig.C > ${tempLocalOCDB}
1514   fi
1515
1516   echo "localOCDBaccessConfig()"                               >  localOCDBaccessConfig.C
1517   echo "{"                                                     >> localOCDBaccessConfig.C
1518   echo "  AliCDBManager* man = AliCDBManager::Instance();"     >> localOCDBaccessConfig.C
1519   spitOutLocalOCDBaccessConfig ${localOCDBpathCPass0}|sort|uniq  >> localOCDBaccessConfig.C
1520   [[ -f "${tempLocalOCDB}" ]] && cat ${tempLocalOCDB}              >> localOCDBaccessConfig.C
1521   echo "}"                                                     >> localOCDBaccessConfig.C
1522
1523   [[ -f "${tempLocalOCDB}" ]] && rm -f ${tempLocalOCDB}
1524
1525   if ! grep SetSpecificStorage localOCDBaccessConfig.C; then 
1526     echo
1527     echo "!!!!!!! CPass0 produced no OCDB entries"
1528     return 1
1529   fi
1530   return 0
1531 }
1532
1533 goMakeFilteredTrees()
1534 (
1535   outputDir=${1}
1536   runNumber=${2}
1537   #get path to input list
1538   inputListfiles=${3}
1539   #get scale number for tracks
1540   filterT=${4}
1541   #get scale number for V0s
1542   filterV=${5}
1543   #get OCDB path (optional)
1544   OCDBpath=${6}
1545   #get max number of files 
1546   maxFiles=${7-"1000000"}
1547   #get offset of first file
1548   offsetFile=${8-"0"}
1549   #get max number of events
1550   maxEvents=${9-"30000000"}
1551   #get offset of first event
1552   offsetEvent=${10-"0"}
1553   configFile=${11-"benchmark.config"}
1554   esdFileName=${12-"AliESDs_Barrel.root"}
1555   shift 12
1556   if ! parseConfig ${configFile} "$@"; then return 1; fi
1557   
1558   #record the working directory provided by the batch system
1559   batchWorkingDirectory=${PWD}
1560
1561   [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
1562   doneFileBase=filtering.cpass1.run${runNumber}.done
1563   doneFileTmp=${batchWorkingDirectory}/${doneFileBase}
1564   doneFile=${commonOutputPath}/meta/${doneFileBase}
1565
1566   cat > filtering.log << EOF
1567   goMakeFilteredTrees config:
1568   runpath=${runpath}
1569   outputDir=${outputDir}
1570   commonOutputPath=${commonOutputPath}
1571   ALICE_ROOT=${ALICE_ROOT}
1572   PATH=${PATH}
1573   offsetEvent=$offsetEvent
1574   configFile=$configFile
1575   esdFileName=$esdFileName
1576   inputListfiles=$inputListfiles
1577   doneFile=$doneFile
1578 EOF
1579
1580   #runpath=${outputDir}
1581   #[[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
1582   #[[ ${reconstructInTemporaryDir} -eq 1 ]] && runpath=$(mktemp -d -t goMakeFilteredTrees.XXXXXX)
1583   #mkdir -p ${outputDir}
1584   #mkdir -p ${runpath}
1585   #if ! cd ${runpath}; then 
1586   #  echo "PWD=$PWD is not the runpath=${runpath}"
1587   #  touch ${doneFile}
1588   #  return 1
1589   #fi
1590   
1591   if [[ -z ${pretend} ]];then
1592     aliroot -l -b -q "${ALICE_ROOT}/PWGPP/macros/runFilteringTask.C(\"${inputListfiles}\",${filterT},${filterV},\"${OCDBpath}\",${maxFiles},${offsetFile},${maxEvents},${offsetEvent},\"${esdFileName}\")" &>> filtering.log
1593   else
1594     sleep ${pretendDelay}
1595     touch filtering.log FilterEvents_Trees.root
1596   fi
1597   pwd
1598   /bin/ls
1599   summarizeLogs >>  ${doneFile}
1600   
1601   #echo mv -f * ${outputDir}
1602   #mv -f * ${outputDir}
1603   #[[ -f ${outputDir}/FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFile}
1604   #cd ${commonOutputPath}
1605   #[[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
1606  
1607   cp "$doneFileTmp" "$doneFile" || rm -f "$doneFileTmp" "$doneFile"
1608   [[ -n ${removeTMPdoneFile} ]] && rm -f ${doneFileTmp}
1609
1610   return 0
1611 )
1612
1613 submit()
1614 {
1615   umask 0002
1616   [[ $# -lt 5 ]] && echo "at least 5 args needed, you supplied $#" && return 1
1617   JobID=${1}
1618   startID=${2}
1619   endID=${3}
1620   waitForJOBID=${4}
1621   command=${5}
1622   shift 5
1623   local commandArgs=("$@")
1624
1625   #add quote strings around the extra arguments
1626   for ((i=0; i<${#commandArgs[@]}; i++)); do 
1627     commandArgs[i]=\"${commandArgs[i]}\"
1628   done
1629
1630   [[ -z ${waitForJOBID} ]] && waitForJOBID=0
1631
1632   newFarm=$(which qsub|grep "^/usr/bin/qsub")
1633   
1634   batchSystem="SGE"
1635
1636   if [[ -z "${newFarm}" ]]
1637   then
1638     #old LSF
1639     # submit it (as job array)
1640     nFiles=$(( ${endID}-${startID}+1 ))
1641     while [ ${startID} -le ${nFiles}  ] ; do
1642       if [ $(expr ${nFiles} - ${startID}) -gt 999 ] ; then 
1643         endID=$(expr ${startID} + 999)
1644       else
1645         endID=${nFiles}
1646       fi      
1647     if [[ ${waitForJOBID} -eq 0 ]]; then
1648         echo ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}"     
1649         ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}"     
1650       else
1651         echo ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -w "ended(${waitForJOBID})" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}"     
1652         ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -w "ended(${waitForJOBID})" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}"     
1653       fi
1654       startID=$(expr ${endID} + 1)
1655     done
1656   else 
1657     #new SGE farm
1658     if [[ ${waitForJOBID} =~ "000" ]]; then
1659       echo ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}"
1660       ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}"
1661     else
1662       echo ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -hold_jid "${waitForJOBID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}"
1663       ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -hold_jid "${waitForJOBID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}"
1664     fi
1665   fi
1666   return 0
1667 }
1668
1669 goSubmitBatch()
1670 {
1671   if [[ $# -lt 3 ]]; then
1672     echo "minimal use:"
1673     echo " ${0} submit fileList productionID configFile"
1674     return 0
1675   fi
1676
1677   productionID=${1}
1678   inputList=${2}
1679   configFile=${3:-"benchmark.config"}
1680   #if which greadlink; then configFile=$(greadlink -f ${configFile}); fi
1681   shift 3
1682   extraOpts=("$@")
1683   if ! parseConfig ${configFile} "${extraOpts[@]}"; then return 1; fi
1684   
1685   #batch systems/makeflow sometimes handle spaces in arguments poorly, so encode them
1686   for (( i=0;i<${#extraOpts[@]};i++ )); do 
1687     extraOpts[i]=$(encSpaces "${extraOpts[i]}")
1688   done
1689   extraOpts+=("encodedSpaces=1")
1690   #this removes the copy of the done file used by makeflow (in the running dir)
1691   extraOpts+=("removeTMPdoneFile=1")
1692
1693   #record the working directory provided by the batch system
1694   batchWorkingDirectory=${PWD}
1695
1696   #redirect all output to submit.log
1697   echo "redirecting all output to ${PWD}/submit_${productionID//"/"/_}.log"
1698   exec 7>&1
1699   exec 1>submit_${productionID//"/"/_}.log 2>&1
1700
1701   umask 0002
1702   echo ${0}" submit $*"
1703   if [[ -z "${inputList}" || -z "${productionID}" ]]
1704   then
1705     echo
1706     echo " Usage: ${0} submit inputList productionID [configFile=benchmark.config]"
1707     echo
1708     return
1709   fi
1710
1711   # check if config file is there
1712   if [ ! -f ${configFile} ]; then
1713     echo "ERROR! Config File '${configFile}' not found" >&2
1714     return
1715   else
1716     echo "Using Config File: '${configFile}'"
1717   fi
1718
1719   [[ ! -f ${alirootEnv} ]] && echo "alirootEnv script ${alirootEnv} not found!..." && return 1
1720
1721   #move the script, config and some other stuff to ${commonOutputPath} first, then use them from there
1722   self=${0}
1723   #if which greadlink; then self=$(greadlink -f "${0}"); fi
1724   configPath=$(dirname ${configFile})
1725   export commonOutputPath=${baseOutputDirectory}/${productionID}
1726   
1727   mkdir -p ${commonOutputPath}
1728   mkdir -p ${commonOutputPath}/logs
1729   mkdir -p ${commonOutputPath}/meta
1730
1731   cp ${self} ${commonOutputPath}
1732   cp ${configFile} ${commonOutputPath}
1733   cp ${inputList} ${commonOutputPath}
1734   self=${commonOutputPath}/${self##*/}
1735   chmod u+x ${self}
1736   configFile=${commonOutputPath}/${configFile##*/}
1737   inputList=${commonOutputPath}/${inputList##*/}
1738
1739   #convert to absolut pathnames
1740   #if which greadlink; then inputList=$(greadlink -f "${inputList}"); fi
1741   #make list of runs
1742   if [[ -z ${runNumber} ]]; then
1743     listOfRuns=($(while read x; do guessRunNumber ${x}; done < ${inputList} | sort | uniq))
1744   else
1745     listOfRuns=${runNumber}
1746   fi
1747
1748   #if which greadlink; then alirootSource=$(greadlink -f "${alirootSource}"); fi
1749
1750   echo ""
1751   echo "### BEGIN CONFIGURATION ###"
1752   echo ""
1753   echo "GENERAL:"
1754   echo ""
1755   echo "    productionID:    ${productionID}"
1756   echo "    batchCommand:    ${batchCommand}"
1757   echo "    batchFlags:      ${batchFlags}"
1758   echo "    alirootEnv:   ${alirootEnv}"
1759   ${alirootEnv} echo '    ALICE_ROOT:      ${ALICE_ROOT}'
1760   ${alirootEnv} echo '    ALIROOT_RELEASE: ${ALICE_RELEASE}'
1761   echo "    inputList:       ${inputList}"
1762   echo "    configPath:      ${configPath}"
1763   echo "    commonOutputPath:      ${commonOutputPath}"
1764   echo "    defaultOCDB:     ${defaultOCDB}"
1765   echo "      autoOCDB: ${autoOCDB}"
1766   echo "    recoTriggerOptions:   ${recoTriggerOptions}"
1767   echo "    runs:"
1768   echo "      ${listOfRuns[*]}"
1769   echo ""
1770   echo "THE TRAIN WILL RUN:"
1771
1772   if [ ${runCPass0reco} -eq 1 ]; then
1773     echo "    Pass0 - Recontruction"
1774   fi
1775
1776   if [ ${runCPass0MergeMakeOCDB} -eq 1 ]; then
1777     echo "    Pass0 - merging and OCDB export"
1778   fi
1779
1780   if [ ${runCPass1reco} -eq 1 ]; then
1781     echo "    Pass1 - Recontruction"
1782   fi
1783   if [ ${runCPass1MergeMakeOCDB} -eq 1 ]; then
1784     echo "    Pass1 - merging and OCDB export"
1785   fi
1786
1787   echo ""
1788   echo "LIMITS:"
1789   echo "    max. Events/Chunk:   ${nEvents}"
1790   echo "    max. Number of Chunks per Run:     ${nMaxChunks}"
1791   echo ""
1792   echo "### END CONFIGURATION ###"
1793   echo ""
1794
1795
1796   # check if input file is there
1797   if [ ! -f ${inputList} ]; then
1798     echo "ERROR! Input List '${inputList}' not found" >&2
1799     return
1800   fi
1801
1802   # define jobid (for dependent jobs)
1803   date=$(date +%Y_%m_%d_%H%M%S)
1804   #for each run we submit one jobarray:
1805   for runNumber in ${listOfRuns[*]}; do
1806     
1807     [[ -z ${runNumber} ]] && continue
1808     [[ ! ${runNumber} =~ ^[0-9]*[0-9]$ ]] && continue
1809
1810     JOBpostfix="${productionID//"/"/_}_${runNumber}_${date}"
1811     JOBID1="p0_${JOBpostfix}"
1812     JOBID1wait="w0_${JOBpostfix}"
1813     JOBID2="m0_${JOBpostfix}"
1814     JOBID2wait="wm0_${JOBpostfix}"
1815     JOBID3="op0_${JOBpostfix}"
1816     JOBID3wait="wop0_${JOBpostfix}"
1817     JOBID4="p1_${JOBpostfix}"
1818     JOBID4wait="w1_${JOBpostfix}"
1819     JOBID5="m1_${JOBpostfix}"
1820     JOBID5wait="wm1_${JOBpostfix}"
1821     JOBID6="s1_${JOBpostfix}"
1822     JOBID6wait="ws1_${JOBpostfix}"
1823     JOBID7="QA_${JOBpostfix}"
1824     JOBmakeESDlistCPass1="lp1_${JOBpostfix}"
1825     JOBfilterESDcpass1="fp1_${JOBpostfix}"
1826     LASTJOB="000"
1827
1828     oneInputFile=$(egrep -m1 "${runNumber}/" ${inputList})
1829
1830     currentDefaultOCDB=${defaultOCDB}
1831     [[ -z ${autoOCDB} ]] && autoOCDB=1
1832     if [[ ${autoOCDB} -ne 0 ]]; then
1833       currentDefaultOCDB=$(setYear ${oneInputFile} ${defaultOCDB})
1834     fi
1835     period=$(guessPeriod ${oneInputFile})
1836     year=$(guessYear ${oneInputFile})
1837
1838     echo "submitting run ${runNumber} with OCDB ${currentDefaultOCDB}"
1839
1840     ###############################################################################
1841     #run one chunk with valgrind:
1842     if [[ -n ${profilingCommand} ]]; then
1843       [[ -z ${nEventsProfiling} ]] && nEventsProfiling=2
1844       [[ -z ${profilingCommand} ]] && profilingCommand="/usr/bin/valgrind --tool=callgrind --num-callers=40 -v --trace-children=yes"
1845       submit "profile-${JOBpostfix}" 1 1 000 "${alirootEnv} ${self}" CPass0 ${commonOutputPath}/${year}/${period}/000${runNumber}/${jobindex} ${oneInputFile} ${nEventsProfiling} ${currentDefaultOCDB} ${configFile} ${runNumber} ${jobindex} useProfilingCommand=$(encSpaces "${profilingCommand}") "${extraOpts[@]}"
1846     fi 
1847
1848     ################################################################################
1849     ################################################################################
1850     # run the CPass0 if requested
1851
1852     if [ ${runCPass0reco} -eq 1 ]; then
1853
1854       echo
1855       echo "starting CPass0... for run ${runNumber}"
1856       echo
1857
1858       # create directory and copy all files that are needed
1859       targetDirectory="${commonOutputPath}/${year}/${period}/000${runNumber}/cpass0"
1860       mkdir -p ${targetDirectory}
1861
1862       filesCPass0=( 
1863                     "${configPath}/runCPass0.sh"
1864                     "${configPath}/recCPass0.C"
1865                     "${configPath}/runCalibTrain.C"
1866                     "${configPath}/localOCDBaccessConfig.C"
1867                     "${configPath}/OCDB*.root"
1868                     "${configPath}/sim.C"
1869                     "${configPath}/rec.C"
1870                     "${configPath}/Config.C"
1871       )
1872       for file in ${filesCPass0[*]}; do
1873         [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
1874       done
1875
1876       localInputList=${targetDirectory}/${inputList##*/}
1877       [[ ! -f ${localInputList} ]] && egrep "\/000${runNumber}\/" ${inputList} > ${localInputList}
1878       # limit nFiles to nMaxChunks
1879       nFiles=$(wc -l < ${localInputList})
1880       [[ ${nFiles} -eq 0 ]] && echo "list contains ZERO files! exiting..." && return 1
1881       echo "raw files in list:    ${nFiles}"
1882       if [[ ${nMaxChunks} -gt 0 && ${nMaxChunks} -le ${nFiles} ]]; then
1883         nFiles=${nMaxChunks}
1884       fi
1885       echo "raw files to process: ${nFiles}"
1886       [[ -z "${percentProcessedFilesToContinue}" ]] && percentProcessedFilesToContinue=100
1887       if [[ ${percentProcessedFilesToContinue} -eq 100 ]]; then
1888         nFilesToWaitFor=${nFiles}
1889       else
1890         nFilesToWaitFor=$(( ${nFiles}-${nFiles}/(100/(100-${percentProcessedFilesToContinue})) ))
1891       fi
1892       echo "requested success rate is ${percentProcessedFilesToContinue}%"
1893       echo "merging will start after ${nFilesToWaitFor} jobs are done"
1894
1895       submit ${JOBID1} 1 ${nFiles} 000 "${alirootEnv} ${self}" CPass0 ${targetDirectory} ${localInputList} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} -1 "${extraOpts[@]}"
1896
1897       ## submit a monitoring job that will run until a certain number of jobs are done with reconstruction
1898       submit "${JOBID1wait}" 1 1 000 "${alirootEnv} ${self}" WaitForOutput ${commonOutputPath} "meta/cpass0.job*.run${runNumber}.done" ${nFilesToWaitFor} ${maxSecondsToWait}
1899       LASTJOB=${JOBID1wait}
1900
1901     fi #end running CPass0
1902     ################################################################################
1903
1904
1905     ################################################################################
1906     # submit merging of CPass0, depends on the reconstruction
1907
1908     if [ ${runCPass0MergeMakeOCDB} -eq 1 ]; then
1909
1910       echo
1911       echo "submit CPass0 merging for run ${runNumber}"
1912       echo
1913
1914       targetDirectory="${commonOutputPath}/${year}/${period}/000${runNumber}/cpass0"
1915       mkdir -p ${targetDirectory}
1916
1917       #copy the scripts
1918       filesMergeCPass0=(
1919                         "${configPath}/OCDB.root"
1920                         "${configPath}/mergeMakeOCDB.byComponent.sh"
1921                         "${configPath}/mergeMakeOCDB.sh"
1922                         "${configPath}/localOCDBaccessConfig.C"
1923                         "${configPath}/mergeByComponent.C"
1924                         "${configPath}/makeOCDB.C"
1925                         "${configPath}/merge.C"
1926       )
1927       for file in ${filesMergeCPass0[*]}; do
1928         [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
1929       done
1930   
1931       submit ${JOBID2} 1 1 "${LASTJOB}" "${alirootEnv} ${self}" MergeCPass0 ${targetDirectory} ${currentDefaultOCDB} ${configFile} ${runNumber} cpass0.calib.run${runNumber}.list "${extraOpts[@]}"
1932       LASTJOB=${JOBID2}
1933
1934       if [[ -n ${generateMC} ]]; then
1935         submit "mrl${JOBpostfix}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" PrintValues sim ${commonOutputPath}/meta/sim.run${runNumber}.list ${commonOutputPath}/meta/cpass0.job*.run${runNumber}.done
1936         LASTJOB="mrl${JOBpostfix}"
1937       fi
1938
1939       echo
1940     fi
1941     # end of merging CPass0
1942     ################################################################################
1943
1944     ################################################################################
1945     ################################################################################
1946     # run the CPass1 if requested
1947
1948     if [ ${runCPass1reco} -eq 1 ]; then
1949
1950       targetDirectory="${commonOutputPath}/${year}/${period}/000${runNumber}/cpass1"
1951       rm -f ${commonOutputPath}/meta/cpass1.job*.run${runNumber}.done
1952
1953       # safety feature: if we are re-running for any reason we want to delete the previous output first.
1954       [[ -d ${targetDirectory} ]] && rm -rf ${targetDirectory}/* && echo "removed old output at ${targetDirectory}/*"
1955
1956       echo
1957       echo "starting CPass1... for run ${runNumber}"
1958       echo
1959
1960       # create directory and copy all files that are needed
1961       mkdir -p ${targetDirectory}
1962       
1963       filesCPass1=( 
1964                     "${configPath}/runCPass1.sh"
1965                     "${configPath}/recCPass1.C"
1966                     "${configPath}/recCPass1_OuterDet.C"
1967                     "${configPath}/runCalibTrain.C"
1968                     "${configPath}/QAtrain_duo.C"
1969                     "${configPath}/localOCDBaccessConfig.C"
1970                     "${configPath}/OCDB.root"
1971       )
1972       for file in ${filesCPass1[*]}; do
1973         [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
1974       done
1975
1976       if [[ -n ${generateMC} ]]; then
1977         localInputList=${commonOutputPath}/meta/sim.run${runNumber}.list
1978       else
1979         localInputList=${targetDirectory}/${inputList##*/}
1980         [[ ! -f ${localInputList} ]] && egrep "\/000${runNumber}\/" ${inputList} > ${localInputList}
1981       fi
1982       # limit nFiles to nMaxChunks
1983       nFiles=$(wc -l < ${localInputList})
1984       [[ ${nFiles} -eq 0 ]] && echo "list contains ZERO files! continuing..." && continue
1985       echo "raw files in list:    ${nFiles}"
1986       if [[ ${nMaxChunks} -gt 0 && ${nMaxChunks} -le ${nFiles} ]]; then
1987         nFiles=${nMaxChunks}
1988       fi
1989       echo "raw files to process: ${nFiles}"
1990       [[ -z "${percentProcessedFilesToContinue}" ]] && percentProcessedFilesToContinue=100
1991       if [[ ${percentProcessedFilesToContinue} -eq 100 ]]; then
1992         nFilesToWaitFor=${nFiles}
1993       else
1994         nFilesToWaitFor=$(( ${nFiles}-${nFiles}/(100/(100-${percentProcessedFilesToContinue})) ))
1995       fi
1996       echo "requested success rate is ${percentProcessedFilesToContinue}%"
1997       echo "merging will start after ${nFilesToWaitFor} jobs are done"
1998
1999       submit ${JOBID4} 1 ${nFiles} "${LASTJOB}" "${alirootEnv} ${self}" CPass1 ${targetDirectory} ${localInputList} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} -1 "${extraOpts[@]}"
2000
2001       ################################################################################
2002       ## submit a monitoring job that will run until a certain number of jobs are done with reconstruction
2003       submit "${JOBID4wait}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" WaitForOutput ${commonOutputPath} "meta/cpass1.job*.run${runNumber}.done" ${nFilesToWaitFor} ${maxSecondsToWait}
2004       LASTJOB=${JOBID4wait}
2005       ################################################################################
2006
2007       echo
2008     fi #end running CPass1
2009
2010     ################################################################################
2011     # submit merging of CPass1, depends on the reconstruction
2012     if [ ${runCPass1MergeMakeOCDB} -eq 1 ]; then
2013
2014       echo
2015       echo "submit CPass1 merging for run ${runNumber}"
2016       echo
2017
2018       targetDirectory="${commonOutputPath}/${year}/${period}/000${runNumber}/cpass1"
2019       rm -f ${commonOutputPath}/meta/merge.cpass1.run${runNumber}.done
2020       mkdir -p ${targetDirectory}
2021
2022       # copy files 
2023       filesMergeCPass1=(
2024                         "${configPath}/OCDB.root"
2025                         "${configPath}/localOCDBaccessConfig.C"
2026                         "${configPath}/mergeMakeOCDB.byComponent.sh"
2027                         "${configPath}/mergeByComponent.C"
2028                         "${configPath}/makeOCDB.C"
2029                         "${configPath}/merge.C"
2030                         "${configPath}/mergeMakeOCDB.sh"
2031                         "${configPath}/QAtrain_duo.C"
2032       )
2033       for file in ${filesMergeCPass1[*]}; do
2034         [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
2035       done
2036
2037       submit "${JOBID5}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" MergeCPass1 ${targetDirectory} ${currentDefaultOCDB} ${configFile} ${runNumber} cpass1.calib.run${runNumber}.list cpass1.QA.run${runNumber}.lastMergingStage.txt.list cpass1.filtered.run${runNumber}.list "${extraOpts[@]}"
2038       LASTJOB=${JOBID5}
2039       echo
2040     fi
2041
2042     ###############################
2043     #if [ ${runESDfiltering} -eq 1 ]; then
2044     #  rm -f ${commonOutputPath}/cpass1.ESD.run${runNumber}.list
2045     #  rm -f ${commonOutputPath}/meta/filtering.cpass1.run*.done
2046     #  echo
2047     #  echo submitting filtering for run ${runNumber}
2048     #  echo
2049     #  submit "${JOBmakeESDlistCPass1}" 1 1 "${LASTJOB}" "${self}" PrintValues esd ${commonOutputPath}/meta/cpass1.ESD.run${runNumber}.list ${commonOutputPath}/meta/cpass1.job*.run${runNumber}.done 
2050     #  submit "${JOBfilterESDcpass1}" 1 1 "${JOBmakeESDlistCPass1}" "${alirootEnv} ${self}" MakeFilteredTrees ${commonOutputPath}/${year}/${period}/000${runNumber}/cpass1 ${runNumber} ${commonOutputPath}/meta/cpass1.ESD.run${runNumber}.list ${filteringFactorHighPt} ${filteringFactorV0s} ${currentDefaultOCDB} 1000000 0 10000000 0 ${configFile} AliESDs_Barrel.root "${extraOpts[@]}"
2051     #  LASTJOB=${JOBfilterESDcpass1}
2052     #fi
2053
2054   done
2055
2056   #################################################################################
2057   #################################################################################
2058   #if [ ${runESDfiltering} -eq 1 ]; then
2059   #  submit "${JOBID5wait}" 1 1 "${LASTJOB}" "${self}" WaitForOutput ${commonOutputPath} "meta/filtering.cpass1.run*.done" "${#listOfRuns[@]}" ${maxSecondsToWait}
2060   #else
2061     submit "${JOBID5wait}" 1 1 "${LASTJOB}" "${self}" WaitForOutput ${commonOutputPath} "meta/merge.cpass1.run*.done" ${#listOfRuns[@]} ${maxSecondsToWait}
2062   #fi
2063   LASTJOB=${JOBID5wait}
2064
2065   #################################################################################
2066   echo
2067   echo "submit make a summary"
2068   echo
2069
2070   [[ -z ${alirootEnvQA} ]] && alirootEnvQA=$(encSpaces "${alirootEnv}")
2071   submit "${JOBID6}" 1 1 "${LASTJOB}" "${alirootEnvQA} ${self}" MakeSummary ${configFile} "commonOutputPath=${commonOutputPath}"
2072   LASTJOB=${JOBID6}
2073   #################################################################################
2074   
2075   #restore stdout
2076   exec 1>&7 7>&-
2077   echo "jobs submitted."
2078   return 0
2079 }
2080
2081 goWaitForOutput()
2082 (
2083   umask 0002
2084   [[ $# -lt 3 ]] && echo "goWaitForOutput() wrong number of arguments, exiting.." && return 1
2085   echo searchPath=${1}
2086   echo fileName=${2}
2087   echo numberOfFiles=${3}
2088   echo maxSecondsToWait=${4}
2089   searchPath=${1}
2090   fileName=${2}
2091   numberOfFiles=${3}
2092   maxSecondsToWait=${4}
2093   echo "command to be executed: /bin/ls -1 ${searchPath}/${fileName}"
2094   [[ -z "${maxSecondsToWait}" ]] && maxSecondsToWait=$(( 3600*12 ))
2095   while true; do
2096     n=$(/bin/ls -1 ${searchPath}/${fileName} 2>/dev/null | wc -l)
2097     [[ ${n} -gt 0 ]] && echo "found ${n} X ${fileName}"
2098     [[ ${n} -ge ${numberOfFiles} ]] && break
2099     [[ ${SECONDS} -gt ${maxSecondsToWait} ]] && echo "timeout of ${maxSecondsToWait}s!" && break
2100     sleep 60
2101   done
2102   echo "DONE! exiting..."
2103   return 0
2104 )
2105
2106 mergeSysLogs()
2107 (
2108   outputFile=${1}
2109   shift
2110   inputFiles="$@"
2111   i=0
2112   if ! ls -1 ${inputFiles} &>/dev/null; then echo "the files dont exist!: ${inputFiles}"; return 1; fi
2113   while read x; do 
2114     runNumber=$(guessRunNumber ${x})
2115     [[ -z ${runNumber} ]] && echo "run number cannot be guessed for ${x}" && continue
2116     awk -v run=${runNumber} -v i=${i} 'NR > 1 {print run" "$0} NR==1 && i==0 {print "run/I:"$0}' ${x}
2117     (( i++ ))
2118   done < <(ls -1 ${inputFiles}) > ${outputFile}
2119   return 0
2120 )
2121
2122 goMakeMergedSummaryTree()
2123 (
2124   # create list of calibration entries
2125   # takes no arguments, just run it in the base output
2126   # directory with the following files in the working directory
2127   # 
2128   #  Calibration file lists:
2129   #       cpass0.dcsTree.list, cpass1.dcsTree.list
2130   #  QA trending root file:
2131   #       trending.root
2132   # 
2133   #  Production infoo ascii files:
2134   #       summary_pass0.tree
2135   #       summary_pass1.tree
2136   #    
2137
2138   [[ ! -f cpass0.dcsTree.list ]] && echo "no cpass0.dcsTree.list" && return 1
2139   [[ ! -f cpass1.dcsTree.list ]] && echo "no cpass1.dcsTree.list" && return 1
2140   [[ ! -f trending.root ]] && echo "no trending.root" && return 1
2141   [[ ! -f summary_pass0.tree ]] && echo "no summary_pass0.tree" && return 1
2142   [[ ! -f summary_pass1.tree ]] && echo "no summary_pass1.tree" && return 1
2143
2144   #first, dump the C macro to file
2145   cat << EOF > mergeTree.C
2146   //
2147   // Merge summary information
2148   // Following files are expected to be in the working directory
2149   //
2150   // Calibration file lists:
2151   //      cpass0.dcsTree.list, cpass1.dcsTree.list
2152   // QA trending root file:
2153   //      trending.root
2154   //
2155   // Production infoo ascii files:
2156   //      summary_pass0.tree
2157   //      summary_pass1.tree
2158   //   
2159   void mergeTree(){
2160     //
2161     //
2162     //
2163     // Calibration values dump
2164     //
2165     //Printf("MakeTreeFromList cpass0.dcsTree.list");
2166     AliXRDPROOFtoolkit::MakeTreeFromList("Calib.TPC.CPass0.root", "dcs","dcs","cpass0.dcsTree.list",1);
2167     //Printf("MakeTreeFromList cpass1.dcsTree.list");
2168     AliXRDPROOFtoolkit::MakeTreeFromList("Calib.TPC.CPass1.root", "dcs","dcs","cpass1.dcsTree.list",1);
2169     //
2170     // Calibration status dump
2171     //
2172     TFile *fprod = TFile::Open("fproduction.root","recreate");
2173     TTree  tree0, tree1;
2174     //Printf("reading summary_pass0.tree");
2175     tree0.ReadFile("summary_pass0.tree");
2176     //Printf("reading summary_pass1.tree");
2177     tree1.ReadFile("summary_pass1.tree");
2178     tree0.Write("CPass0");
2179     tree1.Write("CPass1");
2180     fprod->Close();
2181     //
2182     //
2183     //
2184     TString stringSetup="";
2185     stringSetup+="1#QA.TPC#run#SummaryTPCQA/tpcQA#trending.root+";  // 
2186     stringSetup+="1#Calib.TPC.CPass0#run#dcs#Calib.TPC.CPass0.root+";  // 
2187     stringSetup+="1#Calib.TPC.CPass1#run#dcs#Calib.TPC.CPass1.root+";  // 
2188     //
2189     stringSetup+="1#CPass0#runnumber#CPass0#fproduction.root+";  // 
2190     stringSetup+="1#CPass1#runnumber#CPass1#fproduction.root+";  // 
2191     //
2192     //Printf("stringSetup: %s", stringSetup.Data());
2193     AliXRDPROOFtoolkit::JoinTreesIndex("outAll.root","joinAll","run",stringSetup.Data(), 1);
2194   }
2195 EOF
2196
2197   aliroot -b -q "mergeTree.C" > mergeTrees.log
2198   return $?
2199 )
2200
2201 stackTraceTree()
2202 (
2203   if [[ $# -lt 1 ]]; then
2204     echo 'make stacktrace processing  in case of standard root crash log'
2205     echo 'input is a (list of) text files with the stack trace (either gdb aoutput'
2206     echo 'produced with e.g. gdb --batch --quiet -ex "bt" -ex "quit" aliroot core,'
2207     echo 'or the root crash log), output is a TTree formatted table.'
2208     echo 'example usage:'
2209     echo 'benchmark.sh stackTraceTree /foo/*/rec.log'
2210     echo 'benchmark.sh stackTraceTree $(cat file.list)'
2211     echo 'benchmark.sh stackTraceTree `cat file.list`'
2212     return 0
2213   fi
2214   gawk '
2215        BEGIN { 
2216                print "frame/I:method/C:line/C:cpass/I:aliroot/I:file/C";
2217                RS="#[0-9]*";
2218                aliroot=0;
2219                read=1;
2220              } 
2221       /There was a crash/ {read=1;}
2222       /The lines below might hint at the cause of the crash/ {read=0;}
2223       read==1 { 
2224                if ($3 ~ /Ali*/) aliroot=1; else aliroot=0;
2225                gsub("#","",RT); 
2226                if ($NF!="" && RT!="" && $3!="") print RT" "$3" "$NF" "0" "aliroot" "FILENAME
2227              }
2228       ' "$@" 2>/dev/null
2229 )
2230
2231 goMakeSummary()
2232 (
2233   #all the final stuff goes in here for ease of use:
2234   # summary logs
2235   # qa plot making
2236   # final file lists
2237   # runs in current dir - in makeflow mode it can run LOCAL, then the QA plots and summaries
2238   # will appear in the submission dir.
2239   #some defaults:
2240   log="summary.log"
2241   productionID="qa"
2242
2243   configFile=${1}
2244   shift 1
2245   extraOpts=("$@")
2246   if ! parseConfig ${configFile} "${extraOpts[@]}"; then return 1; fi
2247   
2248   #if which greadlink; then configFile=$(greadlink -f ${configFile}); fi
2249   
2250   #record the working directory provided by the batch system
2251   batchWorkingDirectory=${PWD}
2252
2253   logTmp=${batchWorkingDirectory}/${log}
2254   logDest=${commonOutputPath}/${log}
2255   
2256   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
2257
2258   [[ ! -f ${configFile} ]] && echo "no config file ${configFile}!" && return
2259
2260   [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
2261
2262   #copy some useful stuff
2263   #and go to the commonOutputPath
2264   cp ${configFile} ${commonOutputPath}
2265
2266   exec &> >(tee ${logTmp})
2267
2268   #summarize the global stuff
2269   echo "env script: ${alirootSource} ${alirootEnv}"
2270   echo "\$ALICE_ROOT=${ALICE_ROOT}"
2271   echo "commonOutputPath=${commonOutputPath}"
2272
2273   #summarize the stacktraces
2274   stackTraceTree ${commonOutputPath}/*/*/000*/cpass0/*/stacktrace* > stacktrace_cpass0.tree
2275   stackTraceTree ${commonOutputPath}/*/*/000*/cpass1/*/stacktrace* > stacktrace_cpass1.tree
2276
2277   echo total numbers for the production:
2278   echo
2279   awk 'BEGIN {nFiles=0;nCore=0;} 
2280   /^calibfile/ {nFiles++;} 
2281   /core dumped/ {nCore++i;}
2282   END {print     "cpass0 produced "nFiles" calib files, "nCore" core files";}' ${commonOutputPath}/meta/cpass0.job*done 2>/dev/null
2283   awk 'BEGIN {nOK=0; nBAD=0; } 
2284   /\/rec.log OK/ {nOK++;} 
2285   /\/rec.log BAD/ {nBAD++;} 
2286   /stderr BAD/ {if ($0 ~ /rec.log/){nBAD++;}}
2287   END {print     "cpass0 reco:  OK: "nOK"\tBAD: "nBAD;}' ${commonOutputPath}/meta/cpass0.job*done 2>/dev/null
2288   awk 'BEGIN {nOK=0; nBAD=0; } 
2289   /\/calib.log OK/ {nOK++;} 
2290   /\/calib.log BAD/ {nBAD++;} 
2291   END {print "cpass0 calib: OK: "nOK"\tBAD: "nBAD;}' ${commonOutputPath}/meta/cpass0.job*done 2>/dev/null
2292
2293   awk 'BEGIN {nOK=0; nBAD=0; } 
2294   /merge.log OK/ {nOK++;} 
2295   /merge.log BAD/ {nBAD++;} 
2296   END {print "cpass0 merge: OK: "nOK"\tBAD: "nBAD;}' ${commonOutputPath}/meta/merge.cpass0*done 2>/dev/null
2297   awk 'BEGIN {nOK=0; nBAD=0; } 
2298   /ocdb.log OK/ {nOK++;} 
2299   /ocdb.log BAD/ {nBAD++;} 
2300   END {print   "cpass0 OCDB:  OK: "nOK"\tBAD: "nBAD;}' ${commonOutputPath}/meta/merge.cpass0*done 2>/dev/null
2301
2302   echo
2303   awk 'BEGIN {nFiles=0;nCore=0;} 
2304   /^calibfile/ {nFiles++;} 
2305   /core dumped/ {nCore++;}
2306   END {print     "cpass1 produced "nFiles" calib files, "nCore" core files";}' ${commonOutputPath}/meta/cpass1.job*done 2>/dev/null
2307   awk 'BEGIN {nOK=0; nBAD=0; } 
2308   /\/rec.log OK/ {nOK++;} 
2309   /\/rec.log BAD/ {nBAD++;} 
2310   /stderr BAD/ {if ($0 ~ /rec.log/){nBAD++;}}
2311   END {print     "cpass1 reco:  OK: "nOK"\tBAD: "nBAD;}' ${commonOutputPath}/meta/cpass1.job*done 2>/dev/null
2312   awk 'BEGIN {nOK=0; nBAD=0; } 
2313   /\/calib.log OK/ {nOK++;} 
2314   /\/calib.log BAD/ {nBAD++;} 
2315   END {print "cpass1 calib: OK: "nOK"\tBAD: "nBAD;}' ${commonOutputPath}/meta/cpass1.job*done 2>/dev/null
2316
2317   awk 'BEGIN {nOK=0; nBAD=0; } 
2318   /merge.log OK/ {nOK++;} 
2319   /merge.log BAD/ {nBAD++;} 
2320   END {print "cpass1 merge: OK: "nOK"\tBAD: "nBAD;}' ${commonOutputPath}/meta/merge.cpass1*done 2>/dev/null
2321   awk 'BEGIN {nOK=0; nBAD=0; } 
2322   /ocdb.log OK/ {nOK++;} 
2323   /ocdb.log BAD/ {nBAD++;} 
2324   END {print   "cpass1 OCDB:  OK: "nOK"\tBAD: "nBAD;}' ${commonOutputPath}/meta/merge.cpass1*done 2>/dev/null
2325
2326   echo
2327   echo per run stats:
2328   /bin/ls -1 ${commonOutputPath}/meta/merge.cpass0.run*.done | while read x 
2329 do
2330   dir=$(goPrintValues dir - ${x})
2331   runNumber=$(guessRunNumber ${dir})
2332   [[ -z ${runNumber} ]] && continue
2333
2334   if $(/bin/ls ${commonOutputPath}/meta/cpass0.job*.run${runNumber}.done &> /dev/null); then
2335     statusCPass0=( $(
2336     awk 'BEGIN {nOKrec=0;nBADrec=0;nOKcalib=0;nBADcalib=0;nOKstderr=0;nBADstderr=0;}
2337     /\/rec.log OK/ {nOKrec++;} 
2338     /\/rec.log BAD/ {nBADrec++;}
2339     /stderr BAD/ {if ($0 ~ /rec.log/) {nBADrec++;} nBADstderr++;}
2340     /stderr OK/ {nOKstderr++;}
2341     /\/calib.log OK/ {nOKcalib++;}
2342     /\/calib.log BAD/ {nBADcalib++}
2343     END {print ""nOKrec" "nBADrec" "nOKstderr" "nBADstderr" "nOKcalib" "nBADcalib;}' ${commonOutputPath}/meta/cpass0.job*.run${runNumber}.done 2>/dev/null
2344     ) ) 
2345   fi
2346
2347   if $(/bin/ls ${commonOutputPath}/meta/cpass1.job*.run${runNumber}.done &>/dev/null); then
2348     statusCPass1=( $(
2349     awk 'BEGIN {nOKrec=0;nBADrec=0;nOKcalib=0;nBADcalib=0;nOKstderr=0;nBADstderr=0;nQAbarrelOK=0;nQAbarrelBAD=0;nQAouterOK=0;nQAouterBAD=0;}
2350     /\/rec.log OK/ {nOKrec++;} 
2351     /\/rec.log BAD/ {nBADrec++;}
2352     /stderr BAD/ {if ($0 ~ /rec.log/) nBADrec++;nBADstderr++;}
2353     /stderr OK/ {nOKstderr++;}
2354     /\/calib.log OK/ {nOKcalib++;}
2355     /\/calib.log BAD/ {nBADcalib++}
2356     /\/qa_barrel.log OK/ {nQAbarrelOK++;}
2357     /\/qa_barrel.log BAD/ {nQAbarrelBAD++;}
2358     /\/qa_outer.log OK/ {nQAouterOK++;}
2359     /\/qa_outer.log BAD/ {nQAouterBAD++;}
2360     END {print ""nOKrec" "nBADrec" "nOKstderr" "nBADstderr" "nOKcalib" "nBADcalib" "nQAbarrelOK" "nQAbarrelBAD" "nQAouterOK" "nQAouterBAD;}' ${commonOutputPath}/meta/cpass1.job*.run${runNumber}.done 2>/dev/null
2361     ) ) 
2362   fi
2363
2364   statusOCDBcpass0=$(awk '/ocdb.log/ {print $2} ' ${x} 2>/dev/null)
2365   statusOCDBcpass1=$(awk '/ocdb.log/ {print $2}' ${x/cpass0/cpass1} 2>/dev/null)
2366   statusQA=$(awk '/mergeMakeOCDB.log/ {print $2}' ${x/cpass0/cpass1} 2>/dev/null)
2367
2368   printf "%s\t ocdb.log cpass0: %s\t ocdb.log cpass1: %s\tqa.log:%s\t| cpass0: rec:%s/%s stderr:%s/%s calib:%s/%s cpass1: rec:%s/%s stderr:%s/%s calib:%s/%s QAbarrel:%s/%s QAouter:%s/%s\n" ${runNumber} ${statusOCDBcpass0} ${statusOCDBcpass1} ${statusQA} ${statusCPass0[0]} ${statusCPass0[1]} ${statusCPass0[2]} ${statusCPass0[3]} ${statusCPass0[4]} ${statusCPass0[5]} ${statusCPass1[0]} ${statusCPass1[1]} ${statusCPass1[2]} ${statusCPass1[3]} ${statusCPass1[4]} ${statusCPass1[5]} ${statusCPass1[6]} ${statusCPass1[7]} ${statusCPass1[8]} ${statusCPass1[9]}
2369 done
2370
2371   #make lists with output files - QA, trending, filtering and calibration
2372   ### wait for the merging of all runs to be over ###
2373   rm -f qa.list
2374   goPrintValues qafile qa.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2375   rm -f calib.list
2376   goPrintValues calibfile calib.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2377   rm -f trending.list
2378   goPrintValues trendingfile trending.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2379   rm -f filtering.list
2380   goPrintValues filteredTree filtering.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2381   rm -f cpass0.dcsTree.list
2382   goPrintValues dcsTree cpass0.dcsTree.list ${commonOutputPath}/meta/merge.cpass0.run*.done &>/dev/null
2383   rm -f cpass1.dcsTree.list
2384   goPrintValues dcsTree cpass1.dcsTree.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2385  
2386   #merge trending
2387   rm -f trending.root
2388   goMerge trending.list trending.root ${configFile} "${extraOpts[@]}" &> mergeTrending.log
2389
2390   goMakeSummaryTree ${commonOutputPath} 0
2391   goMakeSummaryTree ${commonOutputPath} 1
2392
2393   goCreateQAplots "${PWD}/qa.list" "${productionID}" "QAplots" "${configFile}" "${extraOpts[@]}" filteringList="${PWD}/filtering.list" &>createQAplots.log
2394
2395   #make a merged summary tree out of the QA trending, dcs trees and log summary trees
2396   goMakeMergedSummaryTree
2397
2398   #if set, email the summary
2399   [[ -n ${MAILTO} ]] && cat ${logTmp} | mail -s "benchmark ${productionID} done" ${MAILTO}
2400
2401   # Copy log to destination (delete all on failure to signal error)
2402   cp "$logTmp" "$logDest" || rm -f "$logTmp" "$logDest"
2403   
2404   #copy output files
2405   exec &> >(tee fileCopy.log)
2406   paranoidCp QAplots ${commonOutputPath}
2407   paranoidCp *.list ${commonOutputPath}
2408   paranoidCp *.root ${commonOutputPath}
2409   paranoidCp *.log ${commonOutputPath}
2410   paranoidCp fileCopy.log ${commonOutputPath}
2411
2412   return 0
2413 )
2414
2415 goMakeSummaryTree()
2416 (
2417   if [[ $# -lt 1 ]] ; then
2418     return
2419   fi
2420   #1. define vars/arrays
2421   DIR=${1} #use input or exec in current dir
2422   pass=${2-"0"} #pass from input
2423   outfile="summary_pass${pass}.tree"
2424   Ncolumns=0
2425   test -f ${outfile} && : >${outfile}
2426   errfile=${outfile/tree/err}
2427   test -f ${errfile} && : >${errfile}
2428
2429   declare -a counterString=(TOFevents TOFtracks TPCevents TPCtracks TRDevents TRDtracks T0events SDDevents SDDtracks MeanVertexevents)
2430   Ncounter=${#counterString[@]}
2431
2432   declare -a statusString=(TRDStatus TOFStatus TPCStatus T0Status MeanVertexStatus)
2433   Nstatus=${#statusString[@]}
2434
2435
2436   declare -a ratesString=(rec stderr calib qa_barrel qa_outer)
2437   Nrates=${#ratesString[@]}
2438
2439   runs=( $(ls -1 ${DIR}/meta/merge.cpass0* | while read x; do guessRunNumber $x; done) )
2440   Nruns=${#runs[@]}
2441
2442   echo -n runnumber/I >>${outfile}
2443   echo -n :cpass${pass}status/I >>${outfile}
2444   echo -n :cpass${pass}QAstatus/I >>${outfile}
2445   for i in ${ratesString[@]}; do
2446     echo -n :${i}OK/I >>${outfile}
2447     echo -n :${i}BAD/I >>${outfile}
2448
2449   done
2450   for i in ${counterString[@]} ${statusString[@]} ; do
2451     echo -n :${i}/I >>${outfile}
2452   done
2453   Ncolumns=$((2 + 2*Nrates + Ncounter + Nstatus))
2454   echo >> ${outfile}
2455
2456   #2. loop runs 
2457
2458   for runnumber in ${runs[@]} ; do 
2459
2460
2461
2462     filejob="${DIR}/meta/cpass${pass}.job*.run${runnumber}.done"
2463     filemerge="${DIR}/meta/merge.cpass${pass}.run${runnumber}.done"
2464     fileOCDB=$(grep /ocdb.log ${filemerge} | awk '{print $1}')
2465     if ! $(/bin/ls ${filemerge} &>/dev/null) ; then
2466       echo "${filemerge} does not exist!" >>${errfile}
2467       continue
2468     elif ! $(/bin/ls ${filejob} &>/dev/null) ; then
2469       echo "${filejob} does not exist!" >>${errfile}
2470       echo -n ${runnumber} >> ${outfile}
2471       for i in $(seq ${Ncolumns}) ; do 
2472         echo -n "-1" >> ${outfile}
2473       done
2474       echo >> ${outfile}
2475       continue
2476     fi
2477     echo -n ${runnumber} >> ${outfile}
2478     #pass0status= grep '/ocdb.log' ${filemerge} |  cut -d' ' -f2 | tr OK x1 | tr BAD xx0 | tr -d 'x'
2479     passStatus=$(grep '/ocdb.log' ${filemerge} | grep OK | wc -l)
2480     echo -n " ${passStatus}" >> ${outfile}
2481     qaStatus=$(grep '/mergeMakeOCDB.log' ${filemerge} | grep OK | wc -l)
2482     echo -n " ${qaStatus}" >> ${outfile}
2483
2484
2485     #fill OK/BAD rates
2486     for i in $(seq 0 $((${Nrates}-1))) ; do
2487       var1=$(grep "/${ratesString[${i}]}.log" ${filejob} | grep OK | wc -l)
2488       var2=$(grep "/${ratesString[${i}]}.log" ${filejob} | grep BAD | wc -l)
2489
2490       if [[ ${ratesString[${i}]} == "stderr" ]] ; then
2491         var1=$(grep "stderr" ${filejob} | grep OK | wc -l)
2492         var2=$(grep "stderr" ${filejob} | grep "rec.log" | grep BAD | wc -l)
2493       fi
2494       echo -n " ${var1}" >> ${outfile}
2495       echo -n " ${var2}" >> ${outfile}
2496     done
2497
2498     if [[ -f ${fileOCDB} ]] ; then
2499       #fill counter
2500       for i in $(seq 0 $((${Ncounter}-1))) ; do
2501         var1=$(grep Monalisa ${fileOCDB} | grep ${counterString[${i}]} | cut -f2)
2502         echo -n " ${var1:-"-1"}" >> ${outfile}
2503       done
2504
2505       #fill status
2506       for i in $(seq 0 $((${Nstatus}-1))) ; do
2507         var1=$(grep "calibration status=" ${fileOCDB} | grep ${statusString[${i}]/Status/} | cut -d'=' -f2)
2508         echo -n " ${var1:-"-1"}" >> ${outfile}
2509       done
2510     fi
2511     echo >> ${outfile}
2512   done
2513
2514   return 0
2515 )
2516
2517 parseConfig()
2518 {
2519   configFile=${1}
2520   shift
2521   args=("$@")
2522
2523
2524   #some defaults
2525   #autoOCDB=0
2526   defaultOCDB="raw://"
2527   #runNumber=167123
2528   #makeflowPath="/hera/alice/aux/cctools/bin"
2529   #makeflowOptions="-T wq -N alice -d all -C ali-copilot.cern.ch:9097"
2530   #makeflowOptions="-T wq -N alice -C ali-copilot.cern.ch:9097"
2531   makeflowOptions=""
2532   #batchCommand="/usr/bin/qsub"
2533   batchFlags=""
2534   baseOutputDirectory="$PWD/output"
2535   #alirootEnv="/cvmfs/alice.cern.ch/bin/alienv setenv AliRoot/v5-04-34-AN -c"
2536   #alirootEnv="/home/mkrzewic/alisoft/balice_master.sh"
2537   #trustedQAtrainMacro='/hera/alice/mkrzewic/gsisvn/Calibration/QAtrain_duo.C'
2538   reconstructInTemporaryDir=0
2539   recoTriggerOptions="\"\""
2540   percentProcessedFilesToContinue=100
2541   maxSecondsToWait=$(( 3600*24 ))
2542   nEvents=-1
2543   nMaxChunks=0
2544   postSetUpActionCPass0=""
2545   postSetUpActionCPass1=""
2546   runCPass0reco=1
2547   runCPass0MergeMakeOCDB=1
2548   runCPass1reco=1
2549   runCPass1MergeMakeOCDB=1
2550   runESDfiltering=1
2551   filteringFactorHighPt=1e2
2552   filteringFactorV0s=1e1
2553   MAILTO=""
2554   #pretend=1
2555   #dontRedirectStdOutToLog=1
2556   logToFinalDestination=1
2557   ALIROOT_FORCE_COREDUMP=1
2558   pretendDelay=0
2559
2560   #first, source the config file
2561   if [ -f ${configFile} ]; then
2562     source ${configFile}
2563   else
2564     echo "config file ${configFile} not found!"
2565     return 1
2566   fi
2567
2568   unset encodedSpaces
2569   for opt in "${args[@]}"; do
2570     [[ "${opt}" =~ encodedSpaces=.* ]] && encodedSpaces=1 && echo "encodedSpaces!" && break
2571   done
2572
2573   #then, parse the options as they override the options from file
2574   for opt in "${args[@]}"; do
2575     [[ -z ${opt} ]] && continue
2576     [[ -n ${encodedSpaces} ]] && opt="$(decSpaces ${opt})"
2577     [[ "${opt}" =~ ^[[:space:]]*$ ]] && continue
2578     if [[ ! "${opt}" =~ .*=.* ]]; then
2579       echo "badly formatted option \"${opt}\" should be: option=value, stopping..."
2580       return 1
2581     fi
2582     local var="${opt%%=*}"
2583     local value="${opt#*=}"
2584     echo "${var}=${value}"
2585     export ${var}="${value}"
2586   done
2587
2588   #do some checking
2589   [[ -z ${alirootEnv} ]] && echo "alirootEnv not defined!" && return 1
2590
2591   #export the aliroot function if defined to override normal behaviour
2592   [[ $(type -t aliroot) =~ "function" ]] && export -f aliroot && echo "exporting aliroot() function..."
2593
2594   return 0
2595 }
2596
2597 aliroot()
2598 {
2599   args=("$@")
2600   if [[ -n ${useProfilingCommand} ]]; then
2601     profilerLogFile="cpu.txt"
2602     [[ "${args[@]}" =~ rec ]] && profilerLogFile="cpu_rec.txt"
2603     [[ "${args[@]}" =~ Calib ]] && profilerLogFile="cpu_calib.txt"
2604     echo running "${useProfilingCommand} aliroot ${args[@]} &> ${profilerLogFile}"
2605     ${useProfilingCommand} aliroot "${args[@]}" &> ${profilerLogFile}
2606   else
2607     #to prevent an infinite recursion use "command aliroot" to disable
2608     #aliases and functions
2609     echo running command aliroot "${args[@]}"
2610     command aliroot "${args[@]}"
2611   fi
2612   return 0
2613 }
2614
2615 paranoidCp()
2616 (
2617   #recursively copy files and directories
2618   #to avoid using find and the like as they kill
2619   #the performance on some cluster file systems
2620   #does not copy links to avoid problems
2621   sourceFiles=("${@}")
2622   destination="${sourceFiles[@]:(-1)}" #last element
2623   unset sourceFiles[${#sourceFiles[@]}-1] #remove last element (dst)
2624   for src in "${sourceFiles[@]}"; do
2625     if [[ -f "${src}" && ! -h  "${src}" ]]; then
2626       paranoidCopyFile "${src}" "${destination}"
2627     elif [[ -d "${src}" && ! -h "${src}" ]]; then
2628       src="${src%/}"
2629       dst="${destination}/${src##*/}"
2630       mkdir -p "${dst}"
2631       paranoidCp "${src}"/* "${dst}"
2632     fi
2633   done
2634 )
2635
2636 paranoidCopyFile()
2637 (
2638   #copy a single file to a target in an existing dir
2639   #repeat a few times if copy fails
2640   src="${1}"
2641   dst="${2}"
2642   [[ -d "${dst}" ]] && dst="${dst}/${src##*/}"
2643   [[ -z "${maxCopyTries}" ]] && maxCopyTries=5
2644   #echo "maxCopyTries=${maxCopyTries}"
2645   echo "cp ${src} ${dst}"
2646   cp "${src}" "${dst}"
2647   i=0
2648   until cmp -s "${src}" "${dst}"; do
2649     echo "try: ${i}"
2650     [[ -f "${dst}" ]] && rm "${dst}"
2651     cp "${src}" "${dst}"
2652     [[ ${i} -gt ${maxCopyTries} ]] && ret=1 && return 1
2653     (( i++ ))
2654   done
2655   return 0
2656 )
2657
2658 guessRunData()
2659 {
2660   #guess the period from the path, pick the rightmost one
2661   period=""
2662   runNumber=""
2663   year=""
2664   pass=""
2665   legoTrainRunNumber=""
2666   dataType=""
2667
2668   local shortRunNumber=""
2669   local IFS="/"
2670   declare -a path=( $1 )
2671   local dirDepth=$(( ${#path[*]}-1 ))
2672   i=0
2673   #for ((x=${dirDepth};x>=0;x--)); do
2674   for ((x=0;x<=${dirDepth};x++)); do
2675
2676     [[ $((x-1)) -ge 0 ]] && local fieldPrev=${path[$((x-1))]}
2677     local field=${path[${x}]}
2678     local fieldNext=${path[$((x+1))]}
2679
2680     [[ ${field} =~ ^[0-9]*$ && ${fieldNext} =~ (.*\.zip$|.*\.root$) ]] && legoTrainRunNumber=${field}
2681     [[ -n ${legoTrainRunNumber} && -z ${pass} ]] && pass=${fieldPrev}
2682     [[ ${field} =~ ^LHC[0-9][0-9][a-z].*$ ]] && period=${field%_*}
2683     [[ ${field} =~ ^000[0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && runNumber=${field#000}
2684     [[ ${field} =~ ^[0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && shortRunNumber=${field}
2685     [[ ${field} =~ ^20[0-9][0-9]$ ]] && year=${field}
2686     [[ ${field} =~ ^(^sim$|^data$) ]] && dataType=${field}
2687     (( i++ ))
2688   done
2689   [[ -z ${legoTrainRunNumber} ]] && pass=${path[$((dirDepth-1))]}
2690   [[ "${dataType}" =~ ^sim$ ]] && pass="passMC" && runNumber=${shortRunNumber}
2691   
2692   #if [[ -z ${dataType} || -z ${year} || -z ${period} || -z ${runNumber}} || -z ${pass} ]];
2693   if [[ -z ${runNumber}} ]];
2694   then
2695     #error condition
2696     return 1
2697   else
2698     #ALL OK
2699     return 0
2700   fi
2701   return 0
2702 }
2703
2704 #these functions encode strings to and from a space-less form
2705 #use when spaces are not well handled (e.g. in arguments to 
2706 #commands in makeflow files, etc.
2707 encSpaces()(echo "${1// /@@@@}")
2708 decSpaces()(echo "${1//@@@@/ }")
2709
2710 main "$@"