5eb1092ab541212a8fc590c680457eabbc63be2c
[u/mrichter/AliRoot.git] / PWGPP / benchmark / benchmark.sh
1 #!/bin/bash
2 #include benchmark.config
3
4 # this script runs the CPass0/CPass1 train
5 # produced OCDB updates are local
6
7 main()
8 {
9   #run in proper mode depending on the selection
10   if [[ $# -lt 1 ]]; then
11     if [[ ! "${0}" =~ "bash" ]]; then
12       echo "uses makeflow:"
13       echo " ${0} \"run\" productionID inputList configFile [extraOpts]"
14       echo "uses a batch system (SGE):"
15       echo " ${0} \"submit\" productionID inputList configFile [extraOpts]"
16       echo "extraOpts if non-empty override the config file, e.g.:"
17       echo " ${0} submit test1 benchmark.list benchmark.config runNumber=169123 nEvents=10"
18     fi
19     return
20   fi
21
22   #define some aliases - default is to call one of the functions directly
23   runMode=${1}
24   umask 0002
25   shift
26   case ${runMode} in
27     "CPass0") goCPass0 "$@";;
28     "CPass1") goCPass1 "$@";;
29     "MakeLocalOCDBaccessConfig") goMakeLocalOCDBaccessConfig "$@";;
30     "MergeCPass0") goMergeCPass0 "$@";;
31     "MergeCPass1") goMergeCPass1 "$@";;
32     "MakeFilteredTrees") goMakeFilteredTrees "$@";;
33     "MakeSummary") goMakeSummary "$@";;
34     "run") goSubmitMakeflow "$@";;
35     "submit") goSubmitBatch "$@";;
36     "test") goTest "$@";;
37     "GenerateMakeflow") goGenerateMakeflow "$@";;
38     "PrintValues") goPrintValues "$@";;
39     "CreateQAplots") goCreateQAplots "$@";;
40     "WaitForOutput") goWaitForOutput "$@";;
41     "Merge") goMerge "$@";;
42     *) 
43       ${runMode} "$@"
44     ;;
45   esac
46   return 0
47 }
48
49 generateMC()
50 {
51   #generate one raw chunk in current directory
52   SEED=${JOB_ID}${SGE_TASK_ID}
53   export CONFIG_SEED=${SEED}
54   runNumber=${1}
55   OCDBpath=${2}
56   nEventsim=${3}
57   if [[ -n ${pretend} ]]; then
58     sleep ${pretendDelay}
59     touch galice.root
60   else
61     if [[ -f sim.C && -f Config.C ]] ; then
62         time aliroot -b -q -x sim.C\(${runNumber},\"${OCDBpath}\",${nEventsim}\) >sim.log 2>&1
63         mv syswatch.log simwatch.log
64     fi
65   fi
66 }
67
68 goCPass0()
69 (
70   umask 0002
71   
72   targetDirectory=${1}
73   inputList=${2}
74   nEvents=${3}
75   ocdbPath=${4}
76   configFile=${5}
77   runNumber=${6}
78   jobindex=${7}
79   shift 7
80   if ! parseConfig ${configFile} "$@"; then return 1; fi
81
82   #record the working directory provided by the batch system
83   batchWorkingDirectory=${PWD}
84
85   #use the jobindex only if set and non-negative
86   if [[ -z ${jobindex} || ${jobindex} -lt 0 ]]; then
87     [[ -n "${LSB_JOBINDEX}" ]] && jobindex=${LSB_JOBINDEX}
88     [[ -n "${SGE_TASK_ID}" ]] && jobindex=${SGE_TASK_ID}
89     if [[ -z ${jobindex} ]]; then 
90       echo "no jobindex!"
91       return 1
92     fi
93   fi
94
95   [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
96   doneFile="cpass0.job${jobindex}.run${runNumber}.done"
97   [[ -n ${useProfilingCommand} ]] && doneFile="profiling.cpass0.job${jobindex}.run${runNumber}.done"
98   [[ -z ${doneFileDest} ]] && doneFileDest="${commonOutputPath}/meta/"
99
100   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
101   
102   if [[ -n ${ALIROOT_FORCE_COREDUMP} ]]; then
103     ulimit -c unlimited 
104     export ALIROOT_FORCE_COREDUMP
105   fi
106
107   #the contents of this is stored in the tree and used later (e.g. AliAnalysisTaskPIDResponse)!
108   #at the QA stage the pass number is guessed from the path stored here.
109   #The Format is:
110   #Packages= ;OutputDir= ;LPMPass= ;TriggerAlias= ;LPMRunNumber= ;LPMProductionType= ;LPMInteractionType= ;LPMProductionTag= ;LPMAnchorRun= ;LPMAnchorProduction= ;LPMAnchorYear= 
111   export PRODUCTION_METADATA="OutputDir=cpass0"
112
113   if [[ "${inputList}" =~ \.root$ ]]; then
114     infile=${inputList}
115   else
116     infile=$(sed -ne "${jobindex}p" ${inputList} | egrep '\s*\w*/\w*')
117   fi
118   chunkName=${infile##*/}
119
120   outputDir=${targetDirectory}/${jobindex}_${chunkName%.*}
121   mkdir -p ${outputDir}
122   if [[ ! -d ${outputDir} ]]; then 
123     touch ${doneFile} 
124     echo "cannot make ${outputDir}" >> ${doneFile}
125     cp ${doneFile} ${doneFileDest}
126     return 1  
127   fi
128   
129   #runpath=${PWD}/rundir_cpass0_${runNumber}_${jobindex}
130   runpath=${outputDir}
131   #[[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
132   [[ ${reconstructInTemporaryDir} -eq 1 ]] && runpath=$(mktemp -d -t cpass0.XXXXXX)
133   mkdir -p ${runpath}
134   if [[ ! -d ${runpath} ]]; then
135     touch ${doneFile} 
136     echo "cannot make runpath ${runpath}" >> ${doneFile}
137     cp ${doneFile} ${doneFileDest}
138     return 1
139   fi
140   if ! cd ${runpath}; then
141     touch ${doneFile}
142     echo "PWD=$PWD is not the runpath=${runpath}" >> ${doneFile}
143     cp ${doneFile} ${doneFileDest}
144     return 1
145   fi
146
147   #runCPassX/C expects the raw chunk to be linked in the run dir
148   #despite it being accessed by the full path
149   ln -s ${infile} ${runpath}/${chunkName}
150
151   #####MC
152   if [[ -n ${generateMC} ]]; then
153     olddir=${PWD}
154     outputDirMC=${commonOutputPath}/000${runNumber}/sim/${jobindex}
155     simrunpath=${outputDirMC}
156     #[[ ${simulateInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && simrunpath=${TMPDIR}
157     [[ ${simulateInTemporaryDir} -eq 1 ]] && simrunpath=$(mktemp -d -t cpass0MC.XXXXXX)
158     mkdir -p ${outputDirMC}
159     mkdir -p ${simrunpath}
160     if cd ${simrunpath}; then
161
162       filesMC=( 
163       "${batchWorkingDirectory}/sim.C"
164       "${batchWorkingDirectory}/rec.C"
165       "${batchWorkingDirectory}/Config.C"
166       "${batchWorkingDirectory}/OCDB_*.root"
167       )
168       for file in ${filesMC[*]}; do
169         [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
170       done
171
172       generateMC ${runNumber} ${ocdbPath} ${nEvents}
173
174       [[ ! "${simrunpath}" =~ "${outputDirMC}" ]] && mv * ${outputDirMC} #TODO check if it works
175       cd ${olddir}
176
177       ln -s ${outputDirMC}/* ${runpath}/ 
178
179       inputList=${outputDirMC}/galice.root #TODO not valid outside shell !!!
180       infile=""
181     fi
182   fi
183   ######
184   
185   if [[ ! -f ${inputList} && -z ${pretend} ]]; then
186     touch ${doneFile} 
187     echo "input file ${inputList} not found, exiting..." >> ${doneFile}
188     cp ${doneFile} ${doneFileDest}
189     return 1
190   fi
191
192   logOutputDir=${runpath}
193   [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir}
194   [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/stdout
195   #[[ -z ${dontRedirectStdOutToLog} ]] && exec 2> ${logOutputDir}/stderr
196   echo "${0} $*"
197
198   echo "#####################"
199   echo CPass0:
200   echo JOB setup
201   echo nEvents            ${nEvents}
202   echo runNumber          ${runNumber}
203   echo ocdbPath           ${ocdbPath}
204   echo infile             ${infile}
205   echo chunkName          ${chunkName}
206   echo jobindex           ${jobindex}
207   echo recoTriggerOptions ${recoTriggerOptions}
208   echo targetDirectory    ${targetDirectory}
209   echo commonOutputPath         ${commonOutputPath}
210   echo doneFile      ${doneFile}
211   echo doneFileDest  ${doneFileDest}
212   echo batchWorkingDirectory=$batchWorkingDirectory
213   echo runpath            ${runpath}  
214   echo outputDir          ${outputDir}
215   echo PWD                ${PWD}
216   echo ALICE_ROOT         ${ALICE_ROOT}
217   echo "########## ###########"
218
219   alirootInfo > ALICE_ROOT.log
220
221   filesCPass0=( 
222                "${batchWorkingDirectory}/runCPass0.sh"
223                "${batchWorkingDirectory}/recCPass0.C"
224                "${batchWorkingDirectory}/runCalibTrain.C"
225                "${batchWorkingDirectory}/localOCDBaccessConfig.C"
226                "${batchWorkingDirectory}/OCDB.root"
227                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/runCPass0.sh"
228                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/recCPass0.C" 
229                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/runCalibTrain.C"
230   )
231
232   for file in ${filesCPass0[*]}; do
233     [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
234     [[ ${file##*/} =~ .*\.sh ]] && chmod +x ${file##*/}
235   done
236
237   echo "this directory (${PWD}) contents:"
238   /bin/ls
239   echo
240   chmod u+x runCPass0.sh
241
242   sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh
243
244   if [[ -n ${postSetUpActionCPass0} ]]; then
245     echo "running ${postSetUpActionCPass0}"
246     eval ${postSetUpActionCPass0}
247   fi
248
249   #run CPass0
250   echo "${runpath}/runCPass0.sh ${infile} ${nEvents} ${runNumber} ${ocdbPath} ${recoTriggerOptions}"
251   if [[ -n ${pretend} ]]; then
252     sleep ${pretendDelay}
253     touch AliESDs.root
254     touch AliESDfriends.root
255     touch AliESDfriends_v1.root
256     touch rec.log
257     touch calib.log
258   else
259     echo ./runCPass0.sh "${infile}" "${nEvents}" "${runNumber}" "${ocdbPath}" "${recoTriggerOptions}"
260     ./runCPass0.sh "${infile}" "${nEvents}" "${runNumber}" "${ocdbPath}" "${recoTriggerOptions}"
261   fi
262   
263   #move stuff to final destination
264   echo "this directory (${PWD}) contents:"
265   /bin/ls
266   echo
267
268   echo rm -f ./${chunkName}
269   rm -f ./${chunkName}
270   echo "cp -R ${runpath}/* ${outputDir}"
271   cp -p -R ${runpath}/* ${outputDir}
272   echo
273   
274   #validate CPass0
275   cd ${outputDir}
276   touch ${doneFile}
277   echo "dir ${outputDir}" >> ${doneFile}
278   if summarizeLogs >> ${doneFile}; then
279     [[ -f ${outputDirMC}/galice.root ]] && echo "sim ${outputDirMC}/galice.root" >> ${doneFile}
280     [[ -f AliESDfriends_v1.root ]] && echo "calibfile ${outputDir}/AliESDfriends_v1.root" >> ${doneFile}
281     [[ -f AliESDs.root ]] && echo "esd ${outputDir}/AliESDs.root" >> ${doneFile}
282   fi
283   cp ${doneFile} ${doneFileDest}
284
285   [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath} && echo "removing ${runpath}"
286   return 0
287 )
288
289 goCPass1()
290 (
291   umask 0002
292   
293   targetDirectory=${1}
294   inputList=${2}
295   nEvents=${3}
296   ocdbPath=${4}
297   configFile=${5}
298   runNumber=${6}
299   jobindex=${7}
300   shift 7
301   extraOpts=("$@")
302   if ! parseConfig ${configFile} "$@"; then return 1; fi
303
304   #record the working directory provided by the batch system
305   batchWorkingDirectory=${PWD}
306
307   #use the jobindex only if set and non-negative
308   if [[ -z ${jobindex} || ${jobindex} -lt 0 ]]; then
309     [[ -n "${LSB_JOBINDEX}" ]] && jobindex=${LSB_JOBINDEX}
310     [[ -n "${SGE_TASK_ID}" ]] && jobindex=${SGE_TASK_ID}
311     if [[ -z ${jobindex} ]]; then 
312       echo "no jobindex!"
313       return 1
314     fi
315   fi
316
317   [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
318   doneFile="cpass1.job${jobindex}.run${runNumber}.done"
319   [[ -n ${useProfilingCommand} ]] && doneFile="profiling.cpass1.job${jobindex}.run${runNumber}.done"
320   [[ -z ${doneFileDest} ]] && doneFileDest="${commonOutputPath}/meta/"
321
322   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
323   
324   if [[ -n ${ALIROOT_FORCE_COREDUMP} ]]; then
325     ulimit -c unlimited 
326     export ALIROOT_FORCE_COREDUMP
327   fi
328
329   #the contents of this is stored in the tree and used later (e.g. AliAnalysisTaskPIDResponse)!
330   #at the QA stage the pass number is guessed from the path stored here.
331   #The Format is:
332   #Packages= ;OutputDir= ;LPMPass= ;TriggerAlias= ;LPMRunNumber= ;LPMProductionType= ;LPMInteractionType= ;LPMProductionTag= ;LPMAnchorRun= ;LPMAnchorProduction= ;LPMAnchorYear= 
333   export PRODUCTION_METADATA="OutputDir=cpass1"
334
335   if [[ ! -f ${inputList} && -z ${pretend} ]]; then
336     touch ${doneFile}
337     echo "input file ${inputList} not found, exiting..." >> ${doneFile}
338     cp ${doneFile} ${doneFileDest}
339     return 1
340   fi
341   if [[ "${inputList}" =~ \.root$ ]]; then
342     infile=${inputList}
343   else
344     infile=$(sed -ne "${jobindex}p" ${inputList} | egrep '\s*\w*/\w*')
345   fi
346   chunkName=${infile##*/}
347
348   outputDir=${targetDirectory}/${jobindex}_${chunkName%.*}
349   mkdir -p ${outputDir}
350   if [[ ! -d ${outputDir} ]];then
351     touch ${doneFile}
352     echo "cannot make ${outputDir}" >> ${doneFile}
353     cp ${doneFile} ${doneFileDest}
354     return 1
355   fi
356   
357   #runpath=${PWD}/rundir_cpass1_${runNumber}_${jobindex}
358   runpath=${outputDir}
359   #[[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
360   [[ ${reconstructInTemporaryDir} -eq 1 ]] && runpath=$(mktemp -d -t cpass1.XXXXXX)
361
362   #MC
363   if [[ "${infile}" =~ galice\.root ]]; then
364     ln -s ${inputList%/*}/* ${runpath}
365     infile=""
366   fi
367
368   #init the running path
369   mkdir -p ${runpath}
370   if [[ ! -d ${runpath} ]]; then
371    touch ${doneFile}
372    echo "cannot make runpath ${runpath}" >> ${doneFile}
373    cp ${doneFile} ${doneFileDest}
374    return 1
375  fi
376   if ! cd ${runpath}; then
377     touch ${doneFile}
378     echo "PWD=$PWD is not the runpath=${runpath}" >> ${doneFile}
379     cp ${doneFile} ${doneFileDest}
380     return 1
381   fi
382
383   #this is needed for runCPass1.sh
384   ln -s ${infile} ${runpath}/${chunkName}
385
386   logOutputDir=${runpath}
387   [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir}
388   [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/stdout
389   #[[ -z ${dontRedirectStdOutToLog} ]] && exec 2> ${logOutputDir}/stderr
390   echo "${0} $*"
391
392   echo "#####################"
393   echo CPass1:
394   echo JOB setup
395   echo nEvents            ${nEvents}
396   echo runNumber          ${runNumber}
397   echo ocdbPath           ${ocdbPath}
398   echo infile             ${infile}
399   echo chunkName          ${chunkName}
400   echo jobindex           ${jobindex}
401   echo recoTriggerOptions ${recoTriggerOptions}
402   echo targetDirectory    ${targetDirectory}
403   echo commonOutputPath         ${commonOutputPath}
404   echo doneFile      ${doneFile}
405   echo doneFileDest  ${doneFileDest}
406   echo runpath            ${runpath}  
407   echo outputDir          ${outputDir}
408   echo batchWorkingDirectory ${batchWorkingDirectory}
409   echo ALICE_ROOT         ${ALICE_ROOT}
410   echo PWD                ${PWD}
411   echo "########## ###########"
412
413   alirootInfo > ALICE_ROOT.log
414
415   filesCPass1=( 
416                "${batchWorkingDirectory}/runCPass1.sh"
417                "${batchWorkingDirectory}/recCPass1.C"
418                "${batchWorkingDirectory}/recCPass1_OuterDet.C"
419                "${batchWorkingDirectory}/runCalibTrain.C"
420                "${batchWorkingDirectory}/QAtrain_duo.C"
421                "${batchWorkingDirectory}/localOCDBaccessConfig.C"
422                "${batchWorkingDirectory}/${configFile}"
423                "${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz"
424                "${batchWorkingDirectory}/OCDB.root"
425                "${trustedQAtrainMacro}"
426                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/runCPass1.sh"
427                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/recCPass1.C" 
428                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/recCPass1_OuterDet.C" 
429                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/runCalibTrain.C"
430                "${ALICE_ROOT}/ANALYSIS/macros/QAtrain_duo.C"
431   )
432
433   for file in "${filesCPass1[@]}"; do
434     [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
435     [[ ${file##*/} =~ .*\.sh ]] && echo "making ${file##*/} executable" && chmod +x ${file##*/}
436   done
437
438   echo "this directory (${PWD}) contents:"
439   /bin/ls
440   echo
441
442   #remove spaces around commas from calls to root
443   sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh
444
445   if [[ -n ${postSetUpActionCPass1} ]]; then
446     echo "running ${postSetUpActionCPass1}"
447     eval ${postSetUpActionCPass1}
448     echo
449   fi
450
451   #configure local OCDB storage from CPass0 (creates the localOCDBaccessConfig.C script)
452   if [[ -f cpass0.localOCDB.${runNumber}.tgz ]]; then
453     echo goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz"
454     goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz"
455   else
456     echo "WARNING: file cpass0.localOCDB.${runNumber}.tgz not found!"
457   fi
458
459   if [[ ! $(/bin/ls -1 OCDB/*/*/*/*.root 2>/dev/null) ]]; then
460     touch ${doneFile}
461     echo "cpass0 produced no calibration! exiting..." >> ${doneFile}
462     cp ${doneFile} ${doneFileDest}
463     return 1
464   fi
465
466   #create the Barrel and OuterDet directories for CPass1 and link the local OCDB directory
467   #there to make the localOCDBaccessConfig.C file work, since it may point to the OCDB
468   #entries using a relative path, e.g. local://./OCDB
469   echo "linking the OCDB/ for Barrel and OuterDet directories"
470   mkdir Barrel OuterDet
471   ls -l
472   ln -s ../OCDB Barrel/OCDB
473   ln -s ../OCDB OuterDet/OCDB
474
475   #setup the filtering
476   #the following option enables the filtering task inside the QAtrain_duo.C
477   [[ -n $runESDfiltering ]] && export QA_TaskFilteredTree=1
478   #set the downscaling factors during the filtering fro expert QA (overrides the previous values)
479   if [[ -n ${filteringFactorHighPt} ]]; then
480     export AliAnalysisTaskFilteredTree_fLowPtTrackDownscaligF=${filteringFactorHighPt}
481   fi
482   if [[ -n ${filteringFactorV0s} ]]; then
483     export AliAnalysisTaskFilteredTree_fLowPtV0DownscaligF=${filteringFactorV0s} 
484   fi
485
486   #run CPass1
487   chmod u+x runCPass1.sh
488   echo "${runpath}/runCPass1.sh ${infile} ${nEvents} ${runNumber} ${ocdbPath} ${recoTriggerOptions}"
489   if [[ -n ${pretend} ]]; then
490     sleep ${pretendDelay}
491     touch AliESDs_Barrel.root
492     touch AliESDfriends_Barrel.root
493     touch AliESDfriends_v1.root
494     touch QAresults_barrel.root
495     touch EventStat_temp_barrel.root
496     touch AODtpITS.root
497     touch AliESDs_Outer.root
498     touch AliESDfriends_Outer.root
499     touch QAresults_outer.root
500     touch EventStat_temp_outer.root
501     touch rec.log
502     touch calib.log
503     touch qa.log
504     touch filtering.log FilterEvents_Trees.root
505   else
506     ./runCPass1.sh "${infile}" "${nEvents}" "${runNumber}" "${ocdbPath}" "${recoTriggerOptions}"
507     
508     [[ ! -f AliESDs_Barrel.root && -f Barrel/AliESDs.root ]] && mv Barrel/AliESDs.root AliESDs_Barrel.root
509     [[ ! -f AliESDfriends_Barrel.root && -f Barrel/AliESDfriends.root ]] && mv Barrel/AliESDfriends.root AliESDfriends_Barrel.root
510     [[ ! -f AliESDfriends_v1.root && -f Barrel/AliESDfriends_v1.root ]] && mv Barrel/AliESDfriends_v1.root .
511     [[ ! -f QAresults_barrel.root && -f Barrel/QAresults_barrel.root ]] && mv Barrel/QAresults_barrel.root .
512     [[ ! -f AliESDs_Outer.root && -f OuterDet/AliESDs.root ]] && mv OuterDet/AliESDs.root AliESDs_Outer.root
513     [[ ! -f AliESDfriends_Outer.root && -f OuterDet/AliESDfriends.root ]] && mv OuterDet/AliESDfriends.root AliESDfriends_Outer.root
514     [[ ! -f QAresults_outer.root && -f OuterDet/QAresults_outer.root ]] && mv OuterDet/QAresults_outer.root .
515     [[ ! -f FilterEvents_Trees.root && -f Barrel/FilterEvents_Trees.root ]] && mv Barrel/FilterEvents_Trees.root .
516
517     #make the filtered tree (if requested and not already produced by QA
518     [[ -f AliESDs_Barrel.root ]] && echo "AliESDs_Barrel.root" > filtered.list
519     if [[ -n ${runESDfiltering} && ! -f FilterEvents_Trees.root && -f filtered.list ]]; then 
520       goMakeFilteredTrees ${PWD} ${runNumber} "${PWD}/filtered.list" ${filteringFactorHighPt} ${filteringFactorV0s} ${ocdbPath} 1000000 0 10000000 0 ${configFile} AliESDs_Barrel.root "${extraOpts[@]}" >filtering.log
521     else
522       echo ""
523     fi
524
525   fi
526
527   ##handle possible crashes in QA (happens often in trunk)
528   ##rerun QA with a trusted aliroot version
529   #if [[ $(validateLog qa_barrel.log) ]]; then
530   #  echo "qa_barrel.log not validated!"
531   #fi
532   #if [[ ! -f QAresults_barrel.root && -f ${setupTrustedAliROOTenvInCurrentShell} || $(validateLog qa_barrel.log) ]]; then
533   #  echo "WARNING: using trusted QA aliroot ${ALICE_ROOT}"
534   #  source ${setupTrustedAliROOTenvInCurrentShell}
535   #  cd Barrel
536   #  rm QAresults_barrel.root
537   #  rm EventStat_temp_barrel.root
538   #  rm AODtpITS.root
539   #  [[ ! -f AliESDs.root ]] && ln -s ../AliESDs_Barrel.root AliESDs.root
540   #  [[ ! -f AliESDfriends.root ]] && ln -s ../AliESDfriends_Barrel.root AliESDfriends.root
541   #  if [[ -n ${trustedQAtrainMacro} ]]; then
542   #    eval "cp ${trustedQAtrainMacro} QAtrain_duo_trusted.C"
543   #  fi
544   #  echo executing aliroot -b -q "QAtrain_duo_trusted.C(\"_barrel\",${runNumber},\"wn.xml\",0,\"${ocdbPath}\")"
545   #  time aliroot -b -q "QAtrain_duo.C(\"_barrel\",${runNumber},\"wn.xml\",0,\"${ocdbPath}\")" &> ../qa_barrel_trusted.log
546   #  cd ../
547   #fi
548
549   #move stuff to final destination
550   echo "this directory (${PWD}) contents:"
551   /bin/ls
552   echo rm -f ./${chunkName}
553   rm -f ./${chunkName}
554   echo "cp -R ${runpath}/* ${outputDir}"
555   cp -pf -R ${runpath}/* ${outputDir}
556   echo
557
558   #validate CPass1
559   cd ${outputDir}
560   touch ${doneFile}
561   echo "dir ${outputDir}" >> ${doneFile}
562   if summarizeLogs >> ${doneFile}; then
563     [[ -f AliESDs_Barrel.root ]] && echo "esd ${outputDir}/AliESDs_Barrel.root" >> ${doneFile}
564     [[ -f AliESDfriends_v1.root ]] && echo "calibfile ${outputDir}/AliESDfriends_v1.root" >> ${doneFile}
565     [[ -f QAresults_Barrel.root ]] && echo "qafile ${outputDir}/QAresults_Barrel.root" >> ${doneFile}
566     [[ -f QAresults_Outer.root ]] && echo "qafile ${outputDir}/QAresults_Outer.root" >> ${doneFile}
567     [[ -f QAresults_barrel.root ]] && echo "qafile ${outputDir}/QAresults_barrel.root" >> ${doneFile}
568     [[ -f QAresults_outer.root ]] && echo "qafile ${outputDir}/QAresults_outer.root" >> ${doneFile}
569     [[ -f FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFile}
570   else
571     if grep "qa_outer.log.*OK" ${doneFile} > /dev/null; then
572       [[ -f QAresults_Outer.root ]] && echo "qafile ${outputDir}/QAresults_Outer.root" >> ${doneFile}
573       [[ -f QAresults_outer.root ]] && echo "qafile ${outputDir}/QAresults_outer.root" >> ${doneFile}
574     fi
575     if grep "qa_barrel.log.*OK" ${doneFile} > /dev/null; then
576       [[ -f QAresults_Barrel.root ]] && echo "qafile ${outputDir}/QAresults_Barrel.root" >> ${doneFile}
577       [[ -f QAresults_barrel.root ]] && echo "qafile ${outputDir}/QAresults_barrel.root" >> ${doneFile}
578     fi
579     if grep "filtering.log.*OK" ${doneFile} > /dev/null; then
580       [[ -f FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFile}
581     fi
582   fi
583   cp ${doneFile} ${doneFileDest}
584
585   [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
586   return 0
587 )
588
589
590 goMergeCPass0()
591 (
592   #
593   # find the output files and merge them
594   #
595
596   outputDir=${1}
597   ocdbStorage=${2}
598   configFile=${3}
599   runNumber=${4}
600   calibrationFilesToMerge=${5}  #can be a non-existent file, will then be produced on the fly
601   shift 5
602   if ! parseConfig ${configFile} "$@"; then return 1; fi
603
604   #record the working directory provided by the batch system
605   batchWorkingDirectory=${PWD}
606
607   [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
608   doneFile="merge.cpass0.run${runNumber}.done"
609   [[ -z ${doneFileDest} ]] && doneFileDest="${commonOutputPath}/meta/"
610
611   umask 0002
612   ulimit -c unlimited 
613
614   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
615
616   #runpath=${PWD}/rundir_cpass0_Merge_${runNumber}
617   runpath=${outputDir}
618   #[[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
619   [[ ${reconstructInTemporaryDir} -eq 1 ]] && runpath=$(mktemp -d -t mergeCPass0.XXXXXX)
620
621   mkdir -p ${runpath}
622   if [[ ! -d ${runpath} ]]; then
623     touch ${doneFile}
624     echo "not able to make the runpath ${runpath}" >> ${doneFile}
625     cp ${doneFile} ${doneFileDest}
626     return 1
627   fi
628   if ! cd ${runpath}; then 
629     touch ${doneFile}
630     echo "PWD=$PWD is not the runpath=${runpath}" >> ${doneFile}
631     cp ${doneFile} ${doneFileDest}
632     return 1
633   fi
634
635   logOutputDir=${runpath}
636   [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir}
637   [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/stdout
638   echo "${0} $*"
639
640   mergingScript="mergeMakeOCDB.byComponent.sh"
641
642   echo goMergeCPass0 SETUP:
643   echo runNumber=${runNumber}
644   echo outputDir=${outputDir}
645   echo ocdbStorage=${ocdbStorage}
646   echo calibrationFilesToMerge=${calibrationFilesToMerge}
647   echo mergingScript=${mergingScript}
648   echo commonOutputPath=${commonOutputPath}
649   echo runpath=${runpath}
650   
651   # copy files in case they are not already there
652   filesMergeCPass0=(
653                     "${batchWorkingDirectory}/${calibrationFilesToMerge}"
654                     "${batchWorkingDirectory}/OCDB.root"
655                     "${batchWorkingDirectory}/localOCDBaccessConfig.C"
656                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/mergeMakeOCDB.byComponent.sh"
657                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/mergeByComponent.C"
658                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/makeOCDB.C"
659                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/merge.C"
660                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/mergeMakeOCDB.sh"
661   )
662   for file in ${filesMergeCPass0[*]}; do
663     [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
664     [[ ${file##*/} =~ .*\.sh ]] && chmod +x ${file##*/}
665   done
666   
667   sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh
668
669   alirootInfo > ALICE_ROOT.log
670
671   #
672   echo "PWD"
673   /bin/ls
674   echo "PWD/.."
675   /bin/ls ../
676
677
678   #merge calibration
679   chmod u+x ${mergingScript}  
680   mkdir -p ./OCDB
681   if [[ ! -f ${calibrationFilesToMerge} ]]; then
682     echo "/bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root > ${calibrationFilesToMerge}"
683     /bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root 2>/dev/null > ${calibrationFilesToMerge}
684   fi
685   
686   echo "${mergingScript} ${calibrationFilesToMerge} ${runNumber} local://./OCDB ${ocdbStorage}"
687   if [[ -n ${pretend} ]]; then
688     sleep ${pretendDelay}
689     touch CalibObjects.root
690     touch ocdb.log
691     touch merge.log
692     touch dcsTime.root
693     mkdir -p ./OCDB/TPC/Calib/TimeGain/
694     mkdir -p ./OCDB/TPC/Calib/TimeDrift/
695     echo "some calibration" >> ./OCDB/TPC/Calib/TimeGain/someCalibObject_0-999999_cpass0.root
696     echo "some calibration" >> ./OCDB/TPC/Calib/TimeDrift/otherCalibObject_0-999999_cpass0.root
697   else
698     ./${mergingScript} ${calibrationFilesToMerge} ${runNumber} "local://./OCDB" ${ocdbStorage} >> "mergeMakeOCDB.log"
699
700     #produce the calib trees for expert QA (dcsTime.root)
701     goMakeLocalOCDBaccessConfig ./OCDB
702     echo aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")"
703     aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")"
704   fi
705   
706   ### produce the output
707   #tar the produced OCDB for reuse
708   tar czf ${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz ./OCDB
709
710   /bin/ls
711
712   #copy all to output dir
713   cp -pf -R ${runpath}/* ${outputDir}
714
715   if [[ -n ${generateMC} ]]; then
716     goPrintValues sim ${commonOutputPath}/meta/sim.run${runNumber}.list ${commonOutputPath}/meta/cpass0.job*.run${runNumber}.done
717   fi
718
719   #validate merging cpass0
720   cd ${outputDir}
721   touch ${doneFile}
722   echo "dir ${outputDir}" >> ${doneFile}
723   if summarizeLogs >> ${doneFile}; then
724     [[ -f CalibObjects.root ]] && echo "calibfile ${outputDir}/CalibObjects.root" >> ${doneFile}
725     [[ -f dcsTime.root ]] && echo "dcsTree ${outputDir}/dcsTime.root" >> ${doneFile}
726   fi
727   cp ${doneFile} ${doneFileDest}
728
729   [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
730   return 0
731 )
732
733 goMergeCPass1()
734 (
735   #
736   # find the output files and merge them
737   #
738
739   outputDir=${1}
740   ocdbStorage=${2}
741   configFile=${3}
742   runNumber=${4}
743   calibrationFilesToMerge=${5}
744   qaFilesToMerge=${6}
745   filteredFilesToMerge=${7}
746   shift 7
747   if ! parseConfig ${configFile} "$@"; then return 1; fi
748
749   #record the working directory provided by the batch system
750   batchWorkingDirectory=${PWD}
751
752   [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
753   doneFile="merge.cpass1.run${runNumber}.done"
754   doneFile="${commonOutputPath}/meta/"
755
756   umask 0002
757   ulimit -c unlimited 
758
759   #clean up first:
760   rm -f ${outputDir}/*.log
761   rm -f ${outputDir}/*.root
762   rm -f ${outputDir}/*done
763
764   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
765
766   #runpath=${PWD}/rundir_cpass1_Merge_${runNumber}
767   runpath=${outputDir}
768   #[[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
769   [[ ${reconstructInTemporaryDir} -eq 1 ]] && runpath=$(mktemp -d -t mergeCPass1.XXXXXX)
770
771   mkdir -p ${runpath}
772   if [[ ! -d ${runpath} ]]; then
773     touch ${doneFile}
774     echo "not able to make the runpath ${runpath}" >> ${doneFile}
775     cp ${doneFile} ${doneFileDest}
776     return 1
777   fi
778   if ! cd ${runpath}; then 
779     touch ${doneFile}
780     echo "PWD=$PWD is not the runpath=${runpath}" >> ${doneFile}
781     cp ${doneFile} ${doneFileDest}
782     return 1
783   fi
784
785   logOutputDir=${runpath}
786   [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir}
787   [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/mergeMakeOCDB.log
788   echo "${0} $*"
789
790   calibrationOutputFileName='AliESDfriends_v1.root'
791   qaOutputFileName='QAresults*.root'
792   mergingScript="mergeMakeOCDB.byComponent.sh"
793   #important to have the string "Stage.txt" in the filename to trigger the merging
794   #it has to be a list of directories containing the files
795   qaMergedOutputFileName="QAresults_merged.root"
796
797   echo goMergeCPass1 SETUP:
798   echo runNumber=${runNumber}
799   echo outputDir=${outputDir}
800   echo ocdbStorage=${ocdbStorage}
801   echo calibrationFilesToMerge=$calibrationFilesToMerge
802   echo qaFilesToMerge=$qaFilesToMerge
803   echo calibrationOutputFileName=${calibrationOutputFileName}
804   echo mergingScript=${mergingScript}
805   
806   # copy files in case they are not already there
807   filesMergeCPass1=(
808                     "${batchWorkingDirectory}/${calibrationFilesToMerge}"
809                     "${batchWorkingDirectory}/${qaFilesToMerge}"
810                     "${batchWorkingDirectory}/OCDB.root"
811                     "${batchWorkingDirectory}/localOCDBaccessConfig.C"
812                     "${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz"
813                     "${batchWorkingDirectory}/QAtrain_duo.C"
814                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/mergeMakeOCDB.byComponent.sh"
815                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/mergeByComponent.C"
816                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/makeOCDB.C"
817                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/merge.C"
818                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/mergeMakeOCDB.sh"
819                     "${trustedQAtrainMacro}"
820                     "${ALICE_ROOT}/ANALYSIS/macros/QAtrain_duo.C"
821   )
822   for file in ${filesMergeCPass1[*]}; do
823     [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
824     [[ ${file##*/} =~ .*\.sh ]] && chmod +x ${file##*/}
825   done
826
827   sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh
828
829   #configure local OCDB storage from CPass0 (creates the localOCDBaccessConfig.C script)
830   if [[ -f cpass0.localOCDB.${runNumber}.tgz ]]; then
831     echo goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz"
832     goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz"
833   else
834     echo "WARNING: file cpass0.localOCDB.${runNumber}.tgz not found!"
835   fi
836
837   alirootInfo > ALICE_ROOT.log
838
839   #
840   /bin/ls
841
842   #merge calibration
843   chmod u+x ${mergingScript}  
844   mkdir -p OCDB
845   
846   #if not provided, create the lists of files to merge
847   if [[ ! -f ${filteredFilesToMerge} ]]; then
848     echo "/bin/ls -1 ${outputDir}/*/FilterEvents_Trees.root > ${filteredFilesToMerge}"
849     /bin/ls -1 ${outputDir}/*/FilterEvents_Trees.root 2>/dev/null > ${filteredFilesToMerge}
850   fi
851   if [[ ! -f ${calibrationFilesToMerge} ]]; then
852     echo "/bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root > ${calibrationFilesToMerge}"
853     /bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root 2>/dev/null > ${calibrationFilesToMerge}
854   fi
855   if [[ ! -f ${qaFilesToMerge} ]]; then
856     #find the files, but only store the directories (QAtrain_duo.C requires this)
857     echo "/bin/ls -1 ${outputDir}/*/QAresults*.root | while read x; do echo ${x%/*}; done | sort | uniq > ${qaFilesToMerge}"
858     /bin/ls -1 ${outputDir}/*/QAresults*.root | while read x; do echo ${x%/*}; done | sort | uniq > ${qaFilesToMerge}
859   fi
860   
861   echo "${mergingScript} ${calibrationFilesToMerge} ${runNumber} local://./OCDB ${ocdbStorage}"
862   if [[ -n ${pretend} ]]; then
863     sleep ${pretendDelay}
864     touch ocdb.log
865     touch cpass1.localOCDB.${runNumber}.tgz
866     touch ${qaMergedOutputFileName}
867     touch merge.log
868     touch trending.root
869     touch FilterEvents_Trees.root
870     touch CalibObjects.root
871     touch dcsTime.root
872     touch ${qaMergedOutputFileName}
873     mkdir -p OCDB
874   else
875     ./${mergingScript} ${calibrationFilesToMerge} ${runNumber} "local://./OCDB" ${ocdbStorage}
876
877     #merge QA (and filtered trees)
878     [[ -n ${AliAnalysisTaskFilteredTree_fLowPtTrackDownscaligF} ]] && export AliAnalysisTaskFilteredTree_fLowPtTrackDownscaligF
879     [[ -n ${AliAnalysisTaskFilteredTree_fLowPtV0DownscaligF} ]] && export AliAnalysisTaskFilteredTree_fLowPtV0DownscaligF
880
881     #echo aliroot -l -b -q "merge.C(\"${qaFilesToMerge}\",\"\",kFALSE,\"${qaMergedOutputFileName}\")"
882     echo aliroot -b -q "QAtrain_duo.C(\"_barrel\",${runNumber},\"${qaFilesToMerge}\",1,\"${ocdbStorage}\")"
883     #aliroot -l -b -q "merge.C(\"${qaFilesToMerge}\",\"\",kFALSE,\"${qaMergedOutputFileName}\")"
884     aliroot -b -q "QAtrain_duo.C(\"_barrel\",${runNumber},\"${qaFilesToMerge}\",1,\"${ocdbStorage}\")" > mergeQA.log
885     mv QAresults_barrel.root ${qaMergedOutputFileName}
886     mv trending_barrel.root trending.root
887
888     #merge filtered trees
889     echo aliroot -l -b -q "merge.C(\"${qaFilesToMerge}\",\"\",kFALSE,\"${qaMergedOutputFileName}\")"
890     aliroot -l -b -q "merge.C(\"${filteredFilesToMerge}\",\"\",kFALSE,\"FilterEvents_Trees.root\")" > mergeFilteredTrees.log
891
892     #produce the calib trees for expert QA
893     echo aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")"
894     aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")" > calibTree.log
895   fi
896
897   tar czf ${commonOutputPath}/meta/cpass1.localOCDB.${runNumber}.tgz ./OCDB
898
899   /bin/ls
900
901   #copy all to output dir
902   cp -pf -R ${runpath}/* ${outputDir}
903   
904   #validate merge cpass1
905   cd ${outputDir}
906   touch ${doneFile}
907   echo "dir ${outputDir}" >> ${doneFile}
908   if summarizeLogs >>  ${doneFile}; then
909     [[ -f CalibObjects.root ]] && echo "calibfile ${outputDir}/CalibObjects.root" >> ${doneFile}
910     [[ -f ${qaMergedOutputFileName} ]] && echo "qafile ${outputDir}/${qaMergedOutputFileName}" >> ${doneFile}
911     [[ -f trending.root ]] && echo "trendingfile ${outputDir}/trending.root" >> ${doneFile}
912     [[ -f dcsTime.root ]] && echo "dcsTree ${outputDir}/dcsTime.root" >> ${doneFile}
913     [[ -f FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFile}
914   else
915     if grep "mergeQA.log.*OK" ${doneFile} > /dev/null; then
916       [[ -f ${qaMergedOutputFileName} ]] && echo "qafile ${outputDir}/${qaMergedOutputFileName}" >> ${doneFile}
917     fi
918     if grep "mergeFilteredTrees.log.*OK" ${doneFile} > /dev/null; then
919       [[ -f FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFile}
920     fi
921   fi
922       
923   cp ${doneFile} ${doneFileDest}
924   
925   [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
926   return 0
927 )
928
929 goMerge()
930 (
931   #generic root merge using CPass1 merge.C script
932   inputList=${1}
933   outputFile=${2}  
934   configFile=${3-"benchmark.config"}
935   shift 3
936   if ! parseConfig ${configFile} "$@"; then return 1; fi
937   
938   #record the working directory provided by the batch system
939   batchWorkingDirectory=${PWD}
940
941   [[ ! -f ${inputList} ]] && echo "inputList ${inputList} does not exist!" && return 1
942   [[ ! -f ${configFile} ]] && echo "configFile ${configFile} does not exist!" && return 1
943   umask 0002
944   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
945   rm -f ${outputFile}
946   aliroot -b -q "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/merge.C(\"${inputList}\",\"\",kFALSE,\"${outputFile}\")" > merge_${inputList}.log
947   return 0
948 )
949
950 goSubmitMakeflow()
951 {
952   #run
953   productionID=${1}
954   inputFileList=${2}
955   configFile=${3}
956   shift 3
957   extraOpts=("$@")
958   if ! parseConfig ${configFile} "${extraOpts[@]}"; then return 1; fi
959  
960   #record the working directory provided by the batch system
961   batchWorkingDirectory=${PWD}
962
963   [[ -z ${configFile} ]] && configFile="benchmark.config"
964   [[ ! -f ${configFile} ]] && echo "no config file found (${configFile})" && return 1
965
966   if [[ ! $(which makeflow &>/dev/null) && -n ${makeflowPath} ]]; then
967     echo "setting the makflow path from the config: "
968     echo "  export PATH=${makeflowPath}:${PATH}"
969     export PATH=${makeflowPath}:${PATH}
970   fi
971
972   #create the common output dir and the meta dir
973   commonOutputPath=${baseOutputDirectory}/${productionID}
974   if [[ -d ${commonOutputPath} ]]; then
975     echo "output dir ${commonOutputPath} exists!"
976     #return 1
977   else
978     mkdir -p ${commonOutputPath}
979   fi
980   mkdir -p ${commonOutputPath}/meta
981   
982   self=${0}
983   #if which greadlink; then self=$(greadlink -f "${0}"); fi
984   
985   #for reference copy the setup to the output dir
986   cp ${self} ${commonOutputPath}
987   cp ${configFile} ${commonOutputPath}
988   cp ${inputFileList} ${commonOutputPath}
989
990   #submit - use makeflow if available, fall back to old stuff when makeflow not there
991   if which makeflow; then
992     goGenerateMakeflow ${productionID} ${inputFileList} ${configFile} "${extraOpts[@]}" commonOutputPath=${commonOutputPath} > benchmark.makeflow
993     cp benchmark.makeflow ${commonOutputPath}
994     makeflow ${makeflowOptions} benchmark.makeflow
995   else 
996     echo "no makeflow!"
997   fi
998   return 0
999 }
1000
1001 goGenerateMakeflow()
1002 (
1003   #generate the makeflow file
1004   [[ $# -lt 3 ]] && echo "args: id inputFileList configFile" && return 1
1005   productionID=${1}
1006   inputFileList=${2}
1007   configFile=${3}
1008   shift 3
1009   extraOpts=("$@")
1010   
1011   #batch systems/makeflow sometimes handle spaces in arguments poorly, so encode them
1012   for (( i=0;i<${#extraOpts[@]};i++ )); do 
1013     extraOpts[i]=$(encSpaces "${extraOpts[i]}")
1014   done
1015   extraOpts+=("encodedSpaces=1")
1016
1017   if ! parseConfig ${configFile} "${extraOpts[@]}" &>/dev/null; then return 1; fi
1018  
1019   #extra safety
1020   if [[ -z ${commonOutputPath} ]]; then
1021     commonOutputPath=${baseOutputDirectory}/${productionID}
1022     extraOpts=( "${extraOpts[@]}" "commonOutputPath=${commonOutputPath}" )
1023   fi
1024
1025   #record the working directory provided by the batch system
1026   batchWorkingDirectory=${PWD}
1027
1028   [[ -z ${configFile} ]] && configFile="benchmark.config"
1029   [[ ! -f ${configFile} ]] && echo "no config file found (${configFile})" && return 1
1030
1031   #these files will be made a dependency - will be copied to the working dir of the jobs
1032   declare -a copyFiles
1033   inputFiles=(
1034               "OCDB.root"
1035               "localOCDBaccessConfig.C"
1036               "QAtrain_duo.C"
1037               "runCPass1.sh"
1038               "recCPass1.C"
1039               "recCPass1_OuterDet.C"
1040               "runCalibTrain.C"
1041               "runCPass0.sh"
1042               "recCPass0.C"
1043   )
1044   for file in ${inputFiles[*]}; do
1045     [[ -f ${file} ]] && copyFiles+=("${file}")
1046   done
1047
1048   #create the makeflow file
1049   [[ -n ${batchFlags} ]] && echo "BATCH_OPTIONS = ${batchFlags}"
1050   declare -A arr_cpass0_merged arr_cpass1_merged
1051   declare -A arr_cpass0_calib_list arr_cpass1_calib_list 
1052   declare -A arr_cpass1_QA_list arr_cpass1_ESD_list arr_cpass1_filtered_list
1053   declare -A arr_cpass0_profiled_outputs
1054   declare -A listOfRuns
1055   [[ -n ${runNumber} ]] && listOfRuns[${runNumber}]=1
1056   while read x; do tmpRun=$(guessRunNumber ${x}); [[ -n ${tmpRun} ]] && listOfRuns[${tmpRun}]=1; done < ${inputFileList}
1057   for runNumber in "${!listOfRuns[@]}"; do
1058     [[ -z ${runNumber} ]] && continue
1059     [[ ! ${runNumber} =~ ^[0-9]*[0-9]$ ]] && continue
1060
1061     unset arr_cpass0_outputs
1062     unset arr_cpass1_outputs
1063     declare -a arr_cpass0_outputs
1064     declare -a arr_cpass1_outputs
1065
1066     jobindex=0
1067     inputFile=""
1068     while read inputFile; do
1069       currentDefaultOCDB=${defaultOCDB}
1070       [[ -z ${autoOCDB} ]] && autoOCDB=1
1071       if [[ ${autoOCDB} -ne 0 ]]; then
1072         currentDefaultOCDB=$(setYear ${inputFile} ${defaultOCDB})
1073       fi
1074
1075       #CPass0
1076       arr_cpass0_outputs[${jobindex}]="${commonOutputPath}/meta/cpass0.job${jobindex}.run${runNumber}.done"
1077       echo "${arr_cpass0_outputs[${jobindex}]} : benchmark.sh ${configFile} ${copyFiles[@]}"
1078       echo " ${alirootEnv} ./benchmark.sh CPass0 ${commonOutputPath}/000${runNumber}/cpass0 ${inputFile} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} ${jobindex} ${extraOpts[@]}"" "
1079
1080       #CPass1
1081       arr_cpass1_outputs[${jobindex}]="${commonOutputPath}/meta/cpass1.job${jobindex}.run${runNumber}.done"
1082       echo "${arr_cpass1_outputs[${jobindex}]} : benchmark.sh ${configFile} ${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz ${copyFiles[@]}"
1083       echo " ${alirootEnv} ./benchmark.sh CPass1 ${commonOutputPath}/000${runNumber}/cpass1 ${inputFile} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} ${jobindex} ${extraOpts[@]}"" "
1084       ((jobindex++))
1085
1086     done< <(grep "/000${runNumber}/" ${inputFileList})
1087     
1088     #CPass0 list of Calib files to merge
1089     arr_cpass0_calib_list[${runNumber}]="${commonOutputPath}/meta/cpass0.calib.run${runNumber}.list"
1090     echo "${arr_cpass0_calib_list[${runNumber}]} : benchmark.sh ${arr_cpass0_outputs[*]}"
1091     echo "  ./benchmark.sh PrintValues calibfile ${arr_cpass0_calib_list[${runNumber}]} ${arr_cpass0_outputs[*]}"
1092     echo
1093
1094     #CPass0 merging
1095     arr_cpass0_merged[${runNumber}]="${commonOutputPath}/meta/merge.cpass0.run${runNumber}.done"
1096     echo "${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz ${arr_cpass0_merged[${runNumber}]} : benchmark.sh ${configFile} ${arr_cpass0_calib_list[${runNumber}]} ${copyFiles[@]}"
1097     echo " ${alirootEnv} ./benchmark.sh MergeCPass0 ${commonOutputPath}/000${runNumber}/cpass0 ${currentDefaultOCDB} ${configFile} ${runNumber} ${arr_cpass0_calib_list[${runNumber}]} ${extraOpts[@]}"" "
1098
1099     #CPass1 list of Calib/QA/ESD/filtered files
1100     # the trick with QA is to have the string "Stage.txt" in the file name of the list of directories with QA output to trigger
1101     # the production of the QA trending tree (only then the task->Finish() will be called in QAtrain_duo.C, on the grid
1102     # this corresponds to the last merging stage)
1103     arr_cpass1_QA_list[${runNumber}]="${commonOutputPath}/meta/cpass1.QA.run${runNumber}.lastMergingStage.txt.list"
1104     echo "${arr_cpass1_QA_list[${runNumber}]}: benchmark.sh ${arr_cpass1_outputs[*]}"
1105     echo "  ./benchmark.sh PrintValues dir ${arr_cpass1_QA_list[${runNumber}]} ${arr_cpass1_outputs[*]}"
1106     echo
1107     arr_cpass1_calib_list[${runNumber}]="${commonOutputPath}/meta/cpass1.calib.run${runNumber}.list"
1108     echo "${arr_cpass1_calib_list[${runNumber}]} : benchmark.sh ${arr_cpass1_outputs[*]}"
1109     echo "  ./benchmark.sh PrintValues calibfile ${arr_cpass1_calib_list[${runNumber}]} ${arr_cpass1_outputs[*]};"
1110     echo
1111     arr_cpass1_ESD_list[${runNumber}]="${commonOutputPath}/meta/cpass1.ESD.run${runNumber}.list"
1112     echo "${arr_cpass1_ESD_list[${runNumber}]} : benchmark.sh ${arr_cpass1_outputs[*]}"
1113     echo "  ./benchmark.sh PrintValues esd ${arr_cpass1_ESD_list[${runNumber}]} ${arr_cpass1_outputs[*]}"
1114     echo
1115     arr_cpass1_filtered_list[${runNumber}]="${commonOutputPath}/meta/cpass1.filtered.run${runNumber}.list"
1116     echo "${arr_cpass1_filtered_list[${runNumber}]} : benchmark.sh ${arr_cpass1_outputs[*]}"
1117     echo "  ./benchmark.sh PrintValues filteredTree ${arr_cpass1_filtered_list[${runNumber}]} ${arr_cpass1_outputs[*]}"
1118     echo
1119   
1120     #CPass1 merging
1121     arr_cpass1_merged[${runNumber}]="${commonOutputPath}/meta/merge.cpass1.run${runNumber}.done"
1122     echo "${commonOutputPath}/meta/cpass1.localOCDB.${runNumber}.tgz ${arr_cpass1_merged[${runNumber}]} :  benchmark.sh ${configFile} ${arr_cpass1_calib_list[${runNumber}]} ${arr_cpass1_QA_list[${runNumber}]} ${copyFiles[@]}"
1123     echo " ${alirootEnv} ./benchmark.sh MergeCPass1 ${commonOutputPath}/000${runNumber}/cpass1 ${currentDefaultOCDB} ${configFile} ${runNumber} ${arr_cpass1_calib_list[${runNumber}]} ${arr_cpass1_QA_list[${runNumber}]} ${arr_cpass1_filtered_list[${runNumber}]} ${extraOpts[@]}"" "
1124
1125     #CPass0 wrapped in a profiling tool (valgrind,....)
1126     if [[ -n ${profilingCommand} ]]; then
1127       inputFile=$(grep -m1 "${runNumber}/" ${inputFileList})
1128       [[ -z ${nEventsProfiling} ]] && nEventsProfiling=2
1129       currentDefaultOCDB=$(setYear ${inputFile} ${defaultOCDB})
1130       jobindex="profiling"
1131
1132       arr_cpass0_profiled_outputs[${runNumber}]="${commonOutputPath}/meta/profiling.cpass0.job${jobindex}.run${runNumber}.done"
1133       echo "${arr_cpass0_profiled_outputs[${runNumber}]} : benchmark.sh ${configFile} ${copyFiles[@]}"
1134       profilingCommand=$(encSpaces "${profilingCommand}")
1135       echo " ${alirootEnv} ./benchmark.sh CPass0 ${commonOutputPath}/000${runNumber}/${jobindex} ${inputFile} ${nEventsProfiling} ${currentDefaultOCDB} ${configFile} ${runNumber} ${jobindex} ${extraOpts[@]} useProfilingCommand=${profilingCommand}"
1136     fi
1137
1138   done #runs
1139
1140   #Summary
1141   echo "${commonOutputPath}/summary.log : benchmark.sh ${configFile} ${arr_cpass1_merged[*]}"
1142   echo "  ${alirootEnv} ./benchmark.sh MakeSummary ${configFile} ${extraOpts[@]}"
1143   echo
1144
1145   return 0
1146 )
1147
1148 goPrintValues()
1149 (
1150   #print the values given the key from any number of files (store in output file on second argument)
1151   if [[ $# -lt 3 ]]; then
1152     echo "goPrintValues key outputFile inputFiles"
1153     echo "if outputFile is \"-\" dont redirect to a file"
1154     return
1155   fi
1156   key=${1}
1157   outputFile=${2}
1158   [[ ${outputFile} =~ "-" ]] && outputFile=""
1159   shift 2 #remove 2 first arguments from arg list to only pass the input files to awk
1160   awk -v key=${key} '$0 ~ key" " {print $2}' "$@" | tee ${outputFile}
1161   return 0
1162 )
1163
1164 goCreateQAplots()
1165 (
1166   umask 0002
1167   mergedQAfileList=${1}
1168   productionID=${2}
1169   outputDir=${3}
1170   configFile=${4}
1171   shift 4
1172   if ! parseConfig ${configFile} "$@"; then return 1; fi
1173   
1174   #record the working directory provided by the batch system
1175   batchWorkingDirectory=${PWD}
1176
1177   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
1178
1179   [[ -z ${logOutputDir} ]] && logOutputDir=${PWD}
1180   [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/makeQAplots.log
1181   echo "${0} $*"
1182
1183   olddir=${PWD}
1184   mkdir -p ${outputDir}
1185   cd ${outputDir}
1186   [[ ! "${PWD}" =~ "${outputDir}" ]] && echo "PWD is not equal to outputDir=${outputDir}" && cd ${olddir} && return 1
1187
1188   if [[ -z ${pretend} ]]; then
1189     ${ALICE_ROOT}/PWGPP/QA/scripts/runQA.sh inputList="${mergedQAfileList}" inputListHighPtTrees="${filteringList}" ocdbStorage=${defaultOCDB}
1190   else
1191     touch pretendCreateQAplots.done
1192   fi
1193   cd ${olddir}
1194   return 0
1195 )
1196
1197 goTest()
1198 (
1199   echo AA
1200 )
1201
1202 alirootInfo()
1203 (
1204   umask 0002
1205   # save aliroot repository info
1206   [[ -z "${ALICE_ROOT}" ]] && return 1
1207   
1208   echo "\${ALICE_ROOT}=${ALICE_ROOT}"
1209   echo "\${ROOTSYS}=${ROOTSYS}"
1210   echo "\${PATH}=${PATH}"
1211   echo "\${LD_LIBRARY_PATH}=${LD_LIBRARY_PATH}"
1212   echo
1213   
1214   pushd ${PWD}
1215   cd ${ALICE_ROOT}
1216
1217   currentBranch=$(git rev-parse --abbrev-ref HEAD)
1218   git status
1219   echo ""
1220   echo ""
1221   git diff ${currentBranch}
1222   popd
1223   return 0
1224 )
1225
1226 setYear()
1227 (
1228   #set the year
1229   #  ${1} - year to be set
1230   #  ${2} - where to set the year
1231   year1=$(guessYear ${1})
1232   year2=$(guessYear ${2})
1233   local path=${2}
1234   [[ ${year1} -ne ${year2} && -n ${year2} && -n ${year1} ]] && path=${2/\/${year2}\//\/${year1}\/}
1235   echo ${path}
1236   return 0
1237 )
1238
1239 guessPeriod()
1240 (
1241   #guess the period from the path, pick the rightmost one
1242   local IFS="/"
1243   declare -a path=( ${1} )
1244   local dirDepth=${#path[*]}
1245   for ((x=${dirDepth}-1;x>=0;x--)); do
1246     local field=${path[${x}]}
1247     [[ ${field} =~ ^LHC[0-9][0-9][a-z]$ ]] && period=${field} && break
1248   done
1249   echo ${period}
1250   return 0
1251 )
1252
1253 guessYear()
1254 (
1255   #guess the year from the path, pick the rightmost one
1256   local IFS="/"
1257   declare -a path=( ${1} )
1258   local dirDepth=${#path[*]}
1259   for ((x=${dirDepth}-1;x>=0;x--)); do
1260     local field=${path[${x}]}
1261     [[ ${field} =~ ^20[0-9][0-9]$ ]] && year=${field} && break
1262   done
1263   echo ${year}
1264   return 0
1265 )
1266
1267 guessRunNumber()
1268 (
1269   #guess the run number from the path, pick the rightmost one
1270   #works for /path/foo/000123456/bar/...
1271   #and       /path/foo.run123456.bar
1272   local IFS="/."
1273   declare -a path=( ${1} )
1274   local dirDepth=${#path[*]}
1275   for ((x=${dirDepth}-1;x>=0;x--)); do
1276     local field=${path[${x}]}
1277     field=${field/run/000}
1278     [[ ${field} =~ [0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && runNumber=${field#000} && break
1279   done
1280   echo ${runNumber}
1281   return 0
1282 )
1283
1284 validateLog()
1285 (
1286   log=${1}
1287   errorConditions=(
1288                     'There was a crash'
1289                     'floating'
1290                     'error while loading shared libraries'
1291                     'std::bad_alloc'
1292                     's_err_syswatch_'
1293                     'Thread [0-9]* (Thread'
1294                     'AliFatal'
1295                     'core dumped'
1296                     '\.C.*error:.*\.h: No such file'
1297                     'line.*Aborted'
1298   )
1299
1300   warningConditions=(
1301                      'This is serious !'
1302                      'rocVoltage out of range:'
1303   )
1304   
1305   local logstatus=0
1306   local errorSummary=""
1307   local warningSummary=""
1308
1309   for ((i=0; i<${#errorConditions[@]};i++)); do
1310     local tmp=$(grep -m1 -e "${errorConditions[${i}]}" ${log})
1311     [[ -n ${tmp} ]] && tmp+=" : "
1312     errorSummary+=${tmp}
1313   done
1314
1315   for ((i=0; i<${#warningConditions[@]};i++)); do
1316     local tmp=$(grep -m1 -e "${warningConditions[${i}]}" ${log})
1317     [[ -n ${tmp} ]] && tmp+=" : "
1318     warningSummary+=${tmp}
1319   done
1320
1321   if [[ -n ${errorSummary} ]]; then 
1322     echo "${errorSummary}"
1323     return 1
1324   fi
1325   
1326   if [[ -n ${warningSummary} ]]; then
1327     echo "${warningSummary}"
1328     return 2
1329   fi
1330
1331   return 0
1332 )
1333
1334 summarizeLogs()
1335 (
1336   #print a summary of logs
1337   logFiles=(
1338             "*.log"
1339             "stdout"
1340             "stderr"
1341   )
1342
1343   #check logs
1344   local logstatus=0
1345   for log in ${logFiles[*]}; do
1346     finallog=${PWD%/}/${log}
1347     [[ ! -f ${log} ]] && continue
1348     errorSummary=$(validateLog ${log})
1349     validationStatus=$?
1350     if [[ ${validationStatus} -eq 0 ]]; then 
1351       #in pretend mode randomly report an error in rec.log some cases
1352       if [[ -n ${pretend} && "${log}" == "rec.log" ]]; then
1353         #[[ $(( ${RANDOM}%2 )) -ge 1 ]] && echo "${finallog} BAD random error" || echo "${finallog} OK"
1354         echo "${finallog} OK"
1355       else
1356         echo "${finallog} OK"
1357       fi
1358     elif [[ ${validationStatus} -eq 1 ]]; then
1359       echo "${finallog} BAD ${errorSummary}"
1360       logstatus=1
1361     elif [[ ${validationStatus} -eq 2 ]]; then
1362       echo "${finallog} OK MWAH ${errorSummary}"
1363     fi
1364   done
1365   
1366   #report core files
1367   while read x; do
1368     echo ${x}
1369     chmod 644 ${x}
1370     gdb --batch --quiet -ex "bt" -ex "quit" aliroot ${x} > stacktrace_${x//\//_}.log
1371   done < <(/bin/ls ${PWD}/*/core 2>/dev/null; /bin/ls ${PWD}/core 2>/dev/null)
1372   
1373   return ${logstatus}
1374 )
1375
1376 spitOutLocalOCDBaccessConfig()
1377 {
1378   umask 0002
1379   #find ${1} -name "*root" | \
1380   /bin/ls -1 ${1}/*/*/*/*.root 2>/dev/null | \
1381   while read line
1382   do 
1383     local tmp=${line#${1}}
1384     echo ${tmp%/*} | \
1385     awk -v ocdb=${1} '{print "  man->SetSpecificStorage(\""$1"\",\"local://"ocdb"\");"}'
1386   done
1387   return 0
1388 }
1389
1390 goMakeLocalOCDBaccessConfig()
1391 {
1392   umask 0002
1393   # make a script that sets the specific storages form all the root files produced by CPass0
1394   local localOCDBpathCPass0=${1}
1395   local OCDBpathPrefix=${2}
1396   [[ -z ${OCDBpathPrefix} ]] && OCDBpathPrefix="."
1397
1398   if [[ -f ${localOCDBpathCPass0} && ${localOCDBpathCPass0} =~ \.tgz$ ]]; then
1399     tar xzf ${localOCDBpathCPass0}
1400     local localOCDBpathCPass0="${OCDBpathPrefix}/OCDB"
1401   fi
1402
1403   echo
1404   echo creating the specific storage script
1405   echo   localOCDBaccessConfig.C
1406   echo   based on OCDB: ${localOCDBaccessConfig}
1407   echo
1408
1409   local tempLocalOCDB=""
1410   if [[ -f localOCDBaccessConfig.C ]]; then
1411     tempLocalOCDB=$(mktemp -t tempLocalOCDB.XXXXXX)
1412     echo "egrep "SetSpecificStorage" localOCDBaccessConfig.C > ${tempLocalOCDB}"
1413     egrep "SetSpecificStorage" localOCDBaccessConfig.C > ${tempLocalOCDB}
1414   fi
1415
1416   echo "localOCDBaccessConfig()"                               >  localOCDBaccessConfig.C
1417   echo "{"                                                     >> localOCDBaccessConfig.C
1418   echo "  AliCDBManager* man = AliCDBManager::Instance();"     >> localOCDBaccessConfig.C
1419   spitOutLocalOCDBaccessConfig ${localOCDBpathCPass0}|sort|uniq  >> localOCDBaccessConfig.C
1420   [[ -f "${tempLocalOCDB}" ]] && cat ${tempLocalOCDB}              >> localOCDBaccessConfig.C
1421   echo "}"                                                     >> localOCDBaccessConfig.C
1422
1423   [[ -f "${tempLocalOCDB}" ]] && rm -f ${tempLocalOCDB}
1424
1425   if ! grep SetSpecificStorage localOCDBaccessConfig.C; then 
1426     echo
1427     echo "!!!!!!! CPass0 produced no OCDB entries"
1428     return 1
1429   fi
1430   return 0
1431 }
1432
1433 goMakeFilteredTrees()
1434 (
1435   outputDir=${1}
1436   runNumber=${2}
1437   #get path to input list
1438   inputListfiles=${3}
1439   #get scale number for tracks
1440   filterT=${4}
1441   #get scale number for V0s
1442   filterV=${5}
1443   #get OCDB path (optional)
1444   OCDBpath=${6}
1445   #get max number of files 
1446   maxFiles=${7-"1000000"}
1447   #get offset of first file
1448   offsetFile=${8-"0"}
1449   #get max number of events
1450   maxEvents=${9-"30000000"}
1451   #get offset of first event
1452   offsetEvent=${10-"0"}
1453   configFile=${11-"benchmark.config"}
1454   esdFileName=${12-"AliESDs_Barrel.root"}
1455   shift 12
1456   if ! parseConfig ${configFile} "$@"; then return 1; fi
1457   
1458   #record the working directory provided by the batch system
1459   batchWorkingDirectory=${PWD}
1460
1461   [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
1462   doneFile=filtering.cpass1.run${runNumber}.done
1463   [[ -z ${doneFileDest} ]] && doneFileDest="${commonOutputPath}/meta/"
1464
1465   cat > filtering.log << EOF
1466   goMakeFilteredTrees config:
1467   runpath=${runpath}
1468   outputDir=${outputDir}
1469   commonOutputPath=${commonOutputPath}
1470   ALICE_ROOT=${ALICE_ROOT}
1471   PATH=${PATH}
1472   offsetEvent=$offsetEvent
1473   configFile=$configFile
1474   esdFileName=$esdFileName
1475   inputListfiles=$inputListfiles
1476   doneFile=$doneFile
1477 EOF
1478
1479   #runpath=${outputDir}
1480   #[[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
1481   #[[ ${reconstructInTemporaryDir} -eq 1 ]] && runpath=$(mktemp -d -t goMakeFilteredTrees.XXXXXX)
1482   #mkdir -p ${outputDir}
1483   #mkdir -p ${runpath}
1484   #if ! cd ${runpath}; then 
1485   #  echo "PWD=$PWD is not the runpath=${runpath}"
1486   #  touch ${doneFile}
1487   #  cp ${doneFile} ${doneFileDest}
1488   #  return 1
1489   #fi
1490   
1491   if [[ -z ${pretend} ]];then
1492     aliroot -l -b -q "${ALICE_ROOT}/PWGPP/macros/runFilteringTask.C(\"${inputListfiles}\",${filterT},${filterV},\"${OCDBpath}\",${maxFiles},${offsetFile},${maxEvents},${offsetEvent},\"${esdFileName}\")" &>> filtering.log
1493   else
1494     sleep ${pretendDelay}
1495     touch filtering.log FilterEvents_Trees.root
1496   fi
1497   pwd
1498   /bin/ls
1499   touch ${doneFile}
1500   summarizeLogs >>  ${doneFile}
1501   
1502   #echo mv -f * ${outputDir}
1503   #mv -f * ${outputDir}
1504   #[[ -f ${outputDir}/FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFile}
1505   #cd ${commonOutputPath}
1506   #[[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
1507  
1508   cp ${doneFile} ${doneFileDest}
1509
1510   return 0
1511 )
1512
1513 submit()
1514 {
1515   umask 0002
1516   [[ $# -lt 5 ]] && echo "at least 5 args needed, you supplied $#" && return 1
1517   JobID=${1}
1518   startID=${2}
1519   endID=${3}
1520   waitForJOBID=${4}
1521   command=${5}
1522   shift 5
1523   local commandArgs=("$@")
1524
1525   #add quote strings around the extra arguments
1526   for ((i=0; i<${#commandArgs[@]}; i++)); do 
1527     commandArgs[i]=\"${commandArgs[i]}\"
1528   done
1529
1530   [[ -z ${waitForJOBID} ]] && waitForJOBID=0
1531
1532   newFarm=$(which qsub|grep "^/usr/bin/qsub")
1533   
1534   batchSystem="SGE"
1535
1536   if [[ -z "${newFarm}" ]]
1537   then
1538     #old LSF
1539     # submit it (as job array)
1540     nFiles=$(( ${endID}-${startID}+1 ))
1541     while [ ${startID} -le ${nFiles}  ] ; do
1542       if [ $(expr ${nFiles} - ${startID}) -gt 999 ] ; then 
1543         endID=$(expr ${startID} + 999)
1544       else
1545         endID=${nFiles}
1546       fi      
1547     if [[ ${waitForJOBID} -eq 0 ]]; then
1548         echo ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}"     
1549         ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}"     
1550       else
1551         echo ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -w "ended(${waitForJOBID})" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}"     
1552         ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -w "ended(${waitForJOBID})" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}"     
1553       fi
1554       startID=$(expr ${endID} + 1)
1555     done
1556   else 
1557     #new SGE farm
1558     if [[ ${waitForJOBID} =~ "000" ]]; then
1559       echo ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}"
1560       ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}"
1561     else
1562       echo ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -hold_jid "${waitForJOBID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}"
1563       ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -hold_jid "${waitForJOBID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}"
1564     fi
1565   fi
1566   return 0
1567 }
1568
1569 goSubmitBatch()
1570 {
1571   if [[ $# -lt 3 ]]; then
1572     echo "minimal use:"
1573     echo " ${0} submit fileList productionID configFile"
1574     return 0
1575   fi
1576
1577   productionID=${1}
1578   inputList=${2}
1579   configFile=${3:-"benchmark.config"}
1580   #if which greadlink; then configFile=$(greadlink -f ${configFile}); fi
1581   shift 3
1582   extraOpts=("$@")
1583   if ! parseConfig ${configFile} "${extraOpts[@]}"; then return 1; fi
1584   
1585   #batch systems/makeflow sometimes handle spaces in arguments poorly, so encode them
1586   for (( i=0;i<${#extraOpts[@]};i++ )); do 
1587     extraOpts[i]=$(encSpaces "${extraOpts[i]}")
1588   done
1589   extraOpts+=("encodedSpaces=1")
1590
1591   #record the working directory provided by the batch system
1592   batchWorkingDirectory=${PWD}
1593
1594   #redirect all output to submit.log
1595   echo "redirecting all output to ${PWD}/submit_${productionID//"/"/_}.log"
1596   exec 7>&1
1597   exec 1>submit_${productionID//"/"/_}.log 2>&1
1598
1599   umask 0002
1600   echo ${0}" submit $*"
1601   if [[ -z "${inputList}" || -z "${productionID}" ]]
1602   then
1603     echo
1604     echo " Usage: ${0} submit inputList productionID [configFile=benchmark.config]"
1605     echo
1606     return
1607   fi
1608
1609   # check if config file is there
1610   if [ ! -f ${configFile} ]; then
1611     echo "ERROR! Config File '${configFile}' not found" >&2
1612     return
1613   else
1614     echo "Using Config File: '${configFile}'"
1615   fi
1616
1617   [[ ! -f ${alirootEnv} ]] && echo "alirootEnv script ${alirootEnv} not found!..." && return 1
1618
1619   #move the script, config and some other stuff to ${commonOutputPath} first, then use them from there
1620   self=${0}
1621   #if which greadlink; then self=$(greadlink -f "${0}"); fi
1622   configPath=$(dirname ${configFile})
1623   export commonOutputPath=${baseOutputDirectory}/${productionID}
1624   
1625   mkdir -p ${commonOutputPath}
1626   mkdir -p ${commonOutputPath}/logs
1627   mkdir -p ${commonOutputPath}/meta
1628
1629   cp ${self} ${commonOutputPath}
1630   cp ${configFile} ${commonOutputPath}
1631   cp ${inputList} ${commonOutputPath}
1632   self=${commonOutputPath}/${self##*/}
1633   chmod u+x ${self}
1634   configFile=${commonOutputPath}/${configFile##*/}
1635   inputList=${commonOutputPath}/${inputList##*/}
1636
1637   #convert to absolut pathnames
1638   #if which greadlink; then inputList=$(greadlink -f "${inputList}"); fi
1639   #make list of runs
1640   if [[ -z ${runNumber} ]]; then
1641     listOfRuns=($(while read x; do guessRunNumber ${x}; done < ${inputList} | sort | uniq))
1642   else
1643     listOfRuns=${runNumber}
1644   fi
1645
1646   #if which greadlink; then alirootSource=$(greadlink -f "${alirootSource}"); fi
1647
1648   echo ""
1649   echo "### BEGIN CONFIGURATION ###"
1650   echo ""
1651   echo "GENERAL:"
1652   echo ""
1653   echo "    productionID:    ${productionID}"
1654   echo "    batchCommand:    ${batchCommand}"
1655   echo "    batchFlags:      ${batchFlags}"
1656   echo "    alirootEnv:   ${alirootEnv}"
1657   ${alirootEnv} echo '    ALICE_ROOT:      ${ALICE_ROOT}'
1658   ${alirootEnv} echo '    ALIROOT_RELEASE: ${ALICE_RELEASE}'
1659   echo "    inputList:       ${inputList}"
1660   echo "    configPath:      ${configPath}"
1661   echo "    commonOutputPath:      ${commonOutputPath}"
1662   echo "    defaultOCDB:     ${defaultOCDB}"
1663   echo "      autoOCDB: ${autoOCDB}"
1664   echo "    recoTriggerOptions:   ${recoTriggerOptions}"
1665   echo "    runs:"
1666   echo "      ${listOfRuns[*]}"
1667   echo ""
1668   echo "THE TRAIN WILL RUN:"
1669
1670   if [ ${runCPass0reco} -eq 1 ]; then
1671     echo "    Pass0 - Recontruction"
1672   fi
1673
1674   if [ ${runCPass0MergeMakeOCDB} -eq 1 ]; then
1675     echo "    Pass0 - merging and OCDB export"
1676   fi
1677
1678   if [ ${runCPass1reco} -eq 1 ]; then
1679     echo "    Pass1 - Recontruction"
1680   fi
1681   if [ ${runCPass1MergeMakeOCDB} -eq 1 ]; then
1682     echo "    Pass1 - merging and OCDB export"
1683   fi
1684
1685   echo ""
1686   echo "LIMITS:"
1687   echo "    max. Events/Chunk:   ${nEvents}"
1688   echo "    max. Number of Chunks per Run:     ${nMaxChunks}"
1689   echo ""
1690   echo "### END CONFIGURATION ###"
1691   echo ""
1692
1693
1694   # check if input file is there
1695   if [ ! -f ${inputList} ]; then
1696     echo "ERROR! Input List '${inputList}' not found" >&2
1697     return
1698   fi
1699
1700   # define jobid (for dependent jobs)
1701   date=$(date +%Y_%m_%d_%H%M%S)
1702   #for each run we submit one jobarray:
1703   for runNumber in ${listOfRuns[*]}; do
1704     
1705     [[ -z ${runNumber} ]] && continue
1706     [[ ! ${runNumber} =~ ^[0-9]*[0-9]$ ]] && continue
1707
1708     JOBpostfix="${productionID//"/"/_}_${runNumber}_${date}"
1709     JOBID1="p0_${JOBpostfix}"
1710     JOBID1wait="w0_${JOBpostfix}"
1711     JOBID2="m0_${JOBpostfix}"
1712     JOBID2wait="wm0_${JOBpostfix}"
1713     JOBID3="op0_${JOBpostfix}"
1714     JOBID3wait="wop0_${JOBpostfix}"
1715     JOBID4="p1_${JOBpostfix}"
1716     JOBID4wait="w1_${JOBpostfix}"
1717     JOBID5="m1_${JOBpostfix}"
1718     JOBID5wait="wm1_${JOBpostfix}"
1719     JOBID6="s1_${JOBpostfix}"
1720     JOBID6wait="ws1_${JOBpostfix}"
1721     JOBID7="QA_${JOBpostfix}"
1722     JOBmakeESDlistCPass1="lp1_${JOBpostfix}"
1723     JOBfilterESDcpass1="fp1_${JOBpostfix}"
1724     LASTJOB="000"
1725
1726     oneInputFile=$(egrep -m1 "${runNumber}/" ${inputList})
1727
1728     currentDefaultOCDB=${defaultOCDB}
1729     [[ -z ${autoOCDB} ]] && autoOCDB=1
1730     if [[ ${autoOCDB} -ne 0 ]]; then
1731       currentDefaultOCDB=$(setYear ${oneInputFile} ${defaultOCDB})
1732     fi
1733
1734     echo "submitting run ${runNumber} with OCDB ${currentDefaultOCDB}"
1735
1736     ###############################################################################
1737     #run one chunk with valgrind:
1738     if [[ -n ${profilingCommand} ]]; then
1739       [[ -z ${nEventsProfiling} ]] && nEventsProfiling=2
1740       [[ -z ${profilingCommand} ]] && profilingCommand="/usr/bin/valgrind --tool=callgrind --num-callers=40 -v --trace-children=yes"
1741       submit "profile-${JOBpostfix}" 1 1 000 "${alirootEnv} ${self}" CPass0 ${commonOutputPath}/000${runNumber}/${jobindex} ${oneInputFile} ${nEventsProfiling} ${currentDefaultOCDB} ${configFile} ${runNumber} ${jobindex} useProfilingCommand=$(encSpaces "${profilingCommand}") "${extraOpts[@]}"
1742     fi 
1743
1744     ################################################################################
1745     ################################################################################
1746     # run the CPass0 if requested
1747
1748     if [ ${runCPass0reco} -eq 1 ]; then
1749
1750       echo
1751       echo "starting CPass0... for run ${runNumber}"
1752       echo
1753
1754       # create directory and copy all files that are needed
1755       targetDirectory="${commonOutputPath}/000${runNumber}/cpass0"
1756       mkdir -p ${targetDirectory}
1757
1758       filesCPass0=( 
1759                     "${configPath}/runCPass0.sh"
1760                     "${configPath}/recCPass0.C"
1761                     "${configPath}/runCalibTrain.C"
1762                     "${configPath}/localOCDBaccessConfig.C"
1763                     "${configPath}/OCDB*.root"
1764                     "${configPath}/sim.C"
1765                     "${configPath}/rec.C"
1766                     "${configPath}/Config.C"
1767       )
1768       for file in ${filesCPass0[*]}; do
1769         [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
1770       done
1771
1772       localInputList=${targetDirectory}/${inputList##*/}
1773       [[ ! -f ${localInputList} ]] && egrep "\/000${runNumber}\/" ${inputList} > ${localInputList}
1774       # limit nFiles to nMaxChunks
1775       nFiles=$(wc -l < ${localInputList})
1776       [[ ${nFiles} -eq 0 ]] && echo "list contains ZERO files! exiting..." && return 1
1777       echo "raw files in list:    ${nFiles}"
1778       if [[ ${nMaxChunks} -gt 0 && ${nMaxChunks} -le ${nFiles} ]]; then
1779         nFiles=${nMaxChunks}
1780       fi
1781       echo "raw files to process: ${nFiles}"
1782       [[ -z "${percentProcessedFilesToContinue}" ]] && percentProcessedFilesToContinue=100
1783       if [[ ${percentProcessedFilesToContinue} -eq 100 ]]; then
1784         nFilesToWaitFor=${nFiles}
1785       else
1786         nFilesToWaitFor=$(( ${nFiles}-${nFiles}/(100/(100-${percentProcessedFilesToContinue})) ))
1787       fi
1788       echo "requested success rate is ${percentProcessedFilesToContinue}%"
1789       echo "merging will start after ${nFilesToWaitFor} jobs are done"
1790
1791       submit ${JOBID1} 1 ${nFiles} 000 "${alirootEnv} ${self}" CPass0 ${targetDirectory} ${localInputList} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} -1 "${extraOpts[@]}"
1792
1793       ## submit a monitoring job that will run until a certain number of jobs are done with reconstruction
1794       submit "${JOBID1wait}" 1 1 000 "${alirootEnv} ${self}" WaitForOutput ${commonOutputPath} "meta/cpass0.job*.run${runNumber}.done" ${nFilesToWaitFor} ${maxSecondsToWait}
1795       LASTJOB=${JOBID1wait}
1796
1797     fi #end running CPass0
1798     ################################################################################
1799
1800
1801     ################################################################################
1802     # submit merging of CPass0, depends on the reconstruction
1803
1804     if [ ${runCPass0MergeMakeOCDB} -eq 1 ]; then
1805
1806       echo
1807       echo "submit CPass0 merging for run ${runNumber}"
1808       echo
1809
1810       targetDirectory="${commonOutputPath}/000${runNumber}/cpass0"
1811       mkdir -p ${targetDirectory}
1812
1813       #copy the scripts
1814       filesMergeCPass0=(
1815                         "${configPath}/OCDB.root"
1816                         "${configPath}/mergeMakeOCDB.byComponent.sh"
1817                         "${configPath}/mergeMakeOCDB.sh"
1818                         "${configPath}/localOCDBaccessConfig.C"
1819                         "${configPath}/mergeByComponent.C"
1820                         "${configPath}/makeOCDB.C"
1821                         "${configPath}/merge.C"
1822       )
1823       for file in ${filesMergeCPass0[*]}; do
1824         [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
1825       done
1826   
1827       submit ${JOBID2} 1 1 "${LASTJOB}" "${alirootEnv} ${self}" MergeCPass0 ${targetDirectory} ${currentDefaultOCDB} ${configFile} ${runNumber} cpass0.calib.run${runNumber}.list "${extraOpts[@]}"
1828       LASTJOB=${JOBID2}
1829
1830       if [[ -n ${generateMC} ]]; then
1831         submit "mrl${JOBpostfix}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" PrintValues sim ${commonOutputPath}/meta/sim.run${runNumber}.list ${commonOutputPath}/meta/cpass0.job*.run${runNumber}.done
1832         LASTJOB="mrl${JOBpostfix}"
1833       fi
1834
1835       echo
1836     fi
1837     # end of merging CPass0
1838     ################################################################################
1839
1840     ################################################################################
1841     ################################################################################
1842     # run the CPass1 if requested
1843
1844     if [ ${runCPass1reco} -eq 1 ]; then
1845
1846       targetDirectory="${commonOutputPath}/000${runNumber}/cpass1"
1847       rm -f ${commonOutputPath}/meta/cpass1.job*.run${runNumber}.done
1848
1849       # safety feature: if we are re-running for any reason we want to delete the previous output first.
1850       [[ -d ${targetDirectory} ]] && rm -rf ${targetDirectory}/* && echo "removed old output at ${targetDirectory}/*"
1851
1852       echo
1853       echo "starting CPass1... for run ${runNumber}"
1854       echo
1855
1856       # create directory and copy all files that are needed
1857       mkdir -p ${targetDirectory}
1858       
1859       filesCPass1=( 
1860                     "${configPath}/runCPass1.sh"
1861                     "${configPath}/recCPass1.C"
1862                     "${configPath}/recCPass1_OuterDet.C"
1863                     "${configPath}/runCalibTrain.C"
1864                     "${configPath}/QAtrain_duo.C"
1865                     "${configPath}/localOCDBaccessConfig.C"
1866                     "${configPath}/OCDB.root"
1867       )
1868       for file in ${filesCPass1[*]}; do
1869         [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
1870       done
1871
1872       if [[ -n ${generateMC} ]]; then
1873         localInputList=${commonOutputPath}/meta/sim.run${runNumber}.list
1874       else
1875         localInputList=${targetDirectory}/${inputList##*/}
1876         [[ ! -f ${localInputList} ]] && egrep "\/000${runNumber}\/" ${inputList} > ${localInputList}
1877       fi
1878       # limit nFiles to nMaxChunks
1879       nFiles=$(wc -l < ${localInputList})
1880       [[ ${nFiles} -eq 0 ]] && echo "list contains ZERO files! continuing..." && continue
1881       echo "raw files in list:    ${nFiles}"
1882       if [[ ${nMaxChunks} -gt 0 && ${nMaxChunks} -le ${nFiles} ]]; then
1883         nFiles=${nMaxChunks}
1884       fi
1885       echo "raw files to process: ${nFiles}"
1886       [[ -z "${percentProcessedFilesToContinue}" ]] && percentProcessedFilesToContinue=100
1887       if [[ ${percentProcessedFilesToContinue} -eq 100 ]]; then
1888         nFilesToWaitFor=${nFiles}
1889       else
1890         nFilesToWaitFor=$(( ${nFiles}-${nFiles}/(100/(100-${percentProcessedFilesToContinue})) ))
1891       fi
1892       echo "requested success rate is ${percentProcessedFilesToContinue}%"
1893       echo "merging will start after ${nFilesToWaitFor} jobs are done"
1894
1895       submit ${JOBID4} 1 ${nFiles} "${LASTJOB}" "${alirootEnv} ${self}" CPass1 ${targetDirectory} ${localInputList} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} -1 "${extraOpts[@]}"
1896
1897       ################################################################################
1898       ## submit a monitoring job that will run until a certain number of jobs are done with reconstruction
1899       submit "${JOBID4wait}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" WaitForOutput ${commonOutputPath} "meta/cpass1.job*.run${runNumber}.done" ${nFilesToWaitFor} ${maxSecondsToWait}
1900       LASTJOB=${JOBID4wait}
1901       ################################################################################
1902
1903       echo
1904     fi #end running CPass1
1905
1906     ################################################################################
1907     # submit merging of CPass1, depends on the reconstruction
1908     if [ ${runCPass1MergeMakeOCDB} -eq 1 ]; then
1909
1910       echo
1911       echo "submit CPass1 merging for run ${runNumber}"
1912       echo
1913
1914       targetDirectory="${commonOutputPath}/000${runNumber}/cpass1"
1915       rm -f ${commonOutputPath}/meta/merge.cpass1.run${runNumber}.done
1916       mkdir -p ${targetDirectory}
1917
1918       # copy files 
1919       filesMergeCPass1=(
1920                         "${configPath}/OCDB.root"
1921                         "${configPath}/localOCDBaccessConfig.C"
1922                         "${configPath}/mergeMakeOCDB.byComponent.sh"
1923                         "${configPath}/mergeByComponent.C"
1924                         "${configPath}/makeOCDB.C"
1925                         "${configPath}/merge.C"
1926                         "${configPath}/mergeMakeOCDB.sh"
1927                         "${configPath}/QAtrain_duo.C"
1928       )
1929       for file in ${filesMergeCPass1[*]}; do
1930         [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
1931       done
1932
1933       submit "${JOBID5}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" MergeCPass1 ${targetDirectory} ${currentDefaultOCDB} ${configFile} ${runNumber} cpass1.calib.run${runNumber}.list cpass1.QA.run${runNumber}.lastMergingStage.txt.list cpass1.filtered.run${runNumber}.list "${extraOpts[@]}"
1934       LASTJOB=${JOBID5}
1935       echo
1936     fi
1937
1938     ###############################
1939     #if [ ${runESDfiltering} -eq 1 ]; then
1940     #  rm -f ${commonOutputPath}/cpass1.ESD.run${runNumber}.list
1941     #  rm -f ${commonOutputPath}/meta/filtering.cpass1.run*.done
1942     #  echo
1943     #  echo submitting filtering for run ${runNumber}
1944     #  echo
1945     #  submit "${JOBmakeESDlistCPass1}" 1 1 "${LASTJOB}" "${self}" PrintValues esd ${commonOutputPath}/meta/cpass1.ESD.run${runNumber}.list ${commonOutputPath}/meta/cpass1.job*.run${runNumber}.done 
1946     #  submit "${JOBfilterESDcpass1}" 1 1 "${JOBmakeESDlistCPass1}" "${alirootEnv} ${self}" MakeFilteredTrees ${commonOutputPath}/000${runNumber}/cpass1 ${runNumber} ${commonOutputPath}/meta/cpass1.ESD.run${runNumber}.list ${filteringFactorHighPt} ${filteringFactorV0s} ${currentDefaultOCDB} 1000000 0 10000000 0 ${configFile} AliESDs_Barrel.root "${extraOpts[@]}"
1947     #  LASTJOB=${JOBfilterESDcpass1}
1948     #fi
1949
1950   done
1951
1952   #################################################################################
1953   #################################################################################
1954   #if [ ${runESDfiltering} -eq 1 ]; then
1955   #  submit "${JOBID5wait}" 1 1 "${LASTJOB}" "${self}" WaitForOutput ${commonOutputPath} "meta/filtering.cpass1.run*.done" "${#listOfRuns[@]}" ${maxSecondsToWait}
1956   #else
1957     submit "${JOBID5wait}" 1 1 "${LASTJOB}" "${self}" WaitForOutput ${commonOutputPath} "meta/merge.cpass1.run*.done" ${#listOfRuns[@]} ${maxSecondsToWait}
1958   #fi
1959   LASTJOB=${JOBID5wait}
1960
1961   #################################################################################
1962   echo
1963   echo "submit make a summary"
1964   echo
1965
1966   submit "${JOBID6}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" MakeSummary ${configFile}
1967   LASTJOB=${JOBID6}
1968   #################################################################################
1969   
1970   #restore stdout
1971   exec 1>&7 7>&-
1972   echo "jobs submitted."
1973   return 0
1974 }
1975
1976 goWaitForOutput()
1977 (
1978   umask 0002
1979   [[ $# -lt 3 ]] && echo "goWaitForOutput() wrong number of arguments, exiting.." && return 1
1980   echo searchPath=${1}
1981   echo fileName=${2}
1982   echo numberOfFiles=${3}
1983   echo maxSecondsToWait=${4}
1984   searchPath=${1}
1985   fileName=${2}
1986   numberOfFiles=${3}
1987   maxSecondsToWait=${4}
1988   echo "command to be executed: /bin/ls -1 ${searchPath}/${fileName}"
1989   [[ -z "${maxSecondsToWait}" ]] && maxSecondsToWait=$(( 3600*12 ))
1990   while true; do
1991     n=$(/bin/ls -1 ${searchPath}/${fileName} 2>/dev/null | wc -l)
1992     [[ ${n} -gt 0 ]] && echo "found ${n} X ${fileName}"
1993     [[ ${n} -ge ${numberOfFiles} ]] && break
1994     [[ ${SECONDS} -gt ${maxSecondsToWait} ]] && echo "timeout of ${maxSecondsToWait}s!" && break
1995     sleep 60
1996   done
1997   echo "DONE! exiting..."
1998   return 0
1999 )
2000
2001 mergeSysLogs()
2002 (
2003   outputFile=${1}
2004   shift
2005   inputFiles="$@"
2006   i=0
2007   if ! ls -1 ${inputFiles} &>/dev/null; then echo "the files dont exist!: ${inputFiles}"; return 1; fi
2008   while read x; do 
2009     runNumber=$(guessRunNumber ${x})
2010     [[ -z ${runNumber} ]] && echo "run number cannot be guessed for ${x}" && continue
2011     awk -v run=${runNumber} -v i=${i} 'NR > 1 {print run" "$0} NR==1 && i==0 {print "run/I:"$0}' ${x}
2012     (( i++ ))
2013   done < <(ls -1 ${inputFiles}) > ${outputFile}
2014   return 0
2015 )
2016
2017 goMakeMergedSummaryTree()
2018 (
2019   # create list of calibration entries
2020   # takes no arguments, just run it in the base output
2021   # directory with the following files in the working directory
2022   # 
2023   #  Calibration file lists:
2024   #       cpass0.dcsTree.list, cpass1.dcsTree.list
2025   #  QA trending root file:
2026   #       trending.root
2027   # 
2028   #  Production infoo ascii files:
2029   #       summary_pass0.tree
2030   #       summary_pass1.tree
2031   #    
2032
2033   [[ ! -f cpass0.dcsTree.list ]] && echo "no cpass0.dcsTree.list" && return 1
2034   [[ ! -f cpass1.dcsTree.list ]] && echo "no cpass1.dcsTree.list" && return 1
2035   [[ ! -f trending.root ]] && echo "no trending.root" && return 1
2036   [[ ! -f summary_pass0.tree ]] && echo "no summary_pass0.tree" && return 1
2037   [[ ! -f summary_pass1.tree ]] && echo "no summary_pass1.tree" && return 1
2038
2039   #first, dump the C macro to file
2040   cat << EOF > mergeTree.C
2041   //
2042   // Merge summary information
2043   // Following files are expected to be in the working directory
2044   //
2045   // Calibration file lists:
2046   //      cpass0.dcsTree.list, cpass1.dcsTree.list
2047   // QA trending root file:
2048   //      trending.root
2049   //
2050   // Production infoo ascii files:
2051   //      summary_pass0.tree
2052   //      summary_pass1.tree
2053   //   
2054   void mergeTree(){
2055     //
2056     //
2057     //
2058     // Calibration values dump
2059     //
2060     //Printf("MakeTreeFromList cpass0.dcsTree.list");
2061     AliXRDPROOFtoolkit::MakeTreeFromList("Calib.TPC.CPass0.root", "dcs","dcs","cpass0.dcsTree.list",1);
2062     //Printf("MakeTreeFromList cpass1.dcsTree.list");
2063     AliXRDPROOFtoolkit::MakeTreeFromList("Calib.TPC.CPass1.root", "dcs","dcs","cpass1.dcsTree.list",1);
2064     //
2065     // Calibration status dump
2066     //
2067     TFile *fprod = TFile::Open("fproduction.root","recreate");
2068     TTree  tree0, tree1;
2069     //Printf("reading summary_pass0.tree");
2070     tree0.ReadFile("summary_pass0.tree");
2071     //Printf("reading summary_pass1.tree");
2072     tree1.ReadFile("summary_pass1.tree");
2073     tree0.Write("CPass0");
2074     tree1.Write("CPass1");
2075     fprod->Close();
2076     //
2077     //
2078     //
2079     TString stringSetup="";
2080     stringSetup+="1#QA.TPC#run#SummaryTPCQA/tpcQA#trending.root+";  // 
2081     stringSetup+="1#Calib.TPC.CPass0#run#dcs#Calib.TPC.CPass0.root+";  // 
2082     stringSetup+="1#Calib.TPC.CPass1#run#dcs#Calib.TPC.CPass1.root+";  // 
2083     //
2084     stringSetup+="1#CPass0#runnumber#CPass0#fproduction.root+";  // 
2085     stringSetup+="1#CPass1#runnumber#CPass1#fproduction.root+";  // 
2086     //
2087     //Printf("stringSetup: %s", stringSetup.Data());
2088     AliXRDPROOFtoolkit::JoinTreesIndex("outAll.root","joinAll","run",stringSetup.Data(), 1);
2089   }
2090 EOF
2091
2092   aliroot -b -q "mergeTree.C" > mergeTrees.log
2093   return $?
2094 )
2095
2096 stackTraceTree()
2097 (
2098   # make stacktrace processing  in case of standard root crash log
2099   # input is a (list of) text files with the stack trace (either gdb aoutput
2100   # produced with e.g. gdb --batch --quiet -ex "bt" -ex "quit" aliroot core, or the root crash log), output is a TTree formatted table.
2101 # example usage:
2102 # benchmark.sh stackTraceTree /foo/*/rec.log
2103   gawk '
2104        BEGIN { 
2105                print "frame/I:method/C:line/C:cpass/I:aliroot/I";
2106                RS="#[0-9]*";
2107                aliroot=0;
2108                read=1;
2109              } 
2110       /There was a crash/ {read=1;}
2111       /The lines below might hint at the cause of the crash/ {read=0;}
2112       read==1 { 
2113                if ($3 ~ /Ali*/) aliroot=1; else aliroot=0;
2114                gsub("#","",RT); 
2115                if ($NF!="" && RT!="" && $3!="") print RT" "$3" "$NF" "0" "aliroot
2116              }
2117       ' "$@" 2>/dev/null
2118 )
2119
2120 goMakeSummary()
2121 (
2122   #all the final stuff goes in here for ease of use:
2123   # summary logs
2124   # qa plot making
2125   # final file lists
2126   #some defaults:
2127   log="summary.log"
2128   productionID="qa"
2129
2130   configFile=${1}
2131   shift 1
2132   extraOpts=("$@")
2133   if ! parseConfig ${configFile} "${extraOpts[@]}"; then return 1; fi
2134   
2135   #if which greadlink; then configFile=$(greadlink -f ${configFile}); fi
2136   
2137   #record the working directory provided by the batch system
2138   batchWorkingDirectory=${PWD}
2139   
2140   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
2141
2142   [[ ! -f ${configFile} ]] && echo "no config file ${configFile}!" && return
2143
2144   [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
2145
2146   #copy some useful stuff
2147   #and go to the commonOutputPath
2148   cp ${configFile} ${commonOutputPath}
2149   cd ${commonOutputPath}
2150
2151   exec &> >(tee ${log})
2152
2153   #summarize the global stuff
2154   echo "env script: ${alirootSource} ${alirootEnv}"
2155   echo "\$ALICE_ROOT=${ALICE_ROOT}"
2156   echo "commonOutputPath=${commonOutputPath}"
2157
2158   #summarize the stacktraces
2159   stackTraceTree 000*/cpass0/*/stacktrace* > stacktrace_cpass0.tree
2160   stackTraceTree 000*/cpass1/*/stacktrace* > stacktrace_cpass1.tree
2161
2162   echo total numbers for the production:
2163   echo
2164   awk 'BEGIN {nFiles=0;nCore=0;} 
2165   /^calibfile/ {nFiles++;} 
2166   /core dumped/ {nCore++i;}
2167   END {print     "cpass0 produced "nFiles" calib files, "nCore" core files";}' meta/cpass0.job*done 2>/dev/null
2168   awk 'BEGIN {nOK=0; nBAD=0; } 
2169   /\/rec.log OK/ {nOK++;} 
2170   /\/rec.log BAD/ {nBAD++;} 
2171   /stderr BAD/ {if ($0 ~ /rec.log/){nBAD++;}}
2172   END {print     "cpass0 reco:  OK: "nOK"\tBAD: "nBAD;}' meta/cpass0.job*done 2>/dev/null
2173   awk 'BEGIN {nOK=0; nBAD=0; } 
2174   /\/calib.log OK/ {nOK++;} 
2175   /\/calib.log BAD/ {nBAD++;} 
2176   END {print "cpass0 calib: OK: "nOK"\tBAD: "nBAD;}' meta/cpass0.job*done 2>/dev/null
2177
2178   awk 'BEGIN {nOK=0; nBAD=0; } 
2179   /merge.log OK/ {nOK++;} 
2180   /merge.log BAD/ {nBAD++;} 
2181   END {print "cpass0 merge: OK: "nOK"\tBAD: "nBAD;}' meta/merge.cpass0*done 2>/dev/null
2182   awk 'BEGIN {nOK=0; nBAD=0; } 
2183   /ocdb.log OK/ {nOK++;} 
2184   /ocdb.log BAD/ {nBAD++;} 
2185   END {print   "cpass0 OCDB:  OK: "nOK"\tBAD: "nBAD;}' meta/merge.cpass0*done 2>/dev/null
2186
2187   echo
2188   awk 'BEGIN {nFiles=0;nCore=0;} 
2189   /^calibfile/ {nFiles++;} 
2190   /core dumped/ {nCore++;}
2191   END {print     "cpass1 produced "nFiles" calib files, "nCore" core files";}' meta/cpass1.job*done 2>/dev/null
2192   awk 'BEGIN {nOK=0; nBAD=0; } 
2193   /\/rec.log OK/ {nOK++;} 
2194   /\/rec.log BAD/ {nBAD++;} 
2195   /stderr BAD/ {if ($0 ~ /rec.log/){nBAD++;}}
2196   END {print     "cpass1 reco:  OK: "nOK"\tBAD: "nBAD;}' meta/cpass1.job*done 2>/dev/null
2197   awk 'BEGIN {nOK=0; nBAD=0; } 
2198   /\/calib.log OK/ {nOK++;} 
2199   /\/calib.log BAD/ {nBAD++;} 
2200   END {print "cpass1 calib: OK: "nOK"\tBAD: "nBAD;}' meta/cpass1.job*done 2>/dev/null
2201
2202   awk 'BEGIN {nOK=0; nBAD=0; } 
2203   /merge.log OK/ {nOK++;} 
2204   /merge.log BAD/ {nBAD++;} 
2205   END {print "cpass1 merge: OK: "nOK"\tBAD: "nBAD;}' meta/merge.cpass1*done 2>/dev/null
2206   awk 'BEGIN {nOK=0; nBAD=0; } 
2207   /ocdb.log OK/ {nOK++;} 
2208   /ocdb.log BAD/ {nBAD++;} 
2209   END {print   "cpass1 OCDB:  OK: "nOK"\tBAD: "nBAD;}' meta/merge.cpass1*done 2>/dev/null
2210
2211   echo
2212   echo per run stats:
2213   /bin/ls -1 meta/merge.cpass0.run*.done | while read x 
2214 do
2215   dir=$(goPrintValues calibfile - ${x})
2216   runNumber=$(guessRunNumber ${dir})
2217   [[ -z ${runNumber} ]] && continue
2218
2219   if $(/bin/ls meta/cpass0.job*.run${runNumber}.done &> /dev/null); then
2220     statusCPass0=( $(
2221     awk 'BEGIN {nOKrec=0;nBADrec=0;nOKcalib=0;nBADcalib=0;nOKstderr=0;nBADstderr=0;}
2222     /\/rec.log OK/ {nOKrec++;} 
2223     /\/rec.log BAD/ {nBADrec++;}
2224     /stderr BAD/ {if ($0 ~ /rec.log/) {nBADrec++;} nBADstderr++;}
2225     /stderr OK/ {nOKstderr++;}
2226     /\/calib.log OK/ {nOKcalib++;}
2227     /\/calib.log BAD/ {nBADcalib++}
2228     END {print ""nOKrec" "nBADrec" "nOKstderr" "nBADstderr" "nOKcalib" "nBADcalib;}' meta/cpass0.job*.run${runNumber}.done 2>/dev/null
2229     ) ) 
2230   fi
2231
2232   if $(/bin/ls meta/cpass1.job*.run${runNumber}.done &>/dev/null); then
2233     statusCPass1=( $(
2234     awk 'BEGIN {nOKrec=0;nBADrec=0;nOKcalib=0;nBADcalib=0;nOKstderr=0;nBADstderr=0;nQAbarrelOK=0;nQAbarrelBAD=0;nQAouterOK=0;nQAouterBAD=0;}
2235     /\/rec.log OK/ {nOKrec++;} 
2236     /\/rec.log BAD/ {nBADrec++;}
2237     /stderr BAD/ {if ($0 ~ /rec.log/) nBADrec++;nBADstderr++;}
2238     /stderr OK/ {nOKstderr++;}
2239     /\/calib.log OK/ {nOKcalib++;}
2240     /\/calib.log BAD/ {nBADcalib++}
2241     /\/qa_barrel.log OK/ {nQAbarrelOK++;}
2242     /\/qa_barrel.log BAD/ {nQAbarrelBAD++;}
2243     /\/qa_outer.log OK/ {nQAouterOK++;}
2244     /\/qa_outer.log BAD/ {nQAouterBAD++;}
2245     END {print ""nOKrec" "nBADrec" "nOKstderr" "nBADstderr" "nOKcalib" "nBADcalib" "nQAbarrelOK" "nQAbarrelBAD" "nQAouterOK" "nQAouterBAD;}' meta/cpass1.job*.run${runNumber}.done 2>/dev/null
2246     ) ) 
2247   fi
2248
2249   statusOCDBcpass0=$(awk '/ocdb.log/ {print $2} ' ${x} 2>/dev/null)
2250   statusOCDBcpass1=$(awk '/ocdb.log/ {print $2}' ${x/cpass0/cpass1} 2>/dev/null)
2251   statusQA=$(awk '/mergeMakeOCDB.log/ {print $2}' ${x/cpass0/cpass1} 2>/dev/null)
2252
2253   printf "%s\t ocdb.log cpass0: %s\t ocdb.log cpass1: %s\tqa.log:%s\t| cpass0: rec:%s/%s stderr:%s/%s calib:%s/%s cpass1: rec:%s/%s stderr:%s/%s calib:%s/%s QAbarrel:%s/%s QAouter:%s/%s\n" ${runNumber} ${statusOCDBcpass0} ${statusOCDBcpass1} ${statusQA} ${statusCPass0[0]} ${statusCPass0[1]} ${statusCPass0[2]} ${statusCPass0[3]} ${statusCPass0[4]} ${statusCPass0[5]} ${statusCPass1[0]} ${statusCPass1[1]} ${statusCPass1[2]} ${statusCPass1[3]} ${statusCPass1[4]} ${statusCPass1[5]} ${statusCPass1[6]} ${statusCPass1[7]} ${statusCPass1[8]} ${statusCPass1[9]}
2254 done
2255
2256   #make lists with output files - QA, trending, filtering and calibration
2257   ### wait for the merging of all runs to be over ###
2258   rm -f qa.list
2259   goPrintValues qafile qa.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2260   rm -f calib.list
2261   goPrintValues calibfile calib.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2262   rm -f trending.list
2263   goPrintValues trendingfile trending.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2264   rm -f filtering.list
2265   goPrintValues filteredTree filtering.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2266   rm -f cpass0.dcsTree.list
2267   goPrintValues dcsTree cpass0.dcsTree.list ${commonOutputPath}/meta/merge.cpass0.run*.done &>/dev/null
2268   rm -f cpass1.dcsTree.list
2269   goPrintValues dcsTree cpass1.dcsTree.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2270  
2271   #merge trending
2272   rm -f ${commonOutputPath}/trending_merged.root
2273   goMerge trending.list ${commonOutputPath}/trending.root ${configFile} "${extraOpts[@]}" &> mergeTrending.log
2274
2275   goMakeSummaryTree ${commonOutputPath} 0
2276   goMakeSummaryTree ${commonOutputPath} 1
2277
2278   goCreateQAplots "${PWD}/qa.list" "${productionID}" "QAplots" "${configFile}" "${extraOpts[@]}" filteringList="${PWD}/filtering.list" &>createQAplots.log
2279
2280   #make a merged summary tree out of the QA trending, dcs trees and log summary trees
2281   goMakeMergedSummaryTree
2282
2283   #if set, email the summary
2284   [[ -n ${MAILTO} ]] && cat ${log} | mail -s "benchmark ${productionID} done" ${MAILTO}
2285
2286   return 0
2287 )
2288
2289 goMakeSummaryTree()
2290 (
2291   if [[ $# -lt 1 ]] ; then
2292     return
2293   fi
2294   #1. define vars/arrays
2295   DIR=${1} #use input or exec in current dir
2296   pass=${2-"0"} #pass from input
2297   outfile="summary_pass${pass}.tree"
2298   Ncolumns=0
2299   test -f ${outfile} && : >${outfile}
2300   errfile=${outfile/tree/err}
2301   test -f ${errfile} && : >${errfile}
2302
2303   declare -a counterString=(TOFevents TOFtracks TPCevents TPCtracks TRDevents TRDtracks T0events SDDevents SDDtracks MeanVertexevents)
2304   Ncounter=${#counterString[@]}
2305
2306   declare -a statusString=(TRDStatus TOFStatus TPCStatus T0Status MeanVertexStatus)
2307   Nstatus=${#statusString[@]}
2308
2309
2310   declare -a ratesString=(rec stderr calib qa_barrel qa_outer)
2311   Nrates=${#ratesString[@]}
2312
2313   runs=( $(ls -1 ${DIR}/meta/merge.cpass0* | while read x; do guessRunNumber $x; done) )
2314   Nruns=${#runs[@]}
2315
2316   echo -n runnumber/I >>${outfile}
2317   echo -n :cpass${pass}status/I >>${outfile}
2318   echo -n :cpass${pass}QAstatus/I >>${outfile}
2319   for i in ${ratesString[@]}; do
2320     echo -n :${i}OK/I >>${outfile}
2321     echo -n :${i}BAD/I >>${outfile}
2322
2323   done
2324   for i in ${counterString[@]} ${statusString[@]} ; do
2325     echo -n :${i}/I >>${outfile}
2326   done
2327   Ncolumns=$((2 + 2*Nrates + Ncounter + Nstatus))
2328   echo >> ${outfile}
2329
2330   #2. loop runs 
2331
2332   for runnumber in ${runs[@]} ; do 
2333
2334
2335
2336     filejob="${DIR}/meta/cpass${pass}.job*.run${runnumber}.done"
2337     filemerge="${DIR}/meta/merge.cpass${pass}.run${runnumber}.done"
2338     fileOCDB=$(grep /ocdb.log ${filemerge} | awk '{print $1}')
2339     if ! $(/bin/ls ${filemerge} &>/dev/null) ; then
2340       echo "${filemerge} does not exist!" >>${errfile}
2341       continue
2342     elif ! $(/bin/ls ${filejob} &>/dev/null) ; then
2343       echo "${filejob} does not exist!" >>${errfile}
2344       echo -n ${runnumber} >> ${outfile}
2345       for i in $(seq ${Ncolumns}) ; do 
2346         echo -n "-1" >> ${outfile}
2347       done
2348       echo >> ${outfile}
2349       continue
2350     fi
2351     echo -n ${runnumber} >> ${outfile}
2352     #pass0status= grep '/ocdb.log' ${filemerge} |  cut -d' ' -f2 | tr OK x1 | tr BAD xx0 | tr -d 'x'
2353     passStatus=$(grep '/ocdb.log' ${filemerge} | grep OK | wc -l)
2354     echo -n " ${passStatus}" >> ${outfile}
2355     qaStatus=$(grep '/mergeMakeOCDB.log' ${filemerge} | grep OK | wc -l)
2356     echo -n " ${qaStatus}" >> ${outfile}
2357
2358
2359     #fill OK/BAD rates
2360     for i in $(seq 0 $((${Nrates}-1))) ; do
2361       var1=$(grep "/${ratesString[${i}]}.log" ${filejob} | grep OK | wc -l)
2362       var2=$(grep "/${ratesString[${i}]}.log" ${filejob} | grep BAD | wc -l)
2363
2364       if [[ ${ratesString[${i}]} == "stderr" ]] ; then
2365         var1=$(grep "stderr" ${filejob} | grep OK | wc -l)
2366         var2=$(grep "stderr" ${filejob} | grep "rec.log" | grep BAD | wc -l)
2367       fi
2368       echo -n " ${var1}" >> ${outfile}
2369       echo -n " ${var2}" >> ${outfile}
2370     done
2371
2372     if [[ -f ${fileOCDB} ]] ; then
2373       #fill counter
2374       for i in $(seq 0 $((${Ncounter}-1))) ; do
2375         var1=$(grep Monalisa ${fileOCDB} | grep ${counterString[${i}]} | cut -f2)
2376         echo -n " ${var1:-"-1"}" >> ${outfile}
2377       done
2378
2379       #fill status
2380       for i in $(seq 0 $((${Nstatus}-1))) ; do
2381         var1=$(grep "calibration status=" ${fileOCDB} | grep ${statusString[${i}]/Status/} | cut -d'=' -f2)
2382         echo -n " ${var1:-"-1"}" >> ${outfile}
2383       done
2384     fi
2385     echo >> ${outfile}
2386   done
2387
2388   return 0
2389 )
2390
2391 parseConfig()
2392 {
2393   configFile=${1}
2394   shift
2395   args=("$@")
2396
2397
2398   #some defaults
2399   #autoOCDB=0
2400   defaultOCDB="raw://"
2401   #runNumber=167123
2402   #makeflowPath="/hera/alice/aux/cctools/bin"
2403   #makeflowOptions="-T wq -N alice -d all -C ali-copilot.cern.ch:9097"
2404   #makeflowOptions="-T wq -N alice -C ali-copilot.cern.ch:9097"
2405   makeflowOptions=""
2406   #batchCommand="/usr/bin/qsub"
2407   batchFlags="-b y -cwd -l h_rt=24:0:0,h_rss=4G "
2408   baseOutputDirectory="$PWD/output"
2409   #alirootEnv="/cvmfs/alice.cern.ch/bin/alienv setenv AliRoot/v5-04-34-AN -c"
2410   #alirootEnv="/home/mkrzewic/alisoft/balice_master.sh"
2411   #trustedQAtrainMacro='/hera/alice/mkrzewic/gsisvn/Calibration/QAtrain_duo.C'
2412   reconstructInTemporaryDir=0
2413   recoTriggerOptions="\"\""
2414   percentProcessedFilesToContinue=100
2415   maxSecondsToWait=$(( 3600*24 ))
2416   nEvents=-1
2417   nMaxChunks=0
2418   postSetUpActionCPass0=""
2419   postSetUpActionCPass1=""
2420   runCPass0reco=1
2421   runCPass0MergeMakeOCDB=1
2422   runCPass1reco=1
2423   runCPass1MergeMakeOCDB=1
2424   runESDfiltering=1
2425   filteringFactorHighPt=1e2
2426   filteringFactorV0s=1e1
2427   MAILTO=""
2428   #pretend=1
2429   #dontRedirectStdOutToLog=1
2430   logToFinalDestination=1
2431   ALIROOT_FORCE_COREDUMP=1
2432   pretendDelay=0
2433
2434   #first, source the config file
2435   if [ -f ${configFile} ]; then
2436     source ${configFile}
2437   else
2438     echo "config file ${configFile} not found!"
2439     return 1
2440   fi
2441
2442   unset encodedSpaces
2443   for opt in "${args[@]}"; do
2444     [[ "${opt}" =~ encodedSpaces=.* ]] && encodedSpaces=1 && echo "encodedSpaces!" && break
2445   done
2446
2447   #then, parse the options as they override the options from file
2448   for opt in "${args[@]}"; do
2449     [[ -z ${opt} ]] && continue
2450     [[ -n ${encodedSpaces} ]] && opt="$(decSpaces ${opt})"
2451     [[ "${opt}" =~ ^[[:space:]]*$ ]] && continue
2452     if [[ ! "${opt}" =~ .*=.* ]]; then
2453       echo "badly formatted option \"${opt}\" should be: option=value, stopping..."
2454       return 1
2455     fi
2456     local var="${opt%%=*}"
2457     local value="${opt#*=}"
2458     echo "${var}=${value}"
2459     export ${var}="${value}"
2460   done
2461
2462   #do some checking
2463   [[ -z ${alirootEnv} ]] && echo "alirootEnv not defined!" && return 1
2464
2465   #export the aliroot function if defined to override normal behaviour
2466   [[ $(type -t aliroot) =~ "function" ]] && export -f aliroot
2467
2468   return 0
2469 }
2470
2471 aliroot()
2472 {
2473   args=("$@")
2474   if [[ -n ${useProfilingCommand} ]]; then
2475     profilerLogFile="cpu.txt"
2476     [[ "${args}" =~ rec ]] && profilerLogFile="cpu_rec.txt"
2477     [[ "${args}}" =~ Calib ]] && profilerLogFile="cpu_calib.txt"
2478     echo running "${useProfilingCommand} aliroot ${args} &> ${profilerLogFile}"
2479     ${useProfilingCommand} aliroot "${args[@]}" &> ${profilerLogFile}
2480   else
2481     #to prevent an infinite recursion use "command aliroot" to disable
2482     #aliases and functions
2483     echo running command aliroot "${args[@]}"
2484     command aliroot "${args[@]}"
2485   fi
2486   return 0
2487 }
2488
2489 guessRunData()
2490 {
2491   #guess the period from the path, pick the rightmost one
2492   period=""
2493   runNumber=""
2494   year=""
2495   pass=""
2496   legoTrainRunNumber=""
2497   dataType=""
2498
2499   local shortRunNumber=""
2500   local IFS="/"
2501   declare -a path=( $1 )
2502   local dirDepth=$(( ${#path[*]}-1 ))
2503   i=0
2504   #for ((x=${dirDepth};x>=0;x--)); do
2505   for ((x=0;x<=${dirDepth};x++)); do
2506
2507     [[ $((x-1)) -ge 0 ]] && local fieldPrev=${path[$((x-1))]}
2508     local field=${path[${x}]}
2509     local fieldNext=${path[$((x+1))]}
2510
2511     [[ ${field} =~ ^[0-9]*$ && ${fieldNext} =~ (.*\.zip$|.*\.root$) ]] && legoTrainRunNumber=${field}
2512     [[ -n ${legoTrainRunNumber} && -z ${pass} ]] && pass=${fieldPrev}
2513     [[ ${field} =~ ^LHC[0-9][0-9][a-z].*$ ]] && period=${field%_*}
2514     [[ ${field} =~ ^000[0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && runNumber=${field#000}
2515     [[ ${field} =~ ^[0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && shortRunNumber=${field}
2516     [[ ${field} =~ ^20[0-9][0-9]$ ]] && year=${field}
2517     [[ ${field} =~ ^(^sim$|^data$) ]] && dataType=${field}
2518     (( i++ ))
2519   done
2520   [[ -z ${legoTrainRunNumber} ]] && pass=${path[$((dirDepth-1))]}
2521   [[ "${dataType}" =~ ^sim$ ]] && pass="passMC" && runNumber=${shortRunNumber}
2522   
2523   #if [[ -z ${dataType} || -z ${year} || -z ${period} || -z ${runNumber}} || -z ${pass} ]];
2524   if [[ -z ${runNumber}} ]];
2525   then
2526     #error condition
2527     return 1
2528   else
2529     #ALL OK
2530     return 0
2531   fi
2532   return 0
2533 }
2534
2535 #these functions encode strings to and from a space-less form
2536 #use when spaces are not well handled (e.g. in arguments to 
2537 #commands in makeflow files, etc.
2538 encSpaces()(echo "${1// /@@@@}")
2539 decSpaces()(echo "${1//@@@@/ }")
2540
2541 main "$@"