9f0642448776b7789fa2e59ee27967f74d62ea79
[u/mrichter/AliRoot.git] / PWGPP / benchmark / benchmark.sh
1 #!/bin/bash
2 #include benchmark.config
3
4 # this script runs the CPass0/CPass1 train
5 # produced OCDB updates are local
6
7 main()
8 {
9   #run in proper mode depending on the selection
10   if [[ $# -lt 1 ]]; then
11     if [[ ! "${0}" =~ "bash" ]]; then
12       echo "uses makeflow:"
13       echo " ${0} \"run\" productionID inputList configFile [extraOpts]"
14       echo "uses a batch system (SGE):"
15       echo " ${0} \"submit\" productionID inputList configFile [extraOpts]"
16       echo "extraOpts if non-empty override the config file, e.g.:"
17       echo " ${0} submit test1 benchmark.list benchmark.config runNumber=169123 nEvents=10"
18     fi
19     return
20   fi
21
22   #define some aliases - default is to call one of the functions directly
23   runMode=${1}
24   umask 0002
25   shift
26   case ${runMode} in
27     "CPass0") goCPass0 "$@";;
28     "CPass1") goCPass1 "$@";;
29     "MakeLocalOCDBaccessConfig") goMakeLocalOCDBaccessConfig "$@";;
30     "MergeCPass0") goMergeCPass0 "$@";;
31     "MergeCPass1") goMergeCPass1 "$@";;
32     "MakeFilteredTrees") goMakeFilteredTrees "$@";;
33     "MakeSummary") goMakeSummary "$@";;
34     "run") goSubmitMakeflow "$@";;
35     "submit") goSubmitBatch "$@";;
36     "test") goTest "$@";;
37     "GenerateMakeflow") goGenerateMakeflow "$@";;
38     "PrintValues") goPrintValues "$@";;
39     "CreateQAplots") goCreateQAplots "$@";;
40     "WaitForOutput") goWaitForOutput "$@";;
41     "Merge") goMerge "$@";;
42     *) 
43       ${runMode} "$@"
44     ;;
45   esac
46   return 0
47 }
48
49 generateMC()
50 {
51   #generate one raw chunk in current directory
52   SEED=${JOB_ID}${SGE_TASK_ID}
53   export CONFIG_SEED=${SEED}
54   runNumber=${1}
55   OCDBpath=${2}
56   nEventsim=${3}
57   if [[ -n ${pretend} ]]; then
58     sleep ${pretendDelay}
59     touch galice.root
60   else
61     if [[ -f sim.C && -f Config.C ]] ; then
62         time aliroot -b -q -x sim.C\(${runNumber},\"${OCDBpath}\",${nEventsim}\) >sim.log 2>&1
63         mv syswatch.log simwatch.log
64     fi
65   fi
66 }
67
68 goCPass0()
69 (
70   umask 0002
71   
72   targetDirectory=${1}
73   inputList=${2}
74   nEvents=${3}
75   ocdbPath=${4}
76   configFile=${5}
77   runNumber=${6}
78   jobindex=${7}
79   shift 7
80   if ! parseConfig ${configFile} "$@"; then return 1; fi
81
82   #record the working directory provided by the batch system
83   batchWorkingDirectory=${PWD}
84
85   #use the jobindex only if set and non-negative
86   if [[ -z ${jobindex} || ${jobindex} -lt 0 ]]; then
87     [[ -n "${LSB_JOBINDEX}" ]] && jobindex=${LSB_JOBINDEX}
88     [[ -n "${SGE_TASK_ID}" ]] && jobindex=${SGE_TASK_ID}
89   fi
90
91   [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
92   doneFile="${commonOutputPath}/meta/cpass0.job${jobindex}.run${runNumber}.done"
93
94   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
95   
96   if [[ -n ${ALIROOT_FORCE_COREDUMP} ]]; then
97     ulimit -c unlimited 
98     export ALIROOT_FORCE_COREDUMP
99   fi
100
101   #the contents of this is stored in the tree and used later (e.g. AliAnalysisTaskPIDResponse)!
102   #at the QA stage the pass number is guessed from the path stored here.
103   #The Format is:
104   #Packages= ;OutputDir= ;LPMPass= ;TriggerAlias= ;LPMRunNumber= ;LPMProductionType= ;LPMInteractionType= ;LPMProductionTag= ;LPMAnchorRun= ;LPMAnchorProduction= ;LPMAnchorYear= 
105   export PRODUCTION_METADATA="OutputDir=cpass0"
106
107   if [[ "${inputList}" =~ \.root$ ]]; then
108     infile=${inputList}
109   else
110     infile=$(sed -ne "${jobindex}p" ${inputList} | egrep '\s*\w*/\w*')
111   fi
112   chunkName=${infile##*/}
113
114   outputDir=${targetDirectory}/${jobindex}_${chunkName%.*}
115   mkdir -p ${outputDir}
116   [[ ! -d ${outputDir} ]] && echo "cannot make ${outputDir}" && touch ${doneFile} && return 1  
117   
118   #runpath=${PWD}/rundir_cpass0_${runNumber}_${jobindex}
119   runpath=${outputDir}
120   #[[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
121   [[ ${reconstructInTemporaryDir} -eq 1 ]] && runpath=$(mktemp -d -t cpass0.XXXXXX)
122   mkdir -p ${runpath}
123   [[ ! -d ${runpath} ]] && echo "cannot make runpath ${runpath}" && touch ${doneFile} && return 1
124   if ! cd ${runpath}; then
125     echo "PWD=$PWD is not the runpath=${runpath}"
126     touch ${doneFile}
127     return 1
128   fi
129
130   #runCPassX/C expects the raw chunk to be linked in the run dir
131   #despite it being accessed by the full path
132   ln -s ${infile} ${runpath}/${chunkName}
133
134   #####MC
135   if [[ -n ${generateMC} ]]; then
136     olddir=${PWD}
137     outputDirMC=${commonOutputPath}/000${runNumber}/sim/${jobindex}
138     simrunpath=${outputDirMC}
139     #[[ ${simulateInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && simrunpath=${TMPDIR}
140     [[ ${simulateInTemporaryDir} -eq 1 ]] && simrunpath=$(mktemp -d -t cpass0MC.XXXXXX)
141     mkdir -p ${outputDirMC}
142     mkdir -p ${simrunpath}
143     if cd ${simrunpath}; then
144
145       filesMC=( 
146       "${batchWorkingDirectory}/sim.C"
147       "${batchWorkingDirectory}/rec.C"
148       "${batchWorkingDirectory}/Config.C"
149       "${batchWorkingDirectory}/OCDB_*.root"
150       )
151       for file in ${filesMC[*]}; do
152         [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
153       done
154
155       generateMC ${runNumber} ${ocdbPath} ${nEvents}
156
157       [[ ! "${simrunpath}" =~ "${outputDirMC}" ]] && mv * ${outputDirMC} #TODO check if it works
158       cd ${olddir}
159
160       ln -s ${outputDirMC}/* ${runpath}/ 
161
162       inputList=${outputDirMC}/galice.root #TODO not valid outside shell !!!
163       infile=""
164     fi
165   fi
166   ######
167   
168   [[ ! -f ${inputList} && -z ${pretend} ]] && echo "input file ${inputList} not found, exiting..." && touch ${doneFile} && return 1
169
170   logOutputDir=${runpath}
171   [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir}
172   [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/stdout
173   #[[ -z ${dontRedirectStdOutToLog} ]] && exec 2> ${logOutputDir}/stderr
174   echo "${0} $*"
175
176   echo "#####################"
177   echo CPass0:
178   echo JOB setup
179   echo nEvents            ${nEvents}
180   echo runNumber          ${runNumber}
181   echo ocdbPath           ${ocdbPath}
182   echo infile             ${infile}
183   echo chunkName          ${chunkName}
184   echo jobindex           ${jobindex}
185   echo recoTriggerOptions ${recoTriggerOptions}
186   echo targetDirectory    ${targetDirectory}
187   echo commonOutputPath         ${commonOutputPath}
188   echo doneFile      ${doneFile}
189   echo runpath            ${runpath}  
190   echo outputDir          ${outputDir}
191   echo ALICE_ROOT         ${ALICE_ROOT}
192   echo PWD                ${PWD}
193   echo "########## ###########"
194
195   alirootInfo > ALICE_ROOT.log
196
197   filesCPass0=( 
198                "${batchWorkingDirectory}/runCPass0.sh"
199                "${batchWorkingDirectory}/recCPass0.C"
200                "${batchWorkingDirectory}/runCalibTrain.C"
201                "${batchWorkingDirectory}/localOCDBaccessConfig.C"
202                "${batchWorkingDirectory}/OCDB.root"
203                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/runCPass0.sh"
204                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/recCPass0.C" 
205                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/runCalibTrain.C"
206   )
207
208   for file in ${filesCPass0[*]}; do
209     [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
210     [[ ${file##*/} =~ .*\.sh ]] && chmod +x ${file##*/}
211   done
212
213   echo "this directory (${PWD}) contents:"
214   /bin/ls
215   echo
216   chmod u+x runCPass0.sh
217
218   sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh
219
220   if [[ -n ${postSetUpActionCPass0} ]]; then
221     echo "running ${postSetUpActionCPass0}"
222     eval ${postSetUpActionCPass0}
223   fi
224
225   #run CPass0
226   echo "${runpath}/runCPass0.sh ${infile} ${nEvents} ${runNumber} ${ocdbPath} ${recoTriggerOptions}"
227   if [[ -n ${pretend} ]]; then
228     sleep ${pretendDelay}
229     touch AliESDs.root
230     touch AliESDfriends.root
231     touch AliESDfriends_v1.root
232     touch rec.log
233     touch calib.log
234   else
235     echo ./runCPass0.sh "${infile}" "${nEvents}" "${runNumber}" "${ocdbPath}" "${recoTriggerOptions}"
236     ./runCPass0.sh "${infile}" "${nEvents}" "${runNumber}" "${ocdbPath}" "${recoTriggerOptions}"
237   fi
238   
239   #move stuff to final destination
240   echo "this directory (${PWD}) contents:"
241   /bin/ls
242   echo
243
244   echo rm -f ./${chunkName}
245   rm -f ./${chunkName}
246   echo "cp -R ${runpath}/* ${outputDir}"
247   cp -p -R ${runpath}/* ${outputDir}
248   echo
249   
250   #validate CPass0
251   cd ${outputDir}
252   touch ${doneFile}
253   echo "dir ${outputDir}" >> ${doneFile}
254   if summarizeLogs >> ${doneFile}; then
255     [[ -f ${outputDirMC}/galice.root ]] && echo "sim ${outputDirMC}/galice.root" >> ${doneFile}
256     [[ -f AliESDfriends_v1.root ]] && echo "calibfile ${outputDir}/AliESDfriends_v1.root" >> ${doneFile}
257     [[ -f AliESDs.root ]] && echo "esd ${outputDir}/AliESDs.root" >> ${doneFile}
258   fi
259
260   [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath} && echo "removing ${runpath}"
261   return 0
262 )
263
264 goCPass1()
265 (
266   umask 0002
267   
268   targetDirectory=${1}
269   inputList=${2}
270   nEvents=${3}
271   ocdbPath=${4}
272   configFile=${5}
273   runNumber=${6}
274   jobindex=${7}
275   shift 7
276   extraOpts=("$@")
277   if ! parseConfig ${configFile} "$@"; then return 1; fi
278
279   #record the working directory provided by the batch system
280   batchWorkingDirectory=${PWD}
281
282   #use the jobindex only if set and non-negative
283   if [[ -z ${jobindex} || ${jobindex} -lt 0 ]]; then
284     [[ -n "${LSB_JOBINDEX}" ]] && jobindex=${LSB_JOBINDEX}
285     [[ -n "${SGE_TASK_ID}" ]] && jobindex=${SGE_TASK_ID}
286   fi
287
288   [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
289   doneFile="${commonOutputPath}/meta/cpass1.job${jobindex}.run${runNumber}.done"
290
291   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
292   
293   if [[ -n ${ALIROOT_FORCE_COREDUMP} ]]; then
294     ulimit -c unlimited 
295     export ALIROOT_FORCE_COREDUMP
296   fi
297
298   #the contents of this is stored in the tree and used later (e.g. AliAnalysisTaskPIDResponse)!
299   #at the QA stage the pass number is guessed from the path stored here.
300   #The Format is:
301   #Packages= ;OutputDir= ;LPMPass= ;TriggerAlias= ;LPMRunNumber= ;LPMProductionType= ;LPMInteractionType= ;LPMProductionTag= ;LPMAnchorRun= ;LPMAnchorProduction= ;LPMAnchorYear= 
302   export PRODUCTION_METADATA="OutputDir=cpass1"
303
304   [[ ! -f ${inputList} && -z ${pretend} ]] && echo "input file ${inputList} not found, exiting..." && touch ${doneFile} && return 1
305   if [[ "${inputList}" =~ \.root$ ]]; then
306     infile=${inputList}
307   else
308     infile=$(sed -ne "${jobindex}p" ${inputList} | egrep '\s*\w*/\w*')
309   fi
310   chunkName=${infile##*/}
311
312   outputDir=${targetDirectory}/${jobindex}_${chunkName%.*}
313   mkdir -p ${outputDir}
314   [[ ! -d ${outputDir} ]] && echo "cannot make ${outputDir}" && touch ${doneFile} && return 1
315   
316   #runpath=${PWD}/rundir_cpass1_${runNumber}_${jobindex}
317   runpath=${outputDir}
318   #[[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
319   [[ ${reconstructInTemporaryDir} -eq 1 ]] && runpath=$(mktemp -d -t cpass1.XXXXXX)
320
321   #MC
322   if [[ "${infile}" =~ galice\.root ]]; then
323     ln -s ${inputList%/*}/* ${runpath}
324     infile=""
325   fi
326
327   #init the running path
328   mkdir -p ${runpath}
329   [[ ! -d ${runpath} ]] && echo "cannot make runpath ${runpath}" && touch ${doneFile} && return 1
330   if ! cd ${runpath}; then
331     echo "PWD=$PWD is not the runpath=${runpath}"
332     touch ${doneFile}
333     return 1
334   fi
335
336   #this is needed for runCPass1.sh
337   ln -s ${infile} ${runpath}/${chunkName}
338
339   logOutputDir=${runpath}
340   [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir}
341   [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/stdout
342   #[[ -z ${dontRedirectStdOutToLog} ]] && exec 2> ${logOutputDir}/stderr
343   echo "${0} $*"
344
345   echo "#####################"
346   echo CPass1:
347   echo JOB setup
348   echo nEvents            ${nEvents}
349   echo runNumber          ${runNumber}
350   echo ocdbPath           ${ocdbPath}
351   echo infile             ${infile}
352   echo chunkName          ${chunkName}
353   echo jobindex           ${jobindex}
354   echo recoTriggerOptions ${recoTriggerOptions}
355   echo targetDirectory    ${targetDirectory}
356   echo commonOutputPath         ${commonOutputPath}
357   echo doneFile      ${doneFile}
358   echo runpath            ${runpath}  
359   echo outputDir          ${outputDir}
360   echo batchWorkingDirectory ${batchWorkingDirectory}
361   echo ALICE_ROOT         ${ALICE_ROOT}
362   echo PWD                ${PWD}
363   echo "########## ###########"
364
365   alirootInfo > ALICE_ROOT.log
366
367   filesCPass1=( 
368                "${batchWorkingDirectory}/runCPass1.sh"
369                "${batchWorkingDirectory}/recCPass1.C"
370                "${batchWorkingDirectory}/recCPass1_OuterDet.C"
371                "${batchWorkingDirectory}/runCalibTrain.C"
372                "${batchWorkingDirectory}/QAtrain_duo.C"
373                "${batchWorkingDirectory}/localOCDBaccessConfig.C"
374                "${batchWorkingDirectory}/${configFile}"
375                "${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz"
376                "${batchWorkingDirectory}/OCDB.root"
377                "${trustedQAtrainMacro}"
378                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/runCPass1.sh"
379                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/recCPass1.C" 
380                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/recCPass1_OuterDet.C" 
381                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/runCalibTrain.C"
382                "${ALICE_ROOT}/ANALYSIS/macros/QAtrain_duo.C"
383   )
384
385   for file in "${filesCPass1[@]}"; do
386     [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
387     [[ ${file##*/} =~ .*\.sh ]] && echo "making ${file##*/} executable" && chmod +x ${file##*/}
388   done
389
390   echo "this directory (${PWD}) contents:"
391   /bin/ls
392   echo
393
394   #remove spaces around commas from calls to root
395   sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh
396
397   if [[ -n ${postSetUpActionCPass1} ]]; then
398     echo "running ${postSetUpActionCPass1}"
399     eval ${postSetUpActionCPass1}
400     echo
401   fi
402
403   #configure local OCDB storage from CPass0 (creates the localOCDBaccessConfig.C script)
404   if [[ -f cpass0.localOCDB.${runNumber}.tgz ]]; then
405     echo goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz"
406     goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz"
407   else
408     echo "WARNING: file cpass0.localOCDB.${runNumber}.tgz not found!"
409   fi
410
411   if [[ ! $(/bin/ls -1 OCDB/*/*/*/*.root 2>/dev/null) ]]; then
412     echo "cpass0 produced no calibration! exiting..."
413     touch ${doneFile}
414     return 1
415   fi
416
417   #create the Barrel and OuterDet directories for CPass1 and link the local OCDB directory
418   #there to make the localOCDBaccessConfig.C file work, since it may point to the OCDB
419   #entries using a relative path, e.g. local://./OCDB
420   echo "linking the OCDB/ for Barrel and OuterDet directories"
421   mkdir Barrel OuterDet
422   ls -l
423   ln -s ../OCDB Barrel/OCDB
424   ln -s ../OCDB OuterDet/OCDB
425
426   #setup the filtering
427   #the following option enables the filtering task inside the QAtrain_duo.C
428   [[ -n $runESDfiltering ]] && export QA_TaskFilteredTree=1
429   #set the downscaling factors during the filtering fro expert QA (overrides the previous values)
430   if [[ -n ${filteringFactorHighPt} ]]; then
431     export AliAnalysisTaskFilteredTree_fLowPtTrackDownscaligF=${filteringFactorHighPt}
432   fi
433   if [[ -n ${filteringFactorV0s} ]]; then
434     export AliAnalysisTaskFilteredTree_fLowPtV0DownscaligF=${filteringFactorV0s} 
435   fi
436
437   #run CPass1
438   chmod u+x runCPass1.sh
439   echo "${runpath}/runCPass1.sh ${infile} ${nEvents} ${runNumber} ${ocdbPath} ${recoTriggerOptions}"
440   if [[ -n ${pretend} ]]; then
441     sleep ${pretendDelay}
442     touch AliESDs_Barrel.root
443     touch AliESDfriends_Barrel.root
444     touch AliESDfriends_v1.root
445     touch QAresults_barrel.root
446     touch EventStat_temp_barrel.root
447     touch AODtpITS.root
448     touch AliESDs_Outer.root
449     touch AliESDfriends_Outer.root
450     touch QAresults_outer.root
451     touch EventStat_temp_outer.root
452     touch rec.log
453     touch calib.log
454     touch qa.log
455     touch filtering.log FilterEvents_Trees.root
456   else
457     ./runCPass1.sh "${infile}" "${nEvents}" "${runNumber}" "${ocdbPath}" "${recoTriggerOptions}"
458     
459     [[ ! -f AliESDs_Barrel.root && -f Barrel/AliESDs.root ]] && mv Barrel/AliESDs.root AliESDs_Barrel.root
460     [[ ! -f AliESDfriends_Barrel.root && -f Barrel/AliESDfriends.root ]] && mv Barrel/AliESDfriends.root AliESDfriends_Barrel.root
461     [[ ! -f AliESDfriends_v1.root && -f Barrel/AliESDfriends_v1.root ]] && mv Barrel/AliESDfriends_v1.root .
462     [[ ! -f QAresults_barrel.root && -f Barrel/QAresults_barrel.root ]] && mv Barrel/QAresults_barrel.root .
463     [[ ! -f AliESDs_Outer.root && -f OuterDet/AliESDs.root ]] && mv OuterDet/AliESDs.root AliESDs_Outer.root
464     [[ ! -f AliESDfriends_Outer.root && -f OuterDet/AliESDfriends.root ]] && mv OuterDet/AliESDfriends.root AliESDfriends_Outer.root
465     [[ ! -f QAresults_outer.root && -f OuterDet/QAresults_outer.root ]] && mv OuterDet/QAresults_outer.root .
466     [[ ! -f FilterEvents_Trees.root && -f Barrel/FilterEvents_Trees.root ]] && mv Barrel/FilterEvents_Trees.root .
467
468     #make the filtered tree (if requested and not already produced by QA
469     [[ -f AliESDs_Barrel.root ]] && echo "AliESDs_Barrel.root" > filtered.list
470     if [[ -n ${runESDfiltering} && ! -f FilterEvents_Trees.root ]]; then 
471       goMakeFilteredTrees ${PWD} ${runNumber} "filtered.list" ${filteringFactorHighPt} ${filteringFactorV0s} ${ocdbPath} 1000000 0 10000000 0 ${configFile} AliESDs_Barrel.root "${extraOpts[@]}" >filtering.log
472     fi
473
474   fi
475
476   ##handle possible crashes in QA (happens often in trunk)
477   ##rerun QA with a trusted aliroot version
478   #if [[ $(validateLog qa_barrel.log) ]]; then
479   #  echo "qa_barrel.log not validated!"
480   #fi
481   #if [[ ! -f QAresults_barrel.root && -f ${setupTrustedAliROOTenvInCurrentShell} || $(validateLog qa_barrel.log) ]]; then
482   #  echo "WARNING: using trusted QA aliroot ${ALICE_ROOT}"
483   #  source ${setupTrustedAliROOTenvInCurrentShell}
484   #  cd Barrel
485   #  rm QAresults_barrel.root
486   #  rm EventStat_temp_barrel.root
487   #  rm AODtpITS.root
488   #  [[ ! -f AliESDs.root ]] && ln -s ../AliESDs_Barrel.root AliESDs.root
489   #  [[ ! -f AliESDfriends.root ]] && ln -s ../AliESDfriends_Barrel.root AliESDfriends.root
490   #  if [[ -n ${trustedQAtrainMacro} ]]; then
491   #    eval "cp ${trustedQAtrainMacro} QAtrain_duo_trusted.C"
492   #  fi
493   #  echo executing aliroot -b -q "QAtrain_duo_trusted.C(\"_barrel\",${runNumber},\"wn.xml\",0,\"${ocdbPath}\")"
494   #  time aliroot -b -q "QAtrain_duo.C(\"_barrel\",${runNumber},\"wn.xml\",0,\"${ocdbPath}\")" &> ../qa_barrel_trusted.log
495   #  cd ../
496   #fi
497
498   #move stuff to final destination
499   echo "this directory (${PWD}) contents:"
500   /bin/ls
501   echo rm -f ./${chunkName}
502   rm -f ./${chunkName}
503   echo "cp -R ${runpath}/* ${outputDir}"
504   cp -pf -R ${runpath}/* ${outputDir}
505   echo
506
507   #validate CPass1
508   cd ${outputDir}
509   touch ${doneFile}
510   echo "dir ${outputDir}" >> ${doneFile}
511   if summarizeLogs >> ${doneFile}; then
512     [[ -f AliESDs_Barrel.root ]] && echo "esd ${outputDir}/AliESDs_Barrel.root" >> ${doneFile}
513     [[ -f AliESDfriends_v1.root ]] && echo "calibfile ${outputDir}/AliESDfriends_v1.root" >> ${doneFile}
514     [[ -f QAresults_Barrel.root ]] && echo "qafile ${outputDir}/QAresults_Barrel.root" >> ${doneFile}
515     [[ -f QAresults_Outer.root ]] && echo "qafile ${outputDir}/QAresults_Outer.root" >> ${doneFile}
516     [[ -f QAresults_barrel.root ]] && echo "qafile ${outputDir}/QAresults_barrel.root" >> ${doneFile}
517     [[ -f QAresults_outer.root ]] && echo "qafile ${outputDir}/QAresults_outer.root" >> ${doneFile}
518     [[ -f FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFile}
519   else
520     if grep "qa_outer.log.*OK" ${doneFile} > /dev/null; then
521       [[ -f QAresults_Outer.root ]] && echo "qafile ${outputDir}/QAresults_Outer.root" >> ${doneFile}
522       [[ -f QAresults_outer.root ]] && echo "qafile ${outputDir}/QAresults_outer.root" >> ${doneFile}
523     fi
524     if grep "qa_barrel.log.*OK" ${doneFile} > /dev/null; then
525       [[ -f QAresults_Barrel.root ]] && echo "qafile ${outputDir}/QAresults_Barrel.root" >> ${doneFile}
526       [[ -f QAresults_barrel.root ]] && echo "qafile ${outputDir}/QAresults_barrel.root" >> ${doneFile}
527     fi
528     if grep "filtering.log.*OK" ${doneFile} > /dev/null; then
529       [[ -f FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFile}
530     fi
531   fi
532
533   [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
534   return 0
535 )
536
537
538 goMergeCPass0()
539 (
540   #
541   # find the output files and merge them
542   #
543
544   outputDir=${1}
545   ocdbStorage=${2}
546   configFile=${3}
547   runNumber=${4}
548   calibrationFilesToMerge=${5}  #can be a non-existent file, will then be produced on the fly
549   shift 5
550   if ! parseConfig ${configFile} "$@"; then return 1; fi
551
552   #record the working directory provided by the batch system
553   batchWorkingDirectory=${PWD}
554
555   [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
556   doneFile="${commonOutputPath}/meta/merge.cpass0.run${runNumber}.done"
557
558   umask 0002
559   ulimit -c unlimited 
560
561   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
562
563   #runpath=${PWD}/rundir_cpass0_Merge_${runNumber}
564   runpath=${outputDir}
565   #[[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
566   [[ ${reconstructInTemporaryDir} -eq 1 ]] && runpath=$(mktemp -d -t mergeCPass0.XXXXXX)
567
568   mkdir -p ${runpath}
569   [[ ! -d ${runpath} ]] && echo "not able to make the runpath ${runpath}" && touch ${doneFile} && return 1
570   if ! cd ${runpath}; then 
571     echo "PWD=$PWD is not the runpath=${runpath}"
572     touch ${doneFile}
573     return 1
574   fi
575
576   logOutputDir=${runpath}
577   [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir}
578   [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/mergeMakeOCDB.log
579   echo "${0} $*"
580
581   mergingScript="mergeMakeOCDB.byComponent.sh"
582
583   echo goMergeCPass0 SETUP:
584   echo runNumber=${runNumber}
585   echo outputDir=${outputDir}
586   echo ocdbStorage=${ocdbStorage}
587   echo calibrationFilesToMerge=${calibrationFilesToMerge}
588   echo mergingScript=${mergingScript}
589   echo commonOutputPath=${commonOutputPath}
590   echo runpath=${runpath}
591   
592   # copy files in case they are not already there
593   filesMergeCPass0=(
594                     "${batchWorkingDirectory}/${calibrationFilesToMerge}"
595                     "${batchWorkingDirectory}/OCDB.root"
596                     "${batchWorkingDirectory}/localOCDBaccessConfig.C"
597                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/mergeMakeOCDB.byComponent.sh"
598                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/mergeByComponent.C"
599                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/makeOCDB.C"
600                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/merge.C"
601                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/mergeMakeOCDB.sh"
602   )
603   for file in ${filesMergeCPass0[*]}; do
604     [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
605     [[ ${file##*/} =~ .*\.sh ]] && chmod +x ${file##*/}
606   done
607   
608   sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh
609
610   alirootInfo > ALICE_ROOT.log
611
612   #
613   echo "PWD"
614   /bin/ls
615   echo "PWD/.."
616   /bin/ls ../
617
618
619   #merge calibration
620   chmod u+x ${mergingScript}  
621   mkdir -p ./OCDB
622   if [[ ! -f ${calibrationFilesToMerge} ]]; then
623     echo "/bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root > ${calibrationFilesToMerge}"
624     /bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root 2>/dev/null > ${calibrationFilesToMerge}
625   fi
626   
627   echo "${mergingScript} ${calibrationFilesToMerge} ${runNumber} local://./OCDB ${ocdbStorage}"
628   if [[ -n ${pretend} ]]; then
629     sleep ${pretendDelay}
630     touch CalibObjects.root
631     touch ocdb.log
632     touch merge.log
633     touch dcsTime.root
634     mkdir -p ./OCDB/TPC/Calib/TimeGain/
635     mkdir -p ./OCDB/TPC/Calib/TimeDrift/
636     touch ./OCDB/TPC/Calib/TimeGain/someCalibObject_0-999999_cpass0.root
637     touch ./OCDB/TPC/Calib/TimeDrift/otherCalibObject_0-999999_cpass0.root
638   else
639     ./${mergingScript} ${calibrationFilesToMerge} ${runNumber} "local://./OCDB" ${ocdbStorage}
640
641     #produce the calib trees for expert QA (dcsTime.root)
642     goMakeLocalOCDBaccessConfig ./OCDB
643     echo aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")"
644     aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")"
645   fi
646   
647   ### produce the output
648   #tar the produced OCDB for reuse
649   tar czf ${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz ./OCDB
650
651   /bin/ls
652
653   #copy all to output dir
654   cp -pf -R ${runpath}/* ${outputDir}
655
656   if [[ -n ${generateMC} ]]; then
657     goPrintValues sim ${commonOutputPath}/meta/sim.run${runNumber}.list ${commonOutputPath}/meta/cpass0.job*.run${runNumber}.done
658   fi
659
660   #validate merging cpass0
661   cd ${outputDir}
662   touch ${doneFile}
663   echo "dir ${outputDir}" >> ${doneFile}
664   if summarizeLogs >> ${doneFile}; then
665     [[ -f CalibObjects.root ]] && echo "calibfile ${outputDir}/CalibObjects.root" >> ${doneFile}
666     [[ -f dcsTime.root ]] && echo "dcsTree ${outputDir}/dcsTime.root" >> ${doneFile}
667   fi
668
669   [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
670   return 0
671 )
672
673 goMergeCPass1()
674 (
675   #
676   # find the output files and merge them
677   #
678
679   outputDir=${1}
680   ocdbStorage=${2}
681   configFile=${3}
682   runNumber=${4}
683   calibrationFilesToMerge=${5}
684   qaFilesToMerge=${6}
685   filteredFilesToMerge=${7}
686   shift 7
687   if ! parseConfig ${configFile} "$@"; then return 1; fi
688
689   #record the working directory provided by the batch system
690   batchWorkingDirectory=${PWD}
691
692   [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
693   doneFile="${commonOutputPath}/meta/merge.cpass1.run${runNumber}.done"
694
695   umask 0002
696   ulimit -c unlimited 
697
698   #clean up first:
699   rm -f ${outputDir}/*.log
700   rm -f ${outputDir}/*.root
701   rm -f ${outputDir}/*done
702
703   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
704
705   #runpath=${PWD}/rundir_cpass1_Merge_${runNumber}
706   runpath=${outputDir}
707   #[[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
708   [[ ${reconstructInTemporaryDir} -eq 1 ]] && runpath=$(mktemp -d -t mergeCPass1.XXXXXX)
709
710   mkdir -p ${runpath}
711   [[ ! -d ${runpath} ]] && echo "not able to make the runpath ${runpath}" && touch ${doneFile} && return 1
712   if ! cd ${runpath}; then 
713     echo "PWD=$PWD is not the runpath=${runpath}"
714     touch ${doneFile}
715     return 1
716   fi
717
718   logOutputDir=${runpath}
719   [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir}
720   [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/mergeMakeOCDB.log
721   echo "${0} $*"
722
723   calibrationOutputFileName='AliESDfriends_v1.root'
724   qaOutputFileName='QAresults*.root'
725   mergingScript="mergeMakeOCDB.byComponent.sh"
726   #important to have the string "Stage.txt" in the filename to trigger the merging
727   #it has to be a list of directories containing the files
728   qaMergedOutputFileName="QAresults_merged.root"
729
730   echo goMergeCPass1 SETUP:
731   echo runNumber=${runNumber}
732   echo outputDir=${outputDir}
733   echo ocdbStorage=${ocdbStorage}
734   echo calibrationFilesToMerge=$calibrationFilesToMerge
735   echo qaFilesToMerge=$qaFilesToMerge
736   echo calibrationOutputFileName=${calibrationOutputFileName}
737   echo mergingScript=${mergingScript}
738   
739   # copy files in case they are not already there
740   filesMergeCPass1=(
741                     "${batchWorkingDirectory}/${calibrationFilesToMerge}"
742                     "${batchWorkingDirectory}/${qaFilesToMerge}"
743                     "${batchWorkingDirectory}/OCDB.root"
744                     "${batchWorkingDirectory}/localOCDBaccessConfig.C"
745                     "${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz"
746                     "${batchWorkingDirectory}/QAtrain_duo.C"
747                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/mergeMakeOCDB.byComponent.sh"
748                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/mergeByComponent.C"
749                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/makeOCDB.C"
750                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/merge.C"
751                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/mergeMakeOCDB.sh"
752                     "${trustedQAtrainMacro}"
753                     "${ALICE_ROOT}/ANALYSIS/macros/QAtrain_duo.C"
754   )
755   for file in ${filesMergeCPass1[*]}; do
756     [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
757     [[ ${file##*/} =~ .*\.sh ]] && chmod +x ${file##*/}
758   done
759
760   sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh
761
762   #configure local OCDB storage from CPass0 (creates the localOCDBaccessConfig.C script)
763   if [[ -f cpass0.localOCDB.${runNumber}.tgz ]]; then
764     echo goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz"
765     goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz"
766   else
767     echo "WARNING: file cpass0.localOCDB.${runNumber}.tgz not found!"
768   fi
769
770   alirootInfo > ALICE_ROOT.log
771
772   #
773   /bin/ls
774
775   #merge calibration
776   chmod u+x ${mergingScript}  
777   mkdir -p OCDB
778   
779   #if not provided, create the lists of files to merge
780   if [[ ! -f ${filteredFilesToMerge} ]]; then
781     echo "/bin/ls -1 ${outputDir}/*/FilterEvents_Trees.root > ${filteredFilesToMerge}"
782     /bin/ls -1 ${outputDir}/*/FilterEvents_Trees.root 2>/dev/null > ${filteredFilesToMerge}
783   fi
784   if [[ ! -f ${calibrationFilesToMerge} ]]; then
785     echo "/bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root > ${calibrationFilesToMerge}"
786     /bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root 2>/dev/null > ${calibrationFilesToMerge}
787   fi
788   if [[ ! -f ${qaFilesToMerge} ]]; then
789     #find the files, but only store the directories (QAtrain_duo.C requires this)
790     echo "/bin/ls -1 ${outputDir}/*/QAresults*.root | while read x; do echo ${x%/*}; done | sort | uniq > ${qaFilesToMerge}"
791     /bin/ls -1 ${outputDir}/*/QAresults*.root | while read x; do echo ${x%/*}; done | sort | uniq > ${qaFilesToMerge}
792   fi
793   
794   echo "${mergingScript} ${calibrationFilesToMerge} ${runNumber} local://./OCDB ${ocdbStorage}"
795   if [[ -n ${pretend} ]]; then
796     sleep ${pretendDelay}
797     touch ocdb.log
798     touch cpass1.localOCDB.${runNumber}.tgz
799     touch ${qaMergedOutputFileName}
800     touch merge.log
801     touch trending.root
802     touch FilterEvents_Trees.root
803     touch CalibObjects.root
804     touch dcsTime.root
805     touch ${qaMergedOutputFileName}
806     mkdir -p OCDB
807   else
808     ./${mergingScript} ${calibrationFilesToMerge} ${runNumber} "local://./OCDB" ${ocdbStorage}
809
810     #merge QA (and filtered trees)
811     [[ -n ${AliAnalysisTaskFilteredTree_fLowPtTrackDownscaligF} ]] && export AliAnalysisTaskFilteredTree_fLowPtTrackDownscaligF
812     [[ -n ${AliAnalysisTaskFilteredTree_fLowPtV0DownscaligF} ]] && export AliAnalysisTaskFilteredTree_fLowPtV0DownscaligF
813
814     #echo aliroot -l -b -q "merge.C(\"${qaFilesToMerge}\",\"\",kFALSE,\"${qaMergedOutputFileName}\")"
815     echo aliroot -b -q "QAtrain_duo.C(\"_barrel\",${runNumber},\"${qaFilesToMerge}\",1,\"${ocdbStorage}\")"
816     #aliroot -l -b -q "merge.C(\"${qaFilesToMerge}\",\"\",kFALSE,\"${qaMergedOutputFileName}\")"
817     aliroot -b -q "QAtrain_duo.C(\"_barrel\",${runNumber},\"${qaFilesToMerge}\",1,\"${ocdbStorage}\")" > mergeQA.log
818     mv QAresults_barrel.root ${qaMergedOutputFileName}
819     mv trending_barrel.root trending.root
820
821     #merge filtered trees
822     echo aliroot -l -b -q "merge.C(\"${qaFilesToMerge}\",\"\",kFALSE,\"${qaMergedOutputFileName}\")"
823     aliroot -l -b -q "merge.C(\"${filteredFilesToMerge}\",\"\",kFALSE,\"FilterEvents_Trees.root\")" > mergeFilteredTrees.log
824
825     #produce the calib trees for expert QA
826     echo aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")"
827     aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")" > calibTree.log
828   fi
829
830   tar czf ${commonOutputPath}/meta/cpass1.localOCDB.${runNumber}.tgz ./OCDB
831
832   /bin/ls
833
834   #copy all to output dir
835   cp -pf -R ${runpath}/* ${outputDir}
836   
837   #validate merge cpass1
838   cd ${outputDir}
839   touch ${doneFile}
840   echo "dir ${outputDir}" >> ${doneFile}
841   if summarizeLogs >>  ${doneFile}; then
842     [[ -f CalibObjects.root ]] && echo "calibfile ${outputDir}/CalibObjects.root" >> ${doneFile}
843     [[ -f ${qaMergedOutputFileName} ]] && echo "qafile ${outputDir}/${qaMergedOutputFileName}" >> ${doneFile}
844     [[ -f trending.root ]] && echo "trendingfile ${outputDir}/trending.root" >> ${doneFile}
845     [[ -f dcsTime.root ]] && echo "dcsTree ${outputDir}/dcsTime.root" >> ${doneFile}
846     [[ -f FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFile}
847   else
848     if grep "mergeQA.log.*OK" ${doneFile} > /dev/null; then
849       [[ -f ${qaMergedOutputFileName} ]] && echo "qafile ${outputDir}/${qaMergedOutputFileName}" >> ${doneFile}
850     fi
851     if grep "mergeFilteredTrees.log.*OK" ${doneFile} > /dev/null; then
852       [[ -f FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFile}
853     fi
854   fi
855       
856   [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
857   return 0
858 )
859
860 goMerge()
861 (
862   #generic root merge using CPass1 merge.C script
863   inputList=${1}
864   outputFile=${2}  
865   configFile=${3-"benchmark.config"}
866   shift 3
867   if ! parseConfig ${configFile} "$@"; then return 1; fi
868   
869   #record the working directory provided by the batch system
870   batchWorkingDirectory=${PWD}
871
872   [[ ! -f ${inputList} ]] && echo "inputList ${inputList} does not exist!" && return 1
873   [[ ! -f ${configFile} ]] && echo "configFile ${configFile} does not exist!" && return 1
874   umask 0002
875   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
876   rm -f ${outputFile}
877   aliroot -b -q "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/merge.C(\"${inputList}\",\"\",kFALSE,\"${outputFile}\")" > merge_${inputList}.log
878   return 0
879 )
880
881 goSubmitMakeflow()
882 {
883   #run
884   productionID=${1}
885   inputFileList=${2}
886   configFile=${3}
887   shift 3
888   extraOpts=("$@")
889   if ! parseConfig ${configFile} "${extraOpts[@]}"; then return 1; fi
890  
891   #record the working directory provided by the batch system
892   batchWorkingDirectory=${PWD}
893
894   [[ -z ${configFile} ]] && configFile="benchmark.config"
895   [[ ! -f ${configFile} ]] && echo "no config file found (${configFile})" && return 1
896
897   if [[ ! $(which makeflow &>/dev/null) && -n ${makeflowPath} ]]; then
898     echo "setting the makflow path from the config: "
899     echo "  export PATH=${makeflowPath}:${PATH}"
900     export PATH=${makeflowPath}:${PATH}
901   fi
902
903   #create the common output dir and the meta dir
904   commonOutputPath=${baseOutputDirectory}/${productionID}
905   if [[ -d ${commonOutputPath} ]]; then
906     echo "output dir ${commonOutputPath} exists!"
907     #return 1
908   else
909     mkdir -p ${commonOutputPath}
910   fi
911   mkdir -p ${commonOutputPath}/meta
912   
913   self=${0}
914   #if which greadlink; then self=$(greadlink -f "${0}"); fi
915   
916   #for reference copy the setup to the output dir
917   cp ${self} ${commonOutputPath}
918   cp ${configFile} ${commonOutputPath}
919   cp ${inputList} ${commonOutputPath}
920
921   #submit - use makeflow if available, fall back to old stuff when makeflow not there
922   if which makeflow; then
923     
924     goGenerateMakeflow ${productionID} ${inputFileList} ${configFile} "${extraOpts[@]}" commonOutputPath=${commonOutputPath} > benchmark.makeflow
925
926     makeflow ${makeflowOptions} benchmark.makeflow
927     cd ../
928   else 
929     echo "no makeflow!"
930   fi
931   return 0
932 }
933
934 goGenerateMakeflow()
935 (
936   #generate the makeflow file
937   [[ $# -lt 3 ]] && echo "args: id inputFileList configFile" && return 1
938   productionID=${1}
939   inputFileList=${2}
940   configFile=${3}
941   shift 3
942   extraOpts=("$@")
943   
944   #batch systems/makeflow sometimes handle spaces in arguments poorly, so encode them
945   for (( i=0;i<${#extraOpts[@]};i++ )); do 
946     extraOpts[i]=$(encSpaces "${extraOpts[i]}")
947   done
948   extraOpts+=("encodedSpaces=1")
949
950   if ! parseConfig ${configFile} "${extraOpts[@]}" &>/dev/null; then return 1; fi
951  
952   #extra safety
953   if [[ -z ${commonOutputPath} ]]; then
954     commonOutputPath=${baseOutputDirectory}/${productionID}
955     extraOpts=( "${extraOpts[@]}" "commonOutputPath=${commonOutputPath}" )
956   fi
957
958   #record the working directory provided by the batch system
959   batchWorkingDirectory=${PWD}
960
961   [[ -z ${configFile} ]] && configFile="benchmark.config"
962   [[ ! -f ${configFile} ]] && echo "no config file found (${configFile})" && return 1
963
964   #these files will be made a dependency - will be copied to the working dir of the jobs
965   declare -a copyFiles
966   inputFiles=(
967               "OCDB.root"
968               "localOCDBaccessConfig.C"
969               "QAtrain_duo.C"
970               "runCPass1.sh"
971               "recCPass1.C"
972               "recCPass1_OuterDet.C"
973               "runCalibTrain.C"
974               "runCPass0.sh"
975               "recCPass0.C"
976   )
977   for file in ${inputFiles[*]}; do
978     [[ -f ${file} ]] && copyFiles+=("${file}")
979   done
980
981   #create the makeflow file
982   [[ -n ${batchFlags} ]] && echo "BATCH_OPTIONS = ${batchFlags}"
983   declare -A arr_cpass0_merged arr_cpass1_merged
984   declare -A arr_cpass0_calib_list arr_cpass1_calib_list 
985   declare -A arr_cpass1_QA_list arr_cpass1_ESD_list arr_cpass1_filtered_list
986   declare -A arr_cpass0_profiled_outputs
987   declare -A listOfRuns
988   [[ -n ${runNumber} ]] && listOfRuns[${runNumber}]=1
989   while read x; do tmpRun=$(guessRunNumber ${x}); [[ -n ${tmpRun} ]] && listOfRuns[${tmpRun}]=1; done < ${inputFileList}
990   for runNumber in "${!listOfRuns[@]}"; do
991     [[ -z ${runNumber} ]] && continue
992     [[ ! ${runNumber} =~ ^[0-9]*[0-9]$ ]] && continue
993
994     unset arr_cpass0_outputs
995     unset arr_cpass1_outputs
996     declare -a arr_cpass0_outputs
997     declare -a arr_cpass1_outputs
998
999     jobindex=0
1000     inputFile=""
1001     while read inputFile; do
1002       currentDefaultOCDB=${defaultOCDB}
1003       [[ -z ${autoOCDB} ]] && autoOCDB=1
1004       if [[ ${autoOCDB} -ne 0 ]]; then
1005         currentDefaultOCDB=$(setYear ${inputFile} ${defaultOCDB})
1006       fi
1007
1008       #CPass0
1009       arr_cpass0_outputs[${jobindex}]="${commonOutputPath}/meta/cpass0.job${jobindex}.run${runNumber}.done"
1010       echo "${arr_cpass0_outputs[${jobindex}]} : benchmark.sh ${configFile} ${copyFiles[@]}"
1011       echo " ${alirootEnv} ./benchmark.sh CPass0 ${commonOutputPath}/000${runNumber}/cpass0 ${inputFile} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} ${jobindex} ${extraOpts[@]}"" "
1012
1013       #CPass1
1014       arr_cpass1_outputs[${jobindex}]="${commonOutputPath}/meta/cpass1.job${jobindex}.run${runNumber}.done"
1015       echo "${arr_cpass1_outputs[${jobindex}]} : benchmark.sh ${configFile} ${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz ${copyFiles[@]}"
1016       echo " ${alirootEnv} ./benchmark.sh CPass1 ${commonOutputPath}/000${runNumber}/cpass1 ${inputFile} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} ${jobindex} ${extraOpts[@]}"" "
1017       ((jobindex++))
1018
1019     done< <(grep "/000${runNumber}/" ${inputFileList})
1020     
1021     #CPass0 list of Calib files to merge
1022     arr_cpass0_calib_list[${runNumber}]="${commonOutputPath}/meta/cpass0.calib.run${runNumber}.list"
1023     echo "${arr_cpass0_calib_list[${runNumber}]} : benchmark.sh ${arr_cpass0_outputs[*]}"
1024     echo "  ./benchmark.sh PrintValues calibfile ${arr_cpass0_calib_list[${runNumber}]} ${arr_cpass0_outputs[*]}"
1025     echo
1026
1027     #CPass0 merging
1028     arr_cpass0_merged[${runNumber}]="${commonOutputPath}/meta/merge.cpass0.run${runNumber}.done"
1029     echo "${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz ${arr_cpass0_merged[${runNumber}]} : benchmark.sh ${configFile} ${arr_cpass0_calib_list[${runNumber}]} ${copyFiles[@]}"
1030     echo " ${alirootEnv} ./benchmark.sh MergeCPass0 ${commonOutputPath}/000${runNumber}/cpass0 ${currentDefaultOCDB} ${configFile} ${runNumber} ${arr_cpass0_calib_list[${runNumber}]} ${extraOpts[@]}"" "
1031
1032     #CPass1 list of Calib/QA/ESD/filtered files
1033     # the trick with QA is to have the string "Stage.txt" in the file name of the list of directories with QA output to trigger
1034     # the production of the QA trending tree (only then the task->Finish() will be called in QAtrain_duo.C, on the grid
1035     # this corresponds to the last merging stage)
1036     arr_cpass1_QA_list[${runNumber}]="${commonOutputPath}/meta/cpass1.QA.run${runNumber}.lastMergingStage.txt.list"
1037     echo "${arr_cpass1_QA_list[${runNumber}]}: benchmark.sh ${arr_cpass1_outputs[*]}"
1038     echo "  ./benchmark.sh PrintValues dir ${arr_cpass1_QA_list[${runNumber}]} ${arr_cpass1_outputs[*]}"
1039     echo
1040     arr_cpass1_calib_list[${runNumber}]="${commonOutputPath}/meta/cpass1.calib.run${runNumber}.list"
1041     echo "${arr_cpass1_calib_list[${runNumber}]} : benchmark.sh ${arr_cpass1_outputs[*]}"
1042     echo "  ./benchmark.sh PrintValues calibfile ${arr_cpass1_calib_list[${runNumber}]} ${arr_cpass1_outputs[*]};"
1043     echo
1044     arr_cpass1_ESD_list[${runNumber}]="${commonOutputPath}/meta/cpass1.ESD.run${runNumber}.list"
1045     echo "${arr_cpass1_ESD_list[${runNumber}]} : benchmark.sh ${arr_cpass1_outputs[*]}"
1046     echo "  ./benchmark.sh PrintValues esd ${arr_cpass1_ESD_list[${runNumber}]} ${arr_cpass1_outputs[*]}"
1047     echo
1048     arr_cpass1_filtered_list[${runNumber}]="${commonOutputPath}/meta/cpass1.filtered.run${runNumber}.list"
1049     echo "${arr_cpass1_filtered_list[${runNumber}]} : benchmark.sh ${arr_cpass1_outputs[*]}"
1050     echo "  ./benchmark.sh PrintValues filteredTree ${arr_cpass1_filtered_list[${runNumber}]} ${arr_cpass1_outputs[*]}"
1051     echo
1052   
1053     #CPass1 merging
1054     arr_cpass1_merged[${runNumber}]="${commonOutputPath}/meta/merge.cpass1.run${runNumber}.done"
1055     echo "${commonOutputPath}/meta/cpass1.localOCDB.${runNumber}.tgz ${arr_cpass1_merged[${runNumber}]} :  benchmark.sh ${configFile} ${arr_cpass1_calib_list[${runNumber}]} ${arr_cpass1_QA_list[${runNumber}]} ${copyFiles[@]}"
1056     echo " ${alirootEnv} ./benchmark.sh MergeCPass1 ${commonOutputPath}/000${runNumber}/cpass1 ${currentDefaultOCDB} ${configFile} ${runNumber} ${arr_cpass1_calib_list[${runNumber}]} ${arr_cpass1_QA_list[${runNumber}]} ${arr_cpass1_filtered_list[${runNumber}]} ${extraOpts[@]}"" "
1057
1058     #CPass0 wrapped in a profiling tool (valgrind,....)
1059     if [[ -n ${profilingCommand} ]]; then
1060       inputFile=$(grep -m1 "${runNumber}/" ${inputFileList})
1061       [[ -z ${nEventsProfiling} ]] && nEventsProfiling=2
1062       currentDefaultOCDB=$(setYear ${inputFile} ${defaultOCDB})
1063       jobindex="profiling"
1064
1065       arr_cpass0_profiled_outputs[${runNumber}]="${commonOutputPath}/meta/cpass0.job${jobindex}.run${runNumber}.done"
1066       echo "${arr_cpass0_profiled_outputs[${runNumber}]} : benchmark.sh ${configFile} ${copyFiles[@]}"
1067       profilingCommand=$(encSpaces "${profilingCommand}")
1068       echo " ${alirootEnv} ./benchmark.sh CPass0 ${commonOutputPath}/000${runNumber}/${jobindex} ${inputFile} ${nEventsProfiling} ${currentDefaultOCDB} ${configFile} ${runNumber} ${jobindex} ${extraOpts[@]} useProfilingCommand=${profilingCommand}"
1069     fi
1070
1071   done #runs
1072
1073   #Summary
1074   echo "${commonOutputPath}/summary.log : benchmark.sh ${configFile} ${arr_cpass1_merged[*]}"
1075   echo "  ${alirootEnv} ./benchmark.sh MakeSummary ${configFile} ${extraOpts[@]}"
1076   echo
1077
1078   return 0
1079 )
1080
1081 goPrintValues()
1082 (
1083   #print the values given the key from any number of files (store in output file on second argument)
1084   if [[ $# -lt 3 ]]; then
1085     echo "goPrintValues key outputFile inputFiles"
1086     echo "if outputFile is \"-\" dont redirect to a file"
1087     return
1088   fi
1089   key=${1}
1090   outputFile=${2}
1091   [[ ${outputFile} =~ "-" ]] && outputFile=""
1092   shift 2 #remove 2 first arguments from arg list to only pass the input files to awk
1093   awk -v key=${key} '$0 ~ key" " {print $2}' "$@" | tee ${outputFile}
1094   return 0
1095 )
1096
1097 goCreateQAplots()
1098 (
1099   umask 0002
1100   mergedQAfileList=${1}
1101   productionID=${2}
1102   outputDir=${3}
1103   configFile=${4}
1104   shift 4
1105   if ! parseConfig ${configFile} ${@}; then return 1; fi
1106   
1107   #record the working directory provided by the batch system
1108   batchWorkingDirectory=${PWD}
1109
1110   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
1111
1112   [[ -z ${logOutputDir} ]] && logOutputDir=${PWD}
1113   [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/makeQAplots.log
1114   echo "${0} $*"
1115
1116   olddir=${PWD}
1117   mkdir -p ${outputDir}
1118   cd ${outputDir}
1119   [[ ! "${PWD}" =~ "${outputDir}" ]] && echo "PWD is not equal to outputDir=${outputDir}" && cd ${olddir} && return 1
1120
1121   ${ALICE_ROOT}/PWGPP/QA/scripts/runQA.sh inputList="${mergedQAfileList}" inputListHighPtTrees="${filteringList}"
1122
1123   cd ${olddir}
1124   return 0
1125 )
1126
1127 goTest()
1128 (
1129   echo AA
1130 )
1131
1132 alirootInfo()
1133 (
1134   umask 0002
1135   # save aliroot repository info
1136   [[ -z "${ALICE_ROOT}" ]] && return 1
1137   
1138   echo "\${ALICE_ROOT}=${ALICE_ROOT}"
1139   echo "\${ROOTSYS}=${ROOTSYS}"
1140   echo "\${PATH}=${PATH}"
1141   echo "\${LD_LIBRARY_PATH}=${LD_LIBRARY_PATH}"
1142   echo
1143   
1144   pushd ${PWD}
1145   cd ${ALICE_ROOT}
1146
1147   currentBranch=$(git rev-parse --abbrev-ref HEAD)
1148   git status
1149   echo ""
1150   echo ""
1151   git diff ${currentBranch}
1152   popd
1153   return 0
1154 )
1155
1156 setYear()
1157 (
1158   #set the year
1159   #  ${1} - year to be set
1160   #  ${2} - where to set the year
1161   year1=$(guessYear ${1})
1162   year2=$(guessYear ${2})
1163   local path=${2}
1164   [[ ${year1} -ne ${year2} && -n ${year2} && -n ${year1} ]] && path=${2/\/${year2}\//\/${year1}\/}
1165   echo ${path}
1166   return 0
1167 )
1168
1169 guessPeriod()
1170 (
1171   #guess the period from the path, pick the rightmost one
1172   local IFS="/"
1173   declare -a path=( ${1} )
1174   local dirDepth=${#path[*]}
1175   for ((x=${dirDepth}-1;x>=0;x--)); do
1176     local field=${path[${x}]}
1177     [[ ${field} =~ ^LHC[0-9][0-9][a-z]$ ]] && period=${field} && break
1178   done
1179   echo ${period}
1180   return 0
1181 )
1182
1183 guessYear()
1184 (
1185   #guess the year from the path, pick the rightmost one
1186   local IFS="/"
1187   declare -a path=( ${1} )
1188   local dirDepth=${#path[*]}
1189   for ((x=${dirDepth}-1;x>=0;x--)); do
1190     local field=${path[${x}]}
1191     [[ ${field} =~ ^20[0-9][0-9]$ ]] && year=${field} && break
1192   done
1193   echo ${year}
1194   return 0
1195 )
1196
1197 guessRunNumber()
1198 (
1199   #guess the run number from the path, pick the rightmost one
1200   #works for /path/foo/000123456/bar/...
1201   #and       /path/foo.run123456.bar
1202   local IFS="/."
1203   declare -a path=( ${1} )
1204   local dirDepth=${#path[*]}
1205   for ((x=${dirDepth}-1;x>=0;x--)); do
1206     local field=${path[${x}]}
1207     field=${field/run/000}
1208     [[ ${field} =~ [0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && runNumber=${field#000} && break
1209   done
1210   echo ${runNumber}
1211   return 0
1212 )
1213
1214 validateLog()
1215 (
1216   log=${1}
1217   errorConditions=(
1218                     'There was a crash'
1219                     'floating'
1220                     'error while loading shared libraries'
1221                     'std::bad_alloc'
1222                     's_err_syswatch_'
1223                     'Thread [0-9]* (Thread'
1224                     'AliFatal'
1225                     'core dumped'
1226                     '\.C.*error:.*\.h: No such file'
1227                     'line.*Aborted'
1228   )
1229
1230   warningConditions=(
1231                      'This is serious !'
1232                      'rocVoltage out of range:'
1233   )
1234   
1235   local logstatus=0
1236   local errorSummary=""
1237   local warningSummary=""
1238
1239   for ((i=0; i<${#errorConditions[@]};i++)); do
1240     local tmp=$(grep -m1 -e "${errorConditions[${i}]}" ${log})
1241     [[ -n ${tmp} ]] && tmp+=" : "
1242     errorSummary+=${tmp}
1243   done
1244
1245   for ((i=0; i<${#warningConditions[@]};i++)); do
1246     local tmp=$(grep -m1 -e "${warningConditions[${i}]}" ${log})
1247     [[ -n ${tmp} ]] && tmp+=" : "
1248     warningSummary+=${tmp}
1249   done
1250
1251   if [[ -n ${errorSummary} ]]; then 
1252     echo "${errorSummary}"
1253     return 1
1254   fi
1255   
1256   if [[ -n ${warningSummary} ]]; then
1257     echo "${warningSummary}"
1258     return 2
1259   fi
1260
1261   return 0
1262 )
1263
1264 summarizeLogs()
1265 (
1266   #print a summary of logs
1267   logFiles=(
1268             "*.log"
1269             "stdout"
1270             "stderr"
1271   )
1272
1273   #check logs
1274   local logstatus=0
1275   for log in ${logFiles[*]}; do
1276     finallog=${PWD%/}/${log}
1277     [[ ! -f ${log} ]] && continue
1278     errorSummary=$(validateLog ${log})
1279     validationStatus=$?
1280     if [[ ${validationStatus} -eq 0 ]]; then 
1281       #in pretend mode randomly report an error in rec.log some cases
1282       if [[ -n ${pretend} && "${log}" == "rec.log" ]]; then
1283         #[[ $(( ${RANDOM}%2 )) -ge 1 ]] && echo "${finallog} BAD random error" || echo "${finallog} OK"
1284         echo "${finallog} OK"
1285       else
1286         echo "${finallog} OK"
1287       fi
1288     elif [[ ${validationStatus} -eq 1 ]]; then
1289       echo "${finallog} BAD ${errorSummary}"
1290       logstatus=1
1291     elif [[ ${validationStatus} -eq 2 ]]; then
1292       echo "${finallog} OK MWAH ${errorSummary}"
1293     fi
1294   done
1295   
1296   #report core files
1297   while read x; do
1298     echo ${x}
1299     chmod 644 ${x}
1300     gdb --batch --quiet -ex "bt" -ex "quit" aliroot ${x} > stacktrace_${x//\//_}.log
1301   done < <(/bin/ls ${PWD}/*/core 2>/dev/null; /bin/ls ${PWD}/core 2>/dev/null)
1302   
1303   return ${logstatus}
1304 )
1305
1306 spitOutLocalOCDBaccessConfig()
1307 {
1308   umask 0002
1309   #find ${1} -name "*root" | \
1310   /bin/ls -1 ${1}/*/*/*/*.root 2>/dev/null | \
1311   while read line
1312   do 
1313     local tmp=${line#${1}}
1314     echo ${tmp%/*} | \
1315     awk -v ocdb=${1} '{print "  man->SetSpecificStorage(\""$1"\",\"local://"ocdb"\");"}'
1316   done
1317   return 0
1318 }
1319
1320 goMakeLocalOCDBaccessConfig()
1321 {
1322   umask 0002
1323   # make a script that sets the specific storages form all the root files produced by CPass0
1324   local localOCDBpathCPass0=${1}
1325   local OCDBpathPrefix=${2}
1326   [[ -z ${OCDBpathPrefix} ]] && OCDBpathPrefix="."
1327
1328   if [[ -f ${localOCDBpathCPass0} && ${localOCDBpathCPass0} =~ \.tgz$ ]]; then
1329     tar xzf ${localOCDBpathCPass0}
1330     local localOCDBpathCPass0="${OCDBpathPrefix}/OCDB"
1331   fi
1332
1333   echo
1334   echo creating the specific storage script
1335   echo   localOCDBaccessConfig.C
1336   echo   based on OCDB: ${localOCDBaccessConfig}
1337   echo
1338
1339   local tempLocalOCDB=""
1340   if [[ -f localOCDBaccessConfig.C ]]; then
1341     tempLocalOCDB=$(mktemp -t tempLocalOCDB.XXXXXX)
1342     echo "egrep "SetSpecificStorage" localOCDBaccessConfig.C > ${tempLocalOCDB}"
1343     egrep "SetSpecificStorage" localOCDBaccessConfig.C > ${tempLocalOCDB}
1344   fi
1345
1346   echo "localOCDBaccessConfig()"                               >  localOCDBaccessConfig.C
1347   echo "{"                                                     >> localOCDBaccessConfig.C
1348   echo "  AliCDBManager* man = AliCDBManager::Instance();"     >> localOCDBaccessConfig.C
1349   spitOutLocalOCDBaccessConfig ${localOCDBpathCPass0}|sort|uniq  >> localOCDBaccessConfig.C
1350   [[ -f "${tempLocalOCDB}" ]] && cat ${tempLocalOCDB}              >> localOCDBaccessConfig.C
1351   echo "}"                                                     >> localOCDBaccessConfig.C
1352
1353   [[ -f "${tempLocalOCDB}" ]] && rm -f ${tempLocalOCDB}
1354
1355   if ! grep SetSpecificStorage localOCDBaccessConfig.C; then 
1356     echo
1357     echo "!!!!!!! CPass0 produced no OCDB entries"
1358     return 1
1359   fi
1360   return 0
1361 }
1362
1363 goMakeFilteredTrees()
1364 (
1365   outputDir=${1}
1366   runNumber=${2}
1367   #get path to input list
1368   inputListfiles=${3}
1369   #get scale number for tracks
1370   filterT=${4}
1371   #get scale number for V0s
1372   filterV=${5}
1373   #get OCDB path (optional)
1374   OCDBpath=${6}
1375   #get max number of files 
1376   maxFiles=${7-"1000000"}
1377   #get offset of first file
1378   offsetFile=${8-"0"}
1379   #get max number of events
1380   maxEvents=${9-"30000000"}
1381   #get offset of first event
1382   offsetEvent=${10-"0"}
1383   configFile=${11-"benchmark.config"}
1384   esdFileName=${12-"AliESDs_Barrel.root"}
1385   shift 12
1386   if ! parseConfig ${configFile} "$@"; then return 1; fi
1387   
1388   #record the working directory provided by the batch system
1389   batchWorkingDirectory=${PWD}
1390
1391   [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
1392   doneFile=${commonOutputPath}/meta/filtering.cpass1.run${runNumber}.done
1393
1394   runpath=${outputDir}
1395   #[[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
1396   [[ ${reconstructInTemporaryDir} -eq 1 ]] && runpath=$(mktemp -d -t goMakeFilteredTrees.XXXXXX)
1397
1398   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
1399
1400   mkdir -p ${outputDir}
1401   mkdir -p ${runpath}
1402
1403   if ! cd ${runpath}; then 
1404     echo "PWD=$PWD is not the runpath=${runpath}"
1405     touch ${doneFile}
1406     return 1
1407   fi
1408   
1409   cat > filtering.log << EOF
1410   goMakeFilteredTrees config:
1411   runpath=${runpath}
1412   outputDir=${outputDir}
1413   commonOutputPath=${commonOutputPath}
1414   ALICE_ROOT=${ALICE_ROOT}
1415   PATH=${PATH}
1416   offsetEvent=$offsetEvent
1417   configFile=$configFile
1418   esdFileName=$esdFileName
1419 EOF
1420
1421   if [[ -z ${pretend} ]];then
1422     aliroot -l -b -q "${ALICE_ROOT}/PWGPP/macros/runFilteringTask.C(\"${inputListfiles}\",${filterT},${filterV},\"${OCDBpath}\",${maxFiles},${offsetFile},${maxEvents},${offsetEvent},\"${esdFileName}\")" &>> filtering.log
1423   else
1424     sleep ${pretendDelay}
1425     touch filtering.log FilterEvents_Trees.root
1426   fi
1427   pwd
1428   /bin/ls
1429   touch ${doneFile}
1430   summarizeLogs >>  ${doneFile}
1431   echo mv -f * ${outputDir}
1432   mv -f * ${outputDir}
1433   [[ -f ${outputDir}/FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFile}
1434   cd ${commonOutputPath}
1435   [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
1436   return 0
1437 )
1438
1439 submit()
1440 {
1441   umask 0002
1442   [[ $# -lt 5 ]] && echo "at least 5 args needed, you supplied $#" && return 1
1443   JobID=${1}
1444   startID=${2}
1445   endID=${3}
1446   waitForJOBID=${4}
1447   command=${5}
1448   shift 5
1449   local commandArgs=("$@")
1450
1451   #add quote strings around the extra arguments
1452   for ((i=0; i<${#commandArgs[@]}; i++)); do 
1453     commandArgs[i]=\"${commandArgs[i]}\"
1454   done
1455
1456   [[ -z ${waitForJOBID} ]] && waitForJOBID=0
1457
1458   newFarm=$(which qsub|grep "^/usr/bin/qsub")
1459   
1460   batchSystem="SGE"
1461
1462   if [[ -z "${newFarm}" ]]
1463   then
1464     #old LSF
1465     # submit it (as job array)
1466     nFiles=$(( ${endID}-${startID}+1 ))
1467     while [ ${startID} -le ${nFiles}  ] ; do
1468       if [ $(expr ${nFiles} - ${startID}) -gt 999 ] ; then 
1469         endID=$(expr ${startID} + 999)
1470       else
1471         endID=${nFiles}
1472       fi      
1473     if [[ ${waitForJOBID} -eq 0 ]]; then
1474         echo ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}"     
1475         ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}"     
1476       else
1477         echo ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -w "ended(${waitForJOBID})" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}"     
1478         ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -w "ended(${waitForJOBID})" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}"     
1479       fi
1480       startID=$(expr ${endID} + 1)
1481     done
1482   else 
1483     #new SGE farm
1484     if [[ ${waitForJOBID} =~ "000" ]]; then
1485       echo ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}"
1486       ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}"
1487     else
1488       echo ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -hold_jid "${waitForJOBID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}"
1489       ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -hold_jid "${waitForJOBID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}"
1490     fi
1491   fi
1492   return 0
1493 }
1494
1495 goSubmitBatch()
1496 {
1497   if [[ $# -lt 3 ]]; then
1498     echo "minimal use:"
1499     echo " ${0} submit fileList productionID configFile"
1500     return 0
1501   fi
1502
1503   productionID=${1}
1504   inputList=${2}
1505   configFile=${3:-"benchmark.config"}
1506   #if which greadlink; then configFile=$(greadlink -f ${configFile}); fi
1507   shift 3
1508   extraOpts=("$@")
1509   if ! parseConfig ${configFile} "${extraOpts[@]}"; then return 1; fi
1510   
1511   #batch systems/makeflow sometimes handle spaces in arguments poorly, so encode them
1512   for (( i=0;i<${#extraOpts[@]};i++ )); do 
1513     extraOpts[i]=$(encSpaces "${extraOpts[i]}")
1514   done
1515   extraOpts+=("encodedSpaces=1")
1516
1517   #record the working directory provided by the batch system
1518   batchWorkingDirectory=${PWD}
1519
1520   #redirect all output to submit.log
1521   echo "redirecting all output to ${PWD}/submit_${productionID//"/"/_}.log"
1522   exec 7>&1
1523   exec 1>submit_${productionID//"/"/_}.log 2>&1
1524
1525   umask 0002
1526   echo ${0}" submit $*"
1527   if [[ -z "${inputList}" || -z "${productionID}" ]]
1528   then
1529     echo
1530     echo " Usage: ${0} submit inputList productionID [configFile=benchmark.config]"
1531     echo
1532     return
1533   fi
1534
1535   # check if config file is there
1536   if [ ! -f ${configFile} ]; then
1537     echo "ERROR! Config File '${configFile}' not found" >&2
1538     return
1539   else
1540     echo "Using Config File: '${configFile}'"
1541   fi
1542
1543   [[ ! -f ${alirootEnv} ]] && echo "alirootEnv script ${alirootEnv} not found!..." && return 1
1544
1545   #move the script, config and some other stuff to ${commonOutputPath} first, then use them from there
1546   self=${0}
1547   #if which greadlink; then self=$(greadlink -f "${0}"); fi
1548   configPath=$(dirname ${configFile})
1549   export commonOutputPath=${baseOutputDirectory}/${productionID}
1550   
1551   mkdir -p ${commonOutputPath}
1552   mkdir -p ${commonOutputPath}/logs
1553   mkdir -p ${commonOutputPath}/meta
1554
1555   cp ${self} ${commonOutputPath}
1556   cp ${configFile} ${commonOutputPath}
1557   cp ${inputList} ${commonOutputPath}
1558   self=${commonOutputPath}/${self##*/}
1559   chmod u+x ${self}
1560   configFile=${commonOutputPath}/${configFile##*/}
1561   inputList=${commonOutputPath}/${inputList##*/}
1562
1563   #convert to absolut pathnames
1564   #if which greadlink; then inputList=$(greadlink -f "${inputList}"); fi
1565   #make list of runs
1566   if [[ -z ${runNumber} ]]; then
1567     listOfRuns=($(while read x; do guessRunNumber ${x}; done < ${inputList} | sort | uniq))
1568   else
1569     listOfRuns=${runNumber}
1570   fi
1571
1572   #if which greadlink; then alirootSource=$(greadlink -f "${alirootSource}"); fi
1573
1574   echo ""
1575   echo "### BEGIN CONFIGURATION ###"
1576   echo ""
1577   echo "GENERAL:"
1578   echo ""
1579   echo "    productionID:    ${productionID}"
1580   echo "    batchCommand:    ${batchCommand}"
1581   echo "    batchFlags:      ${batchFlags}"
1582   echo "    alirootEnv:   ${alirootEnv}"
1583   ${alirootEnv} echo '    ALICE_ROOT:      ${ALICE_ROOT}'
1584   ${alirootEnv} echo '    ALIROOT_RELEASE: ${ALICE_RELEASE}'
1585   echo "    inputList:       ${inputList}"
1586   echo "    configPath:      ${configPath}"
1587   echo "    commonOutputPath:      ${commonOutputPath}"
1588   echo "    defaultOCDB:     ${defaultOCDB}"
1589   echo "      autoOCDB: ${autoOCDB}"
1590   echo "    recoTriggerOptions:   ${recoTriggerOptions}"
1591   echo "    runs:"
1592   echo "      ${listOfRuns[*]}"
1593   echo ""
1594   echo "THE TRAIN WILL RUN:"
1595
1596   if [ ${runCPass0reco} -eq 1 ]; then
1597     echo "    Pass0 - Recontruction"
1598   fi
1599
1600   if [ ${runCPass0MergeMakeOCDB} -eq 1 ]; then
1601     echo "    Pass0 - merging and OCDB export"
1602   fi
1603
1604   if [ ${runCPass1reco} -eq 1 ]; then
1605     echo "    Pass1 - Recontruction"
1606   fi
1607   if [ ${runCPass1MergeMakeOCDB} -eq 1 ]; then
1608     echo "    Pass1 - merging and OCDB export"
1609   fi
1610
1611   echo ""
1612   echo "LIMITS:"
1613   echo "    max. Events/Chunk:   ${nEvents}"
1614   echo "    max. Number of Chunks per Run:     ${nMaxChunks}"
1615   echo ""
1616   echo "### END CONFIGURATION ###"
1617   echo ""
1618
1619
1620   # check if input file is there
1621   if [ ! -f ${inputList} ]; then
1622     echo "ERROR! Input List '${inputList}' not found" >&2
1623     return
1624   fi
1625
1626   # define jobid (for dependent jobs)
1627   date=$(date +%Y_%m_%d_%H%M%S)
1628   #for each run we submit one jobarray:
1629   for runNumber in ${listOfRuns[*]}; do
1630     
1631     [[ -z ${runNumber} ]] && continue
1632     [[ ! ${runNumber} =~ ^[0-9]*[0-9]$ ]] && continue
1633
1634     JOBpostfix="${productionID//"/"/_}_${runNumber}_${date}"
1635     JOBID1="p0_${JOBpostfix}"
1636     JOBID1wait="w0_${JOBpostfix}"
1637     JOBID2="m0_${JOBpostfix}"
1638     JOBID2wait="wm0_${JOBpostfix}"
1639     JOBID3="op0_${JOBpostfix}"
1640     JOBID3wait="wop0_${JOBpostfix}"
1641     JOBID4="p1_${JOBpostfix}"
1642     JOBID4wait="w1_${JOBpostfix}"
1643     JOBID5="m1_${JOBpostfix}"
1644     JOBID5wait="wm1_${JOBpostfix}"
1645     JOBID6="s1_${JOBpostfix}"
1646     JOBID6wait="ws1_${JOBpostfix}"
1647     JOBID7="QA_${JOBpostfix}"
1648     JOBmakeESDlistCPass1="lp1_${JOBpostfix}"
1649     JOBfilterESDcpass1="fp1_${JOBpostfix}"
1650     LASTJOB="000"
1651
1652     oneInputFile=$(egrep -m1 "${runNumber}/" ${inputList})
1653
1654     currentDefaultOCDB=${defaultOCDB}
1655     [[ -z ${autoOCDB} ]] && autoOCDB=1
1656     if [[ ${autoOCDB} -ne 0 ]]; then
1657       currentDefaultOCDB=$(setYear ${oneInputFile} ${defaultOCDB})
1658     fi
1659
1660     echo "submitting run ${runNumber} with OCDB ${currentDefaultOCDB}"
1661
1662     ###############################################################################
1663     #run one chunk with valgrind:
1664     if [[ -n ${profilingCommand} ]]; then
1665       [[ -z ${nEventsProfiling} ]] && nEventsProfiling=2
1666       [[ -z ${profilingCommand} ]] && profilingCommand="/usr/bin/valgrind --tool=callgrind --num-callers=40 -v --trace-children=yes"
1667       submit "valgrind" 1 1 000 "${alirootEnv} ${self}" CPass0 ${commonOutputPath}/000${runNumber}/valgrind ${oneInputFile} ${nEventsProfiling} ${currentDefaultOCDB} ${configFile} ${runNumber} valgrind useProfilingCommand=${profilingCommand} "${extraOpts[@]}"
1668     fi 
1669
1670     ################################################################################
1671     ################################################################################
1672     # run the CPass0 if requested
1673
1674     if [ ${runCPass0reco} -eq 1 ]; then
1675
1676       echo
1677       echo "starting CPass0... for run ${runNumber}"
1678       echo
1679
1680       # create directory and copy all files that are needed
1681       targetDirectory="${commonOutputPath}/000${runNumber}/cpass0"
1682       mkdir -p ${targetDirectory}
1683
1684       filesCPass0=( 
1685                     "${configPath}/runCPass0.sh"
1686                     "${configPath}/recCPass0.C"
1687                     "${configPath}/runCalibTrain.C"
1688                     "${configPath}/localOCDBaccessConfig.C"
1689                     "${configPath}/OCDB*.root"
1690                     "${configPath}/sim.C"
1691                     "${configPath}/rec.C"
1692                     "${configPath}/Config.C"
1693       )
1694       for file in ${filesCPass0[*]}; do
1695         [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
1696       done
1697
1698       localInputList=${targetDirectory}/${inputList##*/}
1699       [[ ! -f ${localInputList} ]] && egrep "\/000${runNumber}\/" ${inputList} > ${localInputList}
1700       # limit nFiles to nMaxChunks
1701       nFiles=$(wc -l < ${localInputList})
1702       [[ ${nFiles} -eq 0 ]] && echo "list contains ZERO files! exiting..." && return 1
1703       echo "raw files in list:    ${nFiles}"
1704       if [[ ${nMaxChunks} -gt 0 && ${nMaxChunks} -le ${nFiles} ]]; then
1705         nFiles=${nMaxChunks}
1706       fi
1707       echo "raw files to process: ${nFiles}"
1708       [[ -z "${percentProcessedFilesToContinue}" ]] && percentProcessedFilesToContinue=100
1709       if [[ ${percentProcessedFilesToContinue} -eq 100 ]]; then
1710         nFilesToWaitFor=${nFiles}
1711       else
1712         nFilesToWaitFor=$(( ${nFiles}-${nFiles}/(100/(100-${percentProcessedFilesToContinue})) ))
1713       fi
1714       echo "requested success rate is ${percentProcessedFilesToContinue}%"
1715       echo "merging will start after ${nFilesToWaitFor} jobs are done"
1716
1717       submit ${JOBID1} 1 ${nFiles} 000 "${alirootEnv} ${self}" CPass0 ${targetDirectory} ${localInputList} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} -1 "${extraOpts[@]}"
1718
1719       ## submit a monitoring job that will run until a certain number of jobs are done with reconstruction
1720       submit "${JOBID1wait}" 1 1 000 "${alirootEnv} ${self}" WaitForOutput ${commonOutputPath} "meta/cpass0.job*.run${runNumber}.done" ${nFilesToWaitFor} ${maxSecondsToWait} '-maxdepth 1'
1721       LASTJOB=${JOBID1wait}
1722
1723     fi #end running CPass0
1724     ################################################################################
1725
1726
1727     ################################################################################
1728     # submit merging of CPass0, depends on the reconstruction
1729
1730     if [ ${runCPass0MergeMakeOCDB} -eq 1 ]; then
1731
1732       echo
1733       echo "submit CPass0 merging for run ${runNumber}"
1734       echo
1735
1736       targetDirectory="${commonOutputPath}/000${runNumber}/cpass0"
1737       mkdir -p ${targetDirectory}
1738
1739       #copy the scripts
1740       filesMergeCPass0=(
1741                         "${configPath}/OCDB.root"
1742                         "${configPath}/mergeMakeOCDB.byComponent.sh"
1743                         "${configPath}/mergeMakeOCDB.sh"
1744                         "${configPath}/localOCDBaccessConfig.C"
1745                         "${configPath}/mergeByComponent.C"
1746                         "${configPath}/makeOCDB.C"
1747                         "${configPath}/merge.C"
1748       )
1749       for file in ${filesMergeCPass0[*]}; do
1750         [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
1751       done
1752   
1753       submit ${JOBID2} 1 1 "${LASTJOB}" "${alirootEnv} ${self}" MergeCPass0 ${targetDirectory} ${currentDefaultOCDB} ${configFile} ${runNumber} cpass0.calib.run${runNumber}.list "${extraOpts[@]}"
1754       LASTJOB=${JOBID2}
1755
1756       if [[ -n ${generateMC} ]]; then
1757         submit "mrl${JOBpostfix}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" PrintValues sim ${commonOutputPath}/meta/sim.run${runNumber}.list ${commonOutputPath}/meta/cpass0.job*.run${runNumber}.done
1758         LASTJOB="mrl${JOBpostfix}"
1759       fi
1760
1761       echo
1762     fi
1763     # end of merging CPass0
1764     ################################################################################
1765
1766     ################################################################################
1767     ################################################################################
1768     # run the CPass1 if requested
1769
1770     if [ ${runCPass1reco} -eq 1 ]; then
1771
1772       targetDirectory="${commonOutputPath}/000${runNumber}/cpass1"
1773       rm -f ${commonOutputPath}/meta/cpass1.job*.run${runNumber}.done
1774
1775       # safety feature: if we are re-running for any reason we want to delete the previous output first.
1776       [[ -d ${targetDirectory} ]] && rm -rf ${targetDirectory}/* && echo "removed old output at ${targetDirectory}/*"
1777
1778       echo
1779       echo "starting CPass1... for run ${runNumber}"
1780       echo
1781
1782       # create directory and copy all files that are needed
1783       mkdir -p ${targetDirectory}
1784       
1785       filesCPass1=( 
1786                     "${configPath}/runCPass1.sh"
1787                     "${configPath}/recCPass1.C"
1788                     "${configPath}/recCPass1_OuterDet.C"
1789                     "${configPath}/runCalibTrain.C"
1790                     "${configPath}/QAtrain_duo.C"
1791                     "${configPath}/localOCDBaccessConfig.C"
1792                     "${configPath}/OCDB.root"
1793       )
1794       for file in ${filesCPass1[*]}; do
1795         [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
1796       done
1797
1798       if [[ -n ${generateMC} ]]; then
1799         localInputList=${commonOutputPath}/meta/sim.run${runNumber}.list
1800       else
1801         localInputList=${targetDirectory}/${inputList##*/}
1802         [[ ! -f ${localInputList} ]] && egrep "\/000${runNumber}\/" ${inputList} > ${localInputList}
1803       fi
1804       # limit nFiles to nMaxChunks
1805       nFiles=$(wc -l < ${localInputList})
1806       [[ ${nFiles} -eq 0 ]] && echo "list contains ZERO files! continuing..." && continue
1807       echo "raw files in list:    ${nFiles}"
1808       if [[ ${nMaxChunks} -gt 0 && ${nMaxChunks} -le ${nFiles} ]]; then
1809         nFiles=${nMaxChunks}
1810       fi
1811       echo "raw files to process: ${nFiles}"
1812       [[ -z "${percentProcessedFilesToContinue}" ]] && percentProcessedFilesToContinue=100
1813       if [[ ${percentProcessedFilesToContinue} -eq 100 ]]; then
1814         nFilesToWaitFor=${nFiles}
1815       else
1816         nFilesToWaitFor=$(( ${nFiles}-${nFiles}/(100/(100-${percentProcessedFilesToContinue})) ))
1817       fi
1818       echo "requested success rate is ${percentProcessedFilesToContinue}%"
1819       echo "merging will start after ${nFilesToWaitFor} jobs are done"
1820
1821       submit ${JOBID4} 1 ${nFiles} "${LASTJOB}" "${alirootEnv} ${self}" CPass1 ${targetDirectory} ${localInputList} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} -1 "${extraOpts[@]}"
1822
1823       ################################################################################
1824       ## submit a monitoring job that will run until a certain number of jobs are done with reconstruction
1825       submit "${JOBID4wait}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" WaitForOutput ${commonOutputPath} "meta/cpass1.job*.run${runNumber}.done" ${nFilesToWaitFor} ${maxSecondsToWait} '-maxdepth 1'
1826       LASTJOB=${JOBID4wait}
1827       ################################################################################
1828
1829       echo
1830     fi #end running CPass1
1831
1832     ################################################################################
1833     # submit merging of CPass1, depends on the reconstruction
1834     if [ ${runCPass1MergeMakeOCDB} -eq 1 ]; then
1835
1836       echo
1837       echo "submit CPass1 merging for run ${runNumber}"
1838       echo
1839
1840       targetDirectory="${commonOutputPath}/000${runNumber}/cpass1"
1841       rm -f ${commonOutputPath}/meta/merge.cpass1.run${runNumber}.done
1842       mkdir -p ${targetDirectory}
1843
1844       # copy files 
1845       filesMergeCPass1=(
1846                         "${configPath}/OCDB.root"
1847                         "${configPath}/localOCDBaccessConfig.C"
1848                         "${configPath}/mergeMakeOCDB.byComponent.sh"
1849                         "${configPath}/mergeByComponent.C"
1850                         "${configPath}/makeOCDB.C"
1851                         "${configPath}/merge.C"
1852                         "${configPath}/mergeMakeOCDB.sh"
1853                         "${configPath}/QAtrain_duo.C"
1854       )
1855       for file in ${filesMergeCPass1[*]}; do
1856         [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
1857       done
1858
1859       submit "${JOBID5}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" MergeCPass1 ${targetDirectory} ${currentDefaultOCDB} ${configFile} ${runNumber} cpass1.calib.run${runNumber}.list cpass1.QA.run${runNumber}.lastMergingStage.txt.list cpass1.filtered.run${runNumber}.list "${extraOpts[@]}"
1860       LASTJOB=${JOBID5}
1861       echo
1862     fi
1863
1864     ###############################
1865     #if [ ${runESDfiltering} -eq 1 ]; then
1866     #  rm -f ${commonOutputPath}/cpass1.ESD.run${runNumber}.list
1867     #  rm -f ${commonOutputPath}/meta/filtering.cpass1.run*.done
1868     #  echo
1869     #  echo submitting filtering for run ${runNumber}
1870     #  echo
1871     #  submit "${JOBmakeESDlistCPass1}" 1 1 "${LASTJOB}" "${self}" PrintValues esd ${commonOutputPath}/meta/cpass1.ESD.run${runNumber}.list ${commonOutputPath}/meta/cpass1.job*.run${runNumber}.done 
1872     #  submit "${JOBfilterESDcpass1}" 1 1 "${JOBmakeESDlistCPass1}" "${alirootEnv} ${self}" MakeFilteredTrees ${commonOutputPath}/000${runNumber}/cpass1 ${runNumber} ${commonOutputPath}/meta/cpass1.ESD.run${runNumber}.list ${filteringFactorHighPt} ${filteringFactorV0s} ${currentDefaultOCDB} 1000000 0 10000000 0 ${configFile} AliESDs_Barrel.root "${extraOpts[@]}"
1873     #  LASTJOB=${JOBfilterESDcpass1}
1874     #fi
1875
1876   done
1877
1878   #################################################################################
1879   #################################################################################
1880   #if [ ${runESDfiltering} -eq 1 ]; then
1881   #  submit "${JOBID5wait}" 1 1 "${LASTJOB}" "${self}" WaitForOutput ${commonOutputPath} "meta/filtering.cpass1.run*.done" "${#listOfRuns[@]}" ${maxSecondsToWait}
1882   #else
1883     submit "${JOBID5wait}" 1 1 "${LASTJOB}" "${self}" WaitForOutput ${commonOutputPath} "meta/merge.cpass1.run*.done" ${#listOfRuns[@]} ${maxSecondsToWait}
1884   #fi
1885   LASTJOB=${JOBID5wait}
1886
1887   #################################################################################
1888   echo
1889   echo "submit make a summary"
1890   echo
1891
1892   submit "${JOBID6}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" MakeSummary ${configFile}
1893   LASTJOB=${JOBID6}
1894   #################################################################################
1895   
1896   #restore stdout
1897   exec 1>&7 7>&-
1898   echo "jobs submitted."
1899   return 0
1900 }
1901
1902 goWaitForOutput()
1903 (
1904   umask 0002
1905   [[ $# -lt 3 ]] && echo "goWaitForOutput() wrong number of arguments, exiting.." && return 1
1906   echo searchPath=${1}
1907   echo fileName=${2}
1908   echo numberOfFiles=${3}
1909   echo maxSecondsToWait=${4}
1910   searchPath=${1}
1911   fileName=${2}
1912   numberOfFiles=${3}
1913   maxSecondsToWait=${4}
1914   extraFindOptions=${5}
1915   echo "command to be executed: /bin/ls -1 ${searchPath}/${fileName} ${extraFindOptions}"
1916   [[ -z "${maxSecondsToWait}" ]] && maxSecondsToWait=$(( 3600*12 ))
1917   while true; do
1918     #n=$(find ${searchPath} ${extraFindOptions} -name "${fileName}" | wc -l)
1919     n=$(/bin/ls -1 ${searchPath}/${fileName} 2>/dev/null | wc -l)
1920     [[ ${n} -gt 0 ]] && echo "found ${n} X ${fileName}"
1921     [[ ${n} -ge ${numberOfFiles} ]] && break
1922     [[ ${SECONDS} -gt ${maxSecondsToWait} ]] && echo "timeout of ${maxSecondsToWait}s!" && break
1923     sleep 60
1924   done
1925   echo "DONE! exiting..."
1926   return 0
1927 )
1928
1929 mergeSysLogs()
1930 (
1931   outputFile=${1}
1932   shift
1933   inputFiles="$@"
1934   i=0
1935   if ! ls -1 ${inputFiles} &>/dev/null; then echo "the files dont exist!: ${inputFiles}"; return 1; fi
1936   while read x; do 
1937     runNumber=$(guessRunNumber ${x})
1938     [[ -z ${runNumber} ]] && echo "run number cannot be guessed for ${x}" && continue
1939     awk -v run=${runNumber} -v i=${i} 'NR > 1 {print run" "$0} NR==1 && i==0 {print "run/I:"$0}' ${x}
1940     (( i++ ))
1941   done < <(ls -1 ${inputFiles}) > ${outputFile}
1942   return 0
1943 )
1944
1945 goMakeMergedSummaryTree()
1946 (
1947   # create list of calibration entries
1948   # takes no arguments, just run it in the base output
1949   # directory with the following files in the working directory
1950   # 
1951   #  Calibration file lists:
1952   #       cpass0.dcsTree.list, cpass1.dcsTree.list
1953   #  QA trending root file:
1954   #       trending.root
1955   # 
1956   #  Production infoo ascii files:
1957   #       summary_pass0.tree
1958   #       summary_pass1.tree
1959   #    
1960
1961   [[ ! -f cpass0.dcsTree.list ]] && echo "no cpass0.dcsTree.list" && return 1
1962   [[ ! -f cpass1.dcsTree.list ]] && echo "no cpass1.dcsTree.list" && return 1
1963   [[ ! -f trending.root ]] && echo "no trending.root" && return 1
1964   [[ ! -f summary_pass0.tree ]] && echo "no summary_pass0.tree" && return 1
1965   [[ ! -f summary_pass1.tree ]] && echo "no summary_pass1.tree" && return 1
1966
1967   #first, dump the C macro to file
1968   cat << EOF > mergeTree.C
1969   //
1970   // Merge summary information
1971   // Following files are expected to be in the working directory
1972   //
1973   // Calibration file lists:
1974   //      cpass0.dcsTree.list, cpass1.dcsTree.list
1975   // QA trending root file:
1976   //      trending.root
1977   //
1978   // Production infoo ascii files:
1979   //      summary_pass0.tree
1980   //      summary_pass1.tree
1981   //   
1982   void mergeTree(){
1983     //
1984     //
1985     //
1986     // Calibration values dump
1987     //
1988     //Printf("MakeTreeFromList cpass0.dcsTree.list");
1989     AliXRDPROOFtoolkit::MakeTreeFromList("Calib.TPC.CPass0.root", "dcs","dcs","cpass0.dcsTree.list",1);
1990     //Printf("MakeTreeFromList cpass1.dcsTree.list");
1991     AliXRDPROOFtoolkit::MakeTreeFromList("Calib.TPC.CPass1.root", "dcs","dcs","cpass1.dcsTree.list",1);
1992     //
1993     // Calibration status dump
1994     //
1995     TFile *fprod = TFile::Open("fproduction.root","recreate");
1996     TTree  tree0, tree1;
1997     //Printf("reading summary_pass0.tree");
1998     tree0.ReadFile("summary_pass0.tree");
1999     //Printf("reading summary_pass1.tree");
2000     tree1.ReadFile("summary_pass1.tree");
2001     tree0.Write("CPass0");
2002     tree1.Write("CPass1");
2003     fprod->Close();
2004     //
2005     //
2006     //
2007     TString stringSetup="";
2008     stringSetup+="1#QA.TPC#run#SummaryTPCQA/tpcQA#trending.root+";  // 
2009     stringSetup+="1#Calib.TPC.CPass0#run#dcs#Calib.TPC.CPass0.root+";  // 
2010     stringSetup+="1#Calib.TPC.CPass1#run#dcs#Calib.TPC.CPass1.root+";  // 
2011     //
2012     stringSetup+="1#CPass0#runnumber#CPass0#fproduction.root+";  // 
2013     stringSetup+="1#CPass1#runnumber#CPass1#fproduction.root+";  // 
2014     //
2015     //Printf("stringSetup: %s", stringSetup.Data());
2016     AliXRDPROOFtoolkit::JoinTreesIndex("outAll.root","joinAll","run",stringSetup.Data(), 1);
2017   }
2018 EOF
2019
2020   aliroot -b -q "mergeTree.C" > mergeTrees.log
2021   return $?
2022 )
2023
2024 goMakeSummary()
2025 (
2026   #all the final stuff goes in here for ease of use:
2027   # summary logs
2028   # qa plot making
2029   # final file lists
2030   #some defaults:
2031   log="summary.log"
2032   productionID="qa"
2033
2034   configFile=${1}
2035   shift 1
2036   extraOpts=("$@")
2037   if ! parseConfig ${configFile} "${extraOpts[@]}"; then return 1; fi
2038   
2039   #if which greadlink; then configFile=$(greadlink -f ${configFile}); fi
2040   
2041   #record the working directory provided by the batch system
2042   batchWorkingDirectory=${PWD}
2043   
2044   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
2045
2046   [[ ! -f ${configFile} ]] && echo "no config file ${configFile}!" && return
2047
2048   [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
2049
2050   #copy some useful stuff
2051   #and go to the commonOutputPath
2052   cp ${configFile} ${commonOutputPath}
2053   cd ${commonOutputPath}
2054
2055   exec &> >(tee ${log})
2056
2057   #summarize the global stuff
2058   echo "env script: ${alirootSource} ${alirootEnv}"
2059   echo "\$ALICE_ROOT=${ALICE_ROOT}"
2060   echo "commonOutputPath=${commonOutputPath}"
2061
2062   #summarize the stacktraces
2063   awk '
2064        BEGIN { 
2065                print "frame/I:method/C:line/C:cpass/I:aliroot/I";
2066                RS="#[0-9]*";
2067                aliroot=0;
2068              } 
2069              { 
2070                if ($3 ~ /Ali*/) aliroot=1; else aliroot=0;
2071                gsub("#","",RT); 
2072                if ($NF!="" && RT!="" && $3!="") print RT" "$3" "$NF" "0" "aliroot
2073              }
2074       ' 000*/cpass0/*/stacktrace* 2>/dev/null > stacktrace.tree
2075   awk '
2076        BEGIN {
2077                RS="#[0-9]*";
2078                aliroot=0;
2079              } 
2080              {
2081                if ($3 ~ /Ali*/) aliroot=1; else aliroot=0;
2082                gsub("#","",RT); 
2083                if ($NF!="" && RT!="" && $3!="") print RT" "$3" "$NF" "1" "aliroot
2084              }
2085       ' 000*/cpass1/*/stacktrace* 2>/dev/null >> stacktrace.tree
2086
2087   echo total numbers for the production:
2088   echo
2089   awk 'BEGIN {nFiles=0;nCore=0;} 
2090   /^calibfile/ {nFiles++;} 
2091   /core dumped/ {nCore++i;}
2092   END {print     "cpass0 produced "nFiles" calib files, "nCore" core files";}' meta/cpass0.job*done 2>/dev/null
2093   awk 'BEGIN {nOK=0; nBAD=0; } 
2094   /\/rec.log OK/ {nOK++;} 
2095   /\/rec.log BAD/ {nBAD++;} 
2096   /stderr BAD/ {if ($0 ~ /rec.log/){nBAD++;}}
2097   END {print     "cpass0 reco:  OK: "nOK"\tBAD: "nBAD;}' meta/cpass0.job*done 2>/dev/null
2098   awk 'BEGIN {nOK=0; nBAD=0; } 
2099   /\/calib.log OK/ {nOK++;} 
2100   /\/calib.log BAD/ {nBAD++;} 
2101   END {print "cpass0 calib: OK: "nOK"\tBAD: "nBAD;}' meta/cpass0.job*done 2>/dev/null
2102
2103   awk 'BEGIN {nOK=0; nBAD=0; } 
2104   /merge.log OK/ {nOK++;} 
2105   /merge.log BAD/ {nBAD++;} 
2106   END {print "cpass0 merge: OK: "nOK"\tBAD: "nBAD;}' meta/merge.cpass0*done 2>/dev/null
2107   awk 'BEGIN {nOK=0; nBAD=0; } 
2108   /ocdb.log OK/ {nOK++;} 
2109   /ocdb.log BAD/ {nBAD++;} 
2110   END {print   "cpass0 OCDB:  OK: "nOK"\tBAD: "nBAD;}' meta/merge.cpass0*done 2>/dev/null
2111
2112   echo
2113   awk 'BEGIN {nFiles=0;nCore=0;} 
2114   /^calibfile/ {nFiles++;} 
2115   /core dumped/ {nCore++;}
2116   END {print     "cpass1 produced "nFiles" calib files, "nCore" core files";}' meta/cpass1.job*done 2>/dev/null
2117   awk 'BEGIN {nOK=0; nBAD=0; } 
2118   /\/rec.log OK/ {nOK++;} 
2119   /\/rec.log BAD/ {nBAD++;} 
2120   /stderr BAD/ {if ($0 ~ /rec.log/){nBAD++;}}
2121   END {print     "cpass1 reco:  OK: "nOK"\tBAD: "nBAD;}' meta/cpass1.job*done 2>/dev/null
2122   awk 'BEGIN {nOK=0; nBAD=0; } 
2123   /\/calib.log OK/ {nOK++;} 
2124   /\/calib.log BAD/ {nBAD++;} 
2125   END {print "cpass1 calib: OK: "nOK"\tBAD: "nBAD;}' meta/cpass1.job*done 2>/dev/null
2126
2127   awk 'BEGIN {nOK=0; nBAD=0; } 
2128   /merge.log OK/ {nOK++;} 
2129   /merge.log BAD/ {nBAD++;} 
2130   END {print "cpass1 merge: OK: "nOK"\tBAD: "nBAD;}' meta/merge.cpass1*done 2>/dev/null
2131   awk 'BEGIN {nOK=0; nBAD=0; } 
2132   /ocdb.log OK/ {nOK++;} 
2133   /ocdb.log BAD/ {nBAD++;} 
2134   END {print   "cpass1 OCDB:  OK: "nOK"\tBAD: "nBAD;}' meta/merge.cpass1*done 2>/dev/null
2135
2136   echo
2137   echo per run stats:
2138   /bin/ls -1 meta/merge.cpass0.run*.done | while read x 
2139 do
2140   dir=$(goPrintValues calibfile - ${x})
2141   runNumber=$(guessRunNumber ${dir})
2142   [[ -z ${runNumber} ]] && continue
2143
2144   if $(/bin/ls meta/cpass0.job*.run${runNumber}.done &> /dev/null); then
2145     statusCPass0=( $(
2146     awk 'BEGIN {nOKrec=0;nBADrec=0;nOKcalib=0;nBADcalib=0;nOKstderr=0;nBADstderr=0;}
2147     /\/rec.log OK/ {nOKrec++;} 
2148     /\/rec.log BAD/ {nBADrec++;}
2149     /stderr BAD/ {if ($0 ~ /rec.log/) {nBADrec++;} nBADstderr++;}
2150     /stderr OK/ {nOKstderr++;}
2151     /\/calib.log OK/ {nOKcalib++;}
2152     /\/calib.log BAD/ {nBADcalib++}
2153     END {print ""nOKrec" "nBADrec" "nOKstderr" "nBADstderr" "nOKcalib" "nBADcalib;}' meta/cpass0.job*.run${runNumber}.done 2>/dev/null
2154     ) ) 
2155   fi
2156
2157   if $(/bin/ls meta/cpass1.job*.run${runNumber}.done &>/dev/null); then
2158     statusCPass1=( $(
2159     awk 'BEGIN {nOKrec=0;nBADrec=0;nOKcalib=0;nBADcalib=0;nOKstderr=0;nBADstderr=0;nQAbarrelOK=0;nQAbarrelBAD=0;nQAouterOK=0;nQAouterBAD=0;}
2160     /\/rec.log OK/ {nOKrec++;} 
2161     /\/rec.log BAD/ {nBADrec++;}
2162     /stderr BAD/ {if ($0 ~ /rec.log/) nBADrec++;nBADstderr++;}
2163     /stderr OK/ {nOKstderr++;}
2164     /\/calib.log OK/ {nOKcalib++;}
2165     /\/calib.log BAD/ {nBADcalib++}
2166     /\/qa_barrel.log OK/ {nQAbarrelOK++;}
2167     /\/qa_barrel.log BAD/ {nQAbarrelBAD++;}
2168     /\/qa_outer.log OK/ {nQAouterOK++;}
2169     /\/qa_outer.log BAD/ {nQAouterBAD++;}
2170     END {print ""nOKrec" "nBADrec" "nOKstderr" "nBADstderr" "nOKcalib" "nBADcalib" "nQAbarrelOK" "nQAbarrelBAD" "nQAouterOK" "nQAouterBAD;}' meta/cpass1.job*.run${runNumber}.done 2>/dev/null
2171     ) ) 
2172   fi
2173
2174   statusOCDBcpass0=$(awk '/ocdb.log/ {print $2} ' ${x} 2>/dev/null)
2175   statusOCDBcpass1=$(awk '/ocdb.log/ {print $2}' ${x/cpass0/cpass1} 2>/dev/null)
2176   statusQA=$(awk '/mergeMakeOCDB.log/ {print $2}' ${x/cpass0/cpass1} 2>/dev/null)
2177
2178   printf "%s\t ocdb.log cpass0: %s\t ocdb.log cpass1: %s\tqa.log:%s\t| cpass0: rec:%s/%s stderr:%s/%s calib:%s/%s cpass1: rec:%s/%s stderr:%s/%s calib:%s/%s QAbarrel:%s/%s QAouter:%s/%s\n" ${runNumber} ${statusOCDBcpass0} ${statusOCDBcpass1} ${statusQA} ${statusCPass0[0]} ${statusCPass0[1]} ${statusCPass0[2]} ${statusCPass0[3]} ${statusCPass0[4]} ${statusCPass0[5]} ${statusCPass1[0]} ${statusCPass1[1]} ${statusCPass1[2]} ${statusCPass1[3]} ${statusCPass1[4]} ${statusCPass1[5]} ${statusCPass1[6]} ${statusCPass1[7]} ${statusCPass1[8]} ${statusCPass1[9]}
2179 done
2180
2181   #make lists with output files - QA, trending, filtering and calibration
2182   ### wait for the merging of all runs to be over ###
2183   rm -f qa.list
2184   goPrintValues qafile qa.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2185   rm -f calib.list
2186   goPrintValues calibfile calib.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2187   rm -f trending.list
2188   goPrintValues trendingfile trending.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2189   rm -f filtering.list
2190   goPrintValues filteredTree filtering.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2191   rm -f cpass0.dcsTree.list
2192   goPrintValues dcsTree cpass0.dcsTree.list ${commonOutputPath}/meta/merge.cpass0.run*.done &>/dev/null
2193   rm -f cpass1.dcsTree.list
2194   goPrintValues dcsTree cpass1.dcsTree.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2195  
2196   #merge trending
2197   rm -f ${commonOutputPath}/trending_merged.root
2198   goMerge trending.list ${commonOutputPath}/trending.root ${configFile} "${extraOpts[@]}" &> mergeTrending.log
2199
2200   goMakeSummaryTree ${commonOutputPath} 0
2201   goMakeSummaryTree ${commonOutputPath} 1
2202
2203   goCreateQAplots "${PWD}/qa.list" "${productionID}" "QAplots" "${configFile}" "${extraOpts[@]}" filteringList="${PWD}/filtering.list" &>createQAplots.log
2204
2205   #make a merged summary tree out of the QA trending, dcs trees and log summary trees
2206   goMakeMergedSummaryTree
2207
2208   #if set, email the summary
2209   [[ -n ${MAILTO} ]] && cat ${log} | mail -s "benchmark ${productionID} done" ${MAILTO}
2210
2211   return 0
2212 )
2213
2214 goMakeSummaryTree()
2215 (
2216   if [[ $# -lt 1 ]] ; then
2217     return
2218   fi
2219   #1. define vars/arrays
2220   DIR=${1} #use input or exec in current dir
2221   pass=${2-"0"} #pass from input
2222   outfile="summary_pass${pass}.tree"
2223   Ncolumns=0
2224   test -f ${outfile} && : >${outfile}
2225   errfile=${outfile/tree/err}
2226   test -f ${errfile} && : >${errfile}
2227
2228   declare -a counterString=(TOFevents TOFtracks TPCevents TPCtracks TRDevents TRDtracks T0events SDDevents SDDtracks MeanVertexevents)
2229   Ncounter=${#counterString[@]}
2230
2231   declare -a statusString=(TRDStatus TOFStatus TPCStatus T0Status MeanVertexStatus)
2232   Nstatus=${#statusString[@]}
2233
2234
2235   declare -a ratesString=(rec stderr calib qa_barrel qa_outer)
2236   Nrates=${#ratesString[@]}
2237
2238   runs=( $(ls -1 ${DIR}/meta/merge.cpass0* | while read x; do guessRunNumber $x; done) )
2239   Nruns=${#runs[@]}
2240
2241   echo -n runnumber/I >>${outfile}
2242   echo -n :cpass${pass}status/I >>${outfile}
2243   echo -n :cpass${pass}QAstatus/I >>${outfile}
2244   for i in ${ratesString[@]}; do
2245     echo -n :${i}OK/I >>${outfile}
2246     echo -n :${i}BAD/I >>${outfile}
2247
2248   done
2249   for i in ${counterString[@]} ${statusString[@]} ; do
2250     echo -n :${i}/I >>${outfile}
2251   done
2252   Ncolumns=$((2 + 2*Nrates + Ncounter + Nstatus))
2253   echo >> ${outfile}
2254
2255   #2. loop runs 
2256
2257   for runnumber in ${runs[@]} ; do 
2258
2259
2260
2261     filejob="${DIR}/meta/cpass${pass}.job*.run${runnumber}.done"
2262     filemerge="${DIR}/meta/merge.cpass${pass}.run${runnumber}.done"
2263     fileOCDB=$(grep /ocdb.log ${filemerge} | awk '{print $1}')
2264     if ! $(/bin/ls ${filemerge} &>/dev/null) ; then
2265       echo "${filemerge} does not exist!" >>${errfile}
2266       continue
2267     elif ! $(/bin/ls ${filejob} &>/dev/null) ; then
2268       echo "${filejob} does not exist!" >>${errfile}
2269       echo -n ${runnumber} >> ${outfile}
2270       for i in $(seq ${Ncolumns}) ; do 
2271         echo -n "-1" >> ${outfile}
2272       done
2273       echo >> ${outfile}
2274       continue
2275     fi
2276     echo -n ${runnumber} >> ${outfile}
2277     #pass0status= grep '/ocdb.log' ${filemerge} |  cut -d' ' -f2 | tr OK x1 | tr BAD xx0 | tr -d 'x'
2278     passStatus=$(grep '/ocdb.log' ${filemerge} | grep OK | wc -l)
2279     echo -n " ${passStatus}" >> ${outfile}
2280     qaStatus=$(grep '/mergeMakeOCDB.log' ${filemerge} | grep OK | wc -l)
2281     echo -n " ${qaStatus}" >> ${outfile}
2282
2283
2284     #fill OK/BAD rates
2285     for i in $(seq 0 $((${Nrates}-1))) ; do
2286       var1=$(grep "/${ratesString[${i}]}.log" ${filejob} | grep OK | wc -l)
2287       var2=$(grep "/${ratesString[${i}]}.log" ${filejob} | grep BAD | wc -l)
2288
2289       if [[ ${ratesString[${i}]} == "stderr" ]] ; then
2290         var1=$(grep "stderr" ${filejob} | grep OK | wc -l)
2291         var2=$(grep "stderr" ${filejob} | grep "rec.log" | grep BAD | wc -l)
2292       fi
2293       echo -n " ${var1}" >> ${outfile}
2294       echo -n " ${var2}" >> ${outfile}
2295     done
2296
2297     if [[ -f ${fileOCDB} ]] ; then
2298       #fill counter
2299       for i in $(seq 0 $((${Ncounter}-1))) ; do
2300         var1=$(grep Monalisa ${fileOCDB} | grep ${counterString[${i}]} | cut -f2)
2301         echo -n " ${var1:-"-1"}" >> ${outfile}
2302       done
2303
2304       #fill status
2305       for i in $(seq 0 $((${Nstatus}-1))) ; do
2306         var1=$(grep "calibration status=" ${fileOCDB} | grep ${statusString[${i}]/Status/} | cut -d'=' -f2)
2307         echo -n " ${var1:-"-1"}" >> ${outfile}
2308       done
2309     fi
2310     echo >> ${outfile}
2311   done
2312
2313   return 0
2314 )
2315
2316 parseConfig()
2317 {
2318   configFile=${1}
2319   shift
2320   args=("$@")
2321
2322
2323   #some defaults
2324   #autoOCDB=0
2325   defaultOCDB="raw://"
2326   #runNumber=167123
2327   #makeflowPath="/hera/alice/aux/cctools/bin"
2328   #makeflowOptions="-T wq -N alice -d all -C ali-copilot.cern.ch:9097"
2329   #makeflowOptions="-T wq -N alice -C ali-copilot.cern.ch:9097"
2330   makeflowOptions=""
2331   #batchCommand="/usr/bin/qsub"
2332   batchFlags="-b y -cwd -l h_rt=24:0:0,h_rss=4G "
2333   baseOutputDirectory="$PWD/output"
2334   #alirootEnv="/cvmfs/alice.cern.ch/bin/alienv setenv AliRoot/v5-04-34-AN -c"
2335   #alirootEnv="/home/mkrzewic/alisoft/balice_master.sh"
2336   #trustedQAtrainMacro='/hera/alice/mkrzewic/gsisvn/Calibration/QAtrain_duo.C'
2337   reconstructInTemporaryDir=0
2338   recoTriggerOptions="\"\""
2339   percentProcessedFilesToContinue=100
2340   maxSecondsToWait=$(( 3600*24 ))
2341   nEvents=-1
2342   nMaxChunks=0
2343   postSetUpActionCPass0=""
2344   postSetUpActionCPass1=""
2345   runCPass0reco=1
2346   runCPass0MergeMakeOCDB=1
2347   runCPass1reco=1
2348   runCPass1MergeMakeOCDB=1
2349   runESDfiltering=1
2350   filteringFactorHighPt=1e2
2351   filteringFactorV0s=1e1
2352   MAILTO=""
2353   #pretend=1
2354   #dontRedirectStdOutToLog=1
2355   logToFinalDestination=1
2356   ALIROOT_FORCE_COREDUMP=1
2357   pretendDelay=0
2358
2359   #first, source the config file
2360   if [ -f ${configFile} ]; then
2361     source ${configFile}
2362   else
2363     echo "config file ${configFile} not found!"
2364     return 1
2365   fi
2366
2367   unset encodedSpaces
2368   for opt in "${args[@]}"; do
2369     [[ "${opt}" =~ encodedSpaces=.* ]] && encodedSpaces=1 && echo "encodedSpaces!" && break
2370   done
2371
2372   #then, parse the options as they override the options from file
2373   for opt in "${args[@]}"; do
2374     [[ -z ${opt} ]] && continue
2375     [[ -n ${encodedSpaces} ]] && opt="$(decSpaces ${opt})"
2376     [[ "${opt}" =~ ^[[:space:]]*$ ]] && continue
2377     if [[ ! "${opt}" =~ .*=.* ]]; then
2378       echo "badly formatted option \"${opt}\" should be: option=value, stopping..."
2379       return 1
2380     fi
2381     local var="${opt%%=*}"
2382     local value="${opt#*=}"
2383     echo "${var} = ${value}"
2384     export ${var}="${value}"
2385   done
2386
2387   #do some checking
2388   [[ -z ${alirootEnv} ]] && echo "alirootEnv not defined!" && return 1
2389
2390   #export the aliroot function if defined to override normal behaviour
2391   [[ $(type -t aliroot) =~ "function" ]] && export -f aliroot
2392
2393   return 0
2394 }
2395
2396 aliroot()
2397 {
2398   args="$@"
2399   if [[ -n ${useProfilingCommand} ]]; then
2400     valgrindLogFile="cpu.txt"
2401     [[ "${args}" =~ rec ]] && valgrindLogFile="cpu_rec.txt"
2402     [[ "${args}}" =~ Calib ]] && valgrindLogFile="cpu_calib.txt"
2403     [[ -n ${useProfilingCommand} ]] && useProfilingCommand="${useProfilingCommand} --log-file=${valgrindLogFile}"
2404     echo running ${useProfilingCommand} aliroot ${args}
2405     ${useProfilingCommand} aliroot ${args}
2406   else
2407     #to prevent an infinite recursion use "command aliroot" to disable
2408     #aliases and functions
2409     echo running command aliroot ${args}
2410     command aliroot "$@"
2411   fi
2412   return 0
2413 }
2414
2415 guessRunData()
2416 {
2417   #guess the period from the path, pick the rightmost one
2418   period=""
2419   runNumber=""
2420   year=""
2421   pass=""
2422   legoTrainRunNumber=""
2423   dataType=""
2424
2425   local shortRunNumber=""
2426   local IFS="/"
2427   declare -a path=( $1 )
2428   local dirDepth=$(( ${#path[*]}-1 ))
2429   i=0
2430   #for ((x=${dirDepth};x>=0;x--)); do
2431   for ((x=0;x<=${dirDepth};x++)); do
2432
2433     [[ $((x-1)) -ge 0 ]] && local fieldPrev=${path[$((x-1))]}
2434     local field=${path[${x}]}
2435     local fieldNext=${path[$((x+1))]}
2436
2437     [[ ${field} =~ ^[0-9]*$ && ${fieldNext} =~ (.*\.zip$|.*\.root$) ]] && legoTrainRunNumber=${field}
2438     [[ -n ${legoTrainRunNumber} && -z ${pass} ]] && pass=${fieldPrev}
2439     [[ ${field} =~ ^LHC[0-9][0-9][a-z].*$ ]] && period=${field%_*}
2440     [[ ${field} =~ ^000[0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && runNumber=${field#000}
2441     [[ ${field} =~ ^[0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && shortRunNumber=${field}
2442     [[ ${field} =~ ^20[0-9][0-9]$ ]] && year=${field}
2443     [[ ${field} =~ ^(^sim$|^data$) ]] && dataType=${field}
2444     (( i++ ))
2445   done
2446   [[ -z ${legoTrainRunNumber} ]] && pass=${path[$((dirDepth-1))]}
2447   [[ "${dataType}" =~ ^sim$ ]] && pass="passMC" && runNumber=${shortRunNumber}
2448   
2449   #if [[ -z ${dataType} || -z ${year} || -z ${period} || -z ${runNumber}} || -z ${pass} ]];
2450   if [[ -z ${runNumber}} ]];
2451   then
2452     #error condition
2453     return 1
2454   else
2455     #ALL OK
2456     return 0
2457   fi
2458   return 0
2459 }
2460
2461 #these functions encode strings to and from a space-less form
2462 #use when spaces are not well handled (e.g. in arguments to 
2463 #commands in makeflow files, etc.
2464 encSpaces()(echo "${1// /@@@@}")
2465 decSpaces()(echo "${1//@@@@/ }")
2466
2467 main "$@"