when will it end??
[u/mrichter/AliRoot.git] / PWGPP / benchmark / benchmark.sh
1 #!/bin/bash
2 #include benchmark.config
3
4 # this script runs the CPass0/CPass1 train
5 # produced OCDB updates are local
6
7 main()
8 {
9   #run in proper mode depending on the selection
10   if [[ $# -lt 1 ]]; then
11     if [[ ! "${0}" =~ "bash" ]]; then
12       echo "uses makeflow:"
13       echo " ${0} \"run\" productionID inputList configFile [extraOpts]"
14       echo "uses a batch system (SGE):"
15       echo " ${0} \"submit\" productionID inputList configFile [extraOpts]"
16       echo "extraOpts if non-empty override the config file, e.g.:"
17       echo " ${0} submit test1 benchmark.list benchmark.config runNumber=169123 nEvents=10"
18     fi
19     return
20   fi
21
22   #define some aliases - default is to call one of the functions directly
23   runMode=${1}
24   umask 0002
25   shift
26   case ${runMode} in
27     "CPass0") goCPass0 "$@";;
28     "CPass1") goCPass1 "$@";;
29     "MakeLocalOCDBaccessConfig") goMakeLocalOCDBaccessConfig "$@";;
30     "MergeCPass0") goMergeCPass0 "$@";;
31     "MergeCPass1") goMergeCPass1 "$@";;
32     "MakeFilteredTrees") goMakeFilteredTrees "$@";;
33     "MakeSummary") goMakeSummary "$@";;
34     "run") goSubmitMakeflow "$@";;
35     "submit") goSubmitBatch "$@";;
36     "test") goTest "$@";;
37     "GenerateMakeflow") goGenerateMakeflow "$@";;
38     "PrintValues") goPrintValues "$@";;
39     "CreateQAplots") goCreateQAplots "$@";;
40     "WaitForOutput") goWaitForOutput "$@";;
41     "Merge") goMerge "$@";;
42     *) 
43       ${runMode} "$@"
44     ;;
45   esac
46   return 0
47 }
48
49 generateMC()
50 {
51   #generate one raw chunk in current directory
52   SEED=${JOB_ID}${SGE_TASK_ID}
53   export CONFIG_SEED=${SEED}
54   runNumber=${1}
55   OCDBpath=${2}
56   nEventsim=${3}
57   if [[ -n ${pretend} ]]; then
58     sleep ${pretendDelay}
59     touch galice.root
60   else
61     if [[ -f sim.C && -f Config.C ]] ; then
62         time aliroot -b -q -x sim.C\(${runNumber},\"${OCDBpath}\",${nEventsim}\) >sim.log 2>&1
63         mv syswatch.log simwatch.log
64     fi
65   fi
66 }
67
68 goCPass0()
69 (
70   umask 0002
71   
72   targetDirectory=${1}
73   inputList=${2}
74   nEvents=${3}
75   ocdbPath=${4}
76   configFile=${5}
77   runNumber=${6}
78   jobindex=${7}
79   shift 7
80   if ! parseConfig ${configFile} "$@"; then return 1; fi
81
82   #record the working directory provided by the batch system
83   batchWorkingDirectory=${PWD}
84
85   #use the jobindex only if set and non-negative
86   if [[ -z ${jobindex} || ${jobindex} -lt 0 ]]; then
87     [[ -n "${LSB_JOBINDEX}" ]] && jobindex=${LSB_JOBINDEX}
88     [[ -n "${SGE_TASK_ID}" ]] && jobindex=${SGE_TASK_ID}
89   fi
90
91   [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
92   doneFile="${commonOutputPath}/meta/cpass0.job${jobindex}.run${runNumber}.done"
93
94   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
95   
96   if [[ -n ${ALIROOT_FORCE_COREDUMP} ]]; then
97     ulimit -c unlimited 
98     export ALIROOT_FORCE_COREDUMP
99   fi
100
101   #the contents of this is stored in the tree and used later (e.g. AliAnalysisTaskPIDResponse)!
102   #at the QA stage the pass number is guessed from the path stored here.
103   #The Format is:
104   #Packages= ;OutputDir= ;LPMPass= ;TriggerAlias= ;LPMRunNumber= ;LPMProductionType= ;LPMInteractionType= ;LPMProductionTag= ;LPMAnchorRun= ;LPMAnchorProduction= ;LPMAnchorYear= 
105   export PRODUCTION_METADATA="OutputDir=cpass0"
106
107   if [[ "${inputList}" =~ \.root$ ]]; then
108     infile=${inputList}
109   else
110     infile=$(sed -ne "${jobindex}p" ${inputList} | egrep '\s*\w*/\w*')
111   fi
112   chunkName=${infile##*/}
113
114   outputDir=${targetDirectory}/${jobindex}_${chunkName%.*}
115   mkdir -p ${outputDir}
116   [[ ! -d ${outputDir} ]] && echo "cannot make ${outputDir}" && touch ${doneFile} && return 1  
117   
118   #runpath=${PWD}/rundir_cpass0_${runNumber}_${jobindex}
119   runpath=${outputDir}
120   #[[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
121   [[ ${reconstructInTemporaryDir} -eq 1 ]] && runpath=$(mktemp -d -t cpass0.XXXXXX)
122   mkdir -p ${runpath}
123   [[ ! -d ${runpath} ]] && echo "cannot make runpath ${runpath}" && touch ${doneFile} && return 1
124   if ! cd ${runpath}; then
125     echo "PWD=$PWD is not the runpath=${runpath}"
126     touch ${doneFile}
127     return 1
128   fi
129
130   #runCPassX/C expects the raw chunk to be linked in the run dir
131   #despite it being accessed by the full path
132   ln -s ${infile} ${runpath}/${chunkName}
133
134   #####MC
135   if [[ -n ${generateMC} ]]; then
136     olddir=${PWD}
137     outputDirMC=${commonOutputPath}/000${runNumber}/sim/${jobindex}
138     simrunpath=${outputDirMC}
139     #[[ ${simulateInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && simrunpath=${TMPDIR}
140     [[ ${simulateInTemporaryDir} -eq 1 ]] && simrunpath=$(mktemp -d -t cpass0MC.XXXXXX)
141     mkdir -p ${outputDirMC}
142     mkdir -p ${simrunpath}
143     if cd ${simrunpath}; then
144
145       filesMC=( 
146       "${batchWorkingDirectory}/sim.C"
147       "${batchWorkingDirectory}/rec.C"
148       "${batchWorkingDirectory}/Config.C"
149       "${batchWorkingDirectory}/OCDB_*.root"
150       )
151       for file in ${filesMC[*]}; do
152         [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
153       done
154
155       generateMC ${runNumber} ${ocdbPath} ${nEvents}
156
157       [[ ! "${simrunpath}" =~ "${outputDirMC}" ]] && mv * ${outputDirMC} #TODO check if it works
158       cd ${olddir}
159
160       ln -s ${outputDirMC}/* ${runpath}/ 
161
162       inputList=${outputDirMC}/galice.root #TODO not valid outside shell !!!
163       infile=""
164     fi
165   fi
166   ######
167   
168   [[ ! -f ${inputList} && -z ${pretend} ]] && echo "input file ${inputList} not found, exiting..." && touch ${doneFile} && return 1
169
170   logOutputDir=${runpath}
171   [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir}
172   [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/stdout
173   #[[ -z ${dontRedirectStdOutToLog} ]] && exec 2> ${logOutputDir}/stderr
174   echo "${0} $*"
175
176   echo "#####################"
177   echo CPass0:
178   echo JOB setup
179   echo nEvents            ${nEvents}
180   echo runNumber          ${runNumber}
181   echo ocdbPath           ${ocdbPath}
182   echo infile             ${infile}
183   echo chunkName          ${chunkName}
184   echo jobindex           ${jobindex}
185   echo recoTriggerOptions ${recoTriggerOptions}
186   echo targetDirectory    ${targetDirectory}
187   echo commonOutputPath         ${commonOutputPath}
188   echo doneFile      ${doneFile}
189   echo runpath            ${runpath}  
190   echo outputDir          ${outputDir}
191   echo ALICE_ROOT         ${ALICE_ROOT}
192   echo PWD                ${PWD}
193   echo "########## ###########"
194
195   alirootInfo > ALICE_ROOT.log
196
197   filesCPass0=( 
198                "${batchWorkingDirectory}/runCPass0.sh"
199                "${batchWorkingDirectory}/recCPass0.C"
200                "${batchWorkingDirectory}/runCalibTrain.C"
201                "${batchWorkingDirectory}/localOCDBaccessConfig.C"
202                "${batchWorkingDirectory}/OCDB.root"
203                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/runCPass0.sh"
204                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/recCPass0.C" 
205                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/runCalibTrain.C"
206   )
207
208   for file in ${filesCPass0[*]}; do
209     [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
210     [[ ${file##*/} =~ .*\.sh ]] && chmod +x ${file##*/}
211   done
212
213   echo "this directory (${PWD}) contents:"
214   /bin/ls
215   echo
216   chmod u+x runCPass0.sh
217
218   sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh
219
220   if [[ -n ${postSetUpActionCPass0} ]]; then
221     echo "running ${postSetUpActionCPass0}"
222     eval ${postSetUpActionCPass0}
223   fi
224
225   #run CPass0
226   echo "${runpath}/runCPass0.sh ${infile} ${nEvents} ${runNumber} ${ocdbPath} ${recoTriggerOptions}"
227   if [[ -n ${pretend} ]]; then
228     sleep ${pretendDelay}
229     touch AliESDs.root
230     touch AliESDfriends.root
231     touch AliESDfriends_v1.root
232     touch rec.log
233     touch calib.log
234   else
235     echo ./runCPass0.sh "${infile}" "${nEvents}" "${runNumber}" "${ocdbPath}" "${recoTriggerOptions}"
236     ./runCPass0.sh "${infile}" "${nEvents}" "${runNumber}" "${ocdbPath}" "${recoTriggerOptions}"
237   fi
238   
239   #move stuff to final destination
240   echo "this directory (${PWD}) contents:"
241   /bin/ls
242   echo
243
244   echo rm -f ./${chunkName}
245   rm -f ./${chunkName}
246   echo "cp -R ${runpath}/* ${outputDir}"
247   cp -p -R ${runpath}/* ${outputDir}
248   echo
249   
250   #validate CPass0
251   cd ${outputDir}
252   touch ${doneFile}
253   echo "dir ${outputDir}" >> ${doneFile}
254   if summarizeLogs >> ${doneFile}; then
255     [[ -f ${outputDirMC}/galice.root ]] && echo "sim ${outputDirMC}/galice.root" >> ${doneFile}
256     [[ -f AliESDfriends_v1.root ]] && echo "calibfile ${outputDir}/AliESDfriends_v1.root" >> ${doneFile}
257     [[ -f AliESDs.root ]] && echo "esd ${outputDir}/AliESDs.root" >> ${doneFile}
258   fi
259
260   [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath} && echo "removing ${runpath}"
261   return 0
262 )
263
264 goCPass1()
265 (
266   umask 0002
267   
268   targetDirectory=${1}
269   inputList=${2}
270   nEvents=${3}
271   ocdbPath=${4}
272   configFile=${5}
273   runNumber=${6}
274   jobindex=${7}
275   shift 7
276   extraOpts=("$@")
277   if ! parseConfig ${configFile} "$@"; then return 1; fi
278
279   #record the working directory provided by the batch system
280   batchWorkingDirectory=${PWD}
281
282   #use the jobindex only if set and non-negative
283   if [[ -z ${jobindex} || ${jobindex} -lt 0 ]]; then
284     [[ -n "${LSB_JOBINDEX}" ]] && jobindex=${LSB_JOBINDEX}
285     [[ -n "${SGE_TASK_ID}" ]] && jobindex=${SGE_TASK_ID}
286   fi
287
288   [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
289   doneFile="${commonOutputPath}/meta/cpass1.job${jobindex}.run${runNumber}.done"
290
291   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
292   
293   if [[ -n ${ALIROOT_FORCE_COREDUMP} ]]; then
294     ulimit -c unlimited 
295     export ALIROOT_FORCE_COREDUMP
296   fi
297
298   #the contents of this is stored in the tree and used later (e.g. AliAnalysisTaskPIDResponse)!
299   #at the QA stage the pass number is guessed from the path stored here.
300   #The Format is:
301   #Packages= ;OutputDir= ;LPMPass= ;TriggerAlias= ;LPMRunNumber= ;LPMProductionType= ;LPMInteractionType= ;LPMProductionTag= ;LPMAnchorRun= ;LPMAnchorProduction= ;LPMAnchorYear= 
302   export PRODUCTION_METADATA="OutputDir=cpass1"
303
304   [[ ! -f ${inputList} && -z ${pretend} ]] && echo "input file ${inputList} not found, exiting..." && touch ${doneFile} && return 1
305   if [[ "${inputList}" =~ \.root$ ]]; then
306     infile=${inputList}
307   else
308     infile=$(sed -ne "${jobindex}p" ${inputList} | egrep '\s*\w*/\w*')
309   fi
310   chunkName=${infile##*/}
311
312   outputDir=${targetDirectory}/${jobindex}_${chunkName%.*}
313   mkdir -p ${outputDir}
314   [[ ! -d ${outputDir} ]] && echo "cannot make ${outputDir}" && touch ${doneFile} && return 1
315   
316   #runpath=${PWD}/rundir_cpass1_${runNumber}_${jobindex}
317   runpath=${outputDir}
318   #[[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
319   [[ ${reconstructInTemporaryDir} -eq 1 ]] && runpath=$(mktemp -d -t cpass1.XXXXXX)
320
321   #MC
322   if [[ "${infile}" =~ galice\.root ]]; then
323     ln -s ${inputList%/*}/* ${runpath}
324     infile=""
325   fi
326
327   #init the running path
328   mkdir -p ${runpath}
329   [[ ! -d ${runpath} ]] && echo "cannot make runpath ${runpath}" && touch ${doneFile} && return 1
330   if ! cd ${runpath}; then
331     echo "PWD=$PWD is not the runpath=${runpath}"
332     touch ${doneFile}
333     return 1
334   fi
335
336   #this is needed for runCPass1.sh
337   ln -s ${infile} ${runpath}/${chunkName}
338
339   logOutputDir=${runpath}
340   [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir}
341   [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/stdout
342   #[[ -z ${dontRedirectStdOutToLog} ]] && exec 2> ${logOutputDir}/stderr
343   echo "${0} $*"
344
345   echo "#####################"
346   echo CPass1:
347   echo JOB setup
348   echo nEvents            ${nEvents}
349   echo runNumber          ${runNumber}
350   echo ocdbPath           ${ocdbPath}
351   echo infile             ${infile}
352   echo chunkName          ${chunkName}
353   echo jobindex           ${jobindex}
354   echo recoTriggerOptions ${recoTriggerOptions}
355   echo targetDirectory    ${targetDirectory}
356   echo commonOutputPath         ${commonOutputPath}
357   echo doneFile      ${doneFile}
358   echo runpath            ${runpath}  
359   echo outputDir          ${outputDir}
360   echo batchWorkingDirectory ${batchWorkingDirectory}
361   echo ALICE_ROOT         ${ALICE_ROOT}
362   echo PWD                ${PWD}
363   echo "########## ###########"
364
365   alirootInfo > ALICE_ROOT.log
366
367   filesCPass1=( 
368                "${batchWorkingDirectory}/runCPass1.sh"
369                "${batchWorkingDirectory}/recCPass1.C"
370                "${batchWorkingDirectory}/recCPass1_OuterDet.C"
371                "${batchWorkingDirectory}/runCalibTrain.C"
372                "${batchWorkingDirectory}/QAtrain_duo.C"
373                "${batchWorkingDirectory}/localOCDBaccessConfig.C"
374                "${batchWorkingDirectory}/${configFile}"
375                "${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz"
376                "${batchWorkingDirectory}/OCDB.root"
377                "${trustedQAtrainMacro}"
378                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/runCPass1.sh"
379                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/recCPass1.C" 
380                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/recCPass1_OuterDet.C" 
381                "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/runCalibTrain.C"
382                "${ALICE_ROOT}/ANALYSIS/macros/QAtrain_duo.C"
383   )
384
385   for file in "${filesCPass1[@]}"; do
386     [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
387     [[ ${file##*/} =~ .*\.sh ]] && echo "making ${file##*/} executable" && chmod +x ${file##*/}
388   done
389
390   echo "this directory (${PWD}) contents:"
391   /bin/ls
392   echo
393
394   #remove spaces around commas from calls to root
395   sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh
396
397   if [[ -n ${postSetUpActionCPass1} ]]; then
398     echo "running ${postSetUpActionCPass1}"
399     eval ${postSetUpActionCPass1}
400     echo
401   fi
402
403   #configure local OCDB storage from CPass0 (creates the localOCDBaccessConfig.C script)
404   if [[ -f cpass0.localOCDB.${runNumber}.tgz ]]; then
405     echo goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz"
406     goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz"
407   else
408     echo "WARNING: file cpass0.localOCDB.${runNumber}.tgz not found!"
409   fi
410
411   if [[ ! $(/bin/ls -1 OCDB/*/*/*/*.root 2>/dev/null) ]]; then
412     echo "cpass0 produced no calibration! exiting..."
413     touch ${doneFile}
414     return 1
415   fi
416
417   #create the Barrel and OuterDet directories for CPass1 and link the local OCDB directory
418   #there to make the localOCDBaccessConfig.C file work, since it may point to the OCDB
419   #entries using a relative path, e.g. local://./OCDB
420   echo "linking the OCDB/ for Barrel and OuterDet directories"
421   mkdir Barrel OuterDet
422   ls -l
423   ln -s ../OCDB Barrel/OCDB
424   ln -s ../OCDB OuterDet/OCDB
425
426   #setup the filtering
427   #the following option enables the filtering task inside the QAtrain_duo.C
428   [[ -n $runESDfiltering ]] && export QA_TaskFilteredTree=1
429   #set the downscaling factors during the filtering fro expert QA (overrides the previous values)
430   if [[ -n ${filteringFactorHighPt} ]]; then
431     export AliAnalysisTaskFilteredTree_fLowPtTrackDownscaligF=${filteringFactorHighPt}
432   fi
433   if [[ -n ${filteringFactorV0s} ]]; then
434     export AliAnalysisTaskFilteredTree_fLowPtV0DownscaligF=${filteringFactorV0s} 
435   fi
436
437   #run CPass1
438   chmod u+x runCPass1.sh
439   echo "${runpath}/runCPass1.sh ${infile} ${nEvents} ${runNumber} ${ocdbPath} ${recoTriggerOptions}"
440   if [[ -n ${pretend} ]]; then
441     sleep ${pretendDelay}
442     touch AliESDs_Barrel.root
443     touch AliESDfriends_Barrel.root
444     touch AliESDfriends_v1.root
445     touch QAresults_barrel.root
446     touch EventStat_temp_barrel.root
447     touch AODtpITS.root
448     touch AliESDs_Outer.root
449     touch AliESDfriends_Outer.root
450     touch QAresults_outer.root
451     touch EventStat_temp_outer.root
452     touch rec.log
453     touch calib.log
454     touch qa.log
455     touch filtering.log FilterEvents_Trees.root
456   else
457     ./runCPass1.sh "${infile}" "${nEvents}" "${runNumber}" "${ocdbPath}" "${recoTriggerOptions}"
458     
459     [[ ! -f AliESDs_Barrel.root && -f Barrel/AliESDs.root ]] && mv Barrel/AliESDs.root AliESDs_Barrel.root
460     [[ ! -f AliESDfriends_Barrel.root && -f Barrel/AliESDfriends.root ]] && mv Barrel/AliESDfriends.root AliESDfriends_Barrel.root
461     [[ ! -f AliESDfriends_v1.root && -f Barrel/AliESDfriends_v1.root ]] && mv Barrel/AliESDfriends_v1.root .
462     [[ ! -f QAresults_barrel.root && -f Barrel/QAresults_barrel.root ]] && mv Barrel/QAresults_barrel.root .
463     [[ ! -f AliESDs_Outer.root && -f OuterDet/AliESDs.root ]] && mv OuterDet/AliESDs.root AliESDs_Outer.root
464     [[ ! -f AliESDfriends_Outer.root && -f OuterDet/AliESDfriends.root ]] && mv OuterDet/AliESDfriends.root AliESDfriends_Outer.root
465     [[ ! -f QAresults_outer.root && -f OuterDet/QAresults_outer.root ]] && mv OuterDet/QAresults_outer.root .
466     [[ ! -f FilterEvents_Trees.root && -f Barrel/FilterEvents_Trees.root ]] && mv Barrel/FilterEvents_Trees.root .
467
468     #make the filtered tree (if requested and not already produced by QA
469     [[ -f AliESDs_Barrel.root ]] && echo "AliESDs_Barrel.root" > filtered.list
470     if [[ -n ${runESDfiltering} && ! -f FilterEvents_Trees.root && -f filtered.list ]]; then 
471       goMakeFilteredTrees ${PWD} ${runNumber} "${PWD}/filtered.list" ${filteringFactorHighPt} ${filteringFactorV0s} ${ocdbPath} 1000000 0 10000000 0 ${configFile} AliESDs_Barrel.root "${extraOpts[@]}" >filtering.log
472     else
473       echo ""
474     fi
475
476   fi
477
478   ##handle possible crashes in QA (happens often in trunk)
479   ##rerun QA with a trusted aliroot version
480   #if [[ $(validateLog qa_barrel.log) ]]; then
481   #  echo "qa_barrel.log not validated!"
482   #fi
483   #if [[ ! -f QAresults_barrel.root && -f ${setupTrustedAliROOTenvInCurrentShell} || $(validateLog qa_barrel.log) ]]; then
484   #  echo "WARNING: using trusted QA aliroot ${ALICE_ROOT}"
485   #  source ${setupTrustedAliROOTenvInCurrentShell}
486   #  cd Barrel
487   #  rm QAresults_barrel.root
488   #  rm EventStat_temp_barrel.root
489   #  rm AODtpITS.root
490   #  [[ ! -f AliESDs.root ]] && ln -s ../AliESDs_Barrel.root AliESDs.root
491   #  [[ ! -f AliESDfriends.root ]] && ln -s ../AliESDfriends_Barrel.root AliESDfriends.root
492   #  if [[ -n ${trustedQAtrainMacro} ]]; then
493   #    eval "cp ${trustedQAtrainMacro} QAtrain_duo_trusted.C"
494   #  fi
495   #  echo executing aliroot -b -q "QAtrain_duo_trusted.C(\"_barrel\",${runNumber},\"wn.xml\",0,\"${ocdbPath}\")"
496   #  time aliroot -b -q "QAtrain_duo.C(\"_barrel\",${runNumber},\"wn.xml\",0,\"${ocdbPath}\")" &> ../qa_barrel_trusted.log
497   #  cd ../
498   #fi
499
500   #move stuff to final destination
501   echo "this directory (${PWD}) contents:"
502   /bin/ls
503   echo rm -f ./${chunkName}
504   rm -f ./${chunkName}
505   echo "cp -R ${runpath}/* ${outputDir}"
506   cp -pf -R ${runpath}/* ${outputDir}
507   echo
508
509   #validate CPass1
510   cd ${outputDir}
511   touch ${doneFile}
512   echo "dir ${outputDir}" >> ${doneFile}
513   if summarizeLogs >> ${doneFile}; then
514     [[ -f AliESDs_Barrel.root ]] && echo "esd ${outputDir}/AliESDs_Barrel.root" >> ${doneFile}
515     [[ -f AliESDfriends_v1.root ]] && echo "calibfile ${outputDir}/AliESDfriends_v1.root" >> ${doneFile}
516     [[ -f QAresults_Barrel.root ]] && echo "qafile ${outputDir}/QAresults_Barrel.root" >> ${doneFile}
517     [[ -f QAresults_Outer.root ]] && echo "qafile ${outputDir}/QAresults_Outer.root" >> ${doneFile}
518     [[ -f QAresults_barrel.root ]] && echo "qafile ${outputDir}/QAresults_barrel.root" >> ${doneFile}
519     [[ -f QAresults_outer.root ]] && echo "qafile ${outputDir}/QAresults_outer.root" >> ${doneFile}
520     [[ -f FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFile}
521   else
522     if grep "qa_outer.log.*OK" ${doneFile} > /dev/null; then
523       [[ -f QAresults_Outer.root ]] && echo "qafile ${outputDir}/QAresults_Outer.root" >> ${doneFile}
524       [[ -f QAresults_outer.root ]] && echo "qafile ${outputDir}/QAresults_outer.root" >> ${doneFile}
525     fi
526     if grep "qa_barrel.log.*OK" ${doneFile} > /dev/null; then
527       [[ -f QAresults_Barrel.root ]] && echo "qafile ${outputDir}/QAresults_Barrel.root" >> ${doneFile}
528       [[ -f QAresults_barrel.root ]] && echo "qafile ${outputDir}/QAresults_barrel.root" >> ${doneFile}
529     fi
530     if grep "filtering.log.*OK" ${doneFile} > /dev/null; then
531       [[ -f FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFile}
532     fi
533   fi
534
535   [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
536   return 0
537 )
538
539
540 goMergeCPass0()
541 (
542   #
543   # find the output files and merge them
544   #
545
546   outputDir=${1}
547   ocdbStorage=${2}
548   configFile=${3}
549   runNumber=${4}
550   calibrationFilesToMerge=${5}  #can be a non-existent file, will then be produced on the fly
551   shift 5
552   if ! parseConfig ${configFile} "$@"; then return 1; fi
553
554   #record the working directory provided by the batch system
555   batchWorkingDirectory=${PWD}
556
557   [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
558   doneFile="${commonOutputPath}/meta/merge.cpass0.run${runNumber}.done"
559
560   umask 0002
561   ulimit -c unlimited 
562
563   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
564
565   #runpath=${PWD}/rundir_cpass0_Merge_${runNumber}
566   runpath=${outputDir}
567   #[[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
568   [[ ${reconstructInTemporaryDir} -eq 1 ]] && runpath=$(mktemp -d -t mergeCPass0.XXXXXX)
569
570   mkdir -p ${runpath}
571   [[ ! -d ${runpath} ]] && echo "not able to make the runpath ${runpath}" && touch ${doneFile} && return 1
572   if ! cd ${runpath}; then 
573     echo "PWD=$PWD is not the runpath=${runpath}"
574     touch ${doneFile}
575     return 1
576   fi
577
578   logOutputDir=${runpath}
579   [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir}
580   [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/mergeMakeOCDB.log
581   echo "${0} $*"
582
583   mergingScript="mergeMakeOCDB.byComponent.sh"
584
585   echo goMergeCPass0 SETUP:
586   echo runNumber=${runNumber}
587   echo outputDir=${outputDir}
588   echo ocdbStorage=${ocdbStorage}
589   echo calibrationFilesToMerge=${calibrationFilesToMerge}
590   echo mergingScript=${mergingScript}
591   echo commonOutputPath=${commonOutputPath}
592   echo runpath=${runpath}
593   
594   # copy files in case they are not already there
595   filesMergeCPass0=(
596                     "${batchWorkingDirectory}/${calibrationFilesToMerge}"
597                     "${batchWorkingDirectory}/OCDB.root"
598                     "${batchWorkingDirectory}/localOCDBaccessConfig.C"
599                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/mergeMakeOCDB.byComponent.sh"
600                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/mergeByComponent.C"
601                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/makeOCDB.C"
602                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/merge.C"
603                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/mergeMakeOCDB.sh"
604   )
605   for file in ${filesMergeCPass0[*]}; do
606     [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
607     [[ ${file##*/} =~ .*\.sh ]] && chmod +x ${file##*/}
608   done
609   
610   sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh
611
612   alirootInfo > ALICE_ROOT.log
613
614   #
615   echo "PWD"
616   /bin/ls
617   echo "PWD/.."
618   /bin/ls ../
619
620
621   #merge calibration
622   chmod u+x ${mergingScript}  
623   mkdir -p ./OCDB
624   if [[ ! -f ${calibrationFilesToMerge} ]]; then
625     echo "/bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root > ${calibrationFilesToMerge}"
626     /bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root 2>/dev/null > ${calibrationFilesToMerge}
627   fi
628   
629   echo "${mergingScript} ${calibrationFilesToMerge} ${runNumber} local://./OCDB ${ocdbStorage}"
630   if [[ -n ${pretend} ]]; then
631     sleep ${pretendDelay}
632     touch CalibObjects.root
633     touch ocdb.log
634     touch merge.log
635     touch dcsTime.root
636     mkdir -p ./OCDB/TPC/Calib/TimeGain/
637     mkdir -p ./OCDB/TPC/Calib/TimeDrift/
638     touch ./OCDB/TPC/Calib/TimeGain/someCalibObject_0-999999_cpass0.root
639     touch ./OCDB/TPC/Calib/TimeDrift/otherCalibObject_0-999999_cpass0.root
640   else
641     ./${mergingScript} ${calibrationFilesToMerge} ${runNumber} "local://./OCDB" ${ocdbStorage}
642
643     #produce the calib trees for expert QA (dcsTime.root)
644     goMakeLocalOCDBaccessConfig ./OCDB
645     echo aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")"
646     aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")"
647   fi
648   
649   ### produce the output
650   #tar the produced OCDB for reuse
651   tar czf ${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz ./OCDB
652
653   /bin/ls
654
655   #copy all to output dir
656   cp -pf -R ${runpath}/* ${outputDir}
657
658   if [[ -n ${generateMC} ]]; then
659     goPrintValues sim ${commonOutputPath}/meta/sim.run${runNumber}.list ${commonOutputPath}/meta/cpass0.job*.run${runNumber}.done
660   fi
661
662   #validate merging cpass0
663   cd ${outputDir}
664   touch ${doneFile}
665   echo "dir ${outputDir}" >> ${doneFile}
666   if summarizeLogs >> ${doneFile}; then
667     [[ -f CalibObjects.root ]] && echo "calibfile ${outputDir}/CalibObjects.root" >> ${doneFile}
668     [[ -f dcsTime.root ]] && echo "dcsTree ${outputDir}/dcsTime.root" >> ${doneFile}
669   fi
670
671   [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
672   return 0
673 )
674
675 goMergeCPass1()
676 (
677   #
678   # find the output files and merge them
679   #
680
681   outputDir=${1}
682   ocdbStorage=${2}
683   configFile=${3}
684   runNumber=${4}
685   calibrationFilesToMerge=${5}
686   qaFilesToMerge=${6}
687   filteredFilesToMerge=${7}
688   shift 7
689   if ! parseConfig ${configFile} "$@"; then return 1; fi
690
691   #record the working directory provided by the batch system
692   batchWorkingDirectory=${PWD}
693
694   [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
695   doneFile="${commonOutputPath}/meta/merge.cpass1.run${runNumber}.done"
696
697   umask 0002
698   ulimit -c unlimited 
699
700   #clean up first:
701   rm -f ${outputDir}/*.log
702   rm -f ${outputDir}/*.root
703   rm -f ${outputDir}/*done
704
705   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
706
707   #runpath=${PWD}/rundir_cpass1_Merge_${runNumber}
708   runpath=${outputDir}
709   #[[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
710   [[ ${reconstructInTemporaryDir} -eq 1 ]] && runpath=$(mktemp -d -t mergeCPass1.XXXXXX)
711
712   mkdir -p ${runpath}
713   [[ ! -d ${runpath} ]] && echo "not able to make the runpath ${runpath}" && touch ${doneFile} && return 1
714   if ! cd ${runpath}; then 
715     echo "PWD=$PWD is not the runpath=${runpath}"
716     touch ${doneFile}
717     return 1
718   fi
719
720   logOutputDir=${runpath}
721   [[ -n ${logToFinalDestination} ]] && logOutputDir=${outputDir}
722   [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/mergeMakeOCDB.log
723   echo "${0} $*"
724
725   calibrationOutputFileName='AliESDfriends_v1.root'
726   qaOutputFileName='QAresults*.root'
727   mergingScript="mergeMakeOCDB.byComponent.sh"
728   #important to have the string "Stage.txt" in the filename to trigger the merging
729   #it has to be a list of directories containing the files
730   qaMergedOutputFileName="QAresults_merged.root"
731
732   echo goMergeCPass1 SETUP:
733   echo runNumber=${runNumber}
734   echo outputDir=${outputDir}
735   echo ocdbStorage=${ocdbStorage}
736   echo calibrationFilesToMerge=$calibrationFilesToMerge
737   echo qaFilesToMerge=$qaFilesToMerge
738   echo calibrationOutputFileName=${calibrationOutputFileName}
739   echo mergingScript=${mergingScript}
740   
741   # copy files in case they are not already there
742   filesMergeCPass1=(
743                     "${batchWorkingDirectory}/${calibrationFilesToMerge}"
744                     "${batchWorkingDirectory}/${qaFilesToMerge}"
745                     "${batchWorkingDirectory}/OCDB.root"
746                     "${batchWorkingDirectory}/localOCDBaccessConfig.C"
747                     "${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz"
748                     "${batchWorkingDirectory}/QAtrain_duo.C"
749                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/mergeMakeOCDB.byComponent.sh"
750                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/mergeByComponent.C"
751                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/makeOCDB.C"
752                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/merge.C"
753                     "${ALICE_ROOT}/PWGPP/CalibMacros/CPass1/mergeMakeOCDB.sh"
754                     "${trustedQAtrainMacro}"
755                     "${ALICE_ROOT}/ANALYSIS/macros/QAtrain_duo.C"
756   )
757   for file in ${filesMergeCPass1[*]}; do
758     [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
759     [[ ${file##*/} =~ .*\.sh ]] && chmod +x ${file##*/}
760   done
761
762   sed -i '/.*root .*\.C/ s|\s*,\s*|,|g' *.sh
763
764   #configure local OCDB storage from CPass0 (creates the localOCDBaccessConfig.C script)
765   if [[ -f cpass0.localOCDB.${runNumber}.tgz ]]; then
766     echo goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz"
767     goMakeLocalOCDBaccessConfig "cpass0.localOCDB.${runNumber}.tgz"
768   else
769     echo "WARNING: file cpass0.localOCDB.${runNumber}.tgz not found!"
770   fi
771
772   alirootInfo > ALICE_ROOT.log
773
774   #
775   /bin/ls
776
777   #merge calibration
778   chmod u+x ${mergingScript}  
779   mkdir -p OCDB
780   
781   #if not provided, create the lists of files to merge
782   if [[ ! -f ${filteredFilesToMerge} ]]; then
783     echo "/bin/ls -1 ${outputDir}/*/FilterEvents_Trees.root > ${filteredFilesToMerge}"
784     /bin/ls -1 ${outputDir}/*/FilterEvents_Trees.root 2>/dev/null > ${filteredFilesToMerge}
785   fi
786   if [[ ! -f ${calibrationFilesToMerge} ]]; then
787     echo "/bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root > ${calibrationFilesToMerge}"
788     /bin/ls -1 ${outputDir}/*/AliESDfriends_v1.root 2>/dev/null > ${calibrationFilesToMerge}
789   fi
790   if [[ ! -f ${qaFilesToMerge} ]]; then
791     #find the files, but only store the directories (QAtrain_duo.C requires this)
792     echo "/bin/ls -1 ${outputDir}/*/QAresults*.root | while read x; do echo ${x%/*}; done | sort | uniq > ${qaFilesToMerge}"
793     /bin/ls -1 ${outputDir}/*/QAresults*.root | while read x; do echo ${x%/*}; done | sort | uniq > ${qaFilesToMerge}
794   fi
795   
796   echo "${mergingScript} ${calibrationFilesToMerge} ${runNumber} local://./OCDB ${ocdbStorage}"
797   if [[ -n ${pretend} ]]; then
798     sleep ${pretendDelay}
799     touch ocdb.log
800     touch cpass1.localOCDB.${runNumber}.tgz
801     touch ${qaMergedOutputFileName}
802     touch merge.log
803     touch trending.root
804     touch FilterEvents_Trees.root
805     touch CalibObjects.root
806     touch dcsTime.root
807     touch ${qaMergedOutputFileName}
808     mkdir -p OCDB
809   else
810     ./${mergingScript} ${calibrationFilesToMerge} ${runNumber} "local://./OCDB" ${ocdbStorage}
811
812     #merge QA (and filtered trees)
813     [[ -n ${AliAnalysisTaskFilteredTree_fLowPtTrackDownscaligF} ]] && export AliAnalysisTaskFilteredTree_fLowPtTrackDownscaligF
814     [[ -n ${AliAnalysisTaskFilteredTree_fLowPtV0DownscaligF} ]] && export AliAnalysisTaskFilteredTree_fLowPtV0DownscaligF
815
816     #echo aliroot -l -b -q "merge.C(\"${qaFilesToMerge}\",\"\",kFALSE,\"${qaMergedOutputFileName}\")"
817     echo aliroot -b -q "QAtrain_duo.C(\"_barrel\",${runNumber},\"${qaFilesToMerge}\",1,\"${ocdbStorage}\")"
818     #aliroot -l -b -q "merge.C(\"${qaFilesToMerge}\",\"\",kFALSE,\"${qaMergedOutputFileName}\")"
819     aliroot -b -q "QAtrain_duo.C(\"_barrel\",${runNumber},\"${qaFilesToMerge}\",1,\"${ocdbStorage}\")" > mergeQA.log
820     mv QAresults_barrel.root ${qaMergedOutputFileName}
821     mv trending_barrel.root trending.root
822
823     #merge filtered trees
824     echo aliroot -l -b -q "merge.C(\"${qaFilesToMerge}\",\"\",kFALSE,\"${qaMergedOutputFileName}\")"
825     aliroot -l -b -q "merge.C(\"${filteredFilesToMerge}\",\"\",kFALSE,\"FilterEvents_Trees.root\")" > mergeFilteredTrees.log
826
827     #produce the calib trees for expert QA
828     echo aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")"
829     aliroot -b -q "${ALICE_ROOT}/PWGPP/TPC/macros/CalibSummary.C(${runNumber},\"${ocdbStorage}\")" > calibTree.log
830   fi
831
832   tar czf ${commonOutputPath}/meta/cpass1.localOCDB.${runNumber}.tgz ./OCDB
833
834   /bin/ls
835
836   #copy all to output dir
837   cp -pf -R ${runpath}/* ${outputDir}
838   
839   #validate merge cpass1
840   cd ${outputDir}
841   touch ${doneFile}
842   echo "dir ${outputDir}" >> ${doneFile}
843   if summarizeLogs >>  ${doneFile}; then
844     [[ -f CalibObjects.root ]] && echo "calibfile ${outputDir}/CalibObjects.root" >> ${doneFile}
845     [[ -f ${qaMergedOutputFileName} ]] && echo "qafile ${outputDir}/${qaMergedOutputFileName}" >> ${doneFile}
846     [[ -f trending.root ]] && echo "trendingfile ${outputDir}/trending.root" >> ${doneFile}
847     [[ -f dcsTime.root ]] && echo "dcsTree ${outputDir}/dcsTime.root" >> ${doneFile}
848     [[ -f FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFile}
849   else
850     if grep "mergeQA.log.*OK" ${doneFile} > /dev/null; then
851       [[ -f ${qaMergedOutputFileName} ]] && echo "qafile ${outputDir}/${qaMergedOutputFileName}" >> ${doneFile}
852     fi
853     if grep "mergeFilteredTrees.log.*OK" ${doneFile} > /dev/null; then
854       [[ -f FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFile}
855     fi
856   fi
857       
858   [[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
859   return 0
860 )
861
862 goMerge()
863 (
864   #generic root merge using CPass1 merge.C script
865   inputList=${1}
866   outputFile=${2}  
867   configFile=${3-"benchmark.config"}
868   shift 3
869   if ! parseConfig ${configFile} "$@"; then return 1; fi
870   
871   #record the working directory provided by the batch system
872   batchWorkingDirectory=${PWD}
873
874   [[ ! -f ${inputList} ]] && echo "inputList ${inputList} does not exist!" && return 1
875   [[ ! -f ${configFile} ]] && echo "configFile ${configFile} does not exist!" && return 1
876   umask 0002
877   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
878   rm -f ${outputFile}
879   aliroot -b -q "${ALICE_ROOT}/PWGPP/CalibMacros/CPass0/merge.C(\"${inputList}\",\"\",kFALSE,\"${outputFile}\")" > merge_${inputList}.log
880   return 0
881 )
882
883 goSubmitMakeflow()
884 {
885   #run
886   productionID=${1}
887   inputFileList=${2}
888   configFile=${3}
889   shift 3
890   extraOpts=("$@")
891   if ! parseConfig ${configFile} "${extraOpts[@]}"; then return 1; fi
892  
893   #record the working directory provided by the batch system
894   batchWorkingDirectory=${PWD}
895
896   [[ -z ${configFile} ]] && configFile="benchmark.config"
897   [[ ! -f ${configFile} ]] && echo "no config file found (${configFile})" && return 1
898
899   if [[ ! $(which makeflow &>/dev/null) && -n ${makeflowPath} ]]; then
900     echo "setting the makflow path from the config: "
901     echo "  export PATH=${makeflowPath}:${PATH}"
902     export PATH=${makeflowPath}:${PATH}
903   fi
904
905   #create the common output dir and the meta dir
906   commonOutputPath=${baseOutputDirectory}/${productionID}
907   if [[ -d ${commonOutputPath} ]]; then
908     echo "output dir ${commonOutputPath} exists!"
909     #return 1
910   else
911     mkdir -p ${commonOutputPath}
912   fi
913   mkdir -p ${commonOutputPath}/meta
914   
915   self=${0}
916   #if which greadlink; then self=$(greadlink -f "${0}"); fi
917   
918   #for reference copy the setup to the output dir
919   cp ${self} ${commonOutputPath}
920   cp ${configFile} ${commonOutputPath}
921   cp ${inputList} ${commonOutputPath}
922
923   #submit - use makeflow if available, fall back to old stuff when makeflow not there
924   if which makeflow; then
925     
926     goGenerateMakeflow ${productionID} ${inputFileList} ${configFile} "${extraOpts[@]}" commonOutputPath=${commonOutputPath} > benchmark.makeflow
927
928     makeflow ${makeflowOptions} benchmark.makeflow
929     cd ../
930   else 
931     echo "no makeflow!"
932   fi
933   return 0
934 }
935
936 goGenerateMakeflow()
937 (
938   #generate the makeflow file
939   [[ $# -lt 3 ]] && echo "args: id inputFileList configFile" && return 1
940   productionID=${1}
941   inputFileList=${2}
942   configFile=${3}
943   shift 3
944   extraOpts=("$@")
945   
946   #batch systems/makeflow sometimes handle spaces in arguments poorly, so encode them
947   for (( i=0;i<${#extraOpts[@]};i++ )); do 
948     extraOpts[i]=$(encSpaces "${extraOpts[i]}")
949   done
950   extraOpts+=("encodedSpaces=1")
951
952   if ! parseConfig ${configFile} "${extraOpts[@]}" &>/dev/null; then return 1; fi
953  
954   #extra safety
955   if [[ -z ${commonOutputPath} ]]; then
956     commonOutputPath=${baseOutputDirectory}/${productionID}
957     extraOpts=( "${extraOpts[@]}" "commonOutputPath=${commonOutputPath}" )
958   fi
959
960   #record the working directory provided by the batch system
961   batchWorkingDirectory=${PWD}
962
963   [[ -z ${configFile} ]] && configFile="benchmark.config"
964   [[ ! -f ${configFile} ]] && echo "no config file found (${configFile})" && return 1
965
966   #these files will be made a dependency - will be copied to the working dir of the jobs
967   declare -a copyFiles
968   inputFiles=(
969               "OCDB.root"
970               "localOCDBaccessConfig.C"
971               "QAtrain_duo.C"
972               "runCPass1.sh"
973               "recCPass1.C"
974               "recCPass1_OuterDet.C"
975               "runCalibTrain.C"
976               "runCPass0.sh"
977               "recCPass0.C"
978   )
979   for file in ${inputFiles[*]}; do
980     [[ -f ${file} ]] && copyFiles+=("${file}")
981   done
982
983   #create the makeflow file
984   [[ -n ${batchFlags} ]] && echo "BATCH_OPTIONS = ${batchFlags}"
985   declare -A arr_cpass0_merged arr_cpass1_merged
986   declare -A arr_cpass0_calib_list arr_cpass1_calib_list 
987   declare -A arr_cpass1_QA_list arr_cpass1_ESD_list arr_cpass1_filtered_list
988   declare -A arr_cpass0_profiled_outputs
989   declare -A listOfRuns
990   [[ -n ${runNumber} ]] && listOfRuns[${runNumber}]=1
991   while read x; do tmpRun=$(guessRunNumber ${x}); [[ -n ${tmpRun} ]] && listOfRuns[${tmpRun}]=1; done < ${inputFileList}
992   for runNumber in "${!listOfRuns[@]}"; do
993     [[ -z ${runNumber} ]] && continue
994     [[ ! ${runNumber} =~ ^[0-9]*[0-9]$ ]] && continue
995
996     unset arr_cpass0_outputs
997     unset arr_cpass1_outputs
998     declare -a arr_cpass0_outputs
999     declare -a arr_cpass1_outputs
1000
1001     jobindex=0
1002     inputFile=""
1003     while read inputFile; do
1004       currentDefaultOCDB=${defaultOCDB}
1005       [[ -z ${autoOCDB} ]] && autoOCDB=1
1006       if [[ ${autoOCDB} -ne 0 ]]; then
1007         currentDefaultOCDB=$(setYear ${inputFile} ${defaultOCDB})
1008       fi
1009
1010       #CPass0
1011       arr_cpass0_outputs[${jobindex}]="${commonOutputPath}/meta/cpass0.job${jobindex}.run${runNumber}.done"
1012       echo "${arr_cpass0_outputs[${jobindex}]} : benchmark.sh ${configFile} ${copyFiles[@]}"
1013       echo " ${alirootEnv} ./benchmark.sh CPass0 ${commonOutputPath}/000${runNumber}/cpass0 ${inputFile} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} ${jobindex} ${extraOpts[@]}"" "
1014
1015       #CPass1
1016       arr_cpass1_outputs[${jobindex}]="${commonOutputPath}/meta/cpass1.job${jobindex}.run${runNumber}.done"
1017       echo "${arr_cpass1_outputs[${jobindex}]} : benchmark.sh ${configFile} ${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz ${copyFiles[@]}"
1018       echo " ${alirootEnv} ./benchmark.sh CPass1 ${commonOutputPath}/000${runNumber}/cpass1 ${inputFile} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} ${jobindex} ${extraOpts[@]}"" "
1019       ((jobindex++))
1020
1021     done< <(grep "/000${runNumber}/" ${inputFileList})
1022     
1023     #CPass0 list of Calib files to merge
1024     arr_cpass0_calib_list[${runNumber}]="${commonOutputPath}/meta/cpass0.calib.run${runNumber}.list"
1025     echo "${arr_cpass0_calib_list[${runNumber}]} : benchmark.sh ${arr_cpass0_outputs[*]}"
1026     echo "  ./benchmark.sh PrintValues calibfile ${arr_cpass0_calib_list[${runNumber}]} ${arr_cpass0_outputs[*]}"
1027     echo
1028
1029     #CPass0 merging
1030     arr_cpass0_merged[${runNumber}]="${commonOutputPath}/meta/merge.cpass0.run${runNumber}.done"
1031     echo "${commonOutputPath}/meta/cpass0.localOCDB.${runNumber}.tgz ${arr_cpass0_merged[${runNumber}]} : benchmark.sh ${configFile} ${arr_cpass0_calib_list[${runNumber}]} ${copyFiles[@]}"
1032     echo " ${alirootEnv} ./benchmark.sh MergeCPass0 ${commonOutputPath}/000${runNumber}/cpass0 ${currentDefaultOCDB} ${configFile} ${runNumber} ${arr_cpass0_calib_list[${runNumber}]} ${extraOpts[@]}"" "
1033
1034     #CPass1 list of Calib/QA/ESD/filtered files
1035     # the trick with QA is to have the string "Stage.txt" in the file name of the list of directories with QA output to trigger
1036     # the production of the QA trending tree (only then the task->Finish() will be called in QAtrain_duo.C, on the grid
1037     # this corresponds to the last merging stage)
1038     arr_cpass1_QA_list[${runNumber}]="${commonOutputPath}/meta/cpass1.QA.run${runNumber}.lastMergingStage.txt.list"
1039     echo "${arr_cpass1_QA_list[${runNumber}]}: benchmark.sh ${arr_cpass1_outputs[*]}"
1040     echo "  ./benchmark.sh PrintValues dir ${arr_cpass1_QA_list[${runNumber}]} ${arr_cpass1_outputs[*]}"
1041     echo
1042     arr_cpass1_calib_list[${runNumber}]="${commonOutputPath}/meta/cpass1.calib.run${runNumber}.list"
1043     echo "${arr_cpass1_calib_list[${runNumber}]} : benchmark.sh ${arr_cpass1_outputs[*]}"
1044     echo "  ./benchmark.sh PrintValues calibfile ${arr_cpass1_calib_list[${runNumber}]} ${arr_cpass1_outputs[*]};"
1045     echo
1046     arr_cpass1_ESD_list[${runNumber}]="${commonOutputPath}/meta/cpass1.ESD.run${runNumber}.list"
1047     echo "${arr_cpass1_ESD_list[${runNumber}]} : benchmark.sh ${arr_cpass1_outputs[*]}"
1048     echo "  ./benchmark.sh PrintValues esd ${arr_cpass1_ESD_list[${runNumber}]} ${arr_cpass1_outputs[*]}"
1049     echo
1050     arr_cpass1_filtered_list[${runNumber}]="${commonOutputPath}/meta/cpass1.filtered.run${runNumber}.list"
1051     echo "${arr_cpass1_filtered_list[${runNumber}]} : benchmark.sh ${arr_cpass1_outputs[*]}"
1052     echo "  ./benchmark.sh PrintValues filteredTree ${arr_cpass1_filtered_list[${runNumber}]} ${arr_cpass1_outputs[*]}"
1053     echo
1054   
1055     #CPass1 merging
1056     arr_cpass1_merged[${runNumber}]="${commonOutputPath}/meta/merge.cpass1.run${runNumber}.done"
1057     echo "${commonOutputPath}/meta/cpass1.localOCDB.${runNumber}.tgz ${arr_cpass1_merged[${runNumber}]} :  benchmark.sh ${configFile} ${arr_cpass1_calib_list[${runNumber}]} ${arr_cpass1_QA_list[${runNumber}]} ${copyFiles[@]}"
1058     echo " ${alirootEnv} ./benchmark.sh MergeCPass1 ${commonOutputPath}/000${runNumber}/cpass1 ${currentDefaultOCDB} ${configFile} ${runNumber} ${arr_cpass1_calib_list[${runNumber}]} ${arr_cpass1_QA_list[${runNumber}]} ${arr_cpass1_filtered_list[${runNumber}]} ${extraOpts[@]}"" "
1059
1060     #CPass0 wrapped in a profiling tool (valgrind,....)
1061     if [[ -n ${profilingCommand} ]]; then
1062       inputFile=$(grep -m1 "${runNumber}/" ${inputFileList})
1063       [[ -z ${nEventsProfiling} ]] && nEventsProfiling=2
1064       currentDefaultOCDB=$(setYear ${inputFile} ${defaultOCDB})
1065       jobindex="profiling"
1066
1067       arr_cpass0_profiled_outputs[${runNumber}]="${commonOutputPath}/meta/cpass0.job${jobindex}.run${runNumber}.done"
1068       echo "${arr_cpass0_profiled_outputs[${runNumber}]} : benchmark.sh ${configFile} ${copyFiles[@]}"
1069       profilingCommand=$(encSpaces "${profilingCommand}")
1070       echo " ${alirootEnv} ./benchmark.sh CPass0 ${commonOutputPath}/000${runNumber}/${jobindex} ${inputFile} ${nEventsProfiling} ${currentDefaultOCDB} ${configFile} ${runNumber} ${jobindex} ${extraOpts[@]} useProfilingCommand=${profilingCommand}"
1071     fi
1072
1073   done #runs
1074
1075   #Summary
1076   echo "${commonOutputPath}/summary.log : benchmark.sh ${configFile} ${arr_cpass1_merged[*]}"
1077   echo "  ${alirootEnv} ./benchmark.sh MakeSummary ${configFile} ${extraOpts[@]}"
1078   echo
1079
1080   return 0
1081 )
1082
1083 goPrintValues()
1084 (
1085   #print the values given the key from any number of files (store in output file on second argument)
1086   if [[ $# -lt 3 ]]; then
1087     echo "goPrintValues key outputFile inputFiles"
1088     echo "if outputFile is \"-\" dont redirect to a file"
1089     return
1090   fi
1091   key=${1}
1092   outputFile=${2}
1093   [[ ${outputFile} =~ "-" ]] && outputFile=""
1094   shift 2 #remove 2 first arguments from arg list to only pass the input files to awk
1095   awk -v key=${key} '$0 ~ key" " {print $2}' "$@" | tee ${outputFile}
1096   return 0
1097 )
1098
1099 goCreateQAplots()
1100 (
1101   umask 0002
1102   mergedQAfileList=${1}
1103   productionID=${2}
1104   outputDir=${3}
1105   configFile=${4}
1106   shift 4
1107   if ! parseConfig ${configFile} ${@}; then return 1; fi
1108   
1109   #record the working directory provided by the batch system
1110   batchWorkingDirectory=${PWD}
1111
1112   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
1113
1114   [[ -z ${logOutputDir} ]] && logOutputDir=${PWD}
1115   [[ -z ${dontRedirectStdOutToLog} ]] && exec &> ${logOutputDir}/makeQAplots.log
1116   echo "${0} $*"
1117
1118   olddir=${PWD}
1119   mkdir -p ${outputDir}
1120   cd ${outputDir}
1121   [[ ! "${PWD}" =~ "${outputDir}" ]] && echo "PWD is not equal to outputDir=${outputDir}" && cd ${olddir} && return 1
1122
1123   ${ALICE_ROOT}/PWGPP/QA/scripts/runQA.sh inputList="${mergedQAfileList}" inputListHighPtTrees="${filteringList}"
1124
1125   cd ${olddir}
1126   return 0
1127 )
1128
1129 goTest()
1130 (
1131   echo AA
1132 )
1133
1134 alirootInfo()
1135 (
1136   umask 0002
1137   # save aliroot repository info
1138   [[ -z "${ALICE_ROOT}" ]] && return 1
1139   
1140   echo "\${ALICE_ROOT}=${ALICE_ROOT}"
1141   echo "\${ROOTSYS}=${ROOTSYS}"
1142   echo "\${PATH}=${PATH}"
1143   echo "\${LD_LIBRARY_PATH}=${LD_LIBRARY_PATH}"
1144   echo
1145   
1146   pushd ${PWD}
1147   cd ${ALICE_ROOT}
1148
1149   currentBranch=$(git rev-parse --abbrev-ref HEAD)
1150   git status
1151   echo ""
1152   echo ""
1153   git diff ${currentBranch}
1154   popd
1155   return 0
1156 )
1157
1158 setYear()
1159 (
1160   #set the year
1161   #  ${1} - year to be set
1162   #  ${2} - where to set the year
1163   year1=$(guessYear ${1})
1164   year2=$(guessYear ${2})
1165   local path=${2}
1166   [[ ${year1} -ne ${year2} && -n ${year2} && -n ${year1} ]] && path=${2/\/${year2}\//\/${year1}\/}
1167   echo ${path}
1168   return 0
1169 )
1170
1171 guessPeriod()
1172 (
1173   #guess the period from the path, pick the rightmost one
1174   local IFS="/"
1175   declare -a path=( ${1} )
1176   local dirDepth=${#path[*]}
1177   for ((x=${dirDepth}-1;x>=0;x--)); do
1178     local field=${path[${x}]}
1179     [[ ${field} =~ ^LHC[0-9][0-9][a-z]$ ]] && period=${field} && break
1180   done
1181   echo ${period}
1182   return 0
1183 )
1184
1185 guessYear()
1186 (
1187   #guess the year from the path, pick the rightmost one
1188   local IFS="/"
1189   declare -a path=( ${1} )
1190   local dirDepth=${#path[*]}
1191   for ((x=${dirDepth}-1;x>=0;x--)); do
1192     local field=${path[${x}]}
1193     [[ ${field} =~ ^20[0-9][0-9]$ ]] && year=${field} && break
1194   done
1195   echo ${year}
1196   return 0
1197 )
1198
1199 guessRunNumber()
1200 (
1201   #guess the run number from the path, pick the rightmost one
1202   #works for /path/foo/000123456/bar/...
1203   #and       /path/foo.run123456.bar
1204   local IFS="/."
1205   declare -a path=( ${1} )
1206   local dirDepth=${#path[*]}
1207   for ((x=${dirDepth}-1;x>=0;x--)); do
1208     local field=${path[${x}]}
1209     field=${field/run/000}
1210     [[ ${field} =~ [0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && runNumber=${field#000} && break
1211   done
1212   echo ${runNumber}
1213   return 0
1214 )
1215
1216 validateLog()
1217 (
1218   log=${1}
1219   errorConditions=(
1220                     'There was a crash'
1221                     'floating'
1222                     'error while loading shared libraries'
1223                     'std::bad_alloc'
1224                     's_err_syswatch_'
1225                     'Thread [0-9]* (Thread'
1226                     'AliFatal'
1227                     'core dumped'
1228                     '\.C.*error:.*\.h: No such file'
1229                     'line.*Aborted'
1230   )
1231
1232   warningConditions=(
1233                      'This is serious !'
1234                      'rocVoltage out of range:'
1235   )
1236   
1237   local logstatus=0
1238   local errorSummary=""
1239   local warningSummary=""
1240
1241   for ((i=0; i<${#errorConditions[@]};i++)); do
1242     local tmp=$(grep -m1 -e "${errorConditions[${i}]}" ${log})
1243     [[ -n ${tmp} ]] && tmp+=" : "
1244     errorSummary+=${tmp}
1245   done
1246
1247   for ((i=0; i<${#warningConditions[@]};i++)); do
1248     local tmp=$(grep -m1 -e "${warningConditions[${i}]}" ${log})
1249     [[ -n ${tmp} ]] && tmp+=" : "
1250     warningSummary+=${tmp}
1251   done
1252
1253   if [[ -n ${errorSummary} ]]; then 
1254     echo "${errorSummary}"
1255     return 1
1256   fi
1257   
1258   if [[ -n ${warningSummary} ]]; then
1259     echo "${warningSummary}"
1260     return 2
1261   fi
1262
1263   return 0
1264 )
1265
1266 summarizeLogs()
1267 (
1268   #print a summary of logs
1269   logFiles=(
1270             "*.log"
1271             "stdout"
1272             "stderr"
1273   )
1274
1275   #check logs
1276   local logstatus=0
1277   for log in ${logFiles[*]}; do
1278     finallog=${PWD%/}/${log}
1279     [[ ! -f ${log} ]] && continue
1280     errorSummary=$(validateLog ${log})
1281     validationStatus=$?
1282     if [[ ${validationStatus} -eq 0 ]]; then 
1283       #in pretend mode randomly report an error in rec.log some cases
1284       if [[ -n ${pretend} && "${log}" == "rec.log" ]]; then
1285         #[[ $(( ${RANDOM}%2 )) -ge 1 ]] && echo "${finallog} BAD random error" || echo "${finallog} OK"
1286         echo "${finallog} OK"
1287       else
1288         echo "${finallog} OK"
1289       fi
1290     elif [[ ${validationStatus} -eq 1 ]]; then
1291       echo "${finallog} BAD ${errorSummary}"
1292       logstatus=1
1293     elif [[ ${validationStatus} -eq 2 ]]; then
1294       echo "${finallog} OK MWAH ${errorSummary}"
1295     fi
1296   done
1297   
1298   #report core files
1299   while read x; do
1300     echo ${x}
1301     chmod 644 ${x}
1302     gdb --batch --quiet -ex "bt" -ex "quit" aliroot ${x} > stacktrace_${x//\//_}.log
1303   done < <(/bin/ls ${PWD}/*/core 2>/dev/null; /bin/ls ${PWD}/core 2>/dev/null)
1304   
1305   return ${logstatus}
1306 )
1307
1308 spitOutLocalOCDBaccessConfig()
1309 {
1310   umask 0002
1311   #find ${1} -name "*root" | \
1312   /bin/ls -1 ${1}/*/*/*/*.root 2>/dev/null | \
1313   while read line
1314   do 
1315     local tmp=${line#${1}}
1316     echo ${tmp%/*} | \
1317     awk -v ocdb=${1} '{print "  man->SetSpecificStorage(\""$1"\",\"local://"ocdb"\");"}'
1318   done
1319   return 0
1320 }
1321
1322 goMakeLocalOCDBaccessConfig()
1323 {
1324   umask 0002
1325   # make a script that sets the specific storages form all the root files produced by CPass0
1326   local localOCDBpathCPass0=${1}
1327   local OCDBpathPrefix=${2}
1328   [[ -z ${OCDBpathPrefix} ]] && OCDBpathPrefix="."
1329
1330   if [[ -f ${localOCDBpathCPass0} && ${localOCDBpathCPass0} =~ \.tgz$ ]]; then
1331     tar xzf ${localOCDBpathCPass0}
1332     local localOCDBpathCPass0="${OCDBpathPrefix}/OCDB"
1333   fi
1334
1335   echo
1336   echo creating the specific storage script
1337   echo   localOCDBaccessConfig.C
1338   echo   based on OCDB: ${localOCDBaccessConfig}
1339   echo
1340
1341   local tempLocalOCDB=""
1342   if [[ -f localOCDBaccessConfig.C ]]; then
1343     tempLocalOCDB=$(mktemp -t tempLocalOCDB.XXXXXX)
1344     echo "egrep "SetSpecificStorage" localOCDBaccessConfig.C > ${tempLocalOCDB}"
1345     egrep "SetSpecificStorage" localOCDBaccessConfig.C > ${tempLocalOCDB}
1346   fi
1347
1348   echo "localOCDBaccessConfig()"                               >  localOCDBaccessConfig.C
1349   echo "{"                                                     >> localOCDBaccessConfig.C
1350   echo "  AliCDBManager* man = AliCDBManager::Instance();"     >> localOCDBaccessConfig.C
1351   spitOutLocalOCDBaccessConfig ${localOCDBpathCPass0}|sort|uniq  >> localOCDBaccessConfig.C
1352   [[ -f "${tempLocalOCDB}" ]] && cat ${tempLocalOCDB}              >> localOCDBaccessConfig.C
1353   echo "}"                                                     >> localOCDBaccessConfig.C
1354
1355   [[ -f "${tempLocalOCDB}" ]] && rm -f ${tempLocalOCDB}
1356
1357   if ! grep SetSpecificStorage localOCDBaccessConfig.C; then 
1358     echo
1359     echo "!!!!!!! CPass0 produced no OCDB entries"
1360     return 1
1361   fi
1362   return 0
1363 }
1364
1365 goMakeFilteredTrees()
1366 (
1367   outputDir=${1}
1368   runNumber=${2}
1369   #get path to input list
1370   inputListfiles=${3}
1371   #get scale number for tracks
1372   filterT=${4}
1373   #get scale number for V0s
1374   filterV=${5}
1375   #get OCDB path (optional)
1376   OCDBpath=${6}
1377   #get max number of files 
1378   maxFiles=${7-"1000000"}
1379   #get offset of first file
1380   offsetFile=${8-"0"}
1381   #get max number of events
1382   maxEvents=${9-"30000000"}
1383   #get offset of first event
1384   offsetEvent=${10-"0"}
1385   configFile=${11-"benchmark.config"}
1386   esdFileName=${12-"AliESDs_Barrel.root"}
1387   shift 12
1388   if ! parseConfig ${configFile} "$@"; then return 1; fi
1389   
1390   #record the working directory provided by the batch system
1391   batchWorkingDirectory=${PWD}
1392
1393   [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
1394   doneFile=${commonOutputPath}/meta/filtering.cpass1.run${runNumber}.done
1395
1396   cat > filtering.log << EOF
1397   goMakeFilteredTrees config:
1398   runpath=${runpath}
1399   outputDir=${outputDir}
1400   commonOutputPath=${commonOutputPath}
1401   ALICE_ROOT=${ALICE_ROOT}
1402   PATH=${PATH}
1403   offsetEvent=$offsetEvent
1404   configFile=$configFile
1405   esdFileName=$esdFileName
1406   inputListfiles=$inputListfiles
1407   doneFile=$doneFile
1408 EOF
1409
1410   #runpath=${outputDir}
1411   #[[ ${reconstructInTemporaryDir} -eq 1 && -n ${TMPDIR} ]] && runpath=${TMPDIR}
1412   #[[ ${reconstructInTemporaryDir} -eq 1 ]] && runpath=$(mktemp -d -t goMakeFilteredTrees.XXXXXX)
1413   #mkdir -p ${outputDir}
1414   #mkdir -p ${runpath}
1415   #if ! cd ${runpath}; then 
1416   #  echo "PWD=$PWD is not the runpath=${runpath}"
1417   #  touch ${doneFile}
1418   #  return 1
1419   #fi
1420   
1421   if [[ -z ${pretend} ]];then
1422     aliroot -l -b -q "${ALICE_ROOT}/PWGPP/macros/runFilteringTask.C(\"${inputListfiles}\",${filterT},${filterV},\"${OCDBpath}\",${maxFiles},${offsetFile},${maxEvents},${offsetEvent},\"${esdFileName}\")" &>> filtering.log
1423   else
1424     sleep ${pretendDelay}
1425     touch filtering.log FilterEvents_Trees.root
1426   fi
1427   pwd
1428   /bin/ls
1429   touch ${doneFile}
1430   summarizeLogs >>  ${doneFile}
1431   
1432   #echo mv -f * ${outputDir}
1433   #mv -f * ${outputDir}
1434   #[[ -f ${outputDir}/FilterEvents_Trees.root ]] && echo "filteredTree ${outputDir}/FilterEvents_Trees.root" >> ${doneFile}
1435   #cd ${commonOutputPath}
1436   #[[ "${runpath}" != "${outputDir}" ]] && rm -rf ${runpath}
1437   
1438   return 0
1439 )
1440
1441 submit()
1442 {
1443   umask 0002
1444   [[ $# -lt 5 ]] && echo "at least 5 args needed, you supplied $#" && return 1
1445   JobID=${1}
1446   startID=${2}
1447   endID=${3}
1448   waitForJOBID=${4}
1449   command=${5}
1450   shift 5
1451   local commandArgs=("$@")
1452
1453   #add quote strings around the extra arguments
1454   for ((i=0; i<${#commandArgs[@]}; i++)); do 
1455     commandArgs[i]=\"${commandArgs[i]}\"
1456   done
1457
1458   [[ -z ${waitForJOBID} ]] && waitForJOBID=0
1459
1460   newFarm=$(which qsub|grep "^/usr/bin/qsub")
1461   
1462   batchSystem="SGE"
1463
1464   if [[ -z "${newFarm}" ]]
1465   then
1466     #old LSF
1467     # submit it (as job array)
1468     nFiles=$(( ${endID}-${startID}+1 ))
1469     while [ ${startID} -le ${nFiles}  ] ; do
1470       if [ $(expr ${nFiles} - ${startID}) -gt 999 ] ; then 
1471         endID=$(expr ${startID} + 999)
1472       else
1473         endID=${nFiles}
1474       fi      
1475     if [[ ${waitForJOBID} -eq 0 ]]; then
1476         echo ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}"     
1477         ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}"     
1478       else
1479         echo ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -w "ended(${waitForJOBID})" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}"     
1480         ${batchCommand} ${batchFlags} -J "${JobID}[${startID}-${endID}]" -w "ended(${waitForJOBID})" -e "${commonOutputPath}/logs/job_%I.err" -o "${commonOutputPath}/logs/job_%I.out" "${command}"     
1481       fi
1482       startID=$(expr ${endID} + 1)
1483     done
1484   else 
1485     #new SGE farm
1486     if [[ ${waitForJOBID} =~ "000" ]]; then
1487       echo ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}"
1488       ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}"
1489     else
1490       echo ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -hold_jid "${waitForJOBID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}"
1491       ${batchCommand} ${batchFlags} -wd ${commonOutputPath} -b y -v commonOutputPath -N "${JobID}" -t "${startID}-${endID}" -hold_jid "${waitForJOBID}" -e "${commonOutputPath}/logs/" -o "${commonOutputPath}/logs/" "${command}" "${commandArgs[@]}"
1492     fi
1493   fi
1494   return 0
1495 }
1496
1497 goSubmitBatch()
1498 {
1499   if [[ $# -lt 3 ]]; then
1500     echo "minimal use:"
1501     echo " ${0} submit fileList productionID configFile"
1502     return 0
1503   fi
1504
1505   productionID=${1}
1506   inputList=${2}
1507   configFile=${3:-"benchmark.config"}
1508   #if which greadlink; then configFile=$(greadlink -f ${configFile}); fi
1509   shift 3
1510   extraOpts=("$@")
1511   if ! parseConfig ${configFile} "${extraOpts[@]}"; then return 1; fi
1512   
1513   #batch systems/makeflow sometimes handle spaces in arguments poorly, so encode them
1514   for (( i=0;i<${#extraOpts[@]};i++ )); do 
1515     extraOpts[i]=$(encSpaces "${extraOpts[i]}")
1516   done
1517   extraOpts+=("encodedSpaces=1")
1518
1519   #record the working directory provided by the batch system
1520   batchWorkingDirectory=${PWD}
1521
1522   #redirect all output to submit.log
1523   echo "redirecting all output to ${PWD}/submit_${productionID//"/"/_}.log"
1524   exec 7>&1
1525   exec 1>submit_${productionID//"/"/_}.log 2>&1
1526
1527   umask 0002
1528   echo ${0}" submit $*"
1529   if [[ -z "${inputList}" || -z "${productionID}" ]]
1530   then
1531     echo
1532     echo " Usage: ${0} submit inputList productionID [configFile=benchmark.config]"
1533     echo
1534     return
1535   fi
1536
1537   # check if config file is there
1538   if [ ! -f ${configFile} ]; then
1539     echo "ERROR! Config File '${configFile}' not found" >&2
1540     return
1541   else
1542     echo "Using Config File: '${configFile}'"
1543   fi
1544
1545   [[ ! -f ${alirootEnv} ]] && echo "alirootEnv script ${alirootEnv} not found!..." && return 1
1546
1547   #move the script, config and some other stuff to ${commonOutputPath} first, then use them from there
1548   self=${0}
1549   #if which greadlink; then self=$(greadlink -f "${0}"); fi
1550   configPath=$(dirname ${configFile})
1551   export commonOutputPath=${baseOutputDirectory}/${productionID}
1552   
1553   mkdir -p ${commonOutputPath}
1554   mkdir -p ${commonOutputPath}/logs
1555   mkdir -p ${commonOutputPath}/meta
1556
1557   cp ${self} ${commonOutputPath}
1558   cp ${configFile} ${commonOutputPath}
1559   cp ${inputList} ${commonOutputPath}
1560   self=${commonOutputPath}/${self##*/}
1561   chmod u+x ${self}
1562   configFile=${commonOutputPath}/${configFile##*/}
1563   inputList=${commonOutputPath}/${inputList##*/}
1564
1565   #convert to absolut pathnames
1566   #if which greadlink; then inputList=$(greadlink -f "${inputList}"); fi
1567   #make list of runs
1568   if [[ -z ${runNumber} ]]; then
1569     listOfRuns=($(while read x; do guessRunNumber ${x}; done < ${inputList} | sort | uniq))
1570   else
1571     listOfRuns=${runNumber}
1572   fi
1573
1574   #if which greadlink; then alirootSource=$(greadlink -f "${alirootSource}"); fi
1575
1576   echo ""
1577   echo "### BEGIN CONFIGURATION ###"
1578   echo ""
1579   echo "GENERAL:"
1580   echo ""
1581   echo "    productionID:    ${productionID}"
1582   echo "    batchCommand:    ${batchCommand}"
1583   echo "    batchFlags:      ${batchFlags}"
1584   echo "    alirootEnv:   ${alirootEnv}"
1585   ${alirootEnv} echo '    ALICE_ROOT:      ${ALICE_ROOT}'
1586   ${alirootEnv} echo '    ALIROOT_RELEASE: ${ALICE_RELEASE}'
1587   echo "    inputList:       ${inputList}"
1588   echo "    configPath:      ${configPath}"
1589   echo "    commonOutputPath:      ${commonOutputPath}"
1590   echo "    defaultOCDB:     ${defaultOCDB}"
1591   echo "      autoOCDB: ${autoOCDB}"
1592   echo "    recoTriggerOptions:   ${recoTriggerOptions}"
1593   echo "    runs:"
1594   echo "      ${listOfRuns[*]}"
1595   echo ""
1596   echo "THE TRAIN WILL RUN:"
1597
1598   if [ ${runCPass0reco} -eq 1 ]; then
1599     echo "    Pass0 - Recontruction"
1600   fi
1601
1602   if [ ${runCPass0MergeMakeOCDB} -eq 1 ]; then
1603     echo "    Pass0 - merging and OCDB export"
1604   fi
1605
1606   if [ ${runCPass1reco} -eq 1 ]; then
1607     echo "    Pass1 - Recontruction"
1608   fi
1609   if [ ${runCPass1MergeMakeOCDB} -eq 1 ]; then
1610     echo "    Pass1 - merging and OCDB export"
1611   fi
1612
1613   echo ""
1614   echo "LIMITS:"
1615   echo "    max. Events/Chunk:   ${nEvents}"
1616   echo "    max. Number of Chunks per Run:     ${nMaxChunks}"
1617   echo ""
1618   echo "### END CONFIGURATION ###"
1619   echo ""
1620
1621
1622   # check if input file is there
1623   if [ ! -f ${inputList} ]; then
1624     echo "ERROR! Input List '${inputList}' not found" >&2
1625     return
1626   fi
1627
1628   # define jobid (for dependent jobs)
1629   date=$(date +%Y_%m_%d_%H%M%S)
1630   #for each run we submit one jobarray:
1631   for runNumber in ${listOfRuns[*]}; do
1632     
1633     [[ -z ${runNumber} ]] && continue
1634     [[ ! ${runNumber} =~ ^[0-9]*[0-9]$ ]] && continue
1635
1636     JOBpostfix="${productionID//"/"/_}_${runNumber}_${date}"
1637     JOBID1="p0_${JOBpostfix}"
1638     JOBID1wait="w0_${JOBpostfix}"
1639     JOBID2="m0_${JOBpostfix}"
1640     JOBID2wait="wm0_${JOBpostfix}"
1641     JOBID3="op0_${JOBpostfix}"
1642     JOBID3wait="wop0_${JOBpostfix}"
1643     JOBID4="p1_${JOBpostfix}"
1644     JOBID4wait="w1_${JOBpostfix}"
1645     JOBID5="m1_${JOBpostfix}"
1646     JOBID5wait="wm1_${JOBpostfix}"
1647     JOBID6="s1_${JOBpostfix}"
1648     JOBID6wait="ws1_${JOBpostfix}"
1649     JOBID7="QA_${JOBpostfix}"
1650     JOBmakeESDlistCPass1="lp1_${JOBpostfix}"
1651     JOBfilterESDcpass1="fp1_${JOBpostfix}"
1652     LASTJOB="000"
1653
1654     oneInputFile=$(egrep -m1 "${runNumber}/" ${inputList})
1655
1656     currentDefaultOCDB=${defaultOCDB}
1657     [[ -z ${autoOCDB} ]] && autoOCDB=1
1658     if [[ ${autoOCDB} -ne 0 ]]; then
1659       currentDefaultOCDB=$(setYear ${oneInputFile} ${defaultOCDB})
1660     fi
1661
1662     echo "submitting run ${runNumber} with OCDB ${currentDefaultOCDB}"
1663
1664     ###############################################################################
1665     #run one chunk with valgrind:
1666     if [[ -n ${profilingCommand} ]]; then
1667       [[ -z ${nEventsProfiling} ]] && nEventsProfiling=2
1668       [[ -z ${profilingCommand} ]] && profilingCommand="/usr/bin/valgrind --tool=callgrind --num-callers=40 -v --trace-children=yes"
1669       submit "valgrind" 1 1 000 "${alirootEnv} ${self}" CPass0 ${commonOutputPath}/000${runNumber}/valgrind ${oneInputFile} ${nEventsProfiling} ${currentDefaultOCDB} ${configFile} ${runNumber} valgrind useProfilingCommand=${profilingCommand} "${extraOpts[@]}"
1670     fi 
1671
1672     ################################################################################
1673     ################################################################################
1674     # run the CPass0 if requested
1675
1676     if [ ${runCPass0reco} -eq 1 ]; then
1677
1678       echo
1679       echo "starting CPass0... for run ${runNumber}"
1680       echo
1681
1682       # create directory and copy all files that are needed
1683       targetDirectory="${commonOutputPath}/000${runNumber}/cpass0"
1684       mkdir -p ${targetDirectory}
1685
1686       filesCPass0=( 
1687                     "${configPath}/runCPass0.sh"
1688                     "${configPath}/recCPass0.C"
1689                     "${configPath}/runCalibTrain.C"
1690                     "${configPath}/localOCDBaccessConfig.C"
1691                     "${configPath}/OCDB*.root"
1692                     "${configPath}/sim.C"
1693                     "${configPath}/rec.C"
1694                     "${configPath}/Config.C"
1695       )
1696       for file in ${filesCPass0[*]}; do
1697         [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
1698       done
1699
1700       localInputList=${targetDirectory}/${inputList##*/}
1701       [[ ! -f ${localInputList} ]] && egrep "\/000${runNumber}\/" ${inputList} > ${localInputList}
1702       # limit nFiles to nMaxChunks
1703       nFiles=$(wc -l < ${localInputList})
1704       [[ ${nFiles} -eq 0 ]] && echo "list contains ZERO files! exiting..." && return 1
1705       echo "raw files in list:    ${nFiles}"
1706       if [[ ${nMaxChunks} -gt 0 && ${nMaxChunks} -le ${nFiles} ]]; then
1707         nFiles=${nMaxChunks}
1708       fi
1709       echo "raw files to process: ${nFiles}"
1710       [[ -z "${percentProcessedFilesToContinue}" ]] && percentProcessedFilesToContinue=100
1711       if [[ ${percentProcessedFilesToContinue} -eq 100 ]]; then
1712         nFilesToWaitFor=${nFiles}
1713       else
1714         nFilesToWaitFor=$(( ${nFiles}-${nFiles}/(100/(100-${percentProcessedFilesToContinue})) ))
1715       fi
1716       echo "requested success rate is ${percentProcessedFilesToContinue}%"
1717       echo "merging will start after ${nFilesToWaitFor} jobs are done"
1718
1719       submit ${JOBID1} 1 ${nFiles} 000 "${alirootEnv} ${self}" CPass0 ${targetDirectory} ${localInputList} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} -1 "${extraOpts[@]}"
1720
1721       ## submit a monitoring job that will run until a certain number of jobs are done with reconstruction
1722       submit "${JOBID1wait}" 1 1 000 "${alirootEnv} ${self}" WaitForOutput ${commonOutputPath} "meta/cpass0.job*.run${runNumber}.done" ${nFilesToWaitFor} ${maxSecondsToWait} '-maxdepth 1'
1723       LASTJOB=${JOBID1wait}
1724
1725     fi #end running CPass0
1726     ################################################################################
1727
1728
1729     ################################################################################
1730     # submit merging of CPass0, depends on the reconstruction
1731
1732     if [ ${runCPass0MergeMakeOCDB} -eq 1 ]; then
1733
1734       echo
1735       echo "submit CPass0 merging for run ${runNumber}"
1736       echo
1737
1738       targetDirectory="${commonOutputPath}/000${runNumber}/cpass0"
1739       mkdir -p ${targetDirectory}
1740
1741       #copy the scripts
1742       filesMergeCPass0=(
1743                         "${configPath}/OCDB.root"
1744                         "${configPath}/mergeMakeOCDB.byComponent.sh"
1745                         "${configPath}/mergeMakeOCDB.sh"
1746                         "${configPath}/localOCDBaccessConfig.C"
1747                         "${configPath}/mergeByComponent.C"
1748                         "${configPath}/makeOCDB.C"
1749                         "${configPath}/merge.C"
1750       )
1751       for file in ${filesMergeCPass0[*]}; do
1752         [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
1753       done
1754   
1755       submit ${JOBID2} 1 1 "${LASTJOB}" "${alirootEnv} ${self}" MergeCPass0 ${targetDirectory} ${currentDefaultOCDB} ${configFile} ${runNumber} cpass0.calib.run${runNumber}.list "${extraOpts[@]}"
1756       LASTJOB=${JOBID2}
1757
1758       if [[ -n ${generateMC} ]]; then
1759         submit "mrl${JOBpostfix}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" PrintValues sim ${commonOutputPath}/meta/sim.run${runNumber}.list ${commonOutputPath}/meta/cpass0.job*.run${runNumber}.done
1760         LASTJOB="mrl${JOBpostfix}"
1761       fi
1762
1763       echo
1764     fi
1765     # end of merging CPass0
1766     ################################################################################
1767
1768     ################################################################################
1769     ################################################################################
1770     # run the CPass1 if requested
1771
1772     if [ ${runCPass1reco} -eq 1 ]; then
1773
1774       targetDirectory="${commonOutputPath}/000${runNumber}/cpass1"
1775       rm -f ${commonOutputPath}/meta/cpass1.job*.run${runNumber}.done
1776
1777       # safety feature: if we are re-running for any reason we want to delete the previous output first.
1778       [[ -d ${targetDirectory} ]] && rm -rf ${targetDirectory}/* && echo "removed old output at ${targetDirectory}/*"
1779
1780       echo
1781       echo "starting CPass1... for run ${runNumber}"
1782       echo
1783
1784       # create directory and copy all files that are needed
1785       mkdir -p ${targetDirectory}
1786       
1787       filesCPass1=( 
1788                     "${configPath}/runCPass1.sh"
1789                     "${configPath}/recCPass1.C"
1790                     "${configPath}/recCPass1_OuterDet.C"
1791                     "${configPath}/runCalibTrain.C"
1792                     "${configPath}/QAtrain_duo.C"
1793                     "${configPath}/localOCDBaccessConfig.C"
1794                     "${configPath}/OCDB.root"
1795       )
1796       for file in ${filesCPass1[*]}; do
1797         [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
1798       done
1799
1800       if [[ -n ${generateMC} ]]; then
1801         localInputList=${commonOutputPath}/meta/sim.run${runNumber}.list
1802       else
1803         localInputList=${targetDirectory}/${inputList##*/}
1804         [[ ! -f ${localInputList} ]] && egrep "\/000${runNumber}\/" ${inputList} > ${localInputList}
1805       fi
1806       # limit nFiles to nMaxChunks
1807       nFiles=$(wc -l < ${localInputList})
1808       [[ ${nFiles} -eq 0 ]] && echo "list contains ZERO files! continuing..." && continue
1809       echo "raw files in list:    ${nFiles}"
1810       if [[ ${nMaxChunks} -gt 0 && ${nMaxChunks} -le ${nFiles} ]]; then
1811         nFiles=${nMaxChunks}
1812       fi
1813       echo "raw files to process: ${nFiles}"
1814       [[ -z "${percentProcessedFilesToContinue}" ]] && percentProcessedFilesToContinue=100
1815       if [[ ${percentProcessedFilesToContinue} -eq 100 ]]; then
1816         nFilesToWaitFor=${nFiles}
1817       else
1818         nFilesToWaitFor=$(( ${nFiles}-${nFiles}/(100/(100-${percentProcessedFilesToContinue})) ))
1819       fi
1820       echo "requested success rate is ${percentProcessedFilesToContinue}%"
1821       echo "merging will start after ${nFilesToWaitFor} jobs are done"
1822
1823       submit ${JOBID4} 1 ${nFiles} "${LASTJOB}" "${alirootEnv} ${self}" CPass1 ${targetDirectory} ${localInputList} ${nEvents} ${currentDefaultOCDB} ${configFile} ${runNumber} -1 "${extraOpts[@]}"
1824
1825       ################################################################################
1826       ## submit a monitoring job that will run until a certain number of jobs are done with reconstruction
1827       submit "${JOBID4wait}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" WaitForOutput ${commonOutputPath} "meta/cpass1.job*.run${runNumber}.done" ${nFilesToWaitFor} ${maxSecondsToWait} '-maxdepth 1'
1828       LASTJOB=${JOBID4wait}
1829       ################################################################################
1830
1831       echo
1832     fi #end running CPass1
1833
1834     ################################################################################
1835     # submit merging of CPass1, depends on the reconstruction
1836     if [ ${runCPass1MergeMakeOCDB} -eq 1 ]; then
1837
1838       echo
1839       echo "submit CPass1 merging for run ${runNumber}"
1840       echo
1841
1842       targetDirectory="${commonOutputPath}/000${runNumber}/cpass1"
1843       rm -f ${commonOutputPath}/meta/merge.cpass1.run${runNumber}.done
1844       mkdir -p ${targetDirectory}
1845
1846       # copy files 
1847       filesMergeCPass1=(
1848                         "${configPath}/OCDB.root"
1849                         "${configPath}/localOCDBaccessConfig.C"
1850                         "${configPath}/mergeMakeOCDB.byComponent.sh"
1851                         "${configPath}/mergeByComponent.C"
1852                         "${configPath}/makeOCDB.C"
1853                         "${configPath}/merge.C"
1854                         "${configPath}/mergeMakeOCDB.sh"
1855                         "${configPath}/QAtrain_duo.C"
1856       )
1857       for file in ${filesMergeCPass1[*]}; do
1858         [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
1859       done
1860
1861       submit "${JOBID5}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" MergeCPass1 ${targetDirectory} ${currentDefaultOCDB} ${configFile} ${runNumber} cpass1.calib.run${runNumber}.list cpass1.QA.run${runNumber}.lastMergingStage.txt.list cpass1.filtered.run${runNumber}.list "${extraOpts[@]}"
1862       LASTJOB=${JOBID5}
1863       echo
1864     fi
1865
1866     ###############################
1867     #if [ ${runESDfiltering} -eq 1 ]; then
1868     #  rm -f ${commonOutputPath}/cpass1.ESD.run${runNumber}.list
1869     #  rm -f ${commonOutputPath}/meta/filtering.cpass1.run*.done
1870     #  echo
1871     #  echo submitting filtering for run ${runNumber}
1872     #  echo
1873     #  submit "${JOBmakeESDlistCPass1}" 1 1 "${LASTJOB}" "${self}" PrintValues esd ${commonOutputPath}/meta/cpass1.ESD.run${runNumber}.list ${commonOutputPath}/meta/cpass1.job*.run${runNumber}.done 
1874     #  submit "${JOBfilterESDcpass1}" 1 1 "${JOBmakeESDlistCPass1}" "${alirootEnv} ${self}" MakeFilteredTrees ${commonOutputPath}/000${runNumber}/cpass1 ${runNumber} ${commonOutputPath}/meta/cpass1.ESD.run${runNumber}.list ${filteringFactorHighPt} ${filteringFactorV0s} ${currentDefaultOCDB} 1000000 0 10000000 0 ${configFile} AliESDs_Barrel.root "${extraOpts[@]}"
1875     #  LASTJOB=${JOBfilterESDcpass1}
1876     #fi
1877
1878   done
1879
1880   #################################################################################
1881   #################################################################################
1882   #if [ ${runESDfiltering} -eq 1 ]; then
1883   #  submit "${JOBID5wait}" 1 1 "${LASTJOB}" "${self}" WaitForOutput ${commonOutputPath} "meta/filtering.cpass1.run*.done" "${#listOfRuns[@]}" ${maxSecondsToWait}
1884   #else
1885     submit "${JOBID5wait}" 1 1 "${LASTJOB}" "${self}" WaitForOutput ${commonOutputPath} "meta/merge.cpass1.run*.done" ${#listOfRuns[@]} ${maxSecondsToWait}
1886   #fi
1887   LASTJOB=${JOBID5wait}
1888
1889   #################################################################################
1890   echo
1891   echo "submit make a summary"
1892   echo
1893
1894   submit "${JOBID6}" 1 1 "${LASTJOB}" "${alirootEnv} ${self}" MakeSummary ${configFile}
1895   LASTJOB=${JOBID6}
1896   #################################################################################
1897   
1898   #restore stdout
1899   exec 1>&7 7>&-
1900   echo "jobs submitted."
1901   return 0
1902 }
1903
1904 goWaitForOutput()
1905 (
1906   umask 0002
1907   [[ $# -lt 3 ]] && echo "goWaitForOutput() wrong number of arguments, exiting.." && return 1
1908   echo searchPath=${1}
1909   echo fileName=${2}
1910   echo numberOfFiles=${3}
1911   echo maxSecondsToWait=${4}
1912   searchPath=${1}
1913   fileName=${2}
1914   numberOfFiles=${3}
1915   maxSecondsToWait=${4}
1916   extraFindOptions=${5}
1917   echo "command to be executed: /bin/ls -1 ${searchPath}/${fileName} ${extraFindOptions}"
1918   [[ -z "${maxSecondsToWait}" ]] && maxSecondsToWait=$(( 3600*12 ))
1919   while true; do
1920     #n=$(find ${searchPath} ${extraFindOptions} -name "${fileName}" | wc -l)
1921     n=$(/bin/ls -1 ${searchPath}/${fileName} 2>/dev/null | wc -l)
1922     [[ ${n} -gt 0 ]] && echo "found ${n} X ${fileName}"
1923     [[ ${n} -ge ${numberOfFiles} ]] && break
1924     [[ ${SECONDS} -gt ${maxSecondsToWait} ]] && echo "timeout of ${maxSecondsToWait}s!" && break
1925     sleep 60
1926   done
1927   echo "DONE! exiting..."
1928   return 0
1929 )
1930
1931 mergeSysLogs()
1932 (
1933   outputFile=${1}
1934   shift
1935   inputFiles="$@"
1936   i=0
1937   if ! ls -1 ${inputFiles} &>/dev/null; then echo "the files dont exist!: ${inputFiles}"; return 1; fi
1938   while read x; do 
1939     runNumber=$(guessRunNumber ${x})
1940     [[ -z ${runNumber} ]] && echo "run number cannot be guessed for ${x}" && continue
1941     awk -v run=${runNumber} -v i=${i} 'NR > 1 {print run" "$0} NR==1 && i==0 {print "run/I:"$0}' ${x}
1942     (( i++ ))
1943   done < <(ls -1 ${inputFiles}) > ${outputFile}
1944   return 0
1945 )
1946
1947 goMakeMergedSummaryTree()
1948 (
1949   # create list of calibration entries
1950   # takes no arguments, just run it in the base output
1951   # directory with the following files in the working directory
1952   # 
1953   #  Calibration file lists:
1954   #       cpass0.dcsTree.list, cpass1.dcsTree.list
1955   #  QA trending root file:
1956   #       trending.root
1957   # 
1958   #  Production infoo ascii files:
1959   #       summary_pass0.tree
1960   #       summary_pass1.tree
1961   #    
1962
1963   [[ ! -f cpass0.dcsTree.list ]] && echo "no cpass0.dcsTree.list" && return 1
1964   [[ ! -f cpass1.dcsTree.list ]] && echo "no cpass1.dcsTree.list" && return 1
1965   [[ ! -f trending.root ]] && echo "no trending.root" && return 1
1966   [[ ! -f summary_pass0.tree ]] && echo "no summary_pass0.tree" && return 1
1967   [[ ! -f summary_pass1.tree ]] && echo "no summary_pass1.tree" && return 1
1968
1969   #first, dump the C macro to file
1970   cat << EOF > mergeTree.C
1971   //
1972   // Merge summary information
1973   // Following files are expected to be in the working directory
1974   //
1975   // Calibration file lists:
1976   //      cpass0.dcsTree.list, cpass1.dcsTree.list
1977   // QA trending root file:
1978   //      trending.root
1979   //
1980   // Production infoo ascii files:
1981   //      summary_pass0.tree
1982   //      summary_pass1.tree
1983   //   
1984   void mergeTree(){
1985     //
1986     //
1987     //
1988     // Calibration values dump
1989     //
1990     //Printf("MakeTreeFromList cpass0.dcsTree.list");
1991     AliXRDPROOFtoolkit::MakeTreeFromList("Calib.TPC.CPass0.root", "dcs","dcs","cpass0.dcsTree.list",1);
1992     //Printf("MakeTreeFromList cpass1.dcsTree.list");
1993     AliXRDPROOFtoolkit::MakeTreeFromList("Calib.TPC.CPass1.root", "dcs","dcs","cpass1.dcsTree.list",1);
1994     //
1995     // Calibration status dump
1996     //
1997     TFile *fprod = TFile::Open("fproduction.root","recreate");
1998     TTree  tree0, tree1;
1999     //Printf("reading summary_pass0.tree");
2000     tree0.ReadFile("summary_pass0.tree");
2001     //Printf("reading summary_pass1.tree");
2002     tree1.ReadFile("summary_pass1.tree");
2003     tree0.Write("CPass0");
2004     tree1.Write("CPass1");
2005     fprod->Close();
2006     //
2007     //
2008     //
2009     TString stringSetup="";
2010     stringSetup+="1#QA.TPC#run#SummaryTPCQA/tpcQA#trending.root+";  // 
2011     stringSetup+="1#Calib.TPC.CPass0#run#dcs#Calib.TPC.CPass0.root+";  // 
2012     stringSetup+="1#Calib.TPC.CPass1#run#dcs#Calib.TPC.CPass1.root+";  // 
2013     //
2014     stringSetup+="1#CPass0#runnumber#CPass0#fproduction.root+";  // 
2015     stringSetup+="1#CPass1#runnumber#CPass1#fproduction.root+";  // 
2016     //
2017     //Printf("stringSetup: %s", stringSetup.Data());
2018     AliXRDPROOFtoolkit::JoinTreesIndex("outAll.root","joinAll","run",stringSetup.Data(), 1);
2019   }
2020 EOF
2021
2022   aliroot -b -q "mergeTree.C" > mergeTrees.log
2023   return $?
2024 )
2025
2026 goMakeSummary()
2027 (
2028   #all the final stuff goes in here for ease of use:
2029   # summary logs
2030   # qa plot making
2031   # final file lists
2032   #some defaults:
2033   log="summary.log"
2034   productionID="qa"
2035
2036   configFile=${1}
2037   shift 1
2038   extraOpts=("$@")
2039   if ! parseConfig ${configFile} "${extraOpts[@]}"; then return 1; fi
2040   
2041   #if which greadlink; then configFile=$(greadlink -f ${configFile}); fi
2042   
2043   #record the working directory provided by the batch system
2044   batchWorkingDirectory=${PWD}
2045   
2046   [[ -f ${alirootSource} && -z ${ALICE_ROOT} ]] && source ${alirootSource}
2047
2048   [[ ! -f ${configFile} ]] && echo "no config file ${configFile}!" && return
2049
2050   [[ -z ${commonOutputPath} ]] && commonOutputPath=${PWD}
2051
2052   #copy some useful stuff
2053   #and go to the commonOutputPath
2054   cp ${configFile} ${commonOutputPath}
2055   cd ${commonOutputPath}
2056
2057   exec &> >(tee ${log})
2058
2059   #summarize the global stuff
2060   echo "env script: ${alirootSource} ${alirootEnv}"
2061   echo "\$ALICE_ROOT=${ALICE_ROOT}"
2062   echo "commonOutputPath=${commonOutputPath}"
2063
2064   #summarize the stacktraces
2065   awk '
2066        BEGIN { 
2067                print "frame/I:method/C:line/C:cpass/I:aliroot/I";
2068                RS="#[0-9]*";
2069                aliroot=0;
2070              } 
2071              { 
2072                if ($3 ~ /Ali*/) aliroot=1; else aliroot=0;
2073                gsub("#","",RT); 
2074                if ($NF!="" && RT!="" && $3!="") print RT" "$3" "$NF" "0" "aliroot
2075              }
2076       ' 000*/cpass0/*/stacktrace* 2>/dev/null > stacktrace.tree
2077   awk '
2078        BEGIN {
2079                RS="#[0-9]*";
2080                aliroot=0;
2081              } 
2082              {
2083                if ($3 ~ /Ali*/) aliroot=1; else aliroot=0;
2084                gsub("#","",RT); 
2085                if ($NF!="" && RT!="" && $3!="") print RT" "$3" "$NF" "1" "aliroot
2086              }
2087       ' 000*/cpass1/*/stacktrace* 2>/dev/null >> stacktrace.tree
2088
2089   echo total numbers for the production:
2090   echo
2091   awk 'BEGIN {nFiles=0;nCore=0;} 
2092   /^calibfile/ {nFiles++;} 
2093   /core dumped/ {nCore++i;}
2094   END {print     "cpass0 produced "nFiles" calib files, "nCore" core files";}' meta/cpass0.job*done 2>/dev/null
2095   awk 'BEGIN {nOK=0; nBAD=0; } 
2096   /\/rec.log OK/ {nOK++;} 
2097   /\/rec.log BAD/ {nBAD++;} 
2098   /stderr BAD/ {if ($0 ~ /rec.log/){nBAD++;}}
2099   END {print     "cpass0 reco:  OK: "nOK"\tBAD: "nBAD;}' meta/cpass0.job*done 2>/dev/null
2100   awk 'BEGIN {nOK=0; nBAD=0; } 
2101   /\/calib.log OK/ {nOK++;} 
2102   /\/calib.log BAD/ {nBAD++;} 
2103   END {print "cpass0 calib: OK: "nOK"\tBAD: "nBAD;}' meta/cpass0.job*done 2>/dev/null
2104
2105   awk 'BEGIN {nOK=0; nBAD=0; } 
2106   /merge.log OK/ {nOK++;} 
2107   /merge.log BAD/ {nBAD++;} 
2108   END {print "cpass0 merge: OK: "nOK"\tBAD: "nBAD;}' meta/merge.cpass0*done 2>/dev/null
2109   awk 'BEGIN {nOK=0; nBAD=0; } 
2110   /ocdb.log OK/ {nOK++;} 
2111   /ocdb.log BAD/ {nBAD++;} 
2112   END {print   "cpass0 OCDB:  OK: "nOK"\tBAD: "nBAD;}' meta/merge.cpass0*done 2>/dev/null
2113
2114   echo
2115   awk 'BEGIN {nFiles=0;nCore=0;} 
2116   /^calibfile/ {nFiles++;} 
2117   /core dumped/ {nCore++;}
2118   END {print     "cpass1 produced "nFiles" calib files, "nCore" core files";}' meta/cpass1.job*done 2>/dev/null
2119   awk 'BEGIN {nOK=0; nBAD=0; } 
2120   /\/rec.log OK/ {nOK++;} 
2121   /\/rec.log BAD/ {nBAD++;} 
2122   /stderr BAD/ {if ($0 ~ /rec.log/){nBAD++;}}
2123   END {print     "cpass1 reco:  OK: "nOK"\tBAD: "nBAD;}' meta/cpass1.job*done 2>/dev/null
2124   awk 'BEGIN {nOK=0; nBAD=0; } 
2125   /\/calib.log OK/ {nOK++;} 
2126   /\/calib.log BAD/ {nBAD++;} 
2127   END {print "cpass1 calib: OK: "nOK"\tBAD: "nBAD;}' meta/cpass1.job*done 2>/dev/null
2128
2129   awk 'BEGIN {nOK=0; nBAD=0; } 
2130   /merge.log OK/ {nOK++;} 
2131   /merge.log BAD/ {nBAD++;} 
2132   END {print "cpass1 merge: OK: "nOK"\tBAD: "nBAD;}' meta/merge.cpass1*done 2>/dev/null
2133   awk 'BEGIN {nOK=0; nBAD=0; } 
2134   /ocdb.log OK/ {nOK++;} 
2135   /ocdb.log BAD/ {nBAD++;} 
2136   END {print   "cpass1 OCDB:  OK: "nOK"\tBAD: "nBAD;}' meta/merge.cpass1*done 2>/dev/null
2137
2138   echo
2139   echo per run stats:
2140   /bin/ls -1 meta/merge.cpass0.run*.done | while read x 
2141 do
2142   dir=$(goPrintValues calibfile - ${x})
2143   runNumber=$(guessRunNumber ${dir})
2144   [[ -z ${runNumber} ]] && continue
2145
2146   if $(/bin/ls meta/cpass0.job*.run${runNumber}.done &> /dev/null); then
2147     statusCPass0=( $(
2148     awk 'BEGIN {nOKrec=0;nBADrec=0;nOKcalib=0;nBADcalib=0;nOKstderr=0;nBADstderr=0;}
2149     /\/rec.log OK/ {nOKrec++;} 
2150     /\/rec.log BAD/ {nBADrec++;}
2151     /stderr BAD/ {if ($0 ~ /rec.log/) {nBADrec++;} nBADstderr++;}
2152     /stderr OK/ {nOKstderr++;}
2153     /\/calib.log OK/ {nOKcalib++;}
2154     /\/calib.log BAD/ {nBADcalib++}
2155     END {print ""nOKrec" "nBADrec" "nOKstderr" "nBADstderr" "nOKcalib" "nBADcalib;}' meta/cpass0.job*.run${runNumber}.done 2>/dev/null
2156     ) ) 
2157   fi
2158
2159   if $(/bin/ls meta/cpass1.job*.run${runNumber}.done &>/dev/null); then
2160     statusCPass1=( $(
2161     awk 'BEGIN {nOKrec=0;nBADrec=0;nOKcalib=0;nBADcalib=0;nOKstderr=0;nBADstderr=0;nQAbarrelOK=0;nQAbarrelBAD=0;nQAouterOK=0;nQAouterBAD=0;}
2162     /\/rec.log OK/ {nOKrec++;} 
2163     /\/rec.log BAD/ {nBADrec++;}
2164     /stderr BAD/ {if ($0 ~ /rec.log/) nBADrec++;nBADstderr++;}
2165     /stderr OK/ {nOKstderr++;}
2166     /\/calib.log OK/ {nOKcalib++;}
2167     /\/calib.log BAD/ {nBADcalib++}
2168     /\/qa_barrel.log OK/ {nQAbarrelOK++;}
2169     /\/qa_barrel.log BAD/ {nQAbarrelBAD++;}
2170     /\/qa_outer.log OK/ {nQAouterOK++;}
2171     /\/qa_outer.log BAD/ {nQAouterBAD++;}
2172     END {print ""nOKrec" "nBADrec" "nOKstderr" "nBADstderr" "nOKcalib" "nBADcalib" "nQAbarrelOK" "nQAbarrelBAD" "nQAouterOK" "nQAouterBAD;}' meta/cpass1.job*.run${runNumber}.done 2>/dev/null
2173     ) ) 
2174   fi
2175
2176   statusOCDBcpass0=$(awk '/ocdb.log/ {print $2} ' ${x} 2>/dev/null)
2177   statusOCDBcpass1=$(awk '/ocdb.log/ {print $2}' ${x/cpass0/cpass1} 2>/dev/null)
2178   statusQA=$(awk '/mergeMakeOCDB.log/ {print $2}' ${x/cpass0/cpass1} 2>/dev/null)
2179
2180   printf "%s\t ocdb.log cpass0: %s\t ocdb.log cpass1: %s\tqa.log:%s\t| cpass0: rec:%s/%s stderr:%s/%s calib:%s/%s cpass1: rec:%s/%s stderr:%s/%s calib:%s/%s QAbarrel:%s/%s QAouter:%s/%s\n" ${runNumber} ${statusOCDBcpass0} ${statusOCDBcpass1} ${statusQA} ${statusCPass0[0]} ${statusCPass0[1]} ${statusCPass0[2]} ${statusCPass0[3]} ${statusCPass0[4]} ${statusCPass0[5]} ${statusCPass1[0]} ${statusCPass1[1]} ${statusCPass1[2]} ${statusCPass1[3]} ${statusCPass1[4]} ${statusCPass1[5]} ${statusCPass1[6]} ${statusCPass1[7]} ${statusCPass1[8]} ${statusCPass1[9]}
2181 done
2182
2183   #make lists with output files - QA, trending, filtering and calibration
2184   ### wait for the merging of all runs to be over ###
2185   rm -f qa.list
2186   goPrintValues qafile qa.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2187   rm -f calib.list
2188   goPrintValues calibfile calib.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2189   rm -f trending.list
2190   goPrintValues trendingfile trending.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2191   rm -f filtering.list
2192   goPrintValues filteredTree filtering.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2193   rm -f cpass0.dcsTree.list
2194   goPrintValues dcsTree cpass0.dcsTree.list ${commonOutputPath}/meta/merge.cpass0.run*.done &>/dev/null
2195   rm -f cpass1.dcsTree.list
2196   goPrintValues dcsTree cpass1.dcsTree.list ${commonOutputPath}/meta/merge.cpass1.run*.done &>/dev/null
2197  
2198   #merge trending
2199   rm -f ${commonOutputPath}/trending_merged.root
2200   goMerge trending.list ${commonOutputPath}/trending.root ${configFile} "${extraOpts[@]}" &> mergeTrending.log
2201
2202   goMakeSummaryTree ${commonOutputPath} 0
2203   goMakeSummaryTree ${commonOutputPath} 1
2204
2205   goCreateQAplots "${PWD}/qa.list" "${productionID}" "QAplots" "${configFile}" "${extraOpts[@]}" filteringList="${PWD}/filtering.list" &>createQAplots.log
2206
2207   #make a merged summary tree out of the QA trending, dcs trees and log summary trees
2208   goMakeMergedSummaryTree
2209
2210   #if set, email the summary
2211   [[ -n ${MAILTO} ]] && cat ${log} | mail -s "benchmark ${productionID} done" ${MAILTO}
2212
2213   return 0
2214 )
2215
2216 goMakeSummaryTree()
2217 (
2218   if [[ $# -lt 1 ]] ; then
2219     return
2220   fi
2221   #1. define vars/arrays
2222   DIR=${1} #use input or exec in current dir
2223   pass=${2-"0"} #pass from input
2224   outfile="summary_pass${pass}.tree"
2225   Ncolumns=0
2226   test -f ${outfile} && : >${outfile}
2227   errfile=${outfile/tree/err}
2228   test -f ${errfile} && : >${errfile}
2229
2230   declare -a counterString=(TOFevents TOFtracks TPCevents TPCtracks TRDevents TRDtracks T0events SDDevents SDDtracks MeanVertexevents)
2231   Ncounter=${#counterString[@]}
2232
2233   declare -a statusString=(TRDStatus TOFStatus TPCStatus T0Status MeanVertexStatus)
2234   Nstatus=${#statusString[@]}
2235
2236
2237   declare -a ratesString=(rec stderr calib qa_barrel qa_outer)
2238   Nrates=${#ratesString[@]}
2239
2240   runs=( $(ls -1 ${DIR}/meta/merge.cpass0* | while read x; do guessRunNumber $x; done) )
2241   Nruns=${#runs[@]}
2242
2243   echo -n runnumber/I >>${outfile}
2244   echo -n :cpass${pass}status/I >>${outfile}
2245   echo -n :cpass${pass}QAstatus/I >>${outfile}
2246   for i in ${ratesString[@]}; do
2247     echo -n :${i}OK/I >>${outfile}
2248     echo -n :${i}BAD/I >>${outfile}
2249
2250   done
2251   for i in ${counterString[@]} ${statusString[@]} ; do
2252     echo -n :${i}/I >>${outfile}
2253   done
2254   Ncolumns=$((2 + 2*Nrates + Ncounter + Nstatus))
2255   echo >> ${outfile}
2256
2257   #2. loop runs 
2258
2259   for runnumber in ${runs[@]} ; do 
2260
2261
2262
2263     filejob="${DIR}/meta/cpass${pass}.job*.run${runnumber}.done"
2264     filemerge="${DIR}/meta/merge.cpass${pass}.run${runnumber}.done"
2265     fileOCDB=$(grep /ocdb.log ${filemerge} | awk '{print $1}')
2266     if ! $(/bin/ls ${filemerge} &>/dev/null) ; then
2267       echo "${filemerge} does not exist!" >>${errfile}
2268       continue
2269     elif ! $(/bin/ls ${filejob} &>/dev/null) ; then
2270       echo "${filejob} does not exist!" >>${errfile}
2271       echo -n ${runnumber} >> ${outfile}
2272       for i in $(seq ${Ncolumns}) ; do 
2273         echo -n "-1" >> ${outfile}
2274       done
2275       echo >> ${outfile}
2276       continue
2277     fi
2278     echo -n ${runnumber} >> ${outfile}
2279     #pass0status= grep '/ocdb.log' ${filemerge} |  cut -d' ' -f2 | tr OK x1 | tr BAD xx0 | tr -d 'x'
2280     passStatus=$(grep '/ocdb.log' ${filemerge} | grep OK | wc -l)
2281     echo -n " ${passStatus}" >> ${outfile}
2282     qaStatus=$(grep '/mergeMakeOCDB.log' ${filemerge} | grep OK | wc -l)
2283     echo -n " ${qaStatus}" >> ${outfile}
2284
2285
2286     #fill OK/BAD rates
2287     for i in $(seq 0 $((${Nrates}-1))) ; do
2288       var1=$(grep "/${ratesString[${i}]}.log" ${filejob} | grep OK | wc -l)
2289       var2=$(grep "/${ratesString[${i}]}.log" ${filejob} | grep BAD | wc -l)
2290
2291       if [[ ${ratesString[${i}]} == "stderr" ]] ; then
2292         var1=$(grep "stderr" ${filejob} | grep OK | wc -l)
2293         var2=$(grep "stderr" ${filejob} | grep "rec.log" | grep BAD | wc -l)
2294       fi
2295       echo -n " ${var1}" >> ${outfile}
2296       echo -n " ${var2}" >> ${outfile}
2297     done
2298
2299     if [[ -f ${fileOCDB} ]] ; then
2300       #fill counter
2301       for i in $(seq 0 $((${Ncounter}-1))) ; do
2302         var1=$(grep Monalisa ${fileOCDB} | grep ${counterString[${i}]} | cut -f2)
2303         echo -n " ${var1:-"-1"}" >> ${outfile}
2304       done
2305
2306       #fill status
2307       for i in $(seq 0 $((${Nstatus}-1))) ; do
2308         var1=$(grep "calibration status=" ${fileOCDB} | grep ${statusString[${i}]/Status/} | cut -d'=' -f2)
2309         echo -n " ${var1:-"-1"}" >> ${outfile}
2310       done
2311     fi
2312     echo >> ${outfile}
2313   done
2314
2315   return 0
2316 )
2317
2318 parseConfig()
2319 {
2320   configFile=${1}
2321   shift
2322   args=("$@")
2323
2324
2325   #some defaults
2326   #autoOCDB=0
2327   defaultOCDB="raw://"
2328   #runNumber=167123
2329   #makeflowPath="/hera/alice/aux/cctools/bin"
2330   #makeflowOptions="-T wq -N alice -d all -C ali-copilot.cern.ch:9097"
2331   #makeflowOptions="-T wq -N alice -C ali-copilot.cern.ch:9097"
2332   makeflowOptions=""
2333   #batchCommand="/usr/bin/qsub"
2334   batchFlags="-b y -cwd -l h_rt=24:0:0,h_rss=4G "
2335   baseOutputDirectory="$PWD/output"
2336   #alirootEnv="/cvmfs/alice.cern.ch/bin/alienv setenv AliRoot/v5-04-34-AN -c"
2337   #alirootEnv="/home/mkrzewic/alisoft/balice_master.sh"
2338   #trustedQAtrainMacro='/hera/alice/mkrzewic/gsisvn/Calibration/QAtrain_duo.C'
2339   reconstructInTemporaryDir=0
2340   recoTriggerOptions="\"\""
2341   percentProcessedFilesToContinue=100
2342   maxSecondsToWait=$(( 3600*24 ))
2343   nEvents=-1
2344   nMaxChunks=0
2345   postSetUpActionCPass0=""
2346   postSetUpActionCPass1=""
2347   runCPass0reco=1
2348   runCPass0MergeMakeOCDB=1
2349   runCPass1reco=1
2350   runCPass1MergeMakeOCDB=1
2351   runESDfiltering=1
2352   filteringFactorHighPt=1e2
2353   filteringFactorV0s=1e1
2354   MAILTO=""
2355   #pretend=1
2356   #dontRedirectStdOutToLog=1
2357   logToFinalDestination=1
2358   ALIROOT_FORCE_COREDUMP=1
2359   pretendDelay=0
2360
2361   #first, source the config file
2362   if [ -f ${configFile} ]; then
2363     source ${configFile}
2364   else
2365     echo "config file ${configFile} not found!"
2366     return 1
2367   fi
2368
2369   unset encodedSpaces
2370   for opt in "${args[@]}"; do
2371     [[ "${opt}" =~ encodedSpaces=.* ]] && encodedSpaces=1 && echo "encodedSpaces!" && break
2372   done
2373
2374   #then, parse the options as they override the options from file
2375   for opt in "${args[@]}"; do
2376     [[ -z ${opt} ]] && continue
2377     [[ -n ${encodedSpaces} ]] && opt="$(decSpaces ${opt})"
2378     [[ "${opt}" =~ ^[[:space:]]*$ ]] && continue
2379     if [[ ! "${opt}" =~ .*=.* ]]; then
2380       echo "badly formatted option \"${opt}\" should be: option=value, stopping..."
2381       return 1
2382     fi
2383     local var="${opt%%=*}"
2384     local value="${opt#*=}"
2385     echo "${var} = ${value}"
2386     export ${var}="${value}"
2387   done
2388
2389   #do some checking
2390   [[ -z ${alirootEnv} ]] && echo "alirootEnv not defined!" && return 1
2391
2392   #export the aliroot function if defined to override normal behaviour
2393   [[ $(type -t aliroot) =~ "function" ]] && export -f aliroot
2394
2395   return 0
2396 }
2397
2398 aliroot()
2399 {
2400   args="$@"
2401   if [[ -n ${useProfilingCommand} ]]; then
2402     valgrindLogFile="cpu.txt"
2403     [[ "${args}" =~ rec ]] && valgrindLogFile="cpu_rec.txt"
2404     [[ "${args}}" =~ Calib ]] && valgrindLogFile="cpu_calib.txt"
2405     [[ -n ${useProfilingCommand} ]] && useProfilingCommand="${useProfilingCommand} --log-file=${valgrindLogFile}"
2406     echo running ${useProfilingCommand} aliroot ${args}
2407     ${useProfilingCommand} aliroot ${args}
2408   else
2409     #to prevent an infinite recursion use "command aliroot" to disable
2410     #aliases and functions
2411     echo running command aliroot ${args}
2412     command aliroot "$@"
2413   fi
2414   return 0
2415 }
2416
2417 guessRunData()
2418 {
2419   #guess the period from the path, pick the rightmost one
2420   period=""
2421   runNumber=""
2422   year=""
2423   pass=""
2424   legoTrainRunNumber=""
2425   dataType=""
2426
2427   local shortRunNumber=""
2428   local IFS="/"
2429   declare -a path=( $1 )
2430   local dirDepth=$(( ${#path[*]}-1 ))
2431   i=0
2432   #for ((x=${dirDepth};x>=0;x--)); do
2433   for ((x=0;x<=${dirDepth};x++)); do
2434
2435     [[ $((x-1)) -ge 0 ]] && local fieldPrev=${path[$((x-1))]}
2436     local field=${path[${x}]}
2437     local fieldNext=${path[$((x+1))]}
2438
2439     [[ ${field} =~ ^[0-9]*$ && ${fieldNext} =~ (.*\.zip$|.*\.root$) ]] && legoTrainRunNumber=${field}
2440     [[ -n ${legoTrainRunNumber} && -z ${pass} ]] && pass=${fieldPrev}
2441     [[ ${field} =~ ^LHC[0-9][0-9][a-z].*$ ]] && period=${field%_*}
2442     [[ ${field} =~ ^000[0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && runNumber=${field#000}
2443     [[ ${field} =~ ^[0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && shortRunNumber=${field}
2444     [[ ${field} =~ ^20[0-9][0-9]$ ]] && year=${field}
2445     [[ ${field} =~ ^(^sim$|^data$) ]] && dataType=${field}
2446     (( i++ ))
2447   done
2448   [[ -z ${legoTrainRunNumber} ]] && pass=${path[$((dirDepth-1))]}
2449   [[ "${dataType}" =~ ^sim$ ]] && pass="passMC" && runNumber=${shortRunNumber}
2450   
2451   #if [[ -z ${dataType} || -z ${year} || -z ${period} || -z ${runNumber}} || -z ${pass} ]];
2452   if [[ -z ${runNumber}} ]];
2453   then
2454     #error condition
2455     return 1
2456   else
2457     #ALL OK
2458     return 0
2459   fi
2460   return 0
2461 }
2462
2463 #these functions encode strings to and from a space-less form
2464 #use when spaces are not well handled (e.g. in arguments to 
2465 #commands in makeflow files, etc.
2466 encSpaces()(echo "${1// /@@@@}")
2467 decSpaces()(echo "${1//@@@@/ }")
2468
2469 main "$@"