]> git.uio.no Git - u/mrichter/AliRoot.git/blame - PWGPP/benchmark/benchmark.sh
git://git.uio.no / u / mrichter / AliRoot.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
adding possibility to select only primary protons for momentum resolution correction...
[u/mrichter/AliRoot.git] / PWGPP / benchmark / benchmark.sh
CommitLineData
fc7e1e45 1#!/bin/bash
2# this script runs the CPass0/CPass1 train
3# produced OCDB updates are local
4
5main()
6{
7 #run in proper mode depending on the selection
8 runMode=$1
cb473870 9 #if [[ $# -gt 0 ]]; then
10 # echo "# $0 $*"
11 #fi
fc7e1e45 12 umask 0002
13 shift
14 case $runMode in
15 "CPass0") goCPass0 "$@";;
16 "CPass1") goCPass1 "$@";;
17 "ConfOCDB") goConfigureOCDBforCPass1 "$@";;
18 "MergeCPass0") goMergeCPass0 "$@";;
19 "MergeCPass1") goMergeCPass1 "$@";;
cb473870 20 "MakeFilteredTrees") goMakeFilteredTrees "$@";;
fc7e1e45 21 "makeSummary") goMakeSummary "$@";;
cb473870 22 "run") goSubmitMakeflow "$@";;
23 "submit") goSubmitBatch "$@";;
fc7e1e45 24 "test") goTest "$@";;
25 "generateMakeflow") goGenerateMakeflow "$@";;
cb473870 26 "printValues") goPrintValues "$@";;
27 "CreateQAplots") goCreateQAplots "$@";;
28 "WaitForOutput") goWaitForOutput "$@";;
29 "merge") goMerge "$@";;
fc7e1e45 30 #and the default: unless sourced print some info
31 *) if [[ ! "$0" =~ "bash" ]]; then
cb473870 32 echo "uses makeflow:"
33 echo " $0 \"run\" inputList productionID [configFile=benchmark.config] [runNumber]"
34 echo "uses a batch system (SGE):"
35 echo " $0 \"submit\" inputList productionID [configFile=benchmark.config] [runNumber]"
fc7e1e45 36 fi
37 ;;
38 esac
39}
40
41goCPass0()
42{
43 umask 0002
44
45 configFile=$5
46 source $configFile
36b5de1a 47 [[ -f ${setupAliROOTenvInCurrentShell} && -z ${alirootEnv} ]] && source $setupAliROOTenvInCurrentShell
fc7e1e45 48
49 targetDirectory=$1
50 inputList=$2
51 nEvents=$3
52 ocdbPath=$4
53 configFile=$5
54 runNumber=$6
55 jobindex=$7
56
cb473870 57 #the contents of this is stored in the tree and used later (e.g. AliAnalysisTaskPIDResponse)!
58 #at the QA stage the pass number is guessed from the path stored here.
59 #The Format is:
60 #Packages= ;OutputDir= ;LPMPass= ;TriggerAlias= ;LPMRunNumber= ;LPMProductionType= ;LPMInteractionType= ;LPMProductionTag= ;LPMAnchorRun= ;LPMAnchorProduction= ;LPMAnchorYear=
61 export PRODUCTION_METADATA="OutputDir=cpass0"
62
fc7e1e45 63 #use the jobindex only if set and non-negative
64 if [[ -z $jobindex || $jobindex -lt 0 ]]; then
65 [[ -n "$LSB_JOBINDEX" ]] && jobindex=$LSB_JOBINDEX
66 [[ -n "$SGE_TASK_ID" ]] && jobindex=$SGE_TASK_ID
67 fi
68
69 [[ ! -f ${inputList} && -z ${pretend} ]] && echo "input file $inputList not found, exiting..." && exit 1
70 if [[ "${inputList}" =~ \.root$ ]]; then
71 infile=$inputList
72 else
73 infile=`sed -ne "${jobindex}p" $inputList`
74 fi
75
76 chunkName=${infile##*/}
cb473870 77 outputDir=${targetDirectory}/${jobindex}_${chunkName}
fc7e1e45 78 mkdir -p $outputDir
79 [[ ! -d $outputDir ]] && echo "cannot make $outputDir" && exit 1
80
81 runpath=${PWD}/rundir_cpass0_${runNumber}_${jobindex}
82 [[ -z $commonOutputPath ]] && commonOutputPath=$PWD
83 [[ $reconstructInTemporaryDir -eq 1 && -n $TMPDIR ]] && runpath=$TMPDIR
84 [[ $reconstructInTemporaryDir -eq 1 && -z $TMPDIR ]] && runpath=$(mktemp -d)
85
86 mkdir -p $runpath
87 [[ ! -d ${runpath} ]] && echo "cannot make runpath ${runpath}" && exit 1
88 cd $runpath
89
90 logOutputDir=$runpath
91 [[ -n $logToFinalDestination ]] && logOutputDir=$outputDir
5ed7c82c 92 [[ -z $dontRedirectStdOutToLog ]] && exec 1> $logOutputDir/stdout
93 [[ -z $dontRedirectStdOutToLog ]] && exec 2> $logOutputDir/stderr
fc7e1e45 94 echo "$0 $*"
95
96 calibDoneFile="$commonOutputPath/cpass0.job${jobindex}.run${runNumber}.done"
97
98 echo "#####################"
99 echo CPass0:
100 echo JOB setup
101 echo nEvents $nEvents
102 echo runNumber $runNumber
103 echo ocdbPath $ocdbPath
104 echo infile $infile
105 echo chunkName $chunkName
106 echo jobindex $jobindex
107 echo recoTriggerOptions $recoTriggerOptions
108 echo targetDirectory $targetDirectory
109 echo commonOutputPath $commonOutputPath
110 echo calibDoneFile $calibDoneFile
111 echo runpath $runpath
112 echo outputDir $outputDir
113 echo ALICE_ROOT $ALICE_ROOT
114 echo PWD $PWD
115 echo "########## ###########"
116
117 alirootInfo > ALICE_ROOT_svn.log
118
119 filesCPass0=(
120 "$commonOutputPath/runCPass0.sh"
121 "$commonOutputPath/recCPass0.C"
122 "$commonOutputPath/runCalibTrain.C"
123 "$commonOutputPath/localOCDBaccessConfig.C"
36b5de1a 124 "$commonOutputPath/OCDB.root"
fc7e1e45 125 "$ALICE_ROOT/PWGPP/CalibMacros/CPass0/runCPass0.sh"
126 "$ALICE_ROOT/PWGPP/CalibMacros/CPass0/recCPass0.C"
127 "$ALICE_ROOT/PWGPP/CalibMacros/CPass0/runCalibTrain.C"
128 )
129
130 for file in ${filesCPass0[*]}; do
131 [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
132 done
133
134 ln -s $infile $runpath/$chunkName
135
136 echo "this directory ($PWD) contents:"
137 ls -lh
138 echo
139 chmod u+x runCPass0.sh
140
141 if [[ -n $postSetUpActionCPass0 ]]; then
142 echo "running $postSetUpActionCPass0"
143 eval $postSetUpActionCPass0
144 fi
145
146 #run CPass0
147 echo "$runpath/runCPass0.sh $infile $nEvents $runNumber $ocdbPath $recoTriggerOptions"
148 if [[ -n $pretend ]]; then
cb473870 149 touch AliESDs.root
150 touch AliESDfriends.root
fc7e1e45 151 touch AliESDfriends_v1.root
152 touch rec.log
153 touch calib.log
154 else
155 ./runCPass0.sh "$infile" "$nEvents" "$runNumber" "$ocdbPath" "$recoTriggerOptions"
156 fi
157
fc7e1e45 158 #move stuff to final destination
159 echo "this directory ($PWD) contents:"
160 ls -lh
161 echo
162
163 echo rm -f ./$chunkName
164 rm -f ./$chunkName
165 echo "cp --recursive $runpath/* ${outputDir}"
cb473870 166 cp -p --recursive $runpath/* $outputDir
fc7e1e45 167 echo
5ed7c82c 168
169 #validate CPass0
170 cd ${outputDir}
171 touch ${calibDoneFile}
172 [[ -f AliESDfriends_v1.root ]] && echo "calibfile ${outputDir}/AliESDfriends_v1.root" > ${calibDoneFile}
cb473870 173 [[ -f AliESDs.root ]] && echo "esd ${outputDir}/AliESDs.root" >> ${calibDoneFile}
174 echo "dir ${outputDir}" >> ${calibDoneFile}
5ed7c82c 175 summarizeLogs >> ${calibDoneFile}
fc7e1e45 176
cb473870 177 [[ "$runpath" != "$outputDir" ]] && rm -rf ${runpath}
fc7e1e45 178}
179
180goCPass1()
181{
182 umask 0002
183
184 configFile=$5
185 source $configFile
36b5de1a 186 [[ -f ${setupAliROOTenvInCurrentShell} && -z ${alirootEnv} ]] && source $setupAliROOTenvInCurrentShell
fc7e1e45 187
188 targetDirectory=$1
189 inputList=$2
190 nEvents=$3
191 ocdbPath=$4
192 configFile=$5
193 runNumber=$6
194 jobindex=$7
195
cb473870 196 #the contents of this is stored in the tree and used later (e.g. AliAnalysisTaskPIDResponse)!
197 #at the QA stage the pass number is guessed from the path stored here.
198 #The Format is:
199 #Packages= ;OutputDir= ;LPMPass= ;TriggerAlias= ;LPMRunNumber= ;LPMProductionType= ;LPMInteractionType= ;LPMProductionTag= ;LPMAnchorRun= ;LPMAnchorProduction= ;LPMAnchorYear=
200 export PRODUCTION_METADATA="OutputDir=cpass1"
201
fc7e1e45 202 #use the jobindex only if set and non-negative
203 if [[ -z $jobindex || $jobindex -lt 0 ]]; then
204 [[ -n "$LSB_JOBINDEX" ]] && jobindex=$LSB_JOBINDEX
205 [[ -n "$SGE_TASK_ID" ]] && jobindex=$SGE_TASK_ID
206 fi
207
208 [[ ! -f ${inputList} && -z ${pretend} ]] && echo "input file $inputList not found, exiting..." && exit 1
209 if [[ "${inputList}" =~ \.root$ ]]; then
210 infile=$inputList
211 else
212 infile=`sed -ne "${jobindex}p" $inputList`
213 fi
214
215 chunkName=${infile##*/}
cb473870 216 outputDir=${targetDirectory}/${jobindex}_${chunkName}
fc7e1e45 217 mkdir -p $outputDir
218 [[ ! -d $outputDir ]] && echo "cannot make $outputDir" && exit 1
219
220 runpath=${PWD}/rundir_cpass1_${runNumber}_${jobindex}
221 [[ -z $commonOutputPath ]] && commonOutputPath=$PWD
222 [[ $reconstructInTemporaryDir -eq 1 && -n $TMPDIR ]] && runpath=$TMPDIR
223 [[ $reconstructInTemporaryDir -eq 1 && -z $TMPDIR ]] && runpath=$(mktemp -d)
224
225 #init the running path
226 mkdir -p $runpath
227 [[ ! -d ${runpath} ]] && echo "cannot make runpath ${runpath}" && exit 1
228 cd $runpath
229
fc7e1e45 230 calibDoneFile="${commonOutputPath}/cpass1.job${jobindex}.run${runNumber}.done"
231
232 logOutputDir=$runpath
233 [[ -n $logToFinalDestination ]] && logOutputDir=$outputDir
5ed7c82c 234 [[ -z $dontRedirectStdOutToLog ]] && exec 1> $logOutputDir/stdout
235 [[ -z $dontRedirectStdOutToLog ]] && exec 2> $logOutputDir/stderr
fc7e1e45 236 echo "$0 $*"
237
238 echo "#####################"
239 echo CPass1:
240 echo JOB setup
241 echo nEvents $nEvents
242 echo runNumber $runNumber
243 echo ocdbPath $ocdbPath
244 echo infile $infile
245 echo chunkName $chunkName
246 echo jobindex $jobindex
247 echo recoTriggerOptions $recoTriggerOptions
248 echo targetDirectory $targetDirectory
249 echo commonOutputPath $commonOutputPath
250 echo calibDoneFile $calibDoneFile
251 echo runpath $runpath
252 echo outputDir $outputDir
253 echo ALICE_ROOT $ALICE_ROOT
254 echo PWD $PWD
255 echo "########## ###########"
256
257 alirootInfo > ALICE_ROOT_svn.log
258
259 filesCPass1=(
260 "$commonOutputPath/runCPass1.sh"
261 "$commonOutputPath/recCPass1.C"
262 "$commonOutputPath/recCPass1_OuterDet.C"
263 "$commonOutputPath/runCalibTrain.C"
264 "$commonOutputPath/QAtrain_duo.C"
265 "$commonOutputPath/localOCDBaccessConfig.C"
266 "$commonOutputPath/cpass0.localOCDB.${runNumber}.tgz"
36b5de1a 267 "$commonOutputPath/OCDB.root"
cb473870 268 "$trustedQAtrainMacro"
fc7e1e45 269 "$ALICE_ROOT/PWGPP/CalibMacros/CPass1/runCPass1.sh"
270 "$ALICE_ROOT/PWGPP/CalibMacros/CPass1/recCPass1.C"
271 "$ALICE_ROOT/PWGPP/CalibMacros/CPass1/recCPass1_OuterDet.C"
272 "$ALICE_ROOT/PWGPP/CalibMacros/CPass1/runCalibTrain.C"
273 "$ALICE_ROOT/ANALYSIS/macros/QAtrain_duo.C"
274 )
275
276 for file in ${filesCPass1[*]}; do
277 [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
278 done
279
280 ln -s $infile $runpath/$chunkName
281
282 echo "this directory ($PWD) contents:"
283 ls -lh
284 echo
285
286 if [[ -n $postSetUpActionCPass1 ]]; then
287 echo "running $postSetUpActionCPass1"
288 eval $postSetUpActionCPass1
289 echo
290 fi
291
292 #configure local OCDB storage from CPass0 (creates the localOCDBaccessConfig.C script)
293 if [[ -f cpass0.localOCDB.${runNumber}.tgz ]]; then
cb473870 294 echo goConfigureOCDBforCPass1 "cpass0.localOCDB.${runNumber}.tgz"
295 goConfigureOCDBforCPass1 "cpass0.localOCDB.${runNumber}.tgz"
296 else
297 echo "WARNING: file cpass0.localOCDB.${runNumber}.tgz not found!"
298 fi
299
300 if [[ ! $(find ./OCDB -name "*.root") ]]; then
301 echo "cpass0 produced no calibration! exiting..."
302 exit 1
303 fi
fc7e1e45 304
305 #run CPass1
306 chmod u+x runCPass1.sh
307 echo "$runpath/runCPass1.sh $infile $nEvents $runNumber $ocdbPath $recoTriggerOptions"
308 if [[ -n $pretend ]]; then
cb473870 309 touch AliESDs_Barrel.root
310 touch AliESDfriends_Barrel.root
fc7e1e45 311 touch AliESDfriends_v1.root
cb473870 312 touch QAresults_barrel.root
313 touch EventStat_temp_barrel.root
314 touch AODtpITS.root
315 touch AliESDs_Outer.root
316 touch AliESDfriends_Outer.root
317 touch QAresults_outer.root
318 touch EventStat_temp_outer.root
fc7e1e45 319 touch rec.log
320 touch calib.log
321 touch qa.log
322 else
323 ./runCPass1.sh "$infile" "$nEvents" "$runNumber" "$ocdbPath" "$recoTriggerOptions"
324 fi
cb473870 325
326 ##handle possible crashes in QA (happens often in trunk)
327 ##rerun QA with a trusted aliroot version
328 #if [[ $(validateLog qa_barrel.log) ]]; then
329 # echo "qa_barrel.log not validated!"
330 #fi
331 #if [[ ! -f QAresults_barrel.root && -f ${setupTrustedAliROOTenvInCurrentShell} || $(validateLog qa_barrel.log) ]]; then
332 # echo "WARNING: using trusted QA aliroot $ALICE_ROOT"
333 # source ${setupTrustedAliROOTenvInCurrentShell}
334 # cd Barrel
335 # rm QAresults_barrel.root
336 # rm EventStat_temp_barrel.root
337 # rm AODtpITS.root
338 # [[ ! -f AliESDs.root ]] && ln -s ../AliESDs_Barrel.root AliESDs.root
339 # [[ ! -f AliESDfriends.root ]] && ln -s ../AliESDfriends_Barrel.root AliESDfriends.root
340 # if [[ -n $trustedQAtrainMacro ]]; then
341 # eval "cp $trustedQAtrainMacro QAtrain_duo_trusted.C"
342 # fi
343 # echo executing aliroot -b -q "QAtrain_duo_trusted.C(\"_barrel\",$runNumber,\"wn.xml\",0,\"$ocdbPath\")"
344 # time aliroot -b -q "QAtrain_duo.C(\"_barrel\",$runNumber,\"wn.xml\",0,\"$ocdbPath\")" &> ../qa_barrel_trusted.log
345 # cd ../
346 #fi
347 [[ ! -f AliESDs_Barrel.root && -f Barrel/AliESDs.root ]] && mv Barrel/AliESDs.root AliESDs_Barrel.root
348 [[ ! -f AliESDfriends_Barrel.root && -f Barrel/AliESDfriends.root ]] && mv Barrel/AliESDfriends.root AliESDfriends_Barrel.root
349 [[ ! -f AliESDfriends_v1.root && -f Barrel/AliESDfriends_v1.root ]] && mv Barrel/AliESDfriends_v1.root .
350 [[ ! -f QAresults_barrel.root && -f Barrel/QAresults_barrel.root ]] && mv Barrel/QAresults_barrel.root .
351 [[ ! -f AliESDs_Outer.root && -f OuterDet/AliESDs.root ]] && mv OuterDet/AliESDs.root AliESDs_Outer.root
352 [[ ! -f AliESDfriends_Outer.root && -f OuterDet/AliESDfriends.root ]] && mv OuterDet/AliESDfriends.root AliESDfriends_Outer.root
353 [[ ! -f QAresults_outer.root && -f OuterDet/QAresults_outer.root ]] && mv OuterDet/QAresults_outer.root .
354
fc7e1e45 355 #move stuff to final destination
356 echo "this directory ($PWD) contents:"
357 ls
fc7e1e45 358 echo rm -f ./$chunkName
359 rm -f ./$chunkName
360 echo "cp --recursive ${runpath}/* ${outputDir}"
cb473870 361 cp -p --recursive ${runpath}/* ${outputDir}
fc7e1e45 362 echo
363
5ed7c82c 364 #validate CPass1
365 cd ${outputDir}
366 touch ${calibDoneFile}
cb473870 367 [[ -f AliESDs_Barrel.root ]] && echo "esd ${outputDir}/AliESDs_Barrel.root" > ${calibDoneFile}
368 [[ -f AliESDfriends_v1.root ]] && echo "calibfile ${outputDir}/AliESDfriends_v1.root" >> ${calibDoneFile}
5ed7c82c 369 [[ -f QAresults_Barrel.root ]] && echo "qafile ${outputDir}/QAresults_Barrel.root" >> ${calibDoneFile}
370 [[ -f QAresults_Outer.root ]] && echo "qafile ${outputDir}/QAresults_Outer.root" >> ${calibDoneFile}
dd755372 371 echo "dir ${outputDir}" >> ${calibDoneFile}
5ed7c82c 372 summarizeLogs >> ${calibDoneFile}
373
cb473870 374 [[ "$runpath" != "$outputDir" ]] && rm -rf ${runpath}
fc7e1e45 375}
376
377
378goMergeCPass0()
379{
380 umask 0002
381 #
382 # find the output files and merge them
383 #
384
385 outputDir=$1
cb473870 386 ocdbStorage=$2
fc7e1e45 387 configFile=$3
388 runNumber=$4
389 calibrationFilesToMergeExternal=$5
390
391 source $configFile
36b5de1a 392 [[ -f ${setupAliROOTenvInCurrentShell} && -z ${alirootEnv} ]] && source $setupAliROOTenvInCurrentShell
fc7e1e45 393
394 runpath=${PWD}/rundir_cpass0_Merge_${runNumber}
395 [[ -z $commonOutputPath ]] && commonOutputPath=$PWD
396 [[ $reconstructInTemporaryDir -eq 1 && -n $TMPDIR ]] && runpath=$TMPDIR
397 [[ $reconstructInTemporaryDir -eq 1 && -z $TMPDIR ]] && runpath=$(mktemp -d)
398
399 mkdir -p $runpath
400 [[ ! -d $runpath ]] && echo "not able to make the runpath $runpath" && exit 1
401 cd $runpath
402
403 logOutputDir=$runpath
404 [[ -n $logToFinalDestination ]] && logOutputDir=$outputDir
cb473870 405 [[ -z $dontRedirectStdOutToLog ]] && exec &> $logOutputDir/mergeMakeOCDB.log
fc7e1e45 406 echo "$0 $*"
407
408 calibrationFilesToMerge=$calibrationFilesToMergeExternal
409 [[ -z $calibrationFilesToMerge ]] && calibrationFilesToMerge="calibrationFilesToMerge.list"
410 calibrationOutputFileName="AliESDfriends_v1.root"
411 mergingScript="mergeMakeOCDB.byComponent.sh"
412 qaFilesToMerge="qaFilesToMerge.list"
413 qaOutputFileName="QAresults*.root"
414 qaMergedOutputFileName="QAresults_merged.root"
415
416 echo goMergeCPass0 SETUP:
417 echo runNumber=$runNumber
418 echo outputDir=$outputDir
cb473870 419 echo ocdbStorage=$ocdbStorage
fc7e1e45 420 echo calibrationFilesToMerge=$calibrationFilesToMerge
421 echo calibrationOutputFileName=$calibrationOutputFileName
422 echo mergingScript=$mergingScript
cb473870 423 echo commonOutputPath=$commonOutputPath
424 echo runpath=$runpath
fc7e1e45 425
426 # copy files in case they are not already there
427 filesMergeCPass0=(
36b5de1a 428 "$commonOutputPath/${calibrationFilesToMerge}"
429 "$commonOutputPath/OCDB.root"
430 "$commonOutputPath/localOCDBaccessConfig.C"
cb473870 431 "$commonOutputPath/cpass0.localOCDB.${runNumber}.tgz"
fc7e1e45 432 "$ALICE_ROOT/PWGPP/CalibMacros/CPass0/mergeMakeOCDB.byComponent.sh"
433 "$ALICE_ROOT/PWGPP/CalibMacros/CPass0/mergeByComponent.C"
434 "$ALICE_ROOT/PWGPP/CalibMacros/CPass0/makeOCDB.C"
435 "$ALICE_ROOT/PWGPP/CalibMacros/CPass0/merge.C"
436 "$ALICE_ROOT/PWGPP/CalibMacros/CPass0/mergeMakeOCDB.sh"
fc7e1e45 437 )
438 for file in ${filesMergeCPass0[*]}; do
439 [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
440 done
441
442 alirootInfo > ALICE_ROOT_svn.log
443
444 #
cb473870 445 echo "PWD"
fc7e1e45 446 ls -lh
cb473870 447 echo "PWD/.."
448 ls -lh ../
449
fc7e1e45 450
451 #merge calibration
452 chmod u+x $mergingScript
453 mkdir -p ./OCDB
454 if [[ -z ${calibrationFilesToMergeExternal} ]]; then
455 echo "find $outputDir -name $calibrationOutputFileName > $calibrationFilesToMerge"
456 find $outputDir -name $calibrationOutputFileName > $calibrationFilesToMerge
457 fi
458
cb473870 459 echo "$mergingScript $calibrationFilesToMerge ${runNumber} local://./OCDB $ocdbStorage"
fc7e1e45 460 if [[ -n $pretend ]]; then
461 touch CalibObjects.root
462 touch ocdb.log
463 touch merge.log
464 mkdir -p ./OCDB/someDetector/
465 mkdir -p ./OCDB/otherDetector/
466 touch ./OCDB/someDetector/someCalibObject_0-999999_cpass0.root
467 touch ./OCDB/otherDetector/otherCalibObject_0-999999_cpass0.root
468 else
cb473870 469 ./$mergingScript $calibrationFilesToMerge ${runNumber} "local://./OCDB" $ocdbStorage
fc7e1e45 470 fi
471
472 ### produce the output
473 #tar the produced OCDB for reuse
474 tar czf $commonOutputPath/cpass0.localOCDB.${runNumber}.tgz ./OCDB
475
5ed7c82c 476 ls -ltrh
477
478 #copy all to output dir
cb473870 479 cp -p --recursive ${runpath}/* $outputDir
5ed7c82c 480
fc7e1e45 481 #validate merging cpass0
5ed7c82c 482 cd ${outputDir}
fc7e1e45 483 calibDoneFile="${commonOutputPath}/merge.cpass0.run${runNumber}.done"
484 touch ${calibDoneFile}
485 [[ -f CalibObjects.root ]] && echo "calibfile $outputDir/CalibObjects.root" > ${calibDoneFile}
486 summarizeLogs >> ${calibDoneFile}
487
cb473870 488 [[ "$runpath" != "$outputDir" ]] && rm -rf ${runpath}
fc7e1e45 489}
490
491goMergeCPass1()
492{
493 umask 0002
494 #
495 # find the output files and merge them
496 #
497
498 outputDir=$1
cb473870 499 ocdbStorage=$2
fc7e1e45 500 configFile=$3
501 runNumber=$4
502 calibrationFilesToMergeExternal=$5
503 qaFilesToMergeExternal=$6
504
505 #clean up first:
506 rm -f $outputDir/*.log
507 rm -f $outputDir/*.root
508 rm -f $outputDir/*done
509
510 source $configFile
36b5de1a 511 [[ -f ${setupAliROOTenvInCurrentShell} && -z ${alirootEnv} ]] && source $setupAliROOTenvInCurrentShell
fc7e1e45 512
513 runpath=${PWD}/rundir_cpass1_Merge_${runNumber}
514 [[ -z $commonOutputPath ]] && commonOutputPath=$PWD
515 [[ $reconstructInTemporaryDir -eq 1 && -n $TMPDIR ]] && runpath=$TMPDIR
516 [[ $reconstructInTemporaryDir -eq 1 && -z $TMPDIR ]] && runpath=$(mktemp -d)
517
518 mkdir -p $runpath
519 [[ ! -d $runpath ]] && echo "not able to make the runpath $runpath" && exit 1
520 cd $runpath
521
522 logOutputDir=$runpath
523 [[ -n $logToFinalDestination ]] && logOutputDir=$outputDir
cb473870 524 [[ -z $dontRedirectStdOutToLog ]] && exec &> $logOutputDir/mergeMakeOCDB.log
fc7e1e45 525 echo "$0 $*"
526
527 calibrationFilesToMerge=$calibrationFilesToMergeExternal
528 [[ -z $calibrationFilesToMerge ]] && calibrationFilesToMerge="calibrationFilesToMerge.list"
529 calibrationOutputFileName="AliESDfriends_v1.root"
530 mergingScript="mergeMakeOCDB.byComponent.sh"
531 qaFilesToMerge=$qaFilesToMergeExternal
cb473870 532 #important to have the string "Stage.txt" in the filename to trigger the merging
533 #it has to be a list of directories containing the files
534 [[ -z $qaFilesToMerge ]] && qaFilesToMerge="qaFilesForMergingStage.txt.list"
fc7e1e45 535 qaOutputFileName="QAresults*.root"
536 qaMergedOutputFileName="QAresults_merged.root"
537
538 echo goMergeCPass1 SETUP:
539 echo runNumber=$runNumber
540 echo outputDir=$outputDir
cb473870 541 echo ocdbStorage=$ocdbStorage
fc7e1e45 542 echo calibrationFilesToMerge=filesToMerge.list
543 echo calibrationOutputFileName=$calibrationOutputFileName
544 echo mergingScript=$mergingScript
545
546 # copy files in case they are not already there
547 filesMergeCPass1=(
36b5de1a 548 "$commonOutputPath/${calibrationFilesToMerge}"
549 "$commonOutputPath/${qaFilesToMerge}"
550 "$commonOutputPath/OCDB.root"
551 "$commonOutputPath/localOCDBaccessConfig.C"
cb473870 552 "$commonOutputPath/cpass0.localOCDB.${runNumber}.tgz"
553 "$commonOutputPath/QAtrain_duo.C"
fc7e1e45 554 "$ALICE_ROOT/PWGPP/CalibMacros/CPass1/mergeMakeOCDB.byComponent.sh"
555 "$ALICE_ROOT/PWGPP/CalibMacros/CPass1/mergeByComponent.C"
556 "$ALICE_ROOT/PWGPP/CalibMacros/CPass1/makeOCDB.C"
557 "$ALICE_ROOT/PWGPP/CalibMacros/CPass1/merge.C"
558 "$ALICE_ROOT/PWGPP/CalibMacros/CPass1/mergeMakeOCDB.sh"
cb473870 559 "$ALICE_ROOT/ANALYSIS/macros/QAtrain_duo.C"
fc7e1e45 560 )
561 for file in ${filesMergeCPass1[*]}; do
562 [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} .
563 done
564
cb473870 565 #configure local OCDB storage from CPass0 (creates the localOCDBaccessConfig.C script)
566 if [[ -f cpass0.localOCDB.${runNumber}.tgz ]]; then
567 echo goConfigureOCDBforCPass1 "cpass0.localOCDB.${runNumber}.tgz"
568 goConfigureOCDBforCPass1 "cpass0.localOCDB.${runNumber}.tgz"
569 else
570 echo "WARNING: file cpass0.localOCDB.${runNumber}.tgz not found!"
571 fi
572
fc7e1e45 573 alirootInfo > ALICE_ROOT_svn.log
574
575 #
576 ls -lh
577
578 #merge calibration
579 chmod u+x $mergingScript
580 mkdir -p OCDB
581 if [[ -z ${calibrationFilesToMergeExternal} ]]; then
582 echo "find $outputDir -name $calibrationOutputFileName > $calibrationFilesToMerge"
583 find $outputDir -name $calibrationOutputFileName > $calibrationFilesToMerge
584 fi
585
cb473870 586 echo "$mergingScript $calibrationFilesToMerge ${runNumber} local://./OCDB $ocdbStorage"
fc7e1e45 587 if [[ -n $pretend ]]; then
588 touch CalibObjects.root
589 touch ocdb.log
590 touch merge.log
591 else
cb473870 592 ./$mergingScript $calibrationFilesToMerge ${runNumber} "local://./OCDB" $ocdbStorage
fc7e1e45 593 fi
594
595 tar czf localCPass1_${runNumber}.tgz ./OCDB
596
597 #merge QA
5ed7c82c 598 [[ -n ${AliAnalysisTaskFilteredTree_fLowPtTrackDownscaligF} ]] && export AliAnalysisTaskFilteredTree_fLowPtTrackDownscaligF
599 [[ -n ${AliAnalysisTaskFilteredTree_fLowPtV0DownscaligF} ]] && export AliAnalysisTaskFilteredTree_fLowPtV0DownscaligF
600
601 if [[ -z $qaFilesToMergeExternal ]]; then
cb473870 602 #find the files, but only store the directories (QAtrain_duo.C requires this)
603 echo "find $outputDir -name $qaOutputFileName | while read x; do echo ${x%/*}; done | sort | uniq > $qaFilesToMerge"
604 find $outputDir -name $qaOutputFileName | while read x; do echo ${x%/*}; done | sort | uniq > $qaFilesToMerge
fc7e1e45 605 fi
606
cb473870 607 #do the merge
608 #echo aliroot -l -b -q "merge.C(\"$qaFilesToMerge\",\"\",kFALSE,\"$qaMergedOutputFileName\")"
609 echo aliroot -b -q "QAtrain_duo.C(\"_barrel\",${runNumber},\"$qaFilesToMerge\",1,\"${ocdbStorage}\")"
fc7e1e45 610 if [[ -n $pretend ]]; then
611 touch $qaMergedOutputFileName
612 touch merge.log
cb473870 613 touch trending.root
fc7e1e45 614 else
cb473870 615 #aliroot -l -b -q "merge.C(\"$qaFilesToMerge\",\"\",kFALSE,\"$qaMergedOutputFileName\")"
616 aliroot -b -q "QAtrain_duo.C(\"_barrel\",${runNumber},\"$qaFilesToMerge\",1,\"${ocdbStorage}\")"
617 mv QAresults_barrel.root $qaMergedOutputFileName
618 mv trending_barrel.root trending.root
fc7e1e45 619 fi
cb473870 620
5ed7c82c 621 ls -ltrh
622
623 #copy all to output dir
cb473870 624 cp -p --recursive ${runpath}/* ${outputDir}
5ed7c82c 625
fc7e1e45 626 #validate merge cpass1
5ed7c82c 627 cd ${outputDir}
fc7e1e45 628 calibDoneFile="${commonOutputPath}/merge.cpass1.run${runNumber}.done"
629 touch ${calibDoneFile}
630 [[ -f CalibObjects.root ]] && echo "calibfile $outputDir/CalibObjects.root" > ${calibDoneFile}
631 [[ -f $qaMergedOutputFileName ]] && echo "qafile $outputDir/$qaMergedOutputFileName" >> ${calibDoneFile}
cb473870 632 [[ -f trending.root ]] && echo "trendingfile $outputDir/trending.root" >> ${calibDoneFile}
dd755372 633 echo "dir ${outputDir}" >> ${calibDoneFile}
fc7e1e45 634 summarizeLogs >> ${calibDoneFile}
635
cb473870 636 [[ "$runpath" != "$outputDir" ]] && rm -rf ${runpath}
637}
638
639goMerge()
640{
641 #generic root merge using CPass1 merge.C script
642 inputList=$1
643 outputFile=$2
644 configFile=${3-"becnhmark.config"}
645 source $configFile
646 source $setupAliROOTenvInCurrentShell
647 aliroot -b -q "$ALICE_ROOT/PWGPP/CalibMacros/CPass0/merge.C(\"${inputList}\",\"\",kFALSE,\"${outputFile}\")" > merge_${inputList}.log
fc7e1e45 648}
649
5dfac970 650goMakeSummary()
fc7e1e45 651{
5dfac970 652 configFile=$1
cb473870 653 outputPath=$2
fc7e1e45 654 source $configFile
5dfac970 655
656 # log filtering, script needs to take the base dir as argument
657 if [[ -x $logFilteringScript ]]; then
cb473870 658 cd ${outputPath}
659 ${logFilteringScript} $outputPath &>/dev/null
660 cd -
fc7e1e45 661 fi
662
cb473870 663 awk 'BEGIN {nFiles=0;nCore=0;}
664 /^calibfile/ {nFiles++;}
665 /core / {nCore++i;}
666 END {print "cpass0 produced "nFiles" calib files, "nCore" core files";}' cpass0.job*done
667 awk 'BEGIN {nOK=0; nBAD=0; }
668 /rec.*log OK/ {nOK++;}
669 /rec.*log BAD/ {nBAD++;}
670 END {print "cpass0 reco: OK: "nOK" BAD: "nBAD;}' cpass0.job*done
671 awk 'BEGIN {nOK=0; nBAD=0; }
672 /calib.*log OK/ {nOK++;}
673 /calib.*log BAD/ {nBAD++;}
674 END {print "cpass0 calib: OK: "nOK" BAD: "nBAD;}' cpass0.job*done
5dfac970 675
cb473870 676 awk 'BEGIN {nOK=0; nBAD=0; }
677 /merge.log OK/ {nOK++;}
678 /merge.log BAD/ {nBAD++;}
679 END {print "cpass0 merge: OK: "nOK" BAD: "nBAD;}' merge.cpass0*done
680 awk 'BEGIN {nOK=0; nBAD=0; }
681 /ocdb.log OK/ {nOK++;}
682 /ocdb.log BAD/ {nBAD++;}
683 END {print "cpass0 OCDB: OK: "nOK" BAD: "nBAD;}' merge.cpass0*done
5dfac970 684
cb473870 685 awk 'BEGIN {nFiles=0;nCore=0;}
686 /^calibfile/ {nFiles++;}
687 /core / {nCore++;}
688 END {print "cpass1 produced "nFiles" calib files, "nCore" core files";}' cpass1.job*done
689 awk 'BEGIN {nOK=0; nBAD=0; }
690 /rec.*log OK/ {nOK++;}
691 /rec.*log BAD/ {nBAD++;}
692 END {print "cpass1 reco: OK: "nOK" BAD: "nBAD;}' cpass1.job*done
693 awk 'BEGIN {nOK=0; nBAD=0; }
694 /calib.*log OK/ {nOK++;}
695 /calib.*log BAD/ {nBAD++;}
696 END {print "cpass1 calib: OK: "nOK" BAD: "nBAD;}' cpass1.job*done
697
698 awk 'BEGIN {nOK=0; nBAD=0; }
699 /merge.log OK/ {nOK++;}
700 /merge.log BAD/ {nBAD++;}
701 END {print "cpass1 merge: OK: "nOK" BAD: "nBAD;}' merge.cpass1*done
702 awk 'BEGIN {nOK=0; nBAD=0; }
703 /ocdb.log OK/ {nOK++;}
704 /ocdb.log BAD/ {nBAD++;}
705 END {print "cpass1 OCDB: OK: "nOK" BAD: "nBAD;}' merge.cpass1*done
5dfac970 706
707 #if set email the summary
708 [[ -n $mailSummaryTo ]] && cat $log | mail -s "benchmark $productionID done" $mailSummaryTo
fc7e1e45 709
5dfac970 710 return 0
711}
fc7e1e45 712
cb473870 713goSubmitMakeflow()
5dfac970 714{
cb473870 715 #run
5dfac970 716 inputFileList=$1
717 productionID=$2
718 configFile=$3
719 runNumber=$4
fc7e1e45 720
5dfac970 721 [[ -z ${configFile} ]] && configFile="benchmark.config"
722 [[ ! -f ${configFile} ]] && echo "no config file found (${configFile})" && return 1
723 source $configFile
fc7e1e45 724
cb473870 725 if [[ ! $(which makeflow &>/dev/null) && -n $makeflowPath ]]; then
726 echo "setting the makflow path from the config: "
727 echo " export PATH=${makeflowPath}:${PATH}"
728 export PATH=${makeflowPath}:${PATH}
729 fi
730
731 #submit - use makeflow if available, fall back to old stuff when makeflow not there
732 if which makeflow; then
733 mkdir -p $productionID
734 cp $0 $productionID
735 cp $configFile $productionID
736 cp $inputFileList $productionID
737 cd $productionID
738 goGenerateMakeflow "$@" > benchmark.makeflow
739
740 [[ -n $createWorkqueue ]] && createWorkqueue=0
741 if [[ $createWorkqueue -eq 1 ]]; then
742 echo "creating workqueue:"
743 echo "${createWorkqueueCommand}"
744 eval ${createWorkqueueCommand}
745 fi
746
747 makeflow ${makeflowOptions} benchmark.makeflow
748 cd ../
749 else
750 echo "no makeflow!"
751 fi
5dfac970 752}
fc7e1e45 753
5dfac970 754goGenerateMakeflow()
755{
756 #generate the makeflow file
757 inputFileList=$1
758 productionID=$2
759 configFile=$3
760 runNumber=$4
fc7e1e45 761
5dfac970 762 [[ -z ${configFile} ]] && configFile="benchmark.config"
763 [[ ! -f ${configFile} ]] && echo "no config file found (${configFile})" && return 1
764 source $configFile
fc7e1e45 765
5dfac970 766 commonOutputPath=${baseOutputDirectory}/${productionID}
fc7e1e45 767
5dfac970 768 #these files will be made a dependency - will be copied to the working dir of the jobs
769 declare -a copyFiles
770 inputFiles=(
771 "OCDB.root"
772 "localOCDBaccessConfig.C"
cb473870 773 "QAtrain_duo.C"
774 "runCPass1.sh"
775 "recCPass1.C"
776 "recCPass1_OuterDet.C"
777 "runCalibTrain.C"
778 "runCPass0.sh"
779 "recCPass0.C"
5dfac970 780 )
781 for file in ${inputFiles[*]}; do
782 [[ -f ${file} ]] && copyFiles+=("${file}")
783 done
784
785 #create the makeflow file
cb473870 786 [[ -n ${batchFlags} ]] && echo "BATCH_OPTIONS = ${batchFlags}"
5dfac970 787 declare -a arr_cpass1_final
cb473870 788 declare -a arr_cleanup
5dfac970 789 listOfRuns=${runNumber}
790 [[ -z ${runNumber} ]] && listOfRuns=($(while read x; do guessRunNumber $x; done < ${inputFileList} | sort | uniq))
791 runindex=0
fc7e1e45 792 for runNumber in ${listOfRuns[*]}; do
793 [[ -z $runNumber ]] && continue
794 [[ ! ${runNumber} =~ ^[0-9]*[0-9]$ ]] && continue
fc7e1e45 795
cb473870 796 unset arr_cpass0_outputs_onerun
797 unset arr_cpass1_outputs_onerun
798 declare -a arr_cpass0_outputs_onerun
799 declare -a arr_cpass1_outputs_onerun
fc7e1e45 800
cb473870 801 jobindex=0
5dfac970 802 while read inputFile; do
803 currentDefaultOCDB=${defaultOCDB}
cb473870 804 [[ -z ${autoOCDB} ]] && autoOCDB=1
805 if [[ ${autoOCDB} -ne 0 ]]; then
806 currentDefaultOCDB=$(setYear $inputFile $defaultOCDB)
807 fi
fc7e1e45 808
5dfac970 809 #CPass0
cb473870 810 arr_cpass0_outputs_onerun[$jobindex]="cpass0.job${jobindex}.run${runNumber}.done"
811 echo "${arr_cpass0_outputs_onerun[$jobindex]} : benchmark.sh ${configFile} ${copyFiles[@]}"
5dfac970 812 echo " ${alirootEnv} ./benchmark.sh CPass0 ${commonOutputPath}/cpass0/000${runNumber} $inputFile $nEvents $currentDefaultOCDB $configFile $runNumber $jobindex"
fc7e1e45 813 echo
5dfac970 814
815 #CPass1
cb473870 816 arr_cpass1_outputs_onerun[$jobindex]="cpass1.job${jobindex}.run${runNumber}.done"
817 echo "${arr_cpass1_outputs_onerun[$jobindex]} : benchmark.sh ${configFile} cpass0.localOCDB.${runNumber}.tgz ${copyFiles[@]}"
5dfac970 818 echo " ${alirootEnv} ./benchmark.sh CPass1 ${commonOutputPath}/cpass1/000${runNumber} $inputFile $nEvents $currentDefaultOCDB $configFile $runNumber $jobindex"
fc7e1e45 819 echo
5dfac970 820 ((jobindex++))
fc7e1e45 821
5dfac970 822 done< <(grep "/000$runNumber/" $inputFileList)
cb473870 823
5dfac970 824 #CPass0 list of Calib files to merge
cb473870 825 arr_cpass0_calib_list[$runindex]="cpass0.calib.run${runNumber}.list"
826 echo "cpass0.calib.run${runNumber}.list: benchmark.sh ${arr_cpass0_outputs_onerun[*]}"
827 echo " ./benchmark.sh printValues calibfile cpass0.calib.run${runNumber}.list ${arr_cpass0_outputs_onerun[*]} "
5dfac970 828 echo
fc7e1e45 829
5dfac970 830 #CPass1 list of Calib/QA files to merge
cb473870 831 # the trick is to have the string "Stage.txt" in the file name of the list of directories with QA output to trigger
8d6a0a43 832 # the production of the trending tree (only then the task->Finish() will be called in QAtrain_duo.C, on the grid
833 # this corresponds to the last merging stage)
cb473870 834 arr_cpass1_calib_list[$runindex]="cpass1.calib.run${runNumber}.list"
835 echo "cpass1.calib.run${runNumber}.list : benchmark.sh ${arr_cpass1_outputs_onerun[*]}"
836 echo " ./benchmark.sh printValues calibfile cpass1.calib.run${runNumber}.list ${arr_cpass1_outputs_onerun[*]};"
837 echo
838 arr_cpass1_QA_list[$runindex]="cpass1.QA.run${runNumber}.lastMergingStage.txt.list"
839 echo "cpass1.QA.run${runNumber}.lastMergingStage.txt.list: benchmark.sh ${arr_cpass1_outputs_onerun[*]}"
840 echo " ./benchmark.sh printValues dir cpass1.QA.run${runNumber}.lastMergingStage.txt.list ${arr_cpass1_outputs_onerun[*]}"
5dfac970 841 echo
fc7e1e45 842
5dfac970 843 #CPass0 merging
844 arr_cpass0_final[$runindex]="merge.cpass0.run${runNumber}.done"
845 echo "cpass0.localOCDB.${runNumber}.tgz ${arr_cpass0_final[$runindex]}: cpass0.calib.run${runNumber}.list benchmark.sh ${configFile} ${copyFiles[@]}"
cb473870 846 echo " ${alirootEnv} ./benchmark.sh MergeCPass0 ${commonOutputPath}/cpass0/000${runNumber} $currentDefaultOCDB ${configFile} $runNumber cpass0.calib.run${runNumber}.list"
5dfac970 847 echo
fc7e1e45 848
5dfac970 849 #CPass1 Calib/QA merging
850 arr_cpass1_final[$runindex]="merge.cpass1.run${runNumber}.done"
cb473870 851 echo "${arr_cpass1_final[$runindex]}: cpass0.localOCDB.${runNumber}.tgz cpass1.calib.run${runNumber}.list cpass1.QA.run${runNumber}.lastMergingStage.txt.list benchmark.sh ${configFile} ${copyFiles[@]}"
852 echo " ${alirootEnv} ./benchmark.sh MergeCPass1 ${commonOutputPath}/cpass1/000${runNumber} $currentDefaultOCDB ${configFile} $runNumber cpass1.calib.run${runNumber}.list cpass1.QA.run${runNumber}.lastMergingStage.txt.list"
5dfac970 853 echo
cb473870 854
855 if [[ -n $runESDfiltering ]]; then
856 #CPass1 list of final ESD files
857 arr_cpass1_ESD_list[$runindex]="cpass1.ESD.run${runNumber}.list"
858 echo "${arr_cpass1_ESD_list[$runindex]} : benchmark.sh ${arr_cpass1_outputs_onerun[*]}"
859 echo " ./benchmark.sh printValues esd ${arr_cpass1_ESD_list[$runindex]} ${arr_cpass1_outputs_onerun[*]}"
860 echo
861
862 #run the filtering
863 arr_cpass1_filtering[${runindex}]="filtering.cpass1.run${runNumber}.done"
864 echo "${arr_cpass1_filtering[${runindex}]} : benchmark.sh ${configFile} ${arr_cpass1_ESD_list[$runindex]} cpass0.localOCDB.${runNumber}.tgz"
865 echo " LOCAL ./benchmark.sh MakeFilteredTrees ${commonOutputPath}/cpass1/000${runNumber}/ ${runNumber} ${arr_cpass1_ESD_list[$runindex]} ${filteringFactorHighPt} ${filteringFactorV0s} ${currentDefaultOCDB}"
866 echo
867 fi
868
869 arr_cleanup[$runindex]="${arr_cpass0_outputs_onerun[*]} ${arr_cpass1_outputs_onerun[*]} cpass0.localOCDB.${runNumber}.tgz"
870
5dfac970 871 ((runindex++))
872 done
fc7e1e45 873
cb473870 874 ## final output:
5dfac970 875 #CPass1 list of final Calib/QA files
cb473870 876 echo "calib.list : benchmark.sh ${arr_cpass1_final[*]}"
877 echo " ./benchmark.sh printValues calibfile calib.list ${arr_cpass1_final[*]};"
878 echo
879 echo "qa.list : benchmark.sh ${arr_cpass1_final[*]}"
880 echo " ./benchmark.sh printValues qafile qa.list ${arr_cpass1_final[*]}"
5dfac970 881 echo
cb473870 882 echo "trending.list : benchmark.sh ${arr_cpass1_final[*]}"
883 echo " ./benchmark.sh printValues trendingfile trending.list ${arr_cpass1_final[*]}"
884 echo
885 if [[ -n $runESDfiltering ]]; then
886 #list of filtered files
887 echo "filtering.list : benchmark.sh ${arr_cpass1_filtering[*]}"
888 echo " ./benchmark.sh printValues filteredTree filtering.list ${arr_cpass1_filtering[*]}"
889 echo
890 fi
891
892 #merge trending
893 echo "trending_merged.root : benchmark.sh benchmark.config trending.list"
894 echo " ./benchmark.sh merge trending.list trending_merged.root ${configFile}"
895
896 #makeQAplots
897 if [[ $runMakeQAplots -eq 1 ]]; then
898 echo "QAplots/ : benchmark.sh benchmark.config qa.list"
899 echo " ./benchmark.sh CreateQAplots qa.list $productionID QAplots $configFile"
900 fi
fc7e1e45 901
5dfac970 902 #Summary
cb473870 903 echo "summary_${productionID}.log : ${arr_cpass0_outputs_onerun[*]} ${arr_cpass1_outputs_onerun[*]} ${arr_cpass1_final[*]} ${arr_cpass0_final[*]} benchmark.sh ${configFile}"
904 echo " LOCAL ./benchmark.sh makeSummary ${configFile} ${commonOutputPath} |tee summary_${productionID}.log"
905 echo
906
907}
908
909goPrintValues()
910{
911 #print the values given the key from any number of files (store in output file on second argument)
912 key=$1
913 outputFile=$2
914 shift 2 #remove 2 first arguments from arg list to only pass the input files to awk
915 awk -v key=${key} '$0 ~ "^"key" " {print $2}' "$@" | tee $outputFile
5dfac970 916}
fc7e1e45 917
5dfac970 918goCreateQAplots()
919{
920 umask 0002
cb473870 921 mergedQAfileList=$1
922 productionID=$2
5dfac970 923 outputDir=$3
cb473870 924 configFile=$4
fc7e1e45 925
5dfac970 926 source $configFile
927 [[ -f ${setupAliROOTenvInCurrentShell} ]] && source $setupAliROOTenvInCurrentShell
fc7e1e45 928
cb473870 929 [[ -z $logOutputDir ]] && logOutputDir=$PWD
5dfac970 930 [[ -z $dontRedirectStdOutToLog ]] && exec 2>&1 > $logOutputDir/makeQAplots.log
931 echo "$0 $*"
fc7e1e45 932
5dfac970 933 [[ -z "$qaPlotsScript" ]] && echo "qaPlotsScript not defined"&&exit 1
934
5dfac970 935 echo $qaPlotsScript "$productionID" "cpass1" $mergedQAfileList $outputDir
936 $qaPlotsScript "$productionID" "cpass1" $mergedQAfileList $outputDir
5dfac970 937}
fc7e1e45 938
5dfac970 939goTest()
940{
941 umask 0002
942 exec 2>&1
943 exec > >(tee test.log)
944 echo "$@"
945 echo something
946}
fc7e1e45 947
5dfac970 948alirootInfo()
949{
950 (
951 umask 0002
952 # save aliroot svn info
953 [[ -z "$ALICE_ROOT" ]] && exit 1
954 prevdir=$PWD
955 cd $ALICE_ROOT
956 echo "\$ALICE_ROOT=$ALICE_ROOT"
957 echo
958 svn info 2>/dev/null
959 echo ""
960 echo ""
961 svn diff 2>/dev/null
962 cd $prevdir
963 )
964}
fc7e1e45 965
5dfac970 966setYear()
967{
968 #set the year
969 # $1 - year to be set
970 # $2 - where to set the year
971 year1=$(guessYear $1)
972 year2=$(guessYear $2)
973 local path=$2
cb473870 974 [[ $year1 -ne $year2 && -n $year2 && -n $year1 ]] && path=${2/\/$year2\//\/$year1\/}
5dfac970 975 echo $path
976}
fc7e1e45 977
5dfac970 978guessPeriod()
979{
980 #guess the period from the path, pick the rightmost one
981 local IFS="/"
982 declare -a path=( $1 )
983 local dirDepth=${#path[*]}
984 for ((x=${dirDepth}-1;x>=0;x--)); do
985 local field=${path[${x}]}
986 [[ ${field} =~ ^LHC[0-9][0-9][a-z]$ ]] && period=${field#000} && break
987 done
988 echo $period
989}
fc7e1e45 990
5dfac970 991guessYear()
992{
993 #guess the year from the path, pick the rightmost one
994 local IFS="/"
995 declare -a path=( $1 )
996 local dirDepth=${#path[*]}
997 for ((x=${dirDepth}-1;x>=0;x--)); do
998 local field=${path[${x}]}
999 [[ ${field} =~ ^20[0-9][0-9]$ ]] && year=${field#000} && break
1000 done
1001 echo $year
1002}
fc7e1e45 1003
5dfac970 1004guessRunNumber()
1005{
1006 #guess the run number from the path, pick the rightmost one
1007 local IFS="/"
1008 declare -a path=( $1 )
1009 local dirDepth=${#path[*]}
1010 for ((x=${dirDepth}-1;x>=0;x--)); do
1011 local field=${path[${x}]}
1012 [[ ${field} =~ ^000[0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && runNumber=${field#000} && break
1013 done
1014 echo $runNumber
1015}
fc7e1e45 1016
cb473870 1017validateLog()
1018{
1019 log=$1
1020 errorConditions=(
1021 "There was a crash"
1022 "floating"
1023 "error while loading shared libraries"
1024 "std::bad_alloc"
1025 "s_err_syswatch_"
1026 "Thread [0-9]* (Thread"
1027 "AliFatal"
1028 "core dumped"
1029 )
1030
1031 local logstatus=0
1032 local errorSummary=""
1033 for ((i=0; i<${#errorConditions[@]};i++)); do
1034 local tmp=$(grep -m1 -e "${errorConditions[$i]}" $log)
1035 [[ -n $tmp ]] && tmp+=" : "
1036 errorSummary+=$tmp
1037 done
1038 if [[ -n $errorSummary ]]; then
1039 logstatus=1
1040 echo "$errorSummary"
1041 fi
1042
1043 return $logstatus
1044}
1045
5dfac970 1046summarizeLogs()
1047{
1048 #print a summary of logs
1049 logFiles=(
1050 "*.log"
cb473870 1051 "stdout"
1052 "stderr"
5dfac970 1053 )
fc7e1e45 1054
5dfac970 1055 errorConditions=(
1056 "There was a crash"
1057 "floating"
1058 "error while loading shared libraries"
1059 "std::bad_alloc"
1060 "s_err_syswatch_"
1061 "Thread [0-9]* (Thread"
cb473870 1062 "AliFatal"
1063 "core dumped"
5dfac970 1064 )
cb473870 1065
1066 [[ -z ${outputDir} ]] && outputDir=$PWD
1067 #check logs
5dfac970 1068 logstatus=0
1069 for log in ${logFiles[*]}; do
1070 finallog=${outputDir%/}/${log}
1071 [[ ! -f $log ]] && continue
cb473870 1072 errorSummary=$(validateLog $log)
5dfac970 1073 if [[ -z $errorSummary ]]; then
1074 #in pretend mode randomly report an error in rec.log some cases
1075 if [[ -n $pretend && "$log" == "rec.log" ]]; then
1076 [[ $(( $RANDOM%2 )) -ge 1 ]] && echo "$finallog BAD random error" || echo "$finallog OK"
fc7e1e45 1077 else
5dfac970 1078 echo "$finallog OK"
fc7e1e45 1079 fi
5dfac970 1080 else
1081 local logstatus=1
1082 echo "$finallog BAD $errorSummary"
1083 fi
1084 done
cb473870 1085
1086 #report core files
1087 find ${PWD} -name core
1088
5dfac970 1089 return $logstatus
1090}
fc7e1e45 1091
5dfac970 1092spitOutLocalOCDBaccessConfig()
1093{
1094 umask 0002
1095 find $1 -name "*root" | \
1096 while read line
1097 do
1098 local tmp=${line#$1}
1099 echo ${tmp%/*} | \
1100 awk -v ocdb=$1 '{print " man->SetSpecificStorage(\""$1"\",\"local://"ocdb"\");"}'
1101 done
1102}
fc7e1e45 1103
5dfac970 1104goConfigureOCDBforCPass1()
1105{
1106 umask 0002
1107 # make a script that sets the specific storages form all the root files produced by CPass0
1108 # in this case the second argument is taken to be the path to the produced OCDB for specific storage
1109 local localOCDBpathCPass0=$1
1110 local outputDir=$2
1111 [[ -d $outputDir ]] && cd $outputDir
fc7e1e45 1112
5dfac970 1113 if [[ -f $localOCDBpathCPass0 && $localOCDBpathCPass0 =~ \.tgz$ ]]; then
1114 tar xzf $localOCDBpathCPass0
cb473870 1115 local localOCDBpathCPass0="${PWD}/OCDB"
5dfac970 1116 fi
fc7e1e45 1117
5dfac970 1118 echo
1119 echo creating the specific storage script
1120 echo localOCDBaccessConfig.C
cb473870 1121 echo based on OCDB: $localOCDBaccessConfig
5dfac970 1122 echo
fc7e1e45 1123
5dfac970 1124 local tempLocalOCDB=""
1125 if [[ -f localOCDBaccessConfig.C ]]; then
1126 tempLocalOCDB=$(mktemp)
1127 echo "egrep "SetSpecificStorage" localOCDBaccessConfig.C > $tempLocalOCDB"
1128 egrep "SetSpecificStorage" localOCDBaccessConfig.C > $tempLocalOCDB
1129 fi
fc7e1e45 1130
5dfac970 1131 echo "localOCDBaccessConfig()" > localOCDBaccessConfig.C
1132 echo "{" >> localOCDBaccessConfig.C
1133 echo " AliCDBManager* man = AliCDBManager::Instance();" >> localOCDBaccessConfig.C
1134 spitOutLocalOCDBaccessConfig $localOCDBpathCPass0|sort|uniq >> localOCDBaccessConfig.C
1135 [[ -f "$tempLocalOCDB" ]] && cat $tempLocalOCDB >> localOCDBaccessConfig.C
1136 echo "}" >> localOCDBaccessConfig.C
fc7e1e45 1137
5dfac970 1138 [[ -f "$tempLocalOCDB" ]] && rm -f $tempLocalOCDB
fc7e1e45 1139
5dfac970 1140 if ! grep SetSpecificStorage localOCDBaccessConfig.C; then
1141 echo
1142 echo "!!!!!!! CPass0 produced no OCDB entries"
1143 return 1
1144 fi
1145}
fc7e1e45 1146
cb473870 1147goMakeFilteredTrees()
1148{
1149 outputDir=$1
1150 runNumber=$2
1151 #get path to input list
1152 inputListfiles=$3
1153 #get scale number for tracks
1154 filterT=$4
1155 #get scale number for V0s
1156 filterV=$5
1157 #get OCDB path (optional)
1158 OCDBpath=${6}
1159 #get max number of files
1160 maxFiles=${7-"1000000"}
1161 #get offset of first file
1162 offsetFile=${8-"0"}
1163 #get max number of events
1164 maxEvents=${9-"30000000"}
1165 #get offset of first event
1166 offsetEvent=${10-"0"}
1167 configFile=${11-"benchmark.config"}
1168 esdFileName=${12-"AliESDs_Barrel.root"}
1169 source ${configFile}
1170 [[ -n ${setupTrustedAliROOTenvInCurrentShell} ]] && source ${setupAliROOTenvInCurrentShell}
1171
1172 commonOutputPath=${PWD}
1173 runpath=${commonOutputPath}/rundir_filtering_run${runNumber}
1174
1175 mkdir -p ${outputDir}
1176 mkdir -p ${runpath}
1177
1178 cd ${runpath}
1179
1180 cat > filtering.log << EOF
1181 goMakeFilteredTrees config:
1182 runpath=$runpath
1183 outputDir=$outputDir
1184 commonOutputPath=$commonOutputPath
1185 ALICE_ROOT=$ALICE_ROOT
1186 PATH=$PATH
1187EOF
1188
1189 if [[ -z ${pretend} ]];then
1190 aliroot -l -b -q "$filteringRunMacro(\"$inputListfiles\",$filterT,$filterV,\"$OCDBpath\",$maxFiles,$offsetFile,$maxEvents,$offsetEvent,\"$esdFileName\")" &>> filtering.log
1191 else
1192 touch filtering.log Filtered.root
1193 fi
1194 pwd
1195 ls -ltrh
1196 echo mv -f * ${outputDir}
1197 mv -f * ${outputDir}
1198 doneFile=${commonOutputPath}/filtering.cpass1.run${runNumber}.done
1199 touch ${doneFile}
1200 [[ -f ${outputDir}/Filtered.root ]] && echo "filteredTree ${outputDir}/Filtered.root" > ${doneFile}
1201 [[ -f ${outputDir}/filtering.log ]] && echo "${outputDir}/filtering.log"
1202 cd ${commonOutputPath}
1203 [[ "$runpath" != "$outputDir" ]] && rm -rf ${runpath}
1204}
1205
1206submit()
1207{
1208 umask 0002
1209 [[ $# -ne 6 ]] && echo "6 args needed, you supplied $#" && exit 1
1210 JobID=$1
1211 startID=$2
1212 endID=$3
1213 waitForJOBID=$4
1214 command=$5
1215 commandArgs=$6
1216
1217 [[ -z $waitForJOBID ]] && waitForJOBID=0
1218
1219 newFarm=$(which qsub|grep "^/usr/bin/qsub")
1220
1221 batchSystem="SGE"
1222
1223 if [[ -z "$newFarm" ]]
1224 then
1225 #old LSF
1226 # submit it (as job array)
1227 nFiles=$(( $endID-$startID+1 ))
1228 while [ $startID -le $nFiles ] ; do
1229 if [ `expr $nFiles - $startID` -gt 999 ] ; then
1230 endID=`expr $startID + 999`
1231 else
1232 endID=$nFiles
1233 fi
1234 if [[ ${waitForJOBID} -eq 0 ]]; then
1235 echo $batchCommand $batchFlags -J "$JobID[$startID-$endID]" -e "$targetDirectory/logs/job_%I.err" -o "$targetDirectory/logs/job_%I.out" "$command"
1236 $batchCommand $batchFlags -J "$JobID[$startID-$endID]" -e "$targetDirectory/logs/job_%I.err" -o "$targetDirectory/logs/job_%I.out" "$command"
1237 else
1238 echo $batchCommand $batchFlags -J "$JobID[$startID-$endID]" -w "ended($waitForJOBID)" -e "$targetDirectory/logs/job_%I.err" -o "$targetDirectory/logs/job_%I.out" "$command"
1239 $batchCommand $batchFlags -J "$JobID[$startID-$endID]" -w "ended($waitForJOBID)" -e "$targetDirectory/logs/job_%I.err" -o "$targetDirectory/logs/job_%I.out" "$command"
1240 fi
1241 startID=`expr $endID + 1`
1242 done
1243 else
1244 #new SGE farm
1245 if [[ ${waitForJOBID} =~ "000" ]]; then
1246 echo $batchCommand $batchFlags -wd ${targetDirectory} -b y -V -N "$JobID" -t "$startID-$endID" -e "$targetDirectory/logs/" -o "$targetDirectory/logs/" "$command" $commandArgs
1247 $batchCommand $batchFlags -wd ${targetDirectory} -b y -V -N "$JobID" -t "$startID-$endID" -e "$targetDirectory/logs/" -o "$targetDirectory/logs/" "$command" $commandArgs
1248 else
1249 echo $batchCommand $batchFlags -wd ${targetDirectory} -b y -V -N "$JobID" -t "$startID-$endID" -hold_jid "$waitForJOBID" -e "$targetDirectory/logs/" -o "$targetDirectory/logs/" "$command" $commandArgs
1250 $batchCommand $batchFlags -wd ${targetDirectory} -b y -V -N "$JobID" -t "$startID-$endID" -hold_jid "$waitForJOBID" -e "$targetDirectory/logs/" -o "$targetDirectory/logs/" "$command" $commandArgs
1251 fi
1252 fi
1253}
1254
1255goSubmitBatch()
5dfac970 1256{
1257 inputList=$1
1258 productionID=$2
1259 configFile="benchmark.config"
1260 [[ -n "$3" ]] && configFile=$3
1261 configFile=$(readlink -f $configFile)
1262 [[ -n $4 ]] && runNumber=$4
fc7e1e45 1263
5dfac970 1264 #redirect all output to submit.log
1265 echo "redirecting all output to ${PWD}/submit_${productionID//"/"/_}.log"
1266 exec 7>&1
1267 exec 1>submit_${productionID//"/"/_}.log 2>&1
fc7e1e45 1268
5dfac970 1269 umask 0002
1270 echo $0" submit $*"
1271 if [[ -z "$inputList" || -z "$productionID" ]]
1272 then
fc7e1e45 1273 echo
5dfac970 1274 echo " Usage: $0 submit inputList productionID [configFile=benchmark.config]"
fc7e1e45 1275 echo
5dfac970 1276 exit
fc7e1e45 1277 fi
fc7e1e45 1278
5dfac970 1279 # check if config file is there
1280 if [ ! -f $configFile ]; then
1281 echo "ERROR! Config File '$configFile' not found" >&2
1282 exit
1283 else
1284 echo "Using Config File: '$configFile'"
fc7e1e45 1285 fi
1286
5dfac970 1287 # source the config file
1288 # and print the configuration
fc7e1e45 1289 source $configFile
5dfac970 1290 [[ -f ${setupAliROOTenvInCurrentShell} && -z ${alirootEnv} ]] && source $setupAliROOTenvInCurrentShell
fc7e1e45 1291
cb473870 1292 #move the script, config and some other stuff to $commonOutputPath first, then use them from there
5dfac970 1293 self=$(readlink -f "$0")
cb473870 1294 configPath=$(dirname $configFile)
5dfac970 1295 export commonOutputPath=${baseOutputDirectory}/${productionID}
cb473870 1296 mkdir -p ${commonOutputPath}
1297 cp $self ${commonOutputPath}
1298 cp $configFile ${commonOutputPath}
1299 cp $inputList ${commonOutputPath}
1300 self=${commonOutputPath}/${self##*/}
1301 chmod u+x $self
1302 configFile=${commonOutputPath}/${configFile##*/}
1303 inputList=${commonOutputPath}/${inputList##*/}
fc7e1e45 1304
5dfac970 1305 #convert to absolut pathnames
1306 inputList=$(readlink -f "$inputList")
1307 #make list of runs
1308 if [[ -z $runNumber ]]; then
1309 listOfRuns=($(while read x; do guessRunNumber $x; done < ${inputList} | sort | uniq))
1310 else
1311 listOfRuns=$runNumber
1312 fi
fc7e1e45 1313
5dfac970 1314 setupAliROOTenvInCurrentShell=$(readlink -f "$setupAliROOTenvInCurrentShell")
fc7e1e45 1315
5dfac970 1316 echo ""
1317 echo "### BEGIN CONFIGURATION ###"
1318 echo ""
1319 echo "GENERAL:"
1320 echo ""
1321 echo " productionID: $productionID"
1322 echo " batchCommand: $batchCommand"
cb473870 1323 echo " batchFlags: $batchFlags"
5dfac970 1324 echo " setupAliROOTenvInCurrentShell: $setupAliROOTenvInCurrentShell"
1325 echo " ALICE_ROOT: $ALICE_ROOT"
1326 echo " ALIROOT_RELEASE: $ALICE_RELEASE"
1327 echo " inputList: $inputList"
1328 echo " configPath: $configPath"
1329 echo " commonOutputPath: $commonOutputPath"
1330 echo " defaultOCDB: $defaultOCDB"
1331 echo " autoOCDB: $autoOCDB"
1332 echo " recoTriggerOptions: $recoTriggerOptions"
1333 echo " runs:"
1334 echo " ${listOfRuns[*]}"
1335 echo ""
1336 echo "THE TRAIN WILL RUN:"
fc7e1e45 1337
5dfac970 1338 if [ $runCPass0reco -eq 1 ]; then
1339 echo " Pass0 - Recontruction"
1340 fi
fc7e1e45 1341
5dfac970 1342 if [ $runCPass0MergeMakeOCDB -eq 1 ]; then
1343 echo " Pass0 - merging and OCDB export"
1344 fi
fc7e1e45 1345
5dfac970 1346 if [ $runCPass1reco -eq 1 ]; then
1347 echo " Pass1 - Recontruction"
1348 fi
1349 if [ $runCPass1MergeMakeOCDB -eq 1 ]; then
1350 echo " Pass1 - merging and OCDB export"
1351 fi
fc7e1e45 1352
5dfac970 1353 echo ""
1354 echo "LIMITS:"
1355 echo " max. Events/Chunk: $nEvents"
1356 echo " max. Number of Chunks per Run: $nMaxChunks"
1357 echo ""
1358 echo "### END CONFIGURATION ###"
1359 echo ""
fc7e1e45 1360
36b5de1a 1361
5dfac970 1362 # check if input file is there
1363 if [ ! -f $inputList ]; then
1364 echo "ERROR! Input List '$inputList' not found" >&2
1365 exit
1366 fi
1367
1368 # define jobid (for dependent jobs)
cb473870 1369 date=$(date +%Y_%m_%d_%H%M%S)
5dfac970 1370 #for each run we submit one jobarray:
fc7e1e45 1371 for runNumber in ${listOfRuns[*]}; do
cb473870 1372
fc7e1e45 1373 [[ -z $runNumber ]] && continue
1374 [[ ! ${runNumber} =~ ^[0-9]*[0-9]$ ]] && continue
cb473870 1375
1376 JOBpostfix="${productionID//"/"/_}_${runNumber}_${date}"
1377 JOBID1="p0_${JOBpostfix}"
1378 JOBID1wait="w0_${JOBpostfix}"
1379 JOBID2="m0_${JOBpostfix}"
1380 JOBID2wait="wm0_${JOBpostfix}"
1381 JOBID3="op0_${JOBpostfix}"
1382 JOBID3wait="wop0_${JOBpostfix}"
1383 JOBID4="p1_${JOBpostfix}"
1384 JOBID4wait="w1_${JOBpostfix}"
1385 JOBID5="m1_${JOBpostfix}"
1386 JOBID5wait="wm1_${JOBpostfix}"
1387 JOBID6="s1_${JOBpostfix}"
1388 JOBID6wait="ws1_${JOBpostfix}"
1389 JOBID7="QA_${JOBpostfix}"
1390 JOBmakeESDlistCPass1="lp1_${JOBpostfix}"
1391 JOBfilterESDcpass1="fp1_${JOBpostfix}"
1392 LASTJOB="000"
1393
5dfac970 1394 oneInputFile=$(egrep -m1 "$runNumber\/" ${inputList})
cb473870 1395
1396 currentDefaultOCDB=${defaultOCDB}
1397 [[ -z ${autoOCDB} ]] && autoOCDB=1
1398 if [[ ${autoOCDB} -ne 0 ]]; then
1399 currentDefaultOCDB=$(setYear $oneInputFile $defaultOCDB)
1400 fi
1401
5dfac970 1402 echo "submitting run $runNumber with OCDB $currentDefaultOCDB"
fc7e1e45 1403
5dfac970 1404 ################################################################################
1405 ################################################################################
1406 # run the CPass0 if requested
fc7e1e45 1407
5dfac970 1408 if [ $runCPass0reco -eq 1 ]; then
fc7e1e45 1409
fc7e1e45 1410 echo
5dfac970 1411 echo "starting CPass0... for run $runNumber"
fc7e1e45 1412 echo
fc7e1e45 1413
5dfac970 1414 # create directory and copy all files that are needed
cb473870 1415 targetDirectory="${commonOutputPath}/000${runNumber}/cpass0"
5dfac970 1416 mkdir -p $targetDirectory
1417 mkdir -p $targetDirectory/logs
cb473870 1418 filesCPass0=(
1419 "$configPath/runCPass0.sh"
1420 "$configPath/recCPass0.C"
1421 "$configPath/runCalibTrain.C"
1422 "$configPath/localOCDBaccessConfig.C"
1423 "$configPath/OCDB.root"
1424 )
1425 for file in ${filesCPass0[*]}; do
1426 [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
1427 done
fc7e1e45 1428
cb473870 1429
1430 localInputList=$targetDirectory/${inputList##*/}
5dfac970 1431 rm -f $localInputList
1432 egrep "\/000$runNumber\/" $inputList >> $localInputList
5dfac970 1433 # limit nFiles to nMaxChunks
1434 nFiles=`wc -l < $localInputList`
1435 [[ $nFiles -eq 0 ]] && echo "list contains ZERO files! exiting..." && exit 1
1436 echo "raw files in list: $nFiles"
1437 if [[ $nMaxChunks -gt 0 && $nMaxChunks -le $nFiles ]]; then
1438 nFiles=$nMaxChunks
1439 fi
1440 echo "raw files to process: $nFiles"
1441 [[ -z "$percentProcessedFilesToContinue" ]] && percentProcessedFilesToContinue=100
1442 if [[ $percentProcessedFilesToContinue -eq 100 ]]; then
1443 nFilesToWaitFor=$nFiles
1444 else
1445 nFilesToWaitFor=$(( $nFiles-$nFiles/(100/(100-$percentProcessedFilesToContinue)) ))
1446 fi
1447 echo "requested success rate is $percentProcessedFilesToContinue%"
1448 echo "merging will start after $nFilesToWaitFor jobs are done"
fc7e1e45 1449
cb473870 1450 submit $JOBID1 1 $nFiles 000 "$self" "CPass0 $targetDirectory $localInputList $nEvents $currentDefaultOCDB $configFile $runNumber"
fc7e1e45 1451
5dfac970 1452 ## submit a monitoring job that will run until a certain number of jobs are done with reconstruction
cb473870 1453 submit "$JOBID1wait" 1 1 000 "$self" "WaitForOutput ${commonOutputPath} 'cpass0.job*.run$runNumber.done' $nFilesToWaitFor $maxSecondsToWait '-maxdepth 1'"
5dfac970 1454 LASTJOB=$JOBID1wait
fc7e1e45 1455
5dfac970 1456 fi #end running CPass0
1457 ################################################################################
fc7e1e45 1458
fc7e1e45 1459
5dfac970 1460 ################################################################################
1461 # submit merging of CPass0, depends on the reconstruction
fc7e1e45 1462
5dfac970 1463 if [ $runCPass0MergeMakeOCDB -eq 1 ]; then
fc7e1e45 1464
5dfac970 1465 echo
1466 echo "submit CPass0 merging for run $runNumber"
1467 echo
fc7e1e45 1468
cb473870 1469 targetDirectory="${commonOutputPath}/000${runNumber}/cpass0"
5dfac970 1470 mkdir -p $targetDirectory
fc7e1e45 1471
cb473870 1472 #copy the scripts
1473 filesMergeCPass0=(
1474 "$configPath/OCDB.root"
1475 "$configPath/mergeMakeOCDB.byComponent.sh"
1476 "$configPath/mergeMakeOCDB.sh"
1477 "$configPath/localOCDBaccessConfig.C"
1478 "$configPath/mergeByComponent.C"
1479 "$configPath/makeOCDB.C"
1480 "$configPath/merge.C"
1481 )
1482 for file in ${filesMergeCPass0[*]}; do
1483 [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
1484 done
1485
fc7e1e45 1486
cb473870 1487 echo submit $JOBID2 1 1 "$LASTJOB" "$self" "MergeCPass0 $targetDirectory $currentDefaultOCDB $configFile $runNumber"
1488 submit $JOBID2 1 1 "$LASTJOB" "$self" "MergeCPass0 $targetDirectory $currentDefaultOCDB $configFile $runNumber"
5dfac970 1489 LASTJOB=$JOBID2
fc7e1e45 1490
5dfac970 1491 echo
fc7e1e45 1492 fi
5dfac970 1493 # end of merging CPass0
1494 ################################################################################
fc7e1e45 1495
5dfac970 1496 ################################################################################
1497 ################################################################################
1498 # run the CPass1 if requested
fc7e1e45 1499
5dfac970 1500 if [ $runCPass1reco -eq 1 ]; then
fc7e1e45 1501
cb473870 1502 targetDirectory="${commonOutputPath}/000${runNumber}/cpass1"
fc7e1e45 1503
5dfac970 1504 # safety feature: if we are re-running for any reason we want to delete the previous output first.
1505 [[ -d $targetDirectory ]] && rm -rf $targetDirectory/* && echo "removed old output at $targetDirectory/*"
fc7e1e45 1506
5dfac970 1507 echo
1508 echo "starting CPass1... for run $runNumber"
1509 echo
1510
1511 # create directory and copy all files that are needed
1512 mkdir -p $targetDirectory
1513 mkdir -p $targetDirectory/logs
1514
cb473870 1515 cp -f $configFile $targetDirectory
1516 filesCPass1=(
1517 "$configPath/runCPass1.sh"
1518 "$configPath/recCPass1.C"
1519 "$configPath/recCPass1_OuterDet.C"
1520 "$configPath/runCalibTrain.C"
1521 "$configPath/QAtrain_duo.C"
1522 "$configPath/localOCDBaccessConfig.C"
1523 "$configPath/OCDB.root"
1524 )
1525 for file in ${filesCPass1[*]}; do
1526 [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
1527 done
5dfac970 1528
cb473870 1529 localInputList=$targetDirectory/${inputList##*/}
5dfac970 1530 rm -f $localInputList
1531 egrep "\/000$runNumber\/" $inputList >> $localInputList
5dfac970 1532 # limit nFiles to nMaxChunks
1533 nFiles=`wc -l < $localInputList`
1534 [[ $nFiles -eq 0 ]] && echo "list contains ZERO files! exiting..." && exit 1
1535 echo "raw files in list: $nFiles"
1536 if [[ $nMaxChunks -gt 0 && $nMaxChunks -le $nFiles ]]; then
1537 nFiles=$nMaxChunks
1538 fi
1539 echo "raw files to process: $nFiles"
1540 [[ -z "$percentProcessedFilesToContinue" ]] && percentProcessedFilesToContinue=100
1541 if [[ $percentProcessedFilesToContinue -eq 100 ]]; then
1542 nFilesToWaitFor=$nFiles
fc7e1e45 1543 else
5dfac970 1544 nFilesToWaitFor=$(( $nFiles-$nFiles/(100/(100-$percentProcessedFilesToContinue)) ))
fc7e1e45 1545 fi
5dfac970 1546 echo "requested success rate is $percentProcessedFilesToContinue%"
1547 echo "merging will start after $nFilesToWaitFor jobs are done"
fc7e1e45 1548
cb473870 1549 submit $JOBID4 1 $nFiles "$LASTJOB" "$self" "CPass1 $targetDirectory $localInputList $nEvents $currentDefaultOCDB $configFile $runNumber"
fc7e1e45 1550
5dfac970 1551 ################################################################################
1552 ## submit a monitoring job that will run until a certain number of jobs are done with reconstruction
cb473870 1553 submit "$JOBID4wait" 1 1 "$LASTJOB" "$self" "WaitForOutput ${commonOutputPath} 'cpass1.job*.run${runNumber}.done' $nFilesToWaitFor $maxSecondsToWait '-maxdepth 1'"
5dfac970 1554 LASTJOB=$JOBID4wait
1555 ################################################################################
fc7e1e45 1556
5dfac970 1557 echo
1558 fi #end running CPass1
fc7e1e45 1559
5dfac970 1560 ################################################################################
1561 # submit merging of CPass1, depends on the reconstruction
1562 if [ $runCPass1MergeMakeOCDB -eq 1 ]; then
fc7e1e45 1563
5dfac970 1564 echo
1565 echo "submit CPass1 merging for run $runNumber"
1566 echo
fc7e1e45 1567
cb473870 1568 targetDirectory="${commonOutputPath}/000${runNumber}/cpass1"
5dfac970 1569 mkdir -p $targetDirectory
fc7e1e45 1570
cb473870 1571 # copy files
1572 filesMergeCPass1=(
1573 "$configPath/OCDB.root"
1574 "$configPath/localOCDBaccessConfig.C"
1575 "$configPath/mergeMakeOCDB.byComponent.sh"
1576 "$configPath/mergeByComponent.C"
1577 "$configPath/makeOCDB.C"
1578 "$configPath/merge.C"
1579 "$configPath/mergeMakeOCDB.sh"
1580 "$configPath/QAtrain_duo.C"
1581 )
1582 for file in ${filesMergeCPass1[*]}; do
1583 [[ -f ${file} ]] && echo "copying ${file}" && cp -f ${file} ${commonOutputPath}
1584 done
1585
1586 echo submit "$JOBID5" 1 1 "$LASTJOB" "$self" "MergeCPass1 $targetDirectory $currentDefaultOCDB $configFile $runNumber"
1587 submit "$JOBID5" 1 1 "$LASTJOB" "$self" "MergeCPass1 $targetDirectory $currentDefaultOCDB $configFile $runNumber"
5dfac970 1588 LASTJOB=$JOBID5
1589 echo
1590 fi
1591
cb473870 1592 ##############################
1593 if [ $runESDfiltering -eq 1 ]; then
1594 export targetDirectory=$commonOutputPath
1595 echo submitting filtering
1596 echo targetDirectory=$targetDirectory
1597 submit "$JOBmakeESDlistCPass1" 1 1 "$LASTJOB" "$self" "printValues esd ${commonOutputPath}/cpass1.ESD.run${runNumber}.list ${commonOutputPath}/cpass1.job*.run${runNumber}.done "
1598 submit "$JOBfilterESDcpass1" 1 1 "$JOBmakeESDlistCPass1" "$self" "MakeFilteredTrees ${commonOutputPath}/000${runNumber}/cpass1 ${runNumber} ${commonOutputPath}/cpass1.ESD.run${runNumber}.list ${filteringFactorHighPt} ${filteringFactorV0s} ${currentDefaultOCDB} 1000000 0 10000000 0 ${configFile}"
1599 fi
1600
5dfac970 1601 done
1602
cb473870 1603 #########################################
1604 #make lists with output files - QA, trending, filtering and calibration
1605 export targetDirectory=$commonOutputPath
1606 submit "flQA_${JOBpostfix}" 1 1 "$LASTJOB" "$self" "printValues qafile ${commonOutputPath}/qa.list ${commonOutputPath}/merge.cpass1.run*.done"
1607 submit "flCal_${JOBpostfix}" 1 1 "$LASTJOB" "$self" "printValues calibfile ${commonOutputPath}/calib.list ${commonOutputPath}/merge.cpass1.run*.done"
1608 submit "flTr_${JOBpostfix}" 1 1 "$LASTJOB" "$self" "printValues trendingfile ${commonOutputPath}/trending.list ${commonOutputPath}/merge.cpass1.run*.done"
1609 if [[ -n $runESDfiltering ]]; then
1610 submit "wFil_${JOBpostfix}" 1 1 "$LASTJOB" "$self" "WaitForOutput ${commonOutputPath} 'filtering.cpass1.run*.done' ${#listOfRuns[*]} $maxSecondsToWait '-maxdepth 1'"
1611 submit "flFil_${JOBpostfix}" 1 1 "wFil_${JOBpostfix}" "$self" "printValues filteredTree ${commonOutputPath}/filtering.list ${commonOutputPath}/filtering.cpass1.run*.done"
1612 fi
1613
1614 #merge trending
1615 submit "mTr_${JOBpostfix}" 1 1 "flTr_${JOBpostfix}" "$self" "merge trending.list trending_merged.root ${configFile}"
1616
1617
5dfac970 1618 ################################################################################
1619 ################################################################################
1620 [[ -z $runMakeSummary ]] && runMakeSummary=0
1621 if [ $runMakeSummary -eq 1 ]; then
fc7e1e45 1622 echo
5dfac970 1623 echo "submit make a summary"
1624 echo
1625
cb473870 1626 export targetDirectory="${commonOutputPath}"
5dfac970 1627 mkdir -p $targetDirectory
1628 mkdir -p $targetDirectory/logs
1629 [[ ! -f $targetDirectory/${self##*/} ]] && cp -f $self $targetDirectory
516b77b8 1630 submit "$JOBID6" 1 1 "$LASTJOB" "$self" "makeSummary $configFile $targetDirectory $commonOutputPath"
5dfac970 1631 fi
1632 ################################################################################
1633
1634 ################################################################################
1635 ################################################################################
1636 if [ $runMakeQAplots -eq 1 ]; then
1637 echo
1638 echo "submit make QA plots"
1639 echo
1640
cb473870 1641 mkdir -p ${commonOutputPath}/logs
5dfac970 1642 targetDirectory="${commonOutputPath}"
5dfac970 1643
cb473870 1644 submit "$JOBID7" 1 1 "flQA_${JOBpostfix}" "$self" "CreateQAplots qa.list $productionID QAplots $configFile"
5dfac970 1645 LASTJOB=$JOBID7
fc7e1e45 1646 fi
cb473870 1647 ################################################################################
5dfac970 1648
1649 #restore stdout
1650 exec 1>&7 7>&-
1651 echo "jobs submitted."
fc7e1e45 1652}
1653
cb473870 1654goWaitForOutput()
1655{
1656 umask 0002
1657 [[ $# -lt 3 ]] && echo "goWaitForOutput() wrong number of arguments, exiting.." && exit 1
1658 echo searchPath=$1
1659 echo fileName=$2
1660 echo numberOfFiles=$3
1661 echo maxSecondsToWait=$4
1662 searchPath=$1
1663 fileName=$2
1664 numberOfFiles=$3
1665 maxSecondsToWait=$4
1666 extraFindOptions=$5
1667 echo "command to be executed: find $searchPath -name "$fileName" ${extraFindOptions}"
1668 [[ -z "$maxSecondsToWait" ]] && maxSecondsToWait=$(( 3600*12 ))
1669 while true; do
1670 #n=$(find $searchPath ${extraFindOptions} -name "$fileName" | wc -l)
1671 n=$(/bin/ls -1 ${searchPath}/${fileName} 2>/dev/null | wc -l)
1672 [[ $n -gt 0 ]] && echo "found $n X $fileName"
1673 [[ $n -ge $numberOfFiles ]] && break
1674 [[ $SECONDS -gt $maxSecondsToWait ]] && break
1675 sleep 60
1676 done
1677 echo "DONE! exiting..."
1678}
1679
fc7e1e45 1680main "$@"