]>
Commit | Line | Data |
---|---|---|
fc7e1e45 | 1 | #!/bin/bash |
2 | # this script runs the CPass0/CPass1 train | |
3 | # produced OCDB updates are local | |
4 | ||
5 | main() | |
6 | { | |
7 | #run in proper mode depending on the selection | |
8 | runMode=$1 | |
9 | if [[ $# -gt 0 ]]; then | |
10 | echo "# $0 $*" | |
11 | fi | |
12 | umask 0002 | |
13 | shift | |
14 | case $runMode in | |
15 | "CPass0") goCPass0 "$@";; | |
16 | "CPass1") goCPass1 "$@";; | |
17 | "ConfOCDB") goConfigureOCDBforCPass1 "$@";; | |
18 | "MergeCPass0") goMergeCPass0 "$@";; | |
19 | "MergeCPass1") goMergeCPass1 "$@";; | |
20 | "CreateQAplots") goCreateQAplots "$@";; | |
21 | "WaitForOutput") goWaitForOutput "$@";; | |
22 | "makeSummary") goMakeSummary "$@";; | |
23 | "submit") goSubmit "$@";; | |
24 | "submitQA") goSubmitQA "$@";; | |
25 | "test") goTest "$@";; | |
26 | "generateMakeflow") goGenerateMakeflow "$@";; | |
27 | "makeflow") goMakeflow "$@";; | |
28 | #and the default: unless sourced print some info | |
29 | *) if [[ ! "$0" =~ "bash" ]]; then | |
30 | echo " batch: $0 \"submit\" inputList productionID [configFile=benchmark.config] [runNumber]" | |
31 | echo " makeflow: $0 \"makeflow\" inputList productionID [configFile=benchmark.config] [runNumber]" | |
32 | fi | |
33 | ;; | |
34 | esac | |
35 | } | |
36 | ||
37 | goCPass0() | |
38 | { | |
39 | umask 0002 | |
40 | ||
41 | configFile=$5 | |
42 | source $configFile | |
36b5de1a | 43 | [[ -f ${setupAliROOTenvInCurrentShell} && -z ${alirootEnv} ]] && source $setupAliROOTenvInCurrentShell |
fc7e1e45 | 44 | |
45 | targetDirectory=$1 | |
46 | inputList=$2 | |
47 | nEvents=$3 | |
48 | ocdbPath=$4 | |
49 | configFile=$5 | |
50 | runNumber=$6 | |
51 | jobindex=$7 | |
52 | ||
53 | #use the jobindex only if set and non-negative | |
54 | if [[ -z $jobindex || $jobindex -lt 0 ]]; then | |
55 | [[ -n "$LSB_JOBINDEX" ]] && jobindex=$LSB_JOBINDEX | |
56 | [[ -n "$SGE_TASK_ID" ]] && jobindex=$SGE_TASK_ID | |
57 | fi | |
58 | ||
59 | [[ ! -f ${inputList} && -z ${pretend} ]] && echo "input file $inputList not found, exiting..." && exit 1 | |
60 | if [[ "${inputList}" =~ \.root$ ]]; then | |
61 | infile=$inputList | |
62 | else | |
63 | infile=`sed -ne "${jobindex}p" $inputList` | |
64 | fi | |
65 | ||
66 | chunkName=${infile##*/} | |
67 | outputDir=${targetDirectory}/${jobindex} | |
68 | mkdir -p $outputDir | |
69 | [[ ! -d $outputDir ]] && echo "cannot make $outputDir" && exit 1 | |
70 | ||
71 | runpath=${PWD}/rundir_cpass0_${runNumber}_${jobindex} | |
72 | [[ -z $commonOutputPath ]] && commonOutputPath=$PWD | |
73 | [[ $reconstructInTemporaryDir -eq 1 && -n $TMPDIR ]] && runpath=$TMPDIR | |
74 | [[ $reconstructInTemporaryDir -eq 1 && -z $TMPDIR ]] && runpath=$(mktemp -d) | |
75 | ||
76 | mkdir -p $runpath | |
77 | [[ ! -d ${runpath} ]] && echo "cannot make runpath ${runpath}" && exit 1 | |
78 | cd $runpath | |
79 | ||
80 | logOutputDir=$runpath | |
81 | [[ -n $logToFinalDestination ]] && logOutputDir=$outputDir | |
82 | [[ -z $dontRedirectStdOutToLog ]] && exec 2>&1 > $logOutputDir/runCPass0.log | |
83 | echo "$0 $*" | |
84 | ||
85 | calibDoneFile="$commonOutputPath/cpass0.job${jobindex}.run${runNumber}.done" | |
86 | ||
87 | echo "#####################" | |
88 | echo CPass0: | |
89 | echo JOB setup | |
90 | echo nEvents $nEvents | |
91 | echo runNumber $runNumber | |
92 | echo ocdbPath $ocdbPath | |
93 | echo infile $infile | |
94 | echo chunkName $chunkName | |
95 | echo jobindex $jobindex | |
96 | echo recoTriggerOptions $recoTriggerOptions | |
97 | echo targetDirectory $targetDirectory | |
98 | echo commonOutputPath $commonOutputPath | |
99 | echo calibDoneFile $calibDoneFile | |
100 | echo runpath $runpath | |
101 | echo outputDir $outputDir | |
102 | echo ALICE_ROOT $ALICE_ROOT | |
103 | echo PWD $PWD | |
104 | echo "########## ###########" | |
105 | ||
106 | alirootInfo > ALICE_ROOT_svn.log | |
107 | ||
108 | filesCPass0=( | |
109 | "$commonOutputPath/runCPass0.sh" | |
110 | "$commonOutputPath/recCPass0.C" | |
111 | "$commonOutputPath/runCalibTrain.C" | |
112 | "$commonOutputPath/localOCDBaccessConfig.C" | |
36b5de1a | 113 | "$commonOutputPath/OCDB.root" |
fc7e1e45 | 114 | "$ALICE_ROOT/PWGPP/CalibMacros/CPass0/runCPass0.sh" |
115 | "$ALICE_ROOT/PWGPP/CalibMacros/CPass0/recCPass0.C" | |
116 | "$ALICE_ROOT/PWGPP/CalibMacros/CPass0/runCalibTrain.C" | |
117 | ) | |
118 | ||
119 | for file in ${filesCPass0[*]}; do | |
120 | [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} . | |
121 | done | |
122 | ||
123 | ln -s $infile $runpath/$chunkName | |
124 | ||
125 | echo "this directory ($PWD) contents:" | |
126 | ls -lh | |
127 | echo | |
128 | chmod u+x runCPass0.sh | |
129 | ||
130 | if [[ -n $postSetUpActionCPass0 ]]; then | |
131 | echo "running $postSetUpActionCPass0" | |
132 | eval $postSetUpActionCPass0 | |
133 | fi | |
134 | ||
135 | #run CPass0 | |
136 | echo "$runpath/runCPass0.sh $infile $nEvents $runNumber $ocdbPath $recoTriggerOptions" | |
137 | if [[ -n $pretend ]]; then | |
138 | touch AliESDfriends_v1.root | |
139 | touch rec.log | |
140 | touch calib.log | |
141 | else | |
142 | ./runCPass0.sh "$infile" "$nEvents" "$runNumber" "$ocdbPath" "$recoTriggerOptions" | |
143 | fi | |
144 | ||
145 | #validate CPass0 | |
146 | touch ${calibDoneFile} | |
147 | [[ -f AliESDfriends_v1.root ]] && echo "calibfile ${outputDir}/AliESDfriends_v1.root" > ${calibDoneFile} | |
148 | summarizeLogs >> ${calibDoneFile} | |
149 | ||
150 | #move stuff to final destination | |
151 | echo "this directory ($PWD) contents:" | |
152 | ls -lh | |
153 | echo | |
154 | ||
155 | echo rm -f ./$chunkName | |
156 | rm -f ./$chunkName | |
157 | echo "cp --recursive $runpath/* ${outputDir}" | |
158 | cp --recursive $runpath/* $outputDir | |
159 | echo | |
160 | ||
161 | rm -rf ${runpath} | |
162 | } | |
163 | ||
164 | goCPass1() | |
165 | { | |
166 | umask 0002 | |
167 | ||
168 | configFile=$5 | |
169 | source $configFile | |
36b5de1a | 170 | [[ -f ${setupAliROOTenvInCurrentShell} && -z ${alirootEnv} ]] && source $setupAliROOTenvInCurrentShell |
fc7e1e45 | 171 | |
172 | targetDirectory=$1 | |
173 | inputList=$2 | |
174 | nEvents=$3 | |
175 | ocdbPath=$4 | |
176 | configFile=$5 | |
177 | runNumber=$6 | |
178 | jobindex=$7 | |
179 | ||
180 | #use the jobindex only if set and non-negative | |
181 | if [[ -z $jobindex || $jobindex -lt 0 ]]; then | |
182 | [[ -n "$LSB_JOBINDEX" ]] && jobindex=$LSB_JOBINDEX | |
183 | [[ -n "$SGE_TASK_ID" ]] && jobindex=$SGE_TASK_ID | |
184 | fi | |
185 | ||
186 | [[ ! -f ${inputList} && -z ${pretend} ]] && echo "input file $inputList not found, exiting..." && exit 1 | |
187 | if [[ "${inputList}" =~ \.root$ ]]; then | |
188 | infile=$inputList | |
189 | else | |
190 | infile=`sed -ne "${jobindex}p" $inputList` | |
191 | fi | |
192 | ||
193 | chunkName=${infile##*/} | |
194 | outputDir=${targetDirectory}/${jobindex} | |
195 | mkdir -p $outputDir | |
196 | [[ ! -d $outputDir ]] && echo "cannot make $outputDir" && exit 1 | |
197 | ||
198 | runpath=${PWD}/rundir_cpass1_${runNumber}_${jobindex} | |
199 | [[ -z $commonOutputPath ]] && commonOutputPath=$PWD | |
200 | [[ $reconstructInTemporaryDir -eq 1 && -n $TMPDIR ]] && runpath=$TMPDIR | |
201 | [[ $reconstructInTemporaryDir -eq 1 && -z $TMPDIR ]] && runpath=$(mktemp -d) | |
202 | ||
203 | #init the running path | |
204 | mkdir -p $runpath | |
205 | [[ ! -d ${runpath} ]] && echo "cannot make runpath ${runpath}" && exit 1 | |
206 | cd $runpath | |
207 | ||
208 | ||
209 | calibDoneFile="${commonOutputPath}/cpass1.job${jobindex}.run${runNumber}.done" | |
210 | ||
211 | logOutputDir=$runpath | |
212 | [[ -n $logToFinalDestination ]] && logOutputDir=$outputDir | |
213 | [[ -z $dontRedirectStdOutToLog ]] && exec 2>&1 > $logOutputDir/runCPass1.log | |
214 | echo "$0 $*" | |
215 | ||
216 | echo "#####################" | |
217 | echo CPass1: | |
218 | echo JOB setup | |
219 | echo nEvents $nEvents | |
220 | echo runNumber $runNumber | |
221 | echo ocdbPath $ocdbPath | |
222 | echo infile $infile | |
223 | echo chunkName $chunkName | |
224 | echo jobindex $jobindex | |
225 | echo recoTriggerOptions $recoTriggerOptions | |
226 | echo targetDirectory $targetDirectory | |
227 | echo commonOutputPath $commonOutputPath | |
228 | echo calibDoneFile $calibDoneFile | |
229 | echo runpath $runpath | |
230 | echo outputDir $outputDir | |
231 | echo ALICE_ROOT $ALICE_ROOT | |
232 | echo PWD $PWD | |
233 | echo "########## ###########" | |
234 | ||
235 | alirootInfo > ALICE_ROOT_svn.log | |
236 | ||
237 | filesCPass1=( | |
238 | "$commonOutputPath/runCPass1.sh" | |
239 | "$commonOutputPath/recCPass1.C" | |
240 | "$commonOutputPath/recCPass1_OuterDet.C" | |
241 | "$commonOutputPath/runCalibTrain.C" | |
242 | "$commonOutputPath/QAtrain_duo.C" | |
243 | "$commonOutputPath/localOCDBaccessConfig.C" | |
244 | "$commonOutputPath/cpass0.localOCDB.${runNumber}.tgz" | |
36b5de1a | 245 | "$commonOutputPath/OCDB.root" |
fc7e1e45 | 246 | "$ALICE_ROOT/PWGPP/CalibMacros/CPass1/runCPass1.sh" |
247 | "$ALICE_ROOT/PWGPP/CalibMacros/CPass1/recCPass1.C" | |
248 | "$ALICE_ROOT/PWGPP/CalibMacros/CPass1/recCPass1_OuterDet.C" | |
249 | "$ALICE_ROOT/PWGPP/CalibMacros/CPass1/runCalibTrain.C" | |
250 | "$ALICE_ROOT/ANALYSIS/macros/QAtrain_duo.C" | |
251 | ) | |
252 | ||
253 | for file in ${filesCPass1[*]}; do | |
254 | [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} . | |
255 | done | |
256 | ||
257 | ln -s $infile $runpath/$chunkName | |
258 | ||
259 | echo "this directory ($PWD) contents:" | |
260 | ls -lh | |
261 | echo | |
262 | ||
263 | if [[ -n $postSetUpActionCPass1 ]]; then | |
264 | echo "running $postSetUpActionCPass1" | |
265 | eval $postSetUpActionCPass1 | |
266 | echo | |
267 | fi | |
268 | ||
269 | #configure local OCDB storage from CPass0 (creates the localOCDBaccessConfig.C script) | |
270 | if [[ -f cpass0.localOCDB.${runNumber}.tgz ]]; then | |
271 | echo goConfigureOCDBforCPass1 "cpass0.localOCDB.${runNumber}.tgz" | |
272 | goConfigureOCDBforCPass1 "cpass0.localOCDB.${runNumber}.tgz" | |
273 | else | |
274 | echo "WARNING: file cpass0.localOCDB.${runNumber}.tgz not found!" | |
275 | fi | |
276 | ||
277 | #run CPass1 | |
278 | chmod u+x runCPass1.sh | |
279 | echo "$runpath/runCPass1.sh $infile $nEvents $runNumber $ocdbPath $recoTriggerOptions" | |
280 | if [[ -n $pretend ]]; then | |
281 | touch AliESDfriends_v1.root | |
282 | touch QAresults_Barrel.root | |
283 | touch QAresults_Outer.root | |
284 | touch rec.log | |
285 | touch calib.log | |
286 | touch qa.log | |
287 | else | |
288 | ./runCPass1.sh "$infile" "$nEvents" "$runNumber" "$ocdbPath" "$recoTriggerOptions" | |
289 | fi | |
290 | ||
291 | #validate CPass1 | |
292 | touch ${calibDoneFile} | |
293 | [[ -f AliESDfriends_v1.root ]] && echo "calibfile ${outputDir}/AliESDfriends_v1.root" > ${calibDoneFile} | |
294 | [[ -f QAresults_Barrel.root ]] && echo "qafile ${outputDir}/QAresults_Barrel.root" >> ${calibDoneFile} | |
295 | [[ -f QAresults_Outer.root ]] && echo "qafile ${outputDir}/QAresults_Outer.root" >> ${calibDoneFile} | |
296 | summarizeLogs >> ${calibDoneFile} | |
297 | ||
298 | #move stuff to final destination | |
299 | echo "this directory ($PWD) contents:" | |
300 | ls | |
301 | echo | |
302 | ||
303 | echo rm -f ./$chunkName | |
304 | rm -f ./$chunkName | |
305 | echo "cp --recursive ${runpath}/* ${outputDir}" | |
306 | cp --recursive ${runpath}/* ${outputDir} | |
307 | echo | |
308 | ||
309 | rm -rf ${runpath} | |
310 | } | |
311 | ||
312 | ||
313 | goMergeCPass0() | |
314 | { | |
315 | umask 0002 | |
316 | # | |
317 | # find the output files and merge them | |
318 | # | |
319 | ||
320 | outputDir=$1 | |
321 | defaultOCDB=$2 | |
322 | configFile=$3 | |
323 | runNumber=$4 | |
324 | calibrationFilesToMergeExternal=$5 | |
325 | ||
326 | source $configFile | |
36b5de1a | 327 | [[ -f ${setupAliROOTenvInCurrentShell} && -z ${alirootEnv} ]] && source $setupAliROOTenvInCurrentShell |
fc7e1e45 | 328 | |
329 | runpath=${PWD}/rundir_cpass0_Merge_${runNumber} | |
330 | [[ -z $commonOutputPath ]] && commonOutputPath=$PWD | |
331 | [[ $reconstructInTemporaryDir -eq 1 && -n $TMPDIR ]] && runpath=$TMPDIR | |
332 | [[ $reconstructInTemporaryDir -eq 1 && -z $TMPDIR ]] && runpath=$(mktemp -d) | |
333 | ||
334 | mkdir -p $runpath | |
335 | [[ ! -d $runpath ]] && echo "not able to make the runpath $runpath" && exit 1 | |
336 | cd $runpath | |
337 | ||
338 | logOutputDir=$runpath | |
339 | [[ -n $logToFinalDestination ]] && logOutputDir=$outputDir | |
340 | [[ -z $dontRedirectStdOutToLog ]] && exec 2>&1 > $logOutputDir/mergeMakeOCDB.log | |
341 | echo "$0 $*" | |
342 | ||
343 | calibrationFilesToMerge=$calibrationFilesToMergeExternal | |
344 | [[ -z $calibrationFilesToMerge ]] && calibrationFilesToMerge="calibrationFilesToMerge.list" | |
345 | calibrationOutputFileName="AliESDfriends_v1.root" | |
346 | mergingScript="mergeMakeOCDB.byComponent.sh" | |
347 | qaFilesToMerge="qaFilesToMerge.list" | |
348 | qaOutputFileName="QAresults*.root" | |
349 | qaMergedOutputFileName="QAresults_merged.root" | |
350 | ||
351 | echo goMergeCPass0 SETUP: | |
352 | echo runNumber=$runNumber | |
353 | echo outputDir=$outputDir | |
354 | echo defaultOCDB=$defaultOCDB | |
355 | echo calibrationFilesToMerge=$calibrationFilesToMerge | |
356 | echo calibrationOutputFileName=$calibrationOutputFileName | |
357 | echo mergingScript=$mergingScript | |
358 | ||
359 | # copy files in case they are not already there | |
360 | filesMergeCPass0=( | |
36b5de1a | 361 | "$commonOutputPath/${calibrationFilesToMerge}" |
362 | "$commonOutputPath/OCDB.root" | |
363 | "$commonOutputPath/localOCDBaccessConfig.C" | |
fc7e1e45 | 364 | "$ALICE_ROOT/PWGPP/CalibMacros/CPass0/mergeMakeOCDB.byComponent.sh" |
365 | "$ALICE_ROOT/PWGPP/CalibMacros/CPass0/mergeByComponent.C" | |
366 | "$ALICE_ROOT/PWGPP/CalibMacros/CPass0/makeOCDB.C" | |
367 | "$ALICE_ROOT/PWGPP/CalibMacros/CPass0/merge.C" | |
368 | "$ALICE_ROOT/PWGPP/CalibMacros/CPass0/mergeMakeOCDB.sh" | |
fc7e1e45 | 369 | ) |
370 | for file in ${filesMergeCPass0[*]}; do | |
371 | [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} . | |
372 | done | |
373 | ||
374 | alirootInfo > ALICE_ROOT_svn.log | |
375 | ||
376 | # | |
377 | ls -lh | |
378 | ||
379 | #merge calibration | |
380 | chmod u+x $mergingScript | |
381 | mkdir -p ./OCDB | |
382 | if [[ -z ${calibrationFilesToMergeExternal} ]]; then | |
383 | echo "find $outputDir -name $calibrationOutputFileName > $calibrationFilesToMerge" | |
384 | find $outputDir -name $calibrationOutputFileName > $calibrationFilesToMerge | |
385 | fi | |
386 | ||
387 | echo "$mergingScript $calibrationFilesToMerge ${runNumber} local://./OCDB $defaultOCDB" | |
388 | if [[ -n $pretend ]]; then | |
389 | touch CalibObjects.root | |
390 | touch ocdb.log | |
391 | touch merge.log | |
392 | mkdir -p ./OCDB/someDetector/ | |
393 | mkdir -p ./OCDB/otherDetector/ | |
394 | touch ./OCDB/someDetector/someCalibObject_0-999999_cpass0.root | |
395 | touch ./OCDB/otherDetector/otherCalibObject_0-999999_cpass0.root | |
396 | else | |
397 | ./$mergingScript $calibrationFilesToMerge ${runNumber} "local://./OCDB" $defaultOCDB | |
398 | fi | |
399 | ||
400 | ### produce the output | |
401 | #tar the produced OCDB for reuse | |
402 | tar czf $commonOutputPath/cpass0.localOCDB.${runNumber}.tgz ./OCDB | |
403 | ||
404 | #validate merging cpass0 | |
405 | calibDoneFile="${commonOutputPath}/merge.cpass0.run${runNumber}.done" | |
406 | touch ${calibDoneFile} | |
407 | [[ -f CalibObjects.root ]] && echo "calibfile $outputDir/CalibObjects.root" > ${calibDoneFile} | |
408 | summarizeLogs >> ${calibDoneFile} | |
409 | ||
410 | ls -ltrh | |
411 | ||
412 | #copy all to output dir | |
413 | cp --recursive ${runpath}/* $outputDir | |
414 | rm -rf ${runpath} | |
415 | } | |
416 | ||
417 | goMergeCPass1() | |
418 | { | |
419 | umask 0002 | |
420 | # | |
421 | # find the output files and merge them | |
422 | # | |
423 | ||
424 | outputDir=$1 | |
425 | defaultOCDB=$2 | |
426 | configFile=$3 | |
427 | runNumber=$4 | |
428 | calibrationFilesToMergeExternal=$5 | |
429 | qaFilesToMergeExternal=$6 | |
430 | ||
431 | #clean up first: | |
432 | rm -f $outputDir/*.log | |
433 | rm -f $outputDir/*.root | |
434 | rm -f $outputDir/*done | |
435 | ||
436 | source $configFile | |
36b5de1a | 437 | [[ -f ${setupAliROOTenvInCurrentShell} && -z ${alirootEnv} ]] && source $setupAliROOTenvInCurrentShell |
fc7e1e45 | 438 | |
439 | runpath=${PWD}/rundir_cpass1_Merge_${runNumber} | |
440 | [[ -z $commonOutputPath ]] && commonOutputPath=$PWD | |
441 | [[ $reconstructInTemporaryDir -eq 1 && -n $TMPDIR ]] && runpath=$TMPDIR | |
442 | [[ $reconstructInTemporaryDir -eq 1 && -z $TMPDIR ]] && runpath=$(mktemp -d) | |
443 | ||
444 | mkdir -p $runpath | |
445 | [[ ! -d $runpath ]] && echo "not able to make the runpath $runpath" && exit 1 | |
446 | cd $runpath | |
447 | ||
448 | logOutputDir=$runpath | |
449 | [[ -n $logToFinalDestination ]] && logOutputDir=$outputDir | |
450 | [[ -z $dontRedirectStdOutToLog ]] && exec 2>&1 > $logOutputDir/mergeMakeOCDB.log | |
451 | echo "$0 $*" | |
452 | ||
453 | calibrationFilesToMerge=$calibrationFilesToMergeExternal | |
454 | [[ -z $calibrationFilesToMerge ]] && calibrationFilesToMerge="calibrationFilesToMerge.list" | |
455 | calibrationOutputFileName="AliESDfriends_v1.root" | |
456 | mergingScript="mergeMakeOCDB.byComponent.sh" | |
457 | qaFilesToMerge=$qaFilesToMergeExternal | |
458 | [[ -z $qaFilesToMerge ]] && qaFilesToMerge="qaFilesToMerge.list" | |
459 | qaOutputFileName="QAresults*.root" | |
460 | qaMergedOutputFileName="QAresults_merged.root" | |
461 | ||
462 | echo goMergeCPass1 SETUP: | |
463 | echo runNumber=$runNumber | |
464 | echo outputDir=$outputDir | |
465 | echo defaultOCDB=$defaultOCDB | |
466 | echo calibrationFilesToMerge=filesToMerge.list | |
467 | echo calibrationOutputFileName=$calibrationOutputFileName | |
468 | echo mergingScript=$mergingScript | |
469 | ||
470 | # copy files in case they are not already there | |
471 | filesMergeCPass1=( | |
36b5de1a | 472 | "$commonOutputPath/${calibrationFilesToMerge}" |
473 | "$commonOutputPath/${qaFilesToMerge}" | |
474 | "$commonOutputPath/OCDB.root" | |
475 | "$commonOutputPath/localOCDBaccessConfig.C" | |
fc7e1e45 | 476 | "$ALICE_ROOT/PWGPP/CalibMacros/CPass1/mergeMakeOCDB.byComponent.sh" |
477 | "$ALICE_ROOT/PWGPP/CalibMacros/CPass1/mergeByComponent.C" | |
478 | "$ALICE_ROOT/PWGPP/CalibMacros/CPass1/makeOCDB.C" | |
479 | "$ALICE_ROOT/PWGPP/CalibMacros/CPass1/merge.C" | |
480 | "$ALICE_ROOT/PWGPP/CalibMacros/CPass1/mergeMakeOCDB.sh" | |
fc7e1e45 | 481 | ) |
482 | for file in ${filesMergeCPass1[*]}; do | |
483 | [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} . | |
484 | done | |
485 | ||
486 | alirootInfo > ALICE_ROOT_svn.log | |
487 | ||
488 | # | |
489 | ls -lh | |
490 | ||
491 | #merge calibration | |
492 | chmod u+x $mergingScript | |
493 | mkdir -p OCDB | |
494 | if [[ -z ${calibrationFilesToMergeExternal} ]]; then | |
495 | echo "find $outputDir -name $calibrationOutputFileName > $calibrationFilesToMerge" | |
496 | find $outputDir -name $calibrationOutputFileName > $calibrationFilesToMerge | |
497 | fi | |
498 | ||
499 | echo "$mergingScript $calibrationFilesToMerge ${runNumber} local://./OCDB $defaultOCDB" | |
500 | if [[ -n $pretend ]]; then | |
501 | touch CalibObjects.root | |
502 | touch ocdb.log | |
503 | touch merge.log | |
504 | else | |
505 | ./$mergingScript $calibrationFilesToMerge ${runNumber} "local://./OCDB" $defaultOCDB | |
506 | fi | |
507 | ||
508 | tar czf localCPass1_${runNumber}.tgz ./OCDB | |
509 | ||
510 | #merge QA | |
511 | if [[ -z qaFilesToMergeExternal ]]; then | |
512 | echo "find $outputDir -name $qaOutputFileName > $qaFilesToMerge" | |
513 | find $outputDir -name $qaOutputFileName > $qaFilesToMerge | |
514 | fi | |
515 | ||
516 | echo aliroot -l -b -q "merge.C(\"$qaFilesToMerge\",\"\",kFALSE,\"$qaMergedOutputFileName\")" | |
517 | if [[ -n $pretend ]]; then | |
518 | touch $qaMergedOutputFileName | |
519 | touch merge.log | |
520 | else | |
521 | aliroot -l -b -q "merge.C(\"$qaFilesToMerge\",\"\",kFALSE,\"$qaMergedOutputFileName\")" | |
522 | fi | |
523 | ||
524 | #validate merge cpass1 | |
525 | calibDoneFile="${commonOutputPath}/merge.cpass1.run${runNumber}.done" | |
526 | touch ${calibDoneFile} | |
527 | [[ -f CalibObjects.root ]] && echo "calibfile $outputDir/CalibObjects.root" > ${calibDoneFile} | |
528 | [[ -f $qaMergedOutputFileName ]] && echo "qafile $outputDir/$qaMergedOutputFileName" >> ${calibDoneFile} | |
529 | summarizeLogs >> ${calibDoneFile} | |
530 | ||
531 | ls -ltrh | |
532 | ||
533 | #copy all to output dir | |
36b5de1a | 534 | cp --recursive ${runpath}/* ${outputDir} |
fc7e1e45 | 535 | rm -rf ${runpath} |
536 | } | |
537 | ||
538 | goSubmit() | |
539 | { | |
540 | inputList=$1 | |
541 | productionID=$2 | |
542 | configFile="benchmark.config" | |
543 | [[ -n "$3" ]] && configFile=$3 | |
544 | configFile=$(readlink -f $configFile) | |
545 | [[ -n $4 ]] && runNumber=$4 | |
546 | ||
547 | #redirect all output to submit.log | |
548 | echo "redirecting all output to ${PWD}/submit_${productionID//"/"/_}.log" | |
549 | exec 7>&1 | |
550 | exec 1>submit_${productionID//"/"/_}.log 2>&1 | |
551 | ||
552 | umask 0002 | |
553 | echo $0" submit $*" | |
554 | if [[ -z "$inputList" || -z "$productionID" ]] | |
555 | then | |
556 | echo | |
557 | echo " Usage: $0 submit inputList productionID [configFile=benchmark.config]" | |
558 | echo | |
559 | exit | |
560 | fi | |
561 | ||
562 | date=`date +%Y-%m-%d_%H%M%S` | |
563 | ||
564 | # check if config file is there | |
565 | if [ ! -f $configFile ]; then | |
566 | echo "ERROR! Config File '$configFile' not found" >&2 | |
567 | exit | |
568 | else | |
569 | echo "Using Config File: '$configFile'" | |
570 | fi | |
571 | ||
572 | # source the config file | |
573 | # and print the configuration | |
574 | source $configFile | |
36b5de1a | 575 | [[ -f ${setupAliROOTenvInCurrentShell} && -z ${alirootEnv} ]] && source $setupAliROOTenvInCurrentShell |
fc7e1e45 | 576 | |
577 | self=$(readlink -f "$0") | |
578 | configPath=`dirname $self` | |
579 | #exporting makes it available in the environment on the nodes - makes the summary output go there | |
580 | export commonOutputPath=${baseOutputDirectory}/${productionID} | |
581 | ||
582 | #[[ -z ${ALIROOT_RELEASE} ]] && ALIROOT_RELEASE=${ALICE_ROOT//"/"/"_"} | |
583 | #ALIROOT_RELEASE_mod=${ALIROOT_RELEASE//"/"/"_"} | |
584 | #ALIROOT_BASEDIR_mod=${ALIROOT_BASEDIR//"/"/"_"} | |
585 | #productionID=${ALIROOT_BASEDIR_mod}_${ALIROOT_RELEASE_mod}/${productionID} | |
586 | ||
587 | #convert to absolut pathnames | |
588 | inputList=$(readlink -f "$inputList") | |
589 | #make list of runs | |
590 | if [[ -z $runNumber ]]; then | |
591 | listOfRuns=($(while read x; do guessRunNumber $x; done < ${inputList} | sort | uniq)) | |
592 | else | |
593 | listOfRuns=$runNumber | |
594 | fi | |
595 | ||
596 | setupAliROOTenvInCurrentShell=$(readlink -f "$setupAliROOTenvInCurrentShell") | |
597 | ||
598 | echo "" | |
599 | echo "### BEGIN CONFIGURATION ###" | |
600 | echo "" | |
601 | echo "GENERAL:" | |
602 | echo "" | |
603 | echo " productionID: $productionID" | |
604 | echo " batchCommand: $batchCommand" | |
605 | echo " setupAliROOTenvInCurrentShell: $setupAliROOTenvInCurrentShell" | |
606 | echo " ALICE_ROOT: $ALICE_ROOT" | |
607 | echo " ALIROOT_RELEASE: $ALICE_RELEASE" | |
608 | echo " inputList: $inputList" | |
609 | echo " configPath: $configPath" | |
610 | echo " commonOutputPath: $commonOutputPath" | |
611 | echo " defaultOCDB: $defaultOCDB" | |
612 | echo " autoOCDB: $autoOCDB" | |
613 | echo " recoTriggerOptions: $recoTriggerOptions" | |
614 | echo " runs:" | |
615 | echo " ${listOfRuns[*]}" | |
616 | echo "" | |
617 | echo "THE TRAIN WILL RUN:" | |
618 | ||
619 | if [ $runCPass0reco -eq 1 ]; then | |
620 | echo " Pass0 - Recontruction" | |
621 | fi | |
622 | ||
623 | if [ $runCPass0MergeMakeOCDB -eq 1 ]; then | |
624 | echo " Pass0 - merging and OCDB export" | |
625 | fi | |
626 | ||
627 | if [ $runCPass1reco -eq 1 ]; then | |
628 | echo " Pass1 - Recontruction" | |
629 | fi | |
630 | if [ $runCPass1MergeMakeOCDB -eq 1 ]; then | |
631 | echo " Pass1 - merging and OCDB export" | |
632 | fi | |
633 | ||
634 | echo "" | |
635 | echo "LIMITS:" | |
636 | echo " max. Events/Chunk: $nEvents" | |
637 | echo " max. Number of Chunks per Run: $nMaxChunks" | |
638 | echo "" | |
639 | echo "### END CONFIGURATION ###" | |
640 | echo "" | |
641 | ||
642 | ||
643 | # check if input file is there | |
644 | if [ ! -f $inputList ]; then | |
645 | echo "ERROR! Input List '$inputList' not found" >&2 | |
646 | exit | |
647 | fi | |
648 | ||
649 | # define jobid (for dependent jobs) | |
650 | JOBID1=p0_${productionID//"/"/_}_${date} | |
651 | JOBID1wait=w0_${productionID//"/"/_}_${date} | |
652 | JOBID2=m0_${productionID//"/"/_}_${date} | |
653 | JOBID2wait=wm0_${productionID//"/"/_}_${date} | |
654 | JOBID3=op0_${productionID//"/"/_}_${date} | |
655 | JOBID3wait=wop0_${productionID//"/"/_}_${date} | |
656 | JOBID4=p1_${productionID//"/"/_}_${date} | |
657 | JOBID4wait=w1_${productionID//"/"/_}_${date} | |
658 | JOBID5=m1_${productionID//"/"/_}_${date} | |
659 | JOBID5wait=wm1_${productionID//"/"/_}_${date} | |
660 | JOBID6=s1_${productionID//"/"/_}_${date} | |
661 | JOBID6wait=ws1_${productionID//"/"/_}_${date} | |
662 | JOBID7=QA_${productionID//"/"/_}_${date} | |
663 | LASTJOB="" | |
664 | ||
665 | #for each run we submit one jobarray: | |
666 | numberOfSubmittedCPass1MergingJobs=0 | |
667 | for runNumber in ${listOfRuns[*]}; do | |
668 | [[ -z $runNumber ]] && continue | |
669 | [[ ! ${runNumber} =~ ^[0-9]*[0-9]$ ]] && continue | |
670 | oneInputFile=$(egrep -m1 "$runNumber\/" ${inputList}) | |
671 | currentDefaultOCDB=$defaultOCDB | |
672 | [[ $autoOCDB -ne 0 ]] && currentDefaultOCDB=$(setYear $oneInputFile $defaultOCDB) | |
673 | echo "submitting run $runNumber with OCDB $currentDefaultOCDB" | |
674 | ||
675 | ################################################################################ | |
676 | ################################################################################ | |
677 | # run the CPass0 if requested | |
678 | ||
679 | if [ $runCPass0reco -eq 1 ]; then | |
680 | ||
681 | echo | |
682 | echo "starting CPass0... for run $runNumber" | |
683 | echo | |
684 | ||
685 | # create directory and copy all files that are needed | |
686 | targetDirectory="${commonOutputPath}/000${runNumber}/CPass0" | |
687 | mkdir -p $targetDirectory | |
688 | mkdir -p $targetDirectory/logs | |
689 | ||
690 | localInputList=$targetDirectory/${inputList##*/} | |
691 | theScript="$targetDirectory/${self##*/}" | |
692 | cp -f $self $theScript | |
693 | chmod u+x $theScript | |
694 | ||
695 | rm -f $localInputList | |
696 | egrep "\/000$runNumber\/" $inputList >> $localInputList | |
697 | cp -f $configFile $targetDirectory | |
698 | [[ -f $configPath/runCPass0.sh ]] && cp -f $configPath/runCPass0.sh $targetDirectory && echo "## using local runCPass0.sh" | |
699 | [[ -f $configPath/recCPass0.C ]] && cp -f $configPath/recCPass0.C $targetDirectory && echo "## using local recCPass0.C" | |
700 | [[ -f $configPath/runCalibTrain.C ]] && cp -f $configPath/runCalibTrain.C $targetDirectory && echo "## using local runCalibTrain.C" | |
701 | [[ -f $configPath/localOCDBaccessConfig.C ]] && cp -f $configPath/localOCDBaccessConfig.C $targetDirectory && echo "## using local localOCDBaccessConfig.C" | |
702 | ||
703 | [[ -f $configPath/CPass0/runCPass0.sh ]] && cp -f $configPath/CPass0/runCPass0.sh $targetDirectory && echo "## using local runCPass0.sh" | |
704 | [[ -f $configPath/CPass0/recCPass0.C ]] && cp -f $configPath/CPass0/recCPass0.C $targetDirectory && echo "## using local recCPass0.C" | |
705 | [[ -f $configPath/CPass0/runCalibTrain.C ]] && cp -f $configPath/CPass0/runCalibTrain.C $targetDirectory && echo "## using local runCalibTrain.C" | |
706 | [[ -f $configPath/CPass0/localOCDBaccessConfig.C ]] && cp -f $configPath/CPass0/localOCDBaccessConfig.C $targetDirectory && echo "## using local localOCDBaccessConfig.C" | |
707 | ||
708 | echo "... files copied." | |
709 | echo | |
710 | ||
711 | # limit nFiles to nMaxChunks | |
712 | nFiles=`wc -l < $localInputList` | |
713 | [[ $nFiles -eq 0 ]] && echo "list contains ZERO files! exiting..." && exit 1 | |
714 | echo "raw files in list: $nFiles" | |
715 | if [[ $nMaxChunks -gt 0 && $nMaxChunks -le $nFiles ]]; then | |
716 | nFiles=$nMaxChunks | |
717 | fi | |
718 | echo "raw files to process: $nFiles" | |
719 | [[ -z "$percentProcessedFilesToContinue" ]] && percentProcessedFilesToContinue=100 | |
720 | if [[ $percentProcessedFilesToContinue -eq 100 ]]; then | |
721 | nFilesToWaitFor=$nFiles | |
722 | else | |
723 | nFilesToWaitFor=$(( $nFiles-$nFiles/(100/(100-$percentProcessedFilesToContinue)) )) | |
724 | fi | |
725 | echo "requested success rate is $percentProcessedFilesToContinue%" | |
726 | echo "merging will start after $nFilesToWaitFor jobs are done" | |
727 | ||
728 | submit $JOBID1 1 $nFiles "" "$theScript" "CPass0 $targetDirectory $localInputList $nEvents $currentDefaultOCDB $configFile $runNumber" | |
729 | ||
730 | ## submit a monitoring job that will run until a certain number of jobs are done with reconstruction | |
731 | submit "$JOBID1wait" 1 1 "" "$theScript" "WaitForOutput ${commonOutputPath} 'cpass0.job*.run$runNumber.done' $nFilesToWaitFor $maxSecondsToWait '-maxdepth 1'" | |
732 | LASTJOB=$JOBID1wait | |
733 | ||
734 | fi #end running CPass0 | |
735 | ################################################################################ | |
736 | ||
737 | ||
738 | ################################################################################ | |
739 | # submit merging of CPass0, depends on the reconstruction | |
740 | ||
741 | if [ $runCPass0MergeMakeOCDB -eq 1 ]; then | |
742 | ||
743 | echo | |
744 | echo "submit CPass0 merging for run $runNumber" | |
745 | echo | |
746 | ||
747 | targetDirectory="${commonOutputPath}/000${runNumber}/CPass0" | |
748 | mkdir -p $targetDirectory | |
749 | ||
750 | # copy the scripts | |
751 | cp -f $self $targetDirectory | |
752 | [[ -f $configPath/mergeMakeOCDB.sh ]] && cp -f $configPath/mergeMakeOCDB.sh $targetDirectory && echo "## using local mergeMakeOCDB.sh" | |
753 | [[ -f $configPath/merge.C ]] && cp -f $configPath/merge.C $targetDirectory && echo "## using local merge.C" | |
754 | [[ -f $configPath/mergeMakeOCDB.byComponent.sh ]] && cp -f $configPath/mergeMakeOCDB.byComponent.sh $targetDirectory && echo "## using local mergeMakeOCDB.byComponent.sh" | |
755 | [[ -f $configPath/mergeByComponent.C ]] && cp -f $configPath/mergeByComponent.C $targetDirectory && echo "## using local mergeByComponent.C" | |
756 | [[ -f $configPath/makeOCDB.C ]] && cp -f $configPath/makeOCDB.C $targetDirectory && echo "## using local makeOCDB.C" | |
757 | ||
758 | theScript="$targetDirectory/${self##*/}" | |
759 | chmod u+x $theScript | |
760 | ||
761 | echo submit $JOBID2 1 1 "$LASTJOB" "$theScript" "MergeCPass0 $targetDirectory $currentDefaultOCDB $configFile $runNumber" | |
762 | submit $JOBID2 1 1 "$LASTJOB" "$theScript" "MergeCPass0 $targetDirectory $currentDefaultOCDB $configFile $runNumber" | |
763 | LASTJOB=$JOBID2 | |
764 | ||
765 | cd $configPath | |
766 | echo | |
767 | fi | |
768 | # end of merging CPass0 | |
769 | ################################################################################ | |
770 | ||
771 | ################################################################################ | |
772 | ################################################################################ | |
773 | # run the CPass1 if requested | |
774 | ||
775 | if [ $runCPass1reco -eq 1 ]; then | |
776 | ||
777 | targetDirectory="${commonOutputPath}/000${runNumber}/CPass1" | |
778 | ||
779 | # safety feature: if we are re-running for any reason we want to delete the previous output first. | |
780 | [[ -d $targetDirectory ]] && rm -rf $targetDirectory/* && echo "removed old output at $targetDirectory/*" | |
781 | ||
782 | ################################################################################ | |
783 | # for the CPass1, based on the OCDB entries produced, we need to create the script | |
784 | # to set the specific storages to the output of CPass0 | |
785 | # submit a job that will execute after merging/OCDB export that will do it. | |
786 | ||
787 | mkdir -p $targetDirectory | |
788 | mkdir -p $targetDirectory/logs | |
789 | cp -f $self $targetDirectory | |
790 | theScript="$targetDirectory/${self##*/}" | |
791 | ||
792 | echo | |
793 | echo submitting the OCDB specific storage config making script for run $runNumber | |
794 | echo | |
795 | ||
796 | [[ -f $configPath/localOCDBaccessConfig.C ]] && cp -f $configPath/localOCDBaccessConfig.C $targetDirectory && echo "## using local localOCDBaccessConfig.C" | |
797 | ||
798 | submit $JOBID3 1 1 "$LASTJOB" "$theScript" "ConfOCDB ${commonOutputPath}/000${runNumber}/CPass0/OCDB ${targetDirectory} " | |
799 | LASTJOB=$JOBID3 | |
800 | ################################################################################ | |
801 | ||
802 | echo | |
803 | echo "starting CPass1... for run $runNumber" | |
804 | echo | |
805 | ||
806 | # create directory and copy all files that are needed | |
807 | mkdir -p $targetDirectory | |
808 | mkdir -p $targetDirectory/logs | |
809 | ||
810 | localInputList=$targetDirectory/${inputList##*/} | |
811 | theScript="$targetDirectory/${self##*/}" | |
812 | cp -f $self $theScript | |
813 | chmod u+x $theScript | |
814 | ||
815 | rm -f $localInputList | |
816 | egrep "\/000$runNumber\/" $inputList >> $localInputList | |
817 | cp -f $configFile $targetDirectory | |
818 | [[ -f $configPath/runCPass1.sh ]] && cp -f $configPath/runCPass1.sh $targetDirectory && echo "## using local runCPass1.sh" | |
819 | [[ -f $configPath/recCPass1.C ]] && cp -f $configPath/recCPass1.C $targetDirectory && echo "## using local recCPass1.C" | |
820 | [[ -f $configPath/recCPass1_OuterDet.C ]] && cp -f $configPath/recCPass1_OuterDet.C $targetDirectory && echo "## using local recCPass1_OuterDet.C" | |
821 | [[ -f $configPath/runCalibTrain.C ]] && cp -f $configPath/runCalibTrain.C $targetDirectory && echo "## using local runCalibTrain.C" | |
822 | [[ -f $configPath/QAtrain.C ]] && cp -f $configPath/QAtrain.C $targetDirectory && echo "## using local QAtrain.C" | |
823 | [[ -f $configPath/QAtrain_duo.C ]] && cp -f $configPath/QAtrain_duo.C $targetDirectory && echo "## using local QAtrain_duo.C" | |
824 | ||
825 | [[ -f $configPath/CPass1/runCPass1.sh ]] && cp -f $configPath/CPass1/runCPass1.sh $targetDirectory && echo "## using local runCPass1.sh" | |
826 | [[ -f $configPath/CPass1/recCPass1.C ]] && cp -f $configPath/CPass1/recCPass1.C $targetDirectory && echo "## using local recCPass1.C" | |
827 | [[ -f $configPath/CPass1/recCPass1_OuterDet.C ]] && cp -f $configPath/CPass1/recCPass1_OuterDet.C $targetDirectory && echo "## using local recCPass1_OuterDet.C" | |
828 | [[ -f $configPath/CPass1/runCalibTrain.C ]] && cp -f $configPath/CPass1/runCalibTrain.C $targetDirectory && echo "## using local runCalibTrain.C" | |
829 | [[ -f $configPath/CPass1/QAtrain.C ]] && cp -f $configPath/CPass1/QAtrain.C $targetDirectory && echo "## using local QAtrain.C" | |
830 | [[ -f $configPath/CPass1/QAtrain_duo.C ]] && cp -f $configPath/CPass1/QAtrain_duo.C $targetDirectory && echo "## using local QAtrain_duo.C" | |
831 | ||
832 | echo "... files copied." | |
833 | echo | |
834 | ||
835 | # limit nFiles to nMaxChunks | |
836 | nFiles=`wc -l < $localInputList` | |
837 | [[ $nFiles -eq 0 ]] && echo "list contains ZERO files! exiting..." && exit 1 | |
838 | echo "raw files in list: $nFiles" | |
839 | if [[ $nMaxChunks -gt 0 && $nMaxChunks -le $nFiles ]]; then | |
840 | nFiles=$nMaxChunks | |
841 | fi | |
842 | echo "raw files to process: $nFiles" | |
843 | [[ -z "$percentProcessedFilesToContinue" ]] && percentProcessedFilesToContinue=100 | |
844 | if [[ $percentProcessedFilesToContinue -eq 100 ]]; then | |
845 | nFilesToWaitFor=$nFiles | |
846 | else | |
847 | nFilesToWaitFor=$(( $nFiles-$nFiles/(100/(100-$percentProcessedFilesToContinue)) )) | |
848 | fi | |
849 | echo "requested success rate is $percentProcessedFilesToContinue%" | |
850 | echo "merging will start after $nFilesToWaitFor jobs are done" | |
851 | ||
852 | submit $JOBID4 1 $nFiles "$LASTJOB" "$theScript" "CPass1 $targetDirectory $localInputList $nEvents $currentDefaultOCDB $configFile $runNumber" | |
853 | ||
854 | ################################################################################ | |
855 | ## submit a monitoring job that will run until a certain number of jobs are done with reconstruction | |
856 | submit "$JOBID4wait" 1 1 "$LASTJOB" "$theScript" "WaitForOutput ${commonOutputPath} 'cpass1.job*.run${runNumber}.done' $nFilesToWaitFor $maxSecondsToWait '-maxdepth 1'" | |
857 | LASTJOB=$JOBID4wait | |
858 | ################################################################################ | |
859 | ||
860 | echo | |
861 | fi #end running CPass1 | |
862 | ||
863 | ################################################################################ | |
864 | # submit merging of CPass1, depends on the reconstruction | |
865 | if [ $runCPass1MergeMakeOCDB -eq 1 ]; then | |
866 | ||
867 | echo | |
868 | echo "submit CPass1 merging for run $runNumber" | |
869 | echo | |
870 | ||
871 | targetDirectory="${commonOutputPath}/000${runNumber}/CPass1" | |
872 | mkdir -p $targetDirectory | |
873 | ||
874 | # copy the scripts | |
875 | cp -f $self $targetDirectory | |
876 | [[ -f $configPath/mergeMakeOCDB.sh ]] && cp -f $configPath/mergeMakeOCDB.sh $targetDirectory && echo "## using local mergeMakeOCDB.sh" | |
877 | [[ -f $configPath/merge.C ]] && cp -f $configPath/merge.C $targetDirectory && echo "## using local merge.C" | |
878 | [[ -f $configPath/mergeMakeOCDB.byComponent.sh ]] && cp -f $configPath/mergeMakeOCDB.byComponent.sh $targetDirectory && echo "## using local mergeMakeOCDB.byComponent.sh" | |
879 | [[ -f $configPath/mergeByComponent.C ]] && cp -f $configPath/mergeByComponent.C $targetDirectory && echo "## using local mergeByComponent.C" | |
880 | [[ -f $configPath/makeOCDB.C ]] && cp -f $configPath/makeOCDB.C $targetDirectory && echo "## using local makeOCDB.C" | |
881 | ||
882 | theScript="$targetDirectory/${self##*/}" | |
883 | chmod u+x $theScript | |
884 | ||
885 | echo submit "$JOBID5" 1 1 "$LASTJOB" "$theScript" "MergeCPass1 $targetDirectory $currentDefaultOCDB $configFile $runNumber" | |
886 | submit "$JOBID5" 1 1 "$LASTJOB" "$theScript" "MergeCPass1 $targetDirectory $currentDefaultOCDB $configFile $runNumber" | |
887 | ((numberOfSubmittedCPass1MergingJobs++)) | |
888 | LASTJOB=$JOBID5 | |
889 | echo | |
890 | fi | |
891 | ||
892 | done | |
893 | ||
894 | ################################################################################ | |
895 | ################################################################################ | |
896 | [[ -z $runMakeSummary ]] && runMakeSummary=0 | |
897 | if [ $runMakeSummary -eq 1 ]; then | |
898 | echo | |
899 | echo "submit make a summary" | |
900 | echo | |
901 | ||
902 | targetDirectory="${commonOutputPath}" | |
903 | mkdir -p $targetDirectory | |
904 | mkdir -p $targetDirectory/logs | |
905 | [[ ! -f $targetDirectory/${self##*/} ]] && cp -f $self $targetDirectory | |
906 | theScript="$targetDirectory/${self##*/}" | |
907 | submit "$JOBID6" 1 1 "$LASTJOB" "$theScript" "MakeSummary $targetDirectory $configFile" | |
908 | fi | |
909 | ################################################################################ | |
910 | ||
911 | ################################################################################ | |
912 | ################################################################################ | |
913 | if [ $runMakeQAplots -eq 1 ]; then | |
914 | echo | |
915 | echo "submit make QA plots" | |
916 | echo | |
917 | ||
918 | ## submit a monitoring job that will wait for a specified number of files to appear somewhere | |
919 | targetDirectory="${commonOutputPath}" | |
920 | [[ ! -f $targetDirectory/${self##*/} ]] && cp -f $self $targetDirectory | |
921 | theScript="$targetDirectory/${self##*/}" | |
922 | echo submit "$JOBID5wait 1 1 $theScript WaitForOutput ${commonOutputPath} 'merge.cpass1.run*.done' $numberOfSubmittedCPass1MergingJobs $maxSecondsToWait '-maxdepth 1'" | |
923 | submit "$JOBID5wait" 1 1 "" "$theScript" "WaitForOutput ${commonOutputPath} 'merge.cpass1.run*.done' $numberOfSubmittedCPass1MergingJobs $maxSecondsToWait '-maxdepth 1'" | |
924 | LASTJOB=$JOBID5wait | |
925 | ||
926 | targetDirectory="${commonOutputPath}/QAplots/" | |
927 | [[ -d $targetDirectory ]] && rm -rf $targetDirectory/* | |
928 | mkdir -p $targetDirectory | |
929 | mkdir -p $targetDirectory/logs | |
930 | qaFilesDirectory="${commonOutputPath}" | |
931 | [[ ! -f $targetDirectory/${self##*/} ]] && cp -f $self $targetDirectory | |
932 | theScript="$targetDirectory/${self##*/}" | |
933 | ||
934 | echo submit "$JOBID7" 1 1 "$LASTJOB" "$theScript" "CreateQAplots QAplots CPass1 $targetDirectory $qaFilesDirectory $qaPlotsScript $configFile" | |
935 | submit "$JOBID7" 1 1 "$LASTJOB" "$theScript" "CreateQAplots QAplots CPass1 $targetDirectory $qaFilesDirectory $qaPlotsScript $configFile" | |
936 | LASTJOB=$JOBID7 | |
937 | fi | |
938 | ||
939 | #restore stdout | |
940 | exec 1>&7 7>&- | |
941 | echo "jobs submitted." | |
942 | } | |
943 | ||
944 | goMakeSummary() | |
945 | { | |
946 | configFile=$1 | |
947 | source $configFile | |
948 | ||
949 | # log filtering, script needs to take the base dir as argument | |
950 | if [[ -x $logFilteringScript ]]; then | |
951 | commonOutputPath=${baseOutputDirectory}/${productionID} | |
952 | ${logFilteringScript} $commonOutputPath | |
953 | fi | |
954 | ||
36b5de1a | 955 | awk 'BEGIN {nFiles=0;} /^calibfile/ {nFiles++;} END {print "cpass0 produced "nFiles" calib files";}' cpass0.job*done |
fc7e1e45 | 956 | awk 'BEGIN {nOK=0; nBAD=0;} /rec.*log OK/ {nOK++;} /rec.*log BAD/ {nBAD++;} END {print "cpass0 reco: OK: "nOK" BAD: "nBAD;}' cpass0.job*done |
957 | awk 'BEGIN {nOK=0; nBAD=0;} /calib.*log OK/ {nOK++;} /calib.*log BAD/ {nBAD++;} END {print "cpass0 calib: OK: "nOK" BAD: "nBAD;}' cpass0.job*done | |
958 | ||
959 | awk 'BEGIN {nOK=0; nBAD=0;} /merge.*log OK/ {nOK++;} /merge.*log BAD/ {nBAD++;} END {print "cpass0 merge: OK: "nOK" BAD: "nBAD;}' merge.cpass0*done | |
960 | awk 'BEGIN {nOK=0; nBAD=0;} /ocdb.*log OK/ {nOK++;} /ocdb.*log BAD/ {nBAD++;} END {print "cpass0 OCDB: OK: "nOK" BAD: "nBAD;}' merge.cpass0*done | |
961 | ||
36b5de1a | 962 | awk 'BEGIN {nFiles=0;} /^calibfile/ {nFiles++;} END {print "cpass1 produced "nFiles" calib files";}' cpass1.job*done |
fc7e1e45 | 963 | awk 'BEGIN {nOK=0; nBAD=0;} /rec.*log OK/ {nOK++;} /rec.*log BAD/ {nBAD++;} END {print "cpass1 reco: OK: "nOK" BAD: "nBAD;}' cpass1.job*done |
964 | awk 'BEGIN {nOK=0; nBAD=0;} /calib.*log OK/ {nOK++;} /calib.*log BAD/ {nBAD++;} END {print "cpass1 calib: OK: "nOK" BAD: "nBAD;}' cpass1.job*done | |
965 | ||
966 | awk 'BEGIN {nOK=0; nBAD=0;} /merge.*log OK/ {nOK++;} /merge.*log BAD/ {nBAD++;} END {print "cpass1 merge: OK: "nOK" BAD: "nBAD;}' merge.cpass1*done | |
967 | awk 'BEGIN {nOK=0; nBAD=0;} /ocdb.*log OK/ {nOK++;} /ocdb.*log BAD/ {nBAD++;} END {print "cpass1 OCDB: OK: "nOK" BAD: "nBAD;}' merge.cpass1*done | |
968 | ||
969 | #if set email the summary | |
970 | [[ -n $mailSummaryTo ]] && cat $log | mail -s "benchmark $productionID done" $mailSummaryTo | |
971 | ||
972 | return 0 | |
973 | } | |
974 | ||
975 | goMakeflow() | |
976 | { | |
977 | #generate the makeflow file and run | |
978 | inputFileList=$1 | |
979 | productionID=$2 | |
980 | configFile=$3 | |
981 | runNumber=$4 | |
982 | ||
983 | [[ -z ${configFile} ]] && configFile="benchmark.config" | |
984 | [[ ! -f ${configFile} ]] && echo "no config file found (${configFile})" && return 1 | |
985 | source $configFile | |
986 | ||
987 | source $configFile | |
988 | goGenerateMakeflow "$@" > benchmark.makeflow | |
989 | makeflow ${makeflowOptions} benchmark.makeflow | |
990 | } | |
991 | ||
992 | goGenerateMakeflow() | |
993 | { | |
994 | #generate the makeflow file | |
995 | inputFileList=$1 | |
996 | productionID=$2 | |
997 | configFile=$3 | |
998 | runNumber=$4 | |
999 | ||
1000 | [[ -z ${configFile} ]] && configFile="benchmark.config" | |
1001 | [[ ! -f ${configFile} ]] && echo "no config file found (${configFile})" && return 1 | |
1002 | source $configFile | |
1003 | ||
1004 | commonOutputPath=${baseOutputDirectory}/${productionID} | |
1005 | ||
36b5de1a | 1006 | #these files will be made a dependency - will be copied to the working dir of the jobs |
1007 | declare -a copyFiles | |
1008 | inputFiles=( | |
1009 | "OCDB.root" | |
1010 | "localOCDBaccessConfig.C" | |
1011 | ) | |
1012 | for file in ${inputFiles[*]}; do | |
1013 | [[ -f ${file} ]] && copyFiles+=("${file}") | |
1014 | done | |
1015 | ||
1016 | #create the makeflow file | |
fc7e1e45 | 1017 | declare -a arr_cpass1_final |
1018 | declare -a arr_cpass1_QA_final | |
fc7e1e45 | 1019 | listOfRuns=${runNumber} |
1020 | [[ -z ${runNumber} ]] && listOfRuns=($(while read x; do guessRunNumber $x; done < ${inputFileList} | sort | uniq)) | |
1021 | runindex=0 | |
1022 | for runNumber in ${listOfRuns[*]}; do | |
1023 | [[ -z $runNumber ]] && continue | |
1024 | [[ ! ${runNumber} =~ ^[0-9]*[0-9]$ ]] && continue | |
1025 | jobindex=0 | |
1026 | ||
1027 | unset arr_cpass0_outputs | |
1028 | unset arr_cpass1_outputs | |
1029 | declare -a arr_cpass0_outputs | |
1030 | declare -a arr_cpass1_outputs | |
1031 | unset arr_cpass0_outputs | |
1032 | ||
1033 | while read inputFile; do | |
1034 | currentDefaultOCDB=${defaultOCDB} | |
1035 | [[ ${autoOCDB} -ne 0 ]] && currentDefaultOCDB=$(setYear $inputFile $defaultOCDB) | |
1036 | ||
1037 | #CPass0 | |
1038 | arr_cpass0_outputs[$jobindex]="cpass0.job${jobindex}.run${runNumber}.done" | |
36b5de1a | 1039 | echo "${arr_cpass0_outputs[$jobindex]}: benchmark.sh ${configFile} ${copyFiles[@]}" |
fc7e1e45 | 1040 | echo " ${alirootEnv} ./benchmark.sh CPass0 ${commonOutputPath}/cpass0/000${runNumber} $inputFile $nEvents $currentDefaultOCDB $configFile $runNumber $jobindex" |
1041 | echo | |
1042 | ||
1043 | #CPass1 | |
1044 | arr_cpass1_outputs[$jobindex]="cpass1.job${jobindex}.run${runNumber}.done" | |
36b5de1a | 1045 | echo "${arr_cpass1_outputs[$jobindex]} : benchmark.sh ${configFile} cpass0.localOCDB.${runNumber}.tgz ${copyFiles[@]}" |
fc7e1e45 | 1046 | echo " ${alirootEnv} ./benchmark.sh CPass1 ${commonOutputPath}/cpass1/000${runNumber} $inputFile $nEvents $currentDefaultOCDB $configFile $runNumber $jobindex" |
1047 | echo | |
1048 | ((jobindex++)) | |
1049 | ||
1050 | done< <(grep "/000$runNumber/" $inputFileList) | |
1051 | ||
1052 | #CPass0 list of Calib files to merge | |
1053 | echo "cpass0.calib.run${runNumber}.list: ${arr_cpass0_outputs[*]}" | |
1054 | echo " awk '/^calibfile / {print "'\$2'"}' ${arr_cpass0_outputs[*]} > cpass0.calib.run${runNumber}.list" | |
1055 | echo | |
1056 | ||
1057 | #CPass1 list of Calib/QA files to merge | |
1058 | echo "cpass1.calib.run${runNumber}.list cpass1.QA.run${runNumber}.list: ${arr_cpass1_outputs[*]}" | |
1059 | echo " awk '/^calibfile / {print "'\$'"2}' ${arr_cpass1_outputs[*]} > cpass1.calib.run${runNumber}.list; awk '/^qafile / {print "'\$'"2}' ${arr_cpass1_outputs[*]} > cpass1.QA.run${runNumber}.list" | |
1060 | echo | |
1061 | ||
1062 | #CPass0 merging | |
1063 | arr_cpass0_final[$runindex]="merge.cpass0.run${runNumber}.done" | |
36b5de1a | 1064 | echo "cpass0.localOCDB.${runNumber}.tgz ${arr_cpass0_final[$runindex]}: cpass0.calib.run${runNumber}.list benchmark.sh ${configFile} ${copyFiles[@]}" |
fc7e1e45 | 1065 | echo " ${alirootEnv} ./benchmark.sh MergeCPass0 ${commonOutputPath}/cpass0/000${runNumber} $currentDefaultOCDB ${configFile} $runNumber cpass0.run${runNumber}.list" |
1066 | echo | |
1067 | ||
1068 | #CPass1 Calib/QA merging | |
1069 | arr_cpass1_final[$runindex]="merge.cpass1.run${runNumber}.done" | |
36b5de1a | 1070 | echo "${arr_cpass1_final[$runindex]}: cpass1.calib.run${runNumber}.list cpass1.QA.run${runNumber}.list benchmark.sh ${configFile} ${copyFiles[@]}" |
fc7e1e45 | 1071 | echo " ${alirootEnv} ./benchmark.sh MergeCPass1 ${commonOutputPath}/cpass1/000${runNumber} $currentDefaultOCDB ${configFile} $runNumber cpass1.calib.run${runNumber}.list cpass1.QA.run${runNumber}.list" |
1072 | echo | |
1073 | ((runindex++)) | |
1074 | done | |
1075 | ||
1076 | #CPass1 list of final Calib/QA files | |
1077 | echo "cpass1.QA.list cpass1.calib.list: ${arr_cpass1_final[*]}" | |
1078 | echo " awk '/^calibfile / {print "'\$'"2}' ${arr_cpass1_final[*]} > cpass1.calib.list; awk '/^qafile / {print "'\$'"2}' ${arr_cpass1_final[*]} > cpass1.QA.list" | |
1079 | echo | |
1080 | ||
1081 | #Summary | |
1082 | echo "summary.log : ${arr_cpass0_outputs[*]} ${arr_cpass1_outputs[*]} ${arr_cpass1_final[*]} ${arr_cpass0_final[*]} benchmark.sh ${configFile}" | |
1083 | echo " LOCAL ./benchmark.sh makeSummary ${configFile} |tee summary.log" | |
1084 | } | |
1085 | ||
1086 | goCreateQAplots() | |
1087 | { | |
1088 | umask 0002 | |
1089 | [[ $# -lt 5 ]] && echo "goCreateQAplots productionID pass outputDir qaFilesDirectory qaPlotScript" && exit 1 | |
1090 | productionID=$1 | |
1091 | pass=$2 | |
1092 | outputDir=$3 | |
1093 | qaFilesDirectory=$4 | |
1094 | qaPlotsScript=$5 | |
1095 | configFile=$6 | |
1096 | ||
1097 | source $configFile | |
1098 | [[ -f ${setupAliROOTenvInCurrentShell} ]] && source $setupAliROOTenvInCurrentShell | |
1099 | ||
1100 | runpath=${PWD}/rundir_cpass0_Merge_${runNumber} | |
1101 | [[ -z $commonOutputPath ]] && commonOutputPath=$PWD | |
1102 | [[ $reconstructInTemporaryDir -eq 1 && -n $TMPDIR ]] && runpath=$TMPDIR | |
1103 | [[ $reconstructInTemporaryDir -eq 1 && -z $TMPDIR ]] && runpath=$(mktemp -d) | |
1104 | ||
1105 | mkdir -p $runpath | |
1106 | [[ ! -d $runpath ]] && echo "not able to make the runpath $runpath" && exit 1 | |
1107 | cd $runpath | |
1108 | ||
1109 | [[ -z $logOutputDir ]] && logOutputDir=$runpath | |
1110 | [[ -z $dontRedirectStdOutToLog ]] && exec 2>&1 > $logOutputDir/makeQAplots.log | |
1111 | echo "$0 $*" | |
1112 | ||
1113 | [[ -z "$qaPlotsScript" ]] && echo "qaPlotsScript not defined"&&exit 1 | |
1114 | ||
1115 | mergedQAfileList=$outputDir/mergedQAfiles.list | |
1116 | echo "MakeListOfQAresults $qaFilesDirectory QAresults.root | grep CPass1 > $mergedQAfileList" | |
1117 | MakeListOfQAresults $qaFilesDirectory QAresults_merged.root | grep CPass1 |tee $mergedQAfileList | |
1118 | echo $qaPlotsScript "$productionID" "cpass1" $mergedQAfileList $outputDir | |
1119 | $qaPlotsScript "$productionID" "cpass1" $mergedQAfileList $outputDir | |
1120 | ||
1121 | mkdir -p $outputDir | |
1122 | [[ ! -d $outputDir ]] && echo "cannot make the output dir $outputDir" && exit 1 | |
1123 | mv -f $runpath/* $outputDir | |
1124 | rm -rf $runpath | |
1125 | } | |
1126 | ||
1127 | goWaitForOutput() | |
1128 | { | |
1129 | umask 0002 | |
1130 | [[ $# -lt 3 ]] && echo "goWaitForOutput() wrong number of arguments, exiting.." && exit 1 | |
1131 | echo searchPath=$1 | |
1132 | echo fileName=$2 | |
1133 | echo numberOfFiles=$3 | |
1134 | echo maxSecondsToWait=$4 | |
1135 | searchPath=$1 | |
1136 | fileName=$2 | |
1137 | numberOfFiles=$3 | |
1138 | maxSecondsToWait=$4 | |
1139 | extraFindOptions=$5 | |
1140 | echo "command to be executed: find $searchPath -name "$fileName" ${extraFindOptions}" | |
1141 | [[ -z "$maxSecondsToWait" ]] && maxSecondsToWait=$(( 3600*12 )) | |
1142 | while sleep 60; do | |
1143 | n=$(find $searchPath -name "$fileName" ${extraFindOptions}| wc -l) | |
1144 | [[ $n -gt 0 ]] && echo "found $n X $fileName" | |
1145 | [[ $n -ge $numberOfFiles ]] && break | |
1146 | [[ $SECONDS -gt $maxSecondsToWait ]] && break | |
1147 | done | |
1148 | echo "DONE! exiting..." | |
1149 | } | |
1150 | ||
1151 | submit() | |
1152 | { | |
1153 | umask 0002 | |
1154 | [[ $# -ne 6 ]] && echo "6 args needed, you supplied $#" && exit 1 | |
1155 | JobID=$1 | |
1156 | startID=$2 | |
1157 | endID=$3 | |
1158 | waitForJOBID=$4 | |
1159 | command=$5 | |
1160 | commandArgs=$6 | |
1161 | ||
1162 | newFarm=$(which qsub|grep "^/usr/bin/qsub") | |
1163 | ||
1164 | batchSystem="SGE" | |
1165 | ||
1166 | if [[ -z "$newFarm" ]] | |
1167 | then | |
1168 | #old LSF | |
1169 | # submit it (as job array) | |
1170 | nFiles=$(( $endID-$startID+1 )) | |
1171 | while [ $startID -le $nFiles ] ; do | |
1172 | if [ `expr $nFiles - $startID` -gt 999 ] ; then | |
1173 | endID=`expr $startID + 999` | |
1174 | else | |
1175 | endID=$nFiles | |
1176 | fi | |
1177 | if [[ -z "$waitForJOBID" ]]; then | |
1178 | echo $batchCommand -J "$JobID[$startID-$endID]" -e "$targetDirectory/logs/job_%I.err" -o "$targetDirectory/logs/job_%I.out" "$command" | |
1179 | $batchCommand -J "$JobID[$startID-$endID]" -e "$targetDirectory/logs/job_%I.err" -o "$targetDirectory/logs/job_%I.out" "$command" | |
1180 | else | |
1181 | echo $batchCommand -J "$JobID[$startID-$endID]" -w "ended($waitForJOBID)" -e "$targetDirectory/logs/job_%I.err" -o "$targetDirectory/logs/job_%I.out" "$command" | |
1182 | $batchCommand -J "$JobID[$startID-$endID]" -w "ended($waitForJOBID)" -e "$targetDirectory/logs/job_%I.err" -o "$targetDirectory/logs/job_%I.out" "$command" | |
1183 | fi | |
1184 | startID=`expr $endID + 1` | |
1185 | done | |
1186 | else | |
1187 | #new SGE farm | |
1188 | if [[ -z "$waitForJOBID" ]]; then | |
1189 | echo $batchCommand -wd ${targetDirectory} -b y -V -N "$JobID" -t "$startID-$endID" -e "$targetDirectory/logs/" -o "$targetDirectory/logs/" "$command" $commandArgs | |
1190 | $batchCommand -wd ${targetDirectory} -b y -V -N "$JobID" -t "$startID-$endID" -e "$targetDirectory/logs/" -o "$targetDirectory/logs/" "$command" $commandArgs | |
1191 | else | |
1192 | echo $batchCommand -wd ${targetDirectory} -b y -V -N "$JobID" -t "$startID-$endID" -hold_jid "$waitForJOBID" -e "$targetDirectory/logs/" -o "$targetDirectory/logs/" "$command" $commandArgs | |
1193 | $batchCommand -wd ${targetDirectory} -b y -V -N "$JobID" -t "$startID-$endID" -hold_jid "$waitForJOBID" -e "$targetDirectory/logs/" -o "$targetDirectory/logs/" "$command" $commandArgs | |
1194 | fi | |
1195 | fi | |
1196 | } | |
1197 | ||
1198 | goTest() | |
1199 | { | |
1200 | umask 0002 | |
1201 | exec 2>&1 | |
1202 | exec > >(tee test.log) | |
1203 | echo "$@" | |
1204 | echo something | |
1205 | } | |
1206 | ||
1207 | alirootInfo() | |
1208 | { | |
1209 | ( | |
1210 | umask 0002 | |
1211 | # save aliroot svn info | |
1212 | [[ -z "$ALICE_ROOT" ]] && exit 1 | |
1213 | prevdir=$PWD | |
1214 | cd $ALICE_ROOT | |
1215 | echo "\$ALICE_ROOT=$ALICE_ROOT" | |
1216 | echo | |
1217 | svn info 2>/dev/null | |
1218 | echo "" | |
1219 | echo "" | |
1220 | svn diff 2>/dev/null | |
1221 | cd $prevdir | |
1222 | ) | |
1223 | } | |
1224 | ||
1225 | MakeListOfQAresults() | |
1226 | { | |
1227 | ( | |
1228 | umask 0002 | |
1229 | if [[ $# -eq 0 ]]; then | |
1230 | echo "make an input file list for the qa script" | |
1231 | echo "Usage: $0 path filename" | |
1232 | return 0 | |
1233 | fi | |
1234 | ||
1235 | path=$1 | |
1236 | fileName=$2 | |
1237 | ||
1238 | while read entry; do | |
1239 | runNumber=$(guessRunNumber $entry) | |
1240 | echo "$runNumber 0 1 10 $entry" | |
1241 | done < <(find $path -name $fileName) | |
1242 | ) | |
1243 | } | |
1244 | ||
1245 | setYear() | |
1246 | { | |
1247 | #set the year | |
1248 | # $1 - year to be set | |
1249 | # $2 - where to set the year | |
1250 | year1=$(guessYear $1) | |
1251 | year2=$(guessYear $2) | |
1252 | local path=$2 | |
1253 | [[ $year1 -ne $year2 && -n $year2 ]] && path=${2/\/$year2\//\/$year1\/} | |
1254 | echo $path | |
1255 | } | |
1256 | ||
1257 | guessPeriod() | |
1258 | { | |
1259 | #guess the period from the path, pick the rightmost one | |
1260 | local IFS="/" | |
1261 | declare -a path=( $1 ) | |
1262 | local dirDepth=${#path[*]} | |
1263 | for ((x=${dirDepth}-1;x>=0;x--)); do | |
1264 | local field=${path[${x}]} | |
1265 | [[ ${field} =~ ^LHC[0-9][0-9][a-z]$ ]] && period=${field#000} && break | |
1266 | done | |
1267 | echo $period | |
1268 | } | |
1269 | ||
1270 | guessYear() | |
1271 | { | |
1272 | #guess the year from the path, pick the rightmost one | |
1273 | local IFS="/" | |
1274 | declare -a path=( $1 ) | |
1275 | local dirDepth=${#path[*]} | |
1276 | for ((x=${dirDepth}-1;x>=0;x--)); do | |
1277 | local field=${path[${x}]} | |
1278 | [[ ${field} =~ ^20[0-9][0-9]$ ]] && year=${field#000} && break | |
1279 | done | |
1280 | echo $year | |
1281 | } | |
1282 | ||
1283 | guessRunNumber() | |
1284 | { | |
1285 | #guess the run number from the path, pick the rightmost one | |
1286 | local IFS="/" | |
1287 | declare -a path=( $1 ) | |
1288 | local dirDepth=${#path[*]} | |
1289 | for ((x=${dirDepth}-1;x>=0;x--)); do | |
1290 | local field=${path[${x}]} | |
1291 | [[ ${field} =~ ^000[0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && runNumber=${field#000} && break | |
1292 | done | |
1293 | echo $runNumber | |
1294 | } | |
1295 | ||
1296 | summarizeLogs() | |
1297 | { | |
1298 | #print a summary of logs | |
1299 | logFiles=( | |
1300 | "*.log" | |
1301 | ) | |
1302 | ||
1303 | errorConditions=( | |
1304 | "There was a crash" | |
1305 | "floating" | |
1306 | "error while loading shared libraries" | |
1307 | "std::bad_alloc" | |
1308 | "s_err_syswatch_" | |
1309 | "Thread [0-9]* (Thread" | |
1310 | ) | |
1311 | logstatus=0 | |
1312 | for log in ${logFiles[*]}; do | |
1313 | finallog=${outputDir%/}/${log} | |
1314 | [[ ! -f $log ]] && continue | |
1315 | errorSummary="" | |
1316 | for ((i=0; i<${#errorConditions[@]};i++)); do | |
1317 | local tmp=$(grep -m1 -e "${errorConditions[$i]}" $log) | |
1318 | [[ -n $tmp ]] && tmp+=" : " | |
1319 | errorSummary+=$tmp | |
1320 | done | |
1321 | if [[ -z $errorSummary ]]; then | |
1322 | #in pretend mode randomly report an error in rec.log some cases | |
1323 | if [[ -n $pretend && "$log" == "rec.log" ]]; then | |
1324 | [[ $(( $RANDOM%2 )) -ge 1 ]] && echo "$finallog BAD random error" || echo "$finallog OK" | |
1325 | else | |
1326 | echo "$finallog OK" | |
1327 | fi | |
1328 | else | |
1329 | local logstatus=1 | |
1330 | echo "$finallog BAD $errorSummary" | |
1331 | fi | |
1332 | done | |
1333 | return $logstatus | |
1334 | } | |
1335 | ||
1336 | spitOutLocalOCDBaccessConfig() | |
1337 | { | |
1338 | umask 0002 | |
1339 | find $1 -name "*root" | \ | |
1340 | while read line | |
1341 | do | |
1342 | local tmp=${line#$1} | |
1343 | echo ${tmp%/*} | \ | |
1344 | awk -v ocdb=$1 '{print " man->SetSpecificStorage(\""$1"\",\"local://"ocdb"\");"}' | |
1345 | done | |
1346 | } | |
1347 | ||
1348 | goConfigureOCDBforCPass1() | |
1349 | { | |
1350 | umask 0002 | |
1351 | # make a script that sets the specific storages form all the root files produced by CPass0 | |
1352 | # in this case the second argument is taken to be the path to the produced OCDB for specific storage | |
1353 | local localOCDBpathCPass0=$1 | |
1354 | local outputDir=$2 | |
1355 | [[ -d $outputDir ]] && cd $outputDir | |
1356 | ||
1357 | if [[ -f $localOCDBpathCPass0 && $localOCDBpathCPass0 =~ \.tgz$ ]]; then | |
1358 | tar xzf $localOCDBpathCPass0 | |
1359 | local localOCDBpathCPass0="./OCDB" | |
1360 | fi | |
1361 | ||
1362 | echo | |
1363 | echo creating the specific storage script | |
1364 | echo localOCDBaccessConfig.C | |
1365 | echo based on OCDB: $fileName | |
1366 | echo | |
1367 | ||
1368 | local tempLocalOCDB="" | |
1369 | if [[ -f localOCDBaccessConfig.C ]]; then | |
1370 | tempLocalOCDB=$(mktemp) | |
1371 | echo "egrep "SetSpecificStorage" localOCDBaccessConfig.C > $tempLocalOCDB" | |
1372 | egrep "SetSpecificStorage" localOCDBaccessConfig.C > $tempLocalOCDB | |
1373 | fi | |
1374 | ||
1375 | echo "localOCDBaccessConfig()" > localOCDBaccessConfig.C | |
1376 | echo "{" >> localOCDBaccessConfig.C | |
1377 | echo " AliCDBManager* man = AliCDBManager::Instance();" >> localOCDBaccessConfig.C | |
1378 | spitOutLocalOCDBaccessConfig $localOCDBpathCPass0|sort|uniq >> localOCDBaccessConfig.C | |
1379 | [[ -f "$tempLocalOCDB" ]] && cat $tempLocalOCDB >> localOCDBaccessConfig.C | |
1380 | echo "}" >> localOCDBaccessConfig.C | |
1381 | ||
1382 | [[ -f "$tempLocalOCDB" ]] && rm -f $tempLocalOCDB | |
1383 | ||
1384 | if ! grep SetSpecificStorage localOCDBaccessConfig.C; then | |
1385 | echo | |
1386 | echo "!!!!!!! CPass0 produced no OCDB entries" | |
1387 | return 1 | |
1388 | fi | |
1389 | } | |
1390 | ||
1391 | main "$@" |