]>
Commit | Line | Data |
---|---|---|
fc7e1e45 | 1 | #!/bin/bash |
2 | # this script runs the CPass0/CPass1 train | |
3 | # produced OCDB updates are local | |
4 | ||
5 | main() | |
6 | { | |
7 | #run in proper mode depending on the selection | |
8 | runMode=$1 | |
9 | if [[ $# -gt 0 ]]; then | |
10 | echo "# $0 $*" | |
11 | fi | |
12 | umask 0002 | |
13 | shift | |
14 | case $runMode in | |
15 | "CPass0") goCPass0 "$@";; | |
16 | "CPass1") goCPass1 "$@";; | |
17 | "ConfOCDB") goConfigureOCDBforCPass1 "$@";; | |
18 | "MergeCPass0") goMergeCPass0 "$@";; | |
19 | "MergeCPass1") goMergeCPass1 "$@";; | |
20 | "CreateQAplots") goCreateQAplots "$@";; | |
21 | "WaitForOutput") goWaitForOutput "$@";; | |
22 | "makeSummary") goMakeSummary "$@";; | |
23 | "submit") goSubmit "$@";; | |
24 | "submitQA") goSubmitQA "$@";; | |
25 | "test") goTest "$@";; | |
26 | "generateMakeflow") goGenerateMakeflow "$@";; | |
27 | "makeflow") goMakeflow "$@";; | |
28 | #and the default: unless sourced print some info | |
29 | *) if [[ ! "$0" =~ "bash" ]]; then | |
30 | echo " batch: $0 \"submit\" inputList productionID [configFile=benchmark.config] [runNumber]" | |
31 | echo " makeflow: $0 \"makeflow\" inputList productionID [configFile=benchmark.config] [runNumber]" | |
32 | fi | |
33 | ;; | |
34 | esac | |
35 | } | |
36 | ||
37 | goCPass0() | |
38 | { | |
39 | umask 0002 | |
40 | ||
41 | configFile=$5 | |
42 | source $configFile | |
36b5de1a | 43 | [[ -f ${setupAliROOTenvInCurrentShell} && -z ${alirootEnv} ]] && source $setupAliROOTenvInCurrentShell |
fc7e1e45 | 44 | |
45 | targetDirectory=$1 | |
46 | inputList=$2 | |
47 | nEvents=$3 | |
48 | ocdbPath=$4 | |
49 | configFile=$5 | |
50 | runNumber=$6 | |
51 | jobindex=$7 | |
52 | ||
53 | #use the jobindex only if set and non-negative | |
54 | if [[ -z $jobindex || $jobindex -lt 0 ]]; then | |
55 | [[ -n "$LSB_JOBINDEX" ]] && jobindex=$LSB_JOBINDEX | |
56 | [[ -n "$SGE_TASK_ID" ]] && jobindex=$SGE_TASK_ID | |
57 | fi | |
58 | ||
59 | [[ ! -f ${inputList} && -z ${pretend} ]] && echo "input file $inputList not found, exiting..." && exit 1 | |
60 | if [[ "${inputList}" =~ \.root$ ]]; then | |
61 | infile=$inputList | |
62 | else | |
63 | infile=`sed -ne "${jobindex}p" $inputList` | |
64 | fi | |
65 | ||
66 | chunkName=${infile##*/} | |
67 | outputDir=${targetDirectory}/${jobindex} | |
68 | mkdir -p $outputDir | |
69 | [[ ! -d $outputDir ]] && echo "cannot make $outputDir" && exit 1 | |
70 | ||
71 | runpath=${PWD}/rundir_cpass0_${runNumber}_${jobindex} | |
72 | [[ -z $commonOutputPath ]] && commonOutputPath=$PWD | |
73 | [[ $reconstructInTemporaryDir -eq 1 && -n $TMPDIR ]] && runpath=$TMPDIR | |
74 | [[ $reconstructInTemporaryDir -eq 1 && -z $TMPDIR ]] && runpath=$(mktemp -d) | |
75 | ||
76 | mkdir -p $runpath | |
77 | [[ ! -d ${runpath} ]] && echo "cannot make runpath ${runpath}" && exit 1 | |
78 | cd $runpath | |
79 | ||
80 | logOutputDir=$runpath | |
81 | [[ -n $logToFinalDestination ]] && logOutputDir=$outputDir | |
5ed7c82c | 82 | [[ -z $dontRedirectStdOutToLog ]] && exec 1> $logOutputDir/stdout |
83 | [[ -z $dontRedirectStdOutToLog ]] && exec 2> $logOutputDir/stderr | |
fc7e1e45 | 84 | echo "$0 $*" |
85 | ||
86 | calibDoneFile="$commonOutputPath/cpass0.job${jobindex}.run${runNumber}.done" | |
87 | ||
88 | echo "#####################" | |
89 | echo CPass0: | |
90 | echo JOB setup | |
91 | echo nEvents $nEvents | |
92 | echo runNumber $runNumber | |
93 | echo ocdbPath $ocdbPath | |
94 | echo infile $infile | |
95 | echo chunkName $chunkName | |
96 | echo jobindex $jobindex | |
97 | echo recoTriggerOptions $recoTriggerOptions | |
98 | echo targetDirectory $targetDirectory | |
99 | echo commonOutputPath $commonOutputPath | |
100 | echo calibDoneFile $calibDoneFile | |
101 | echo runpath $runpath | |
102 | echo outputDir $outputDir | |
103 | echo ALICE_ROOT $ALICE_ROOT | |
104 | echo PWD $PWD | |
105 | echo "########## ###########" | |
106 | ||
107 | alirootInfo > ALICE_ROOT_svn.log | |
108 | ||
109 | filesCPass0=( | |
110 | "$commonOutputPath/runCPass0.sh" | |
111 | "$commonOutputPath/recCPass0.C" | |
112 | "$commonOutputPath/runCalibTrain.C" | |
113 | "$commonOutputPath/localOCDBaccessConfig.C" | |
36b5de1a | 114 | "$commonOutputPath/OCDB.root" |
fc7e1e45 | 115 | "$ALICE_ROOT/PWGPP/CalibMacros/CPass0/runCPass0.sh" |
116 | "$ALICE_ROOT/PWGPP/CalibMacros/CPass0/recCPass0.C" | |
117 | "$ALICE_ROOT/PWGPP/CalibMacros/CPass0/runCalibTrain.C" | |
118 | ) | |
119 | ||
120 | for file in ${filesCPass0[*]}; do | |
121 | [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} . | |
122 | done | |
123 | ||
124 | ln -s $infile $runpath/$chunkName | |
125 | ||
126 | echo "this directory ($PWD) contents:" | |
127 | ls -lh | |
128 | echo | |
129 | chmod u+x runCPass0.sh | |
130 | ||
131 | if [[ -n $postSetUpActionCPass0 ]]; then | |
132 | echo "running $postSetUpActionCPass0" | |
133 | eval $postSetUpActionCPass0 | |
134 | fi | |
135 | ||
136 | #run CPass0 | |
137 | echo "$runpath/runCPass0.sh $infile $nEvents $runNumber $ocdbPath $recoTriggerOptions" | |
138 | if [[ -n $pretend ]]; then | |
139 | touch AliESDfriends_v1.root | |
140 | touch rec.log | |
141 | touch calib.log | |
142 | else | |
143 | ./runCPass0.sh "$infile" "$nEvents" "$runNumber" "$ocdbPath" "$recoTriggerOptions" | |
144 | fi | |
145 | ||
fc7e1e45 | 146 | #move stuff to final destination |
147 | echo "this directory ($PWD) contents:" | |
148 | ls -lh | |
149 | echo | |
150 | ||
151 | echo rm -f ./$chunkName | |
152 | rm -f ./$chunkName | |
153 | echo "cp --recursive $runpath/* ${outputDir}" | |
154 | cp --recursive $runpath/* $outputDir | |
155 | echo | |
5ed7c82c | 156 | |
157 | #validate CPass0 | |
158 | cd ${outputDir} | |
159 | touch ${calibDoneFile} | |
160 | [[ -f AliESDfriends_v1.root ]] && echo "calibfile ${outputDir}/AliESDfriends_v1.root" > ${calibDoneFile} | |
161 | summarizeLogs >> ${calibDoneFile} | |
fc7e1e45 | 162 | |
163 | rm -rf ${runpath} | |
164 | } | |
165 | ||
166 | goCPass1() | |
167 | { | |
168 | umask 0002 | |
169 | ||
170 | configFile=$5 | |
171 | source $configFile | |
36b5de1a | 172 | [[ -f ${setupAliROOTenvInCurrentShell} && -z ${alirootEnv} ]] && source $setupAliROOTenvInCurrentShell |
fc7e1e45 | 173 | |
174 | targetDirectory=$1 | |
175 | inputList=$2 | |
176 | nEvents=$3 | |
177 | ocdbPath=$4 | |
178 | configFile=$5 | |
179 | runNumber=$6 | |
180 | jobindex=$7 | |
181 | ||
182 | #use the jobindex only if set and non-negative | |
183 | if [[ -z $jobindex || $jobindex -lt 0 ]]; then | |
184 | [[ -n "$LSB_JOBINDEX" ]] && jobindex=$LSB_JOBINDEX | |
185 | [[ -n "$SGE_TASK_ID" ]] && jobindex=$SGE_TASK_ID | |
186 | fi | |
187 | ||
188 | [[ ! -f ${inputList} && -z ${pretend} ]] && echo "input file $inputList not found, exiting..." && exit 1 | |
189 | if [[ "${inputList}" =~ \.root$ ]]; then | |
190 | infile=$inputList | |
191 | else | |
192 | infile=`sed -ne "${jobindex}p" $inputList` | |
193 | fi | |
194 | ||
195 | chunkName=${infile##*/} | |
196 | outputDir=${targetDirectory}/${jobindex} | |
197 | mkdir -p $outputDir | |
198 | [[ ! -d $outputDir ]] && echo "cannot make $outputDir" && exit 1 | |
199 | ||
200 | runpath=${PWD}/rundir_cpass1_${runNumber}_${jobindex} | |
201 | [[ -z $commonOutputPath ]] && commonOutputPath=$PWD | |
202 | [[ $reconstructInTemporaryDir -eq 1 && -n $TMPDIR ]] && runpath=$TMPDIR | |
203 | [[ $reconstructInTemporaryDir -eq 1 && -z $TMPDIR ]] && runpath=$(mktemp -d) | |
204 | ||
205 | #init the running path | |
206 | mkdir -p $runpath | |
207 | [[ ! -d ${runpath} ]] && echo "cannot make runpath ${runpath}" && exit 1 | |
208 | cd $runpath | |
209 | ||
210 | ||
211 | calibDoneFile="${commonOutputPath}/cpass1.job${jobindex}.run${runNumber}.done" | |
212 | ||
213 | logOutputDir=$runpath | |
214 | [[ -n $logToFinalDestination ]] && logOutputDir=$outputDir | |
5ed7c82c | 215 | [[ -z $dontRedirectStdOutToLog ]] && exec 1> $logOutputDir/stdout |
216 | [[ -z $dontRedirectStdOutToLog ]] && exec 2> $logOutputDir/stderr | |
fc7e1e45 | 217 | echo "$0 $*" |
218 | ||
219 | echo "#####################" | |
220 | echo CPass1: | |
221 | echo JOB setup | |
222 | echo nEvents $nEvents | |
223 | echo runNumber $runNumber | |
224 | echo ocdbPath $ocdbPath | |
225 | echo infile $infile | |
226 | echo chunkName $chunkName | |
227 | echo jobindex $jobindex | |
228 | echo recoTriggerOptions $recoTriggerOptions | |
229 | echo targetDirectory $targetDirectory | |
230 | echo commonOutputPath $commonOutputPath | |
231 | echo calibDoneFile $calibDoneFile | |
232 | echo runpath $runpath | |
233 | echo outputDir $outputDir | |
234 | echo ALICE_ROOT $ALICE_ROOT | |
235 | echo PWD $PWD | |
236 | echo "########## ###########" | |
237 | ||
238 | alirootInfo > ALICE_ROOT_svn.log | |
239 | ||
240 | filesCPass1=( | |
241 | "$commonOutputPath/runCPass1.sh" | |
242 | "$commonOutputPath/recCPass1.C" | |
243 | "$commonOutputPath/recCPass1_OuterDet.C" | |
244 | "$commonOutputPath/runCalibTrain.C" | |
245 | "$commonOutputPath/QAtrain_duo.C" | |
246 | "$commonOutputPath/localOCDBaccessConfig.C" | |
247 | "$commonOutputPath/cpass0.localOCDB.${runNumber}.tgz" | |
36b5de1a | 248 | "$commonOutputPath/OCDB.root" |
fc7e1e45 | 249 | "$ALICE_ROOT/PWGPP/CalibMacros/CPass1/runCPass1.sh" |
250 | "$ALICE_ROOT/PWGPP/CalibMacros/CPass1/recCPass1.C" | |
251 | "$ALICE_ROOT/PWGPP/CalibMacros/CPass1/recCPass1_OuterDet.C" | |
252 | "$ALICE_ROOT/PWGPP/CalibMacros/CPass1/runCalibTrain.C" | |
253 | "$ALICE_ROOT/ANALYSIS/macros/QAtrain_duo.C" | |
254 | ) | |
255 | ||
256 | for file in ${filesCPass1[*]}; do | |
257 | [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} . | |
258 | done | |
259 | ||
260 | ln -s $infile $runpath/$chunkName | |
261 | ||
262 | echo "this directory ($PWD) contents:" | |
263 | ls -lh | |
264 | echo | |
265 | ||
266 | if [[ -n $postSetUpActionCPass1 ]]; then | |
267 | echo "running $postSetUpActionCPass1" | |
268 | eval $postSetUpActionCPass1 | |
269 | echo | |
270 | fi | |
271 | ||
272 | #configure local OCDB storage from CPass0 (creates the localOCDBaccessConfig.C script) | |
273 | if [[ -f cpass0.localOCDB.${runNumber}.tgz ]]; then | |
274 | echo goConfigureOCDBforCPass1 "cpass0.localOCDB.${runNumber}.tgz" | |
275 | goConfigureOCDBforCPass1 "cpass0.localOCDB.${runNumber}.tgz" | |
276 | else | |
277 | echo "WARNING: file cpass0.localOCDB.${runNumber}.tgz not found!" | |
278 | fi | |
279 | ||
280 | #run CPass1 | |
281 | chmod u+x runCPass1.sh | |
282 | echo "$runpath/runCPass1.sh $infile $nEvents $runNumber $ocdbPath $recoTriggerOptions" | |
283 | if [[ -n $pretend ]]; then | |
284 | touch AliESDfriends_v1.root | |
285 | touch QAresults_Barrel.root | |
286 | touch QAresults_Outer.root | |
287 | touch rec.log | |
288 | touch calib.log | |
289 | touch qa.log | |
290 | else | |
291 | ./runCPass1.sh "$infile" "$nEvents" "$runNumber" "$ocdbPath" "$recoTriggerOptions" | |
292 | fi | |
293 | ||
fc7e1e45 | 294 | #move stuff to final destination |
295 | echo "this directory ($PWD) contents:" | |
296 | ls | |
297 | echo | |
298 | ||
299 | echo rm -f ./$chunkName | |
300 | rm -f ./$chunkName | |
301 | echo "cp --recursive ${runpath}/* ${outputDir}" | |
302 | cp --recursive ${runpath}/* ${outputDir} | |
303 | echo | |
304 | ||
5ed7c82c | 305 | #validate CPass1 |
306 | cd ${outputDir} | |
307 | touch ${calibDoneFile} | |
308 | [[ -f AliESDfriends_v1.root ]] && echo "calibfile ${outputDir}/AliESDfriends_v1.root" > ${calibDoneFile} | |
309 | [[ -f QAresults_Barrel.root ]] && echo "qafile ${outputDir}/QAresults_Barrel.root" >> ${calibDoneFile} | |
310 | [[ -f QAresults_Outer.root ]] && echo "qafile ${outputDir}/QAresults_Outer.root" >> ${calibDoneFile} | |
dd755372 | 311 | echo "dir ${outputDir}" >> ${calibDoneFile} |
5ed7c82c | 312 | summarizeLogs >> ${calibDoneFile} |
313 | ||
fc7e1e45 | 314 | rm -rf ${runpath} |
315 | } | |
316 | ||
317 | ||
318 | goMergeCPass0() | |
319 | { | |
320 | umask 0002 | |
321 | # | |
322 | # find the output files and merge them | |
323 | # | |
324 | ||
325 | outputDir=$1 | |
326 | defaultOCDB=$2 | |
327 | configFile=$3 | |
328 | runNumber=$4 | |
329 | calibrationFilesToMergeExternal=$5 | |
330 | ||
331 | source $configFile | |
36b5de1a | 332 | [[ -f ${setupAliROOTenvInCurrentShell} && -z ${alirootEnv} ]] && source $setupAliROOTenvInCurrentShell |
fc7e1e45 | 333 | |
334 | runpath=${PWD}/rundir_cpass0_Merge_${runNumber} | |
335 | [[ -z $commonOutputPath ]] && commonOutputPath=$PWD | |
336 | [[ $reconstructInTemporaryDir -eq 1 && -n $TMPDIR ]] && runpath=$TMPDIR | |
337 | [[ $reconstructInTemporaryDir -eq 1 && -z $TMPDIR ]] && runpath=$(mktemp -d) | |
338 | ||
339 | mkdir -p $runpath | |
340 | [[ ! -d $runpath ]] && echo "not able to make the runpath $runpath" && exit 1 | |
341 | cd $runpath | |
342 | ||
343 | logOutputDir=$runpath | |
344 | [[ -n $logToFinalDestination ]] && logOutputDir=$outputDir | |
345 | [[ -z $dontRedirectStdOutToLog ]] && exec 2>&1 > $logOutputDir/mergeMakeOCDB.log | |
346 | echo "$0 $*" | |
347 | ||
348 | calibrationFilesToMerge=$calibrationFilesToMergeExternal | |
349 | [[ -z $calibrationFilesToMerge ]] && calibrationFilesToMerge="calibrationFilesToMerge.list" | |
350 | calibrationOutputFileName="AliESDfriends_v1.root" | |
351 | mergingScript="mergeMakeOCDB.byComponent.sh" | |
352 | qaFilesToMerge="qaFilesToMerge.list" | |
353 | qaOutputFileName="QAresults*.root" | |
354 | qaMergedOutputFileName="QAresults_merged.root" | |
355 | ||
356 | echo goMergeCPass0 SETUP: | |
357 | echo runNumber=$runNumber | |
358 | echo outputDir=$outputDir | |
359 | echo defaultOCDB=$defaultOCDB | |
360 | echo calibrationFilesToMerge=$calibrationFilesToMerge | |
361 | echo calibrationOutputFileName=$calibrationOutputFileName | |
362 | echo mergingScript=$mergingScript | |
363 | ||
364 | # copy files in case they are not already there | |
365 | filesMergeCPass0=( | |
36b5de1a | 366 | "$commonOutputPath/${calibrationFilesToMerge}" |
367 | "$commonOutputPath/OCDB.root" | |
368 | "$commonOutputPath/localOCDBaccessConfig.C" | |
fc7e1e45 | 369 | "$ALICE_ROOT/PWGPP/CalibMacros/CPass0/mergeMakeOCDB.byComponent.sh" |
370 | "$ALICE_ROOT/PWGPP/CalibMacros/CPass0/mergeByComponent.C" | |
371 | "$ALICE_ROOT/PWGPP/CalibMacros/CPass0/makeOCDB.C" | |
372 | "$ALICE_ROOT/PWGPP/CalibMacros/CPass0/merge.C" | |
373 | "$ALICE_ROOT/PWGPP/CalibMacros/CPass0/mergeMakeOCDB.sh" | |
fc7e1e45 | 374 | ) |
375 | for file in ${filesMergeCPass0[*]}; do | |
376 | [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} . | |
377 | done | |
378 | ||
379 | alirootInfo > ALICE_ROOT_svn.log | |
380 | ||
381 | # | |
382 | ls -lh | |
383 | ||
384 | #merge calibration | |
385 | chmod u+x $mergingScript | |
386 | mkdir -p ./OCDB | |
387 | if [[ -z ${calibrationFilesToMergeExternal} ]]; then | |
388 | echo "find $outputDir -name $calibrationOutputFileName > $calibrationFilesToMerge" | |
389 | find $outputDir -name $calibrationOutputFileName > $calibrationFilesToMerge | |
390 | fi | |
391 | ||
392 | echo "$mergingScript $calibrationFilesToMerge ${runNumber} local://./OCDB $defaultOCDB" | |
393 | if [[ -n $pretend ]]; then | |
394 | touch CalibObjects.root | |
395 | touch ocdb.log | |
396 | touch merge.log | |
397 | mkdir -p ./OCDB/someDetector/ | |
398 | mkdir -p ./OCDB/otherDetector/ | |
399 | touch ./OCDB/someDetector/someCalibObject_0-999999_cpass0.root | |
400 | touch ./OCDB/otherDetector/otherCalibObject_0-999999_cpass0.root | |
401 | else | |
402 | ./$mergingScript $calibrationFilesToMerge ${runNumber} "local://./OCDB" $defaultOCDB | |
403 | fi | |
404 | ||
405 | ### produce the output | |
406 | #tar the produced OCDB for reuse | |
407 | tar czf $commonOutputPath/cpass0.localOCDB.${runNumber}.tgz ./OCDB | |
408 | ||
5ed7c82c | 409 | ls -ltrh |
410 | ||
411 | #copy all to output dir | |
412 | cp --recursive ${runpath}/* $outputDir | |
413 | ||
fc7e1e45 | 414 | #validate merging cpass0 |
5ed7c82c | 415 | cd ${outputDir} |
fc7e1e45 | 416 | calibDoneFile="${commonOutputPath}/merge.cpass0.run${runNumber}.done" |
417 | touch ${calibDoneFile} | |
418 | [[ -f CalibObjects.root ]] && echo "calibfile $outputDir/CalibObjects.root" > ${calibDoneFile} | |
419 | summarizeLogs >> ${calibDoneFile} | |
420 | ||
fc7e1e45 | 421 | rm -rf ${runpath} |
422 | } | |
423 | ||
424 | goMergeCPass1() | |
425 | { | |
426 | umask 0002 | |
427 | # | |
428 | # find the output files and merge them | |
429 | # | |
430 | ||
431 | outputDir=$1 | |
432 | defaultOCDB=$2 | |
433 | configFile=$3 | |
434 | runNumber=$4 | |
435 | calibrationFilesToMergeExternal=$5 | |
436 | qaFilesToMergeExternal=$6 | |
437 | ||
438 | #clean up first: | |
439 | rm -f $outputDir/*.log | |
440 | rm -f $outputDir/*.root | |
441 | rm -f $outputDir/*done | |
442 | ||
443 | source $configFile | |
36b5de1a | 444 | [[ -f ${setupAliROOTenvInCurrentShell} && -z ${alirootEnv} ]] && source $setupAliROOTenvInCurrentShell |
fc7e1e45 | 445 | |
446 | runpath=${PWD}/rundir_cpass1_Merge_${runNumber} | |
447 | [[ -z $commonOutputPath ]] && commonOutputPath=$PWD | |
448 | [[ $reconstructInTemporaryDir -eq 1 && -n $TMPDIR ]] && runpath=$TMPDIR | |
449 | [[ $reconstructInTemporaryDir -eq 1 && -z $TMPDIR ]] && runpath=$(mktemp -d) | |
450 | ||
451 | mkdir -p $runpath | |
452 | [[ ! -d $runpath ]] && echo "not able to make the runpath $runpath" && exit 1 | |
453 | cd $runpath | |
454 | ||
455 | logOutputDir=$runpath | |
456 | [[ -n $logToFinalDestination ]] && logOutputDir=$outputDir | |
457 | [[ -z $dontRedirectStdOutToLog ]] && exec 2>&1 > $logOutputDir/mergeMakeOCDB.log | |
458 | echo "$0 $*" | |
459 | ||
460 | calibrationFilesToMerge=$calibrationFilesToMergeExternal | |
461 | [[ -z $calibrationFilesToMerge ]] && calibrationFilesToMerge="calibrationFilesToMerge.list" | |
462 | calibrationOutputFileName="AliESDfriends_v1.root" | |
463 | mergingScript="mergeMakeOCDB.byComponent.sh" | |
464 | qaFilesToMerge=$qaFilesToMergeExternal | |
465 | [[ -z $qaFilesToMerge ]] && qaFilesToMerge="qaFilesToMerge.list" | |
466 | qaOutputFileName="QAresults*.root" | |
467 | qaMergedOutputFileName="QAresults_merged.root" | |
468 | ||
469 | echo goMergeCPass1 SETUP: | |
470 | echo runNumber=$runNumber | |
471 | echo outputDir=$outputDir | |
472 | echo defaultOCDB=$defaultOCDB | |
473 | echo calibrationFilesToMerge=filesToMerge.list | |
474 | echo calibrationOutputFileName=$calibrationOutputFileName | |
475 | echo mergingScript=$mergingScript | |
476 | ||
477 | # copy files in case they are not already there | |
478 | filesMergeCPass1=( | |
36b5de1a | 479 | "$commonOutputPath/${calibrationFilesToMerge}" |
480 | "$commonOutputPath/${qaFilesToMerge}" | |
481 | "$commonOutputPath/OCDB.root" | |
482 | "$commonOutputPath/localOCDBaccessConfig.C" | |
fc7e1e45 | 483 | "$ALICE_ROOT/PWGPP/CalibMacros/CPass1/mergeMakeOCDB.byComponent.sh" |
484 | "$ALICE_ROOT/PWGPP/CalibMacros/CPass1/mergeByComponent.C" | |
485 | "$ALICE_ROOT/PWGPP/CalibMacros/CPass1/makeOCDB.C" | |
486 | "$ALICE_ROOT/PWGPP/CalibMacros/CPass1/merge.C" | |
487 | "$ALICE_ROOT/PWGPP/CalibMacros/CPass1/mergeMakeOCDB.sh" | |
fc7e1e45 | 488 | ) |
489 | for file in ${filesMergeCPass1[*]}; do | |
490 | [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} . | |
491 | done | |
492 | ||
493 | alirootInfo > ALICE_ROOT_svn.log | |
494 | ||
495 | # | |
496 | ls -lh | |
497 | ||
498 | #merge calibration | |
499 | chmod u+x $mergingScript | |
500 | mkdir -p OCDB | |
501 | if [[ -z ${calibrationFilesToMergeExternal} ]]; then | |
502 | echo "find $outputDir -name $calibrationOutputFileName > $calibrationFilesToMerge" | |
503 | find $outputDir -name $calibrationOutputFileName > $calibrationFilesToMerge | |
504 | fi | |
505 | ||
506 | echo "$mergingScript $calibrationFilesToMerge ${runNumber} local://./OCDB $defaultOCDB" | |
507 | if [[ -n $pretend ]]; then | |
508 | touch CalibObjects.root | |
509 | touch ocdb.log | |
510 | touch merge.log | |
511 | else | |
512 | ./$mergingScript $calibrationFilesToMerge ${runNumber} "local://./OCDB" $defaultOCDB | |
513 | fi | |
514 | ||
515 | tar czf localCPass1_${runNumber}.tgz ./OCDB | |
516 | ||
517 | #merge QA | |
5ed7c82c | 518 | [[ -n ${AliAnalysisTaskFilteredTree_fLowPtTrackDownscaligF} ]] && export AliAnalysisTaskFilteredTree_fLowPtTrackDownscaligF |
519 | [[ -n ${AliAnalysisTaskFilteredTree_fLowPtV0DownscaligF} ]] && export AliAnalysisTaskFilteredTree_fLowPtV0DownscaligF | |
520 | ||
521 | if [[ -z $qaFilesToMergeExternal ]]; then | |
fc7e1e45 | 522 | echo "find $outputDir -name $qaOutputFileName > $qaFilesToMerge" |
523 | find $outputDir -name $qaOutputFileName > $qaFilesToMerge | |
524 | fi | |
525 | ||
526 | echo aliroot -l -b -q "merge.C(\"$qaFilesToMerge\",\"\",kFALSE,\"$qaMergedOutputFileName\")" | |
527 | if [[ -n $pretend ]]; then | |
528 | touch $qaMergedOutputFileName | |
529 | touch merge.log | |
530 | else | |
531 | aliroot -l -b -q "merge.C(\"$qaFilesToMerge\",\"\",kFALSE,\"$qaMergedOutputFileName\")" | |
532 | fi | |
533 | ||
5ed7c82c | 534 | ls -ltrh |
535 | ||
536 | #copy all to output dir | |
537 | cp --recursive ${runpath}/* ${outputDir} | |
538 | ||
fc7e1e45 | 539 | #validate merge cpass1 |
5ed7c82c | 540 | cd ${outputDir} |
fc7e1e45 | 541 | calibDoneFile="${commonOutputPath}/merge.cpass1.run${runNumber}.done" |
542 | touch ${calibDoneFile} | |
543 | [[ -f CalibObjects.root ]] && echo "calibfile $outputDir/CalibObjects.root" > ${calibDoneFile} | |
544 | [[ -f $qaMergedOutputFileName ]] && echo "qafile $outputDir/$qaMergedOutputFileName" >> ${calibDoneFile} | |
dd755372 | 545 | echo "dir ${outputDir}" >> ${calibDoneFile} |
fc7e1e45 | 546 | summarizeLogs >> ${calibDoneFile} |
547 | ||
fc7e1e45 | 548 | rm -rf ${runpath} |
549 | } | |
550 | ||
5dfac970 | 551 | goMakeSummary() |
fc7e1e45 | 552 | { |
5dfac970 | 553 | configFile=$1 |
fc7e1e45 | 554 | source $configFile |
5dfac970 | 555 | |
556 | # log filtering, script needs to take the base dir as argument | |
557 | if [[ -x $logFilteringScript ]]; then | |
558 | commonOutputPath=${baseOutputDirectory}/${productionID} | |
559 | ${logFilteringScript} $commonOutputPath | |
fc7e1e45 | 560 | fi |
561 | ||
5dfac970 | 562 | awk 'BEGIN {nFiles=0;} /^calibfile/ {nFiles++;} END {print "cpass0 produced "nFiles" calib files";}' cpass0.job*done |
563 | awk 'BEGIN {nOK=0; nBAD=0;} /rec.*log OK/ {nOK++;} /rec.*log BAD/ {nBAD++;} END {print "cpass0 reco: OK: "nOK" BAD: "nBAD;}' cpass0.job*done | |
564 | awk 'BEGIN {nOK=0; nBAD=0;} /calib.*log OK/ {nOK++;} /calib.*log BAD/ {nBAD++;} END {print "cpass0 calib: OK: "nOK" BAD: "nBAD;}' cpass0.job*done | |
565 | ||
566 | awk 'BEGIN {nOK=0; nBAD=0;} /merge.*log OK/ {nOK++;} /merge.*log BAD/ {nBAD++;} END {print "cpass0 merge: OK: "nOK" BAD: "nBAD;}' merge.cpass0*done | |
567 | awk 'BEGIN {nOK=0; nBAD=0;} /ocdb.*log OK/ {nOK++;} /ocdb.*log BAD/ {nBAD++;} END {print "cpass0 OCDB: OK: "nOK" BAD: "nBAD;}' merge.cpass0*done | |
568 | ||
569 | awk 'BEGIN {nFiles=0;} /^calibfile/ {nFiles++;} END {print "cpass1 produced "nFiles" calib files";}' cpass1.job*done | |
570 | awk 'BEGIN {nOK=0; nBAD=0;} /rec.*log OK/ {nOK++;} /rec.*log BAD/ {nBAD++;} END {print "cpass1 reco: OK: "nOK" BAD: "nBAD;}' cpass1.job*done | |
571 | awk 'BEGIN {nOK=0; nBAD=0;} /calib.*log OK/ {nOK++;} /calib.*log BAD/ {nBAD++;} END {print "cpass1 calib: OK: "nOK" BAD: "nBAD;}' cpass1.job*done | |
fc7e1e45 | 572 | |
5dfac970 | 573 | awk 'BEGIN {nOK=0; nBAD=0;} /merge.*log OK/ {nOK++;} /merge.*log BAD/ {nBAD++;} END {print "cpass1 merge: OK: "nOK" BAD: "nBAD;}' merge.cpass1*done |
574 | awk 'BEGIN {nOK=0; nBAD=0;} /ocdb.*log OK/ {nOK++;} /ocdb.*log BAD/ {nBAD++;} END {print "cpass1 OCDB: OK: "nOK" BAD: "nBAD;}' merge.cpass1*done | |
575 | ||
576 | #if set email the summary | |
577 | [[ -n $mailSummaryTo ]] && cat $log | mail -s "benchmark $productionID done" $mailSummaryTo | |
fc7e1e45 | 578 | |
5dfac970 | 579 | return 0 |
580 | } | |
fc7e1e45 | 581 | |
5dfac970 | 582 | goMakeflow() |
583 | { | |
584 | #generate the makeflow file and run | |
585 | inputFileList=$1 | |
586 | productionID=$2 | |
587 | configFile=$3 | |
588 | runNumber=$4 | |
fc7e1e45 | 589 | |
5dfac970 | 590 | [[ -z ${configFile} ]] && configFile="benchmark.config" |
591 | [[ ! -f ${configFile} ]] && echo "no config file found (${configFile})" && return 1 | |
592 | source $configFile | |
fc7e1e45 | 593 | |
5dfac970 | 594 | source $configFile |
595 | goGenerateMakeflow "$@" > benchmark.makeflow | |
596 | makeflow ${makeflowOptions} benchmark.makeflow | |
597 | } | |
fc7e1e45 | 598 | |
5dfac970 | 599 | goGenerateMakeflow() |
600 | { | |
601 | #generate the makeflow file | |
602 | inputFileList=$1 | |
603 | productionID=$2 | |
604 | configFile=$3 | |
605 | runNumber=$4 | |
fc7e1e45 | 606 | |
5dfac970 | 607 | [[ -z ${configFile} ]] && configFile="benchmark.config" |
608 | [[ ! -f ${configFile} ]] && echo "no config file found (${configFile})" && return 1 | |
609 | source $configFile | |
fc7e1e45 | 610 | |
5dfac970 | 611 | commonOutputPath=${baseOutputDirectory}/${productionID} |
fc7e1e45 | 612 | |
5dfac970 | 613 | #these files will be made a dependency - will be copied to the working dir of the jobs |
614 | declare -a copyFiles | |
615 | inputFiles=( | |
616 | "OCDB.root" | |
617 | "localOCDBaccessConfig.C" | |
618 | ) | |
619 | for file in ${inputFiles[*]}; do | |
620 | [[ -f ${file} ]] && copyFiles+=("${file}") | |
621 | done | |
622 | ||
623 | #create the makeflow file | |
624 | declare -a arr_cpass1_final | |
625 | declare -a arr_cpass1_QA_final | |
626 | listOfRuns=${runNumber} | |
627 | [[ -z ${runNumber} ]] && listOfRuns=($(while read x; do guessRunNumber $x; done < ${inputFileList} | sort | uniq)) | |
628 | runindex=0 | |
fc7e1e45 | 629 | for runNumber in ${listOfRuns[*]}; do |
630 | [[ -z $runNumber ]] && continue | |
631 | [[ ! ${runNumber} =~ ^[0-9]*[0-9]$ ]] && continue | |
5dfac970 | 632 | jobindex=0 |
fc7e1e45 | 633 | |
5dfac970 | 634 | unset arr_cpass0_outputs |
635 | unset arr_cpass1_outputs | |
636 | declare -a arr_cpass0_outputs | |
637 | declare -a arr_cpass1_outputs | |
638 | unset arr_cpass0_outputs | |
fc7e1e45 | 639 | |
5dfac970 | 640 | while read inputFile; do |
641 | currentDefaultOCDB=${defaultOCDB} | |
642 | [[ ${autoOCDB} -ne 0 ]] && currentDefaultOCDB=$(setYear $inputFile $defaultOCDB) | |
fc7e1e45 | 643 | |
5dfac970 | 644 | #CPass0 |
645 | arr_cpass0_outputs[$jobindex]="cpass0.job${jobindex}.run${runNumber}.done" | |
646 | echo "${arr_cpass0_outputs[$jobindex]}: benchmark.sh ${configFile} ${copyFiles[@]}" | |
647 | echo " ${alirootEnv} ./benchmark.sh CPass0 ${commonOutputPath}/cpass0/000${runNumber} $inputFile $nEvents $currentDefaultOCDB $configFile $runNumber $jobindex" | |
fc7e1e45 | 648 | echo |
5dfac970 | 649 | |
650 | #CPass1 | |
651 | arr_cpass1_outputs[$jobindex]="cpass1.job${jobindex}.run${runNumber}.done" | |
652 | echo "${arr_cpass1_outputs[$jobindex]} : benchmark.sh ${configFile} cpass0.localOCDB.${runNumber}.tgz ${copyFiles[@]}" | |
653 | echo " ${alirootEnv} ./benchmark.sh CPass1 ${commonOutputPath}/cpass1/000${runNumber} $inputFile $nEvents $currentDefaultOCDB $configFile $runNumber $jobindex" | |
fc7e1e45 | 654 | echo |
5dfac970 | 655 | ((jobindex++)) |
fc7e1e45 | 656 | |
5dfac970 | 657 | done< <(grep "/000$runNumber/" $inputFileList) |
fc7e1e45 | 658 | |
5dfac970 | 659 | #CPass0 list of Calib files to merge |
660 | echo "cpass0.calib.run${runNumber}.list: ${arr_cpass0_outputs[*]}" | |
661 | echo " awk '/^calibfile / {print "'\$2'"}' ${arr_cpass0_outputs[*]} > cpass0.calib.run${runNumber}.list" | |
662 | echo | |
fc7e1e45 | 663 | |
5dfac970 | 664 | #CPass1 list of Calib/QA files to merge |
665 | echo "cpass1.calib.run${runNumber}.list cpass1.QA.run${runNumber}.list: ${arr_cpass1_outputs[*]}" | |
666 | echo " awk '/^calibfile / {print "'\$'"2}' ${arr_cpass1_outputs[*]} > cpass1.calib.run${runNumber}.list; awk '/^qafile / {print "'\$'"2}' ${arr_cpass1_outputs[*]} > cpass1.QA.run${runNumber}.list" | |
667 | echo | |
fc7e1e45 | 668 | |
5dfac970 | 669 | #CPass0 merging |
670 | arr_cpass0_final[$runindex]="merge.cpass0.run${runNumber}.done" | |
671 | echo "cpass0.localOCDB.${runNumber}.tgz ${arr_cpass0_final[$runindex]}: cpass0.calib.run${runNumber}.list benchmark.sh ${configFile} ${copyFiles[@]}" | |
672 | echo " ${alirootEnv} ./benchmark.sh MergeCPass0 ${commonOutputPath}/cpass0/000${runNumber} $currentDefaultOCDB ${configFile} $runNumber cpass0.run${runNumber}.list" | |
673 | echo | |
fc7e1e45 | 674 | |
5dfac970 | 675 | #CPass1 Calib/QA merging |
676 | arr_cpass1_final[$runindex]="merge.cpass1.run${runNumber}.done" | |
677 | echo "${arr_cpass1_final[$runindex]}: cpass1.calib.run${runNumber}.list cpass1.QA.run${runNumber}.list benchmark.sh ${configFile} ${copyFiles[@]}" | |
678 | echo " ${alirootEnv} ./benchmark.sh MergeCPass1 ${commonOutputPath}/cpass1/000${runNumber} $currentDefaultOCDB ${configFile} $runNumber cpass1.calib.run${runNumber}.list cpass1.QA.run${runNumber}.list" | |
679 | echo | |
680 | ((runindex++)) | |
681 | done | |
fc7e1e45 | 682 | |
5dfac970 | 683 | #CPass1 list of final Calib/QA files |
684 | echo "cpass1.QA.list cpass1.calib.list: ${arr_cpass1_final[*]}" | |
685 | echo " awk '/^calibfile / {print "'\$'"2}' ${arr_cpass1_final[*]} > cpass1.calib.list; awk '/^qafile / {print "'\$'"2}' ${arr_cpass1_final[*]} > cpass1.QA.list" | |
686 | echo | |
fc7e1e45 | 687 | |
5dfac970 | 688 | #Summary |
689 | echo "summary.log : ${arr_cpass0_outputs[*]} ${arr_cpass1_outputs[*]} ${arr_cpass1_final[*]} ${arr_cpass0_final[*]} benchmark.sh ${configFile}" | |
690 | echo " LOCAL ./benchmark.sh makeSummary ${configFile} |tee summary.log" | |
691 | } | |
fc7e1e45 | 692 | |
5dfac970 | 693 | goCreateQAplots() |
694 | { | |
695 | umask 0002 | |
696 | [[ $# -lt 5 ]] && echo "goCreateQAplots productionID pass outputDir qaFilesDirectory qaPlotScript" && exit 1 | |
697 | productionID=$1 | |
698 | pass=$2 | |
699 | outputDir=$3 | |
700 | qaFilesDirectory=$4 | |
701 | qaPlotsScript=$5 | |
702 | configFile=$6 | |
fc7e1e45 | 703 | |
5dfac970 | 704 | source $configFile |
705 | [[ -f ${setupAliROOTenvInCurrentShell} ]] && source $setupAliROOTenvInCurrentShell | |
fc7e1e45 | 706 | |
5dfac970 | 707 | runpath=${PWD}/rundir_cpass0_Merge_${runNumber} |
708 | [[ -z $commonOutputPath ]] && commonOutputPath=$PWD | |
709 | [[ $reconstructInTemporaryDir -eq 1 && -n $TMPDIR ]] && runpath=$TMPDIR | |
710 | [[ $reconstructInTemporaryDir -eq 1 && -z $TMPDIR ]] && runpath=$(mktemp -d) | |
fc7e1e45 | 711 | |
5dfac970 | 712 | mkdir -p $runpath |
713 | [[ ! -d $runpath ]] && echo "not able to make the runpath $runpath" && exit 1 | |
714 | cd $runpath | |
fc7e1e45 | 715 | |
5dfac970 | 716 | [[ -z $logOutputDir ]] && logOutputDir=$runpath |
717 | [[ -z $dontRedirectStdOutToLog ]] && exec 2>&1 > $logOutputDir/makeQAplots.log | |
718 | echo "$0 $*" | |
fc7e1e45 | 719 | |
5dfac970 | 720 | [[ -z "$qaPlotsScript" ]] && echo "qaPlotsScript not defined"&&exit 1 |
721 | ||
722 | mergedQAfileList=$outputDir/mergedQAfiles.list | |
723 | echo "MakeListOfQAresults $qaFilesDirectory QAresults.root | grep CPass1 > $mergedQAfileList" | |
724 | MakeListOfQAresults $qaFilesDirectory QAresults_merged.root | grep CPass1 |tee $mergedQAfileList | |
725 | echo $qaPlotsScript "$productionID" "cpass1" $mergedQAfileList $outputDir | |
726 | $qaPlotsScript "$productionID" "cpass1" $mergedQAfileList $outputDir | |
fc7e1e45 | 727 | |
5dfac970 | 728 | mkdir -p $outputDir |
729 | [[ ! -d $outputDir ]] && echo "cannot make the output dir $outputDir" && exit 1 | |
730 | mv -f $runpath/* $outputDir | |
731 | rm -rf $runpath | |
732 | } | |
fc7e1e45 | 733 | |
5dfac970 | 734 | goWaitForOutput() |
735 | { | |
736 | umask 0002 | |
737 | [[ $# -lt 3 ]] && echo "goWaitForOutput() wrong number of arguments, exiting.." && exit 1 | |
738 | echo searchPath=$1 | |
739 | echo fileName=$2 | |
740 | echo numberOfFiles=$3 | |
741 | echo maxSecondsToWait=$4 | |
742 | searchPath=$1 | |
743 | fileName=$2 | |
744 | numberOfFiles=$3 | |
745 | maxSecondsToWait=$4 | |
746 | extraFindOptions=$5 | |
747 | echo "command to be executed: find $searchPath -name "$fileName" ${extraFindOptions}" | |
748 | [[ -z "$maxSecondsToWait" ]] && maxSecondsToWait=$(( 3600*12 )) | |
749 | while sleep 60; do | |
750 | n=$(find $searchPath -name "$fileName" ${extraFindOptions}| wc -l) | |
751 | [[ $n -gt 0 ]] && echo "found $n X $fileName" | |
752 | [[ $n -ge $numberOfFiles ]] && break | |
753 | [[ $SECONDS -gt $maxSecondsToWait ]] && break | |
754 | done | |
755 | echo "DONE! exiting..." | |
756 | } | |
fc7e1e45 | 757 | |
5dfac970 | 758 | submit() |
759 | { | |
760 | umask 0002 | |
761 | [[ $# -ne 6 ]] && echo "6 args needed, you supplied $#" && exit 1 | |
762 | JobID=$1 | |
763 | startID=$2 | |
764 | endID=$3 | |
765 | waitForJOBID=$4 | |
766 | command=$5 | |
767 | commandArgs=$6 | |
fc7e1e45 | 768 | |
5dfac970 | 769 | newFarm=$(which qsub|grep "^/usr/bin/qsub") |
770 | ||
771 | batchSystem="SGE" | |
fc7e1e45 | 772 | |
5dfac970 | 773 | if [[ -z "$newFarm" ]] |
774 | then | |
775 | #old LSF | |
776 | # submit it (as job array) | |
777 | nFiles=$(( $endID-$startID+1 )) | |
778 | while [ $startID -le $nFiles ] ; do | |
779 | if [ `expr $nFiles - $startID` -gt 999 ] ; then | |
780 | endID=`expr $startID + 999` | |
781 | else | |
782 | endID=$nFiles | |
783 | fi | |
784 | if [[ -z "$waitForJOBID" ]]; then | |
785 | echo $batchCommand -J "$JobID[$startID-$endID]" -e "$targetDirectory/logs/job_%I.err" -o "$targetDirectory/logs/job_%I.out" "$command" | |
786 | $batchCommand -J "$JobID[$startID-$endID]" -e "$targetDirectory/logs/job_%I.err" -o "$targetDirectory/logs/job_%I.out" "$command" | |
787 | else | |
788 | echo $batchCommand -J "$JobID[$startID-$endID]" -w "ended($waitForJOBID)" -e "$targetDirectory/logs/job_%I.err" -o "$targetDirectory/logs/job_%I.out" "$command" | |
789 | $batchCommand -J "$JobID[$startID-$endID]" -w "ended($waitForJOBID)" -e "$targetDirectory/logs/job_%I.err" -o "$targetDirectory/logs/job_%I.out" "$command" | |
790 | fi | |
791 | startID=`expr $endID + 1` | |
792 | done | |
793 | else | |
794 | #new SGE farm | |
795 | if [[ -z "$waitForJOBID" ]]; then | |
796 | echo $batchCommand -wd ${targetDirectory} -b y -V -N "$JobID" -t "$startID-$endID" -e "$targetDirectory/logs/" -o "$targetDirectory/logs/" "$command" $commandArgs | |
797 | $batchCommand -wd ${targetDirectory} -b y -V -N "$JobID" -t "$startID-$endID" -e "$targetDirectory/logs/" -o "$targetDirectory/logs/" "$command" $commandArgs | |
798 | else | |
799 | echo $batchCommand -wd ${targetDirectory} -b y -V -N "$JobID" -t "$startID-$endID" -hold_jid "$waitForJOBID" -e "$targetDirectory/logs/" -o "$targetDirectory/logs/" "$command" $commandArgs | |
800 | $batchCommand -wd ${targetDirectory} -b y -V -N "$JobID" -t "$startID-$endID" -hold_jid "$waitForJOBID" -e "$targetDirectory/logs/" -o "$targetDirectory/logs/" "$command" $commandArgs | |
801 | fi | |
802 | fi | |
803 | } | |
fc7e1e45 | 804 | |
5dfac970 | 805 | goTest() |
806 | { | |
807 | umask 0002 | |
808 | exec 2>&1 | |
809 | exec > >(tee test.log) | |
810 | echo "$@" | |
811 | echo something | |
812 | } | |
fc7e1e45 | 813 | |
5dfac970 | 814 | alirootInfo() |
815 | { | |
816 | ( | |
817 | umask 0002 | |
818 | # save aliroot svn info | |
819 | [[ -z "$ALICE_ROOT" ]] && exit 1 | |
820 | prevdir=$PWD | |
821 | cd $ALICE_ROOT | |
822 | echo "\$ALICE_ROOT=$ALICE_ROOT" | |
823 | echo | |
824 | svn info 2>/dev/null | |
825 | echo "" | |
826 | echo "" | |
827 | svn diff 2>/dev/null | |
828 | cd $prevdir | |
829 | ) | |
830 | } | |
fc7e1e45 | 831 | |
5dfac970 | 832 | MakeListOfQAresults() |
833 | { | |
834 | ( | |
835 | umask 0002 | |
836 | if [[ $# -eq 0 ]]; then | |
837 | echo "make an input file list for the qa script" | |
838 | echo "Usage: $0 path filename" | |
839 | return 0 | |
840 | fi | |
fc7e1e45 | 841 | |
5dfac970 | 842 | path=$1 |
843 | fileName=$2 | |
844 | ||
845 | while read entry; do | |
846 | runNumber=$(guessRunNumber $entry) | |
847 | echo "$runNumber 0 1 10 $entry" | |
848 | done < <(find $path -name $fileName) | |
849 | ) | |
850 | } | |
fc7e1e45 | 851 | |
5dfac970 | 852 | setYear() |
853 | { | |
854 | #set the year | |
855 | # $1 - year to be set | |
856 | # $2 - where to set the year | |
857 | year1=$(guessYear $1) | |
858 | year2=$(guessYear $2) | |
859 | local path=$2 | |
860 | [[ $year1 -ne $year2 && -n $year2 ]] && path=${2/\/$year2\//\/$year1\/} | |
861 | echo $path | |
862 | } | |
fc7e1e45 | 863 | |
5dfac970 | 864 | guessPeriod() |
865 | { | |
866 | #guess the period from the path, pick the rightmost one | |
867 | local IFS="/" | |
868 | declare -a path=( $1 ) | |
869 | local dirDepth=${#path[*]} | |
870 | for ((x=${dirDepth}-1;x>=0;x--)); do | |
871 | local field=${path[${x}]} | |
872 | [[ ${field} =~ ^LHC[0-9][0-9][a-z]$ ]] && period=${field#000} && break | |
873 | done | |
874 | echo $period | |
875 | } | |
fc7e1e45 | 876 | |
5dfac970 | 877 | guessYear() |
878 | { | |
879 | #guess the year from the path, pick the rightmost one | |
880 | local IFS="/" | |
881 | declare -a path=( $1 ) | |
882 | local dirDepth=${#path[*]} | |
883 | for ((x=${dirDepth}-1;x>=0;x--)); do | |
884 | local field=${path[${x}]} | |
885 | [[ ${field} =~ ^20[0-9][0-9]$ ]] && year=${field#000} && break | |
886 | done | |
887 | echo $year | |
888 | } | |
fc7e1e45 | 889 | |
5dfac970 | 890 | guessRunNumber() |
891 | { | |
892 | #guess the run number from the path, pick the rightmost one | |
893 | local IFS="/" | |
894 | declare -a path=( $1 ) | |
895 | local dirDepth=${#path[*]} | |
896 | for ((x=${dirDepth}-1;x>=0;x--)); do | |
897 | local field=${path[${x}]} | |
898 | [[ ${field} =~ ^000[0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && runNumber=${field#000} && break | |
899 | done | |
900 | echo $runNumber | |
901 | } | |
fc7e1e45 | 902 | |
5dfac970 | 903 | summarizeLogs() |
904 | { | |
905 | #print a summary of logs | |
906 | logFiles=( | |
907 | "*.log" | |
908 | ) | |
fc7e1e45 | 909 | |
5dfac970 | 910 | errorConditions=( |
911 | "There was a crash" | |
912 | "floating" | |
913 | "error while loading shared libraries" | |
914 | "std::bad_alloc" | |
915 | "s_err_syswatch_" | |
916 | "Thread [0-9]* (Thread" | |
917 | ) | |
918 | logstatus=0 | |
919 | for log in ${logFiles[*]}; do | |
920 | finallog=${outputDir%/}/${log} | |
921 | [[ ! -f $log ]] && continue | |
922 | errorSummary="" | |
923 | for ((i=0; i<${#errorConditions[@]};i++)); do | |
924 | local tmp=$(grep -m1 -e "${errorConditions[$i]}" $log) | |
925 | [[ -n $tmp ]] && tmp+=" : " | |
926 | errorSummary+=$tmp | |
927 | done | |
928 | if [[ -z $errorSummary ]]; then | |
929 | #in pretend mode randomly report an error in rec.log some cases | |
930 | if [[ -n $pretend && "$log" == "rec.log" ]]; then | |
931 | [[ $(( $RANDOM%2 )) -ge 1 ]] && echo "$finallog BAD random error" || echo "$finallog OK" | |
fc7e1e45 | 932 | else |
5dfac970 | 933 | echo "$finallog OK" |
fc7e1e45 | 934 | fi |
5dfac970 | 935 | else |
936 | local logstatus=1 | |
937 | echo "$finallog BAD $errorSummary" | |
938 | fi | |
939 | done | |
940 | return $logstatus | |
941 | } | |
fc7e1e45 | 942 | |
5dfac970 | 943 | spitOutLocalOCDBaccessConfig() |
944 | { | |
945 | umask 0002 | |
946 | find $1 -name "*root" | \ | |
947 | while read line | |
948 | do | |
949 | local tmp=${line#$1} | |
950 | echo ${tmp%/*} | \ | |
951 | awk -v ocdb=$1 '{print " man->SetSpecificStorage(\""$1"\",\"local://"ocdb"\");"}' | |
952 | done | |
953 | } | |
fc7e1e45 | 954 | |
5dfac970 | 955 | goConfigureOCDBforCPass1() |
956 | { | |
957 | umask 0002 | |
958 | # make a script that sets the specific storages form all the root files produced by CPass0 | |
959 | # in this case the second argument is taken to be the path to the produced OCDB for specific storage | |
960 | local localOCDBpathCPass0=$1 | |
961 | local outputDir=$2 | |
962 | [[ -d $outputDir ]] && cd $outputDir | |
fc7e1e45 | 963 | |
5dfac970 | 964 | if [[ -f $localOCDBpathCPass0 && $localOCDBpathCPass0 =~ \.tgz$ ]]; then |
965 | tar xzf $localOCDBpathCPass0 | |
966 | local localOCDBpathCPass0="./OCDB" | |
967 | fi | |
fc7e1e45 | 968 | |
5dfac970 | 969 | echo |
970 | echo creating the specific storage script | |
971 | echo localOCDBaccessConfig.C | |
972 | echo based on OCDB: $fileName | |
973 | echo | |
fc7e1e45 | 974 | |
5dfac970 | 975 | local tempLocalOCDB="" |
976 | if [[ -f localOCDBaccessConfig.C ]]; then | |
977 | tempLocalOCDB=$(mktemp) | |
978 | echo "egrep "SetSpecificStorage" localOCDBaccessConfig.C > $tempLocalOCDB" | |
979 | egrep "SetSpecificStorage" localOCDBaccessConfig.C > $tempLocalOCDB | |
980 | fi | |
fc7e1e45 | 981 | |
5dfac970 | 982 | echo "localOCDBaccessConfig()" > localOCDBaccessConfig.C |
983 | echo "{" >> localOCDBaccessConfig.C | |
984 | echo " AliCDBManager* man = AliCDBManager::Instance();" >> localOCDBaccessConfig.C | |
985 | spitOutLocalOCDBaccessConfig $localOCDBpathCPass0|sort|uniq >> localOCDBaccessConfig.C | |
986 | [[ -f "$tempLocalOCDB" ]] && cat $tempLocalOCDB >> localOCDBaccessConfig.C | |
987 | echo "}" >> localOCDBaccessConfig.C | |
fc7e1e45 | 988 | |
5dfac970 | 989 | [[ -f "$tempLocalOCDB" ]] && rm -f $tempLocalOCDB |
fc7e1e45 | 990 | |
5dfac970 | 991 | if ! grep SetSpecificStorage localOCDBaccessConfig.C; then |
992 | echo | |
993 | echo "!!!!!!! CPass0 produced no OCDB entries" | |
994 | return 1 | |
995 | fi | |
996 | } | |
fc7e1e45 | 997 | |
5dfac970 | 998 | goSubmit() |
999 | { | |
1000 | inputList=$1 | |
1001 | productionID=$2 | |
1002 | configFile="benchmark.config" | |
1003 | [[ -n "$3" ]] && configFile=$3 | |
1004 | configFile=$(readlink -f $configFile) | |
1005 | [[ -n $4 ]] && runNumber=$4 | |
fc7e1e45 | 1006 | |
5dfac970 | 1007 | #redirect all output to submit.log |
1008 | echo "redirecting all output to ${PWD}/submit_${productionID//"/"/_}.log" | |
1009 | exec 7>&1 | |
1010 | exec 1>submit_${productionID//"/"/_}.log 2>&1 | |
fc7e1e45 | 1011 | |
5dfac970 | 1012 | umask 0002 |
1013 | echo $0" submit $*" | |
1014 | if [[ -z "$inputList" || -z "$productionID" ]] | |
1015 | then | |
fc7e1e45 | 1016 | echo |
5dfac970 | 1017 | echo " Usage: $0 submit inputList productionID [configFile=benchmark.config]" |
fc7e1e45 | 1018 | echo |
5dfac970 | 1019 | exit |
fc7e1e45 | 1020 | fi |
fc7e1e45 | 1021 | |
5dfac970 | 1022 | date=`date +%Y-%m-%d_%H%M%S` |
fc7e1e45 | 1023 | |
5dfac970 | 1024 | # check if config file is there |
1025 | if [ ! -f $configFile ]; then | |
1026 | echo "ERROR! Config File '$configFile' not found" >&2 | |
1027 | exit | |
1028 | else | |
1029 | echo "Using Config File: '$configFile'" | |
fc7e1e45 | 1030 | fi |
1031 | ||
5dfac970 | 1032 | # source the config file |
1033 | # and print the configuration | |
fc7e1e45 | 1034 | source $configFile |
5dfac970 | 1035 | [[ -f ${setupAliROOTenvInCurrentShell} && -z ${alirootEnv} ]] && source $setupAliROOTenvInCurrentShell |
fc7e1e45 | 1036 | |
5dfac970 | 1037 | self=$(readlink -f "$0") |
1038 | configPath=`dirname $self` | |
1039 | #exporting makes it available in the environment on the nodes - makes the summary output go there | |
1040 | export commonOutputPath=${baseOutputDirectory}/${productionID} | |
fc7e1e45 | 1041 | |
5dfac970 | 1042 | #[[ -z ${ALIROOT_RELEASE} ]] && ALIROOT_RELEASE=${ALICE_ROOT//"/"/"_"} |
1043 | #ALIROOT_RELEASE_mod=${ALIROOT_RELEASE//"/"/"_"} | |
1044 | #ALIROOT_BASEDIR_mod=${ALIROOT_BASEDIR//"/"/"_"} | |
1045 | #productionID=${ALIROOT_BASEDIR_mod}_${ALIROOT_RELEASE_mod}/${productionID} | |
fc7e1e45 | 1046 | |
5dfac970 | 1047 | #convert to absolut pathnames |
1048 | inputList=$(readlink -f "$inputList") | |
1049 | #make list of runs | |
1050 | if [[ -z $runNumber ]]; then | |
1051 | listOfRuns=($(while read x; do guessRunNumber $x; done < ${inputList} | sort | uniq)) | |
1052 | else | |
1053 | listOfRuns=$runNumber | |
1054 | fi | |
fc7e1e45 | 1055 | |
5dfac970 | 1056 | setupAliROOTenvInCurrentShell=$(readlink -f "$setupAliROOTenvInCurrentShell") |
fc7e1e45 | 1057 | |
5dfac970 | 1058 | echo "" |
1059 | echo "### BEGIN CONFIGURATION ###" | |
1060 | echo "" | |
1061 | echo "GENERAL:" | |
1062 | echo "" | |
1063 | echo " productionID: $productionID" | |
1064 | echo " batchCommand: $batchCommand" | |
1065 | echo " setupAliROOTenvInCurrentShell: $setupAliROOTenvInCurrentShell" | |
1066 | echo " ALICE_ROOT: $ALICE_ROOT" | |
1067 | echo " ALIROOT_RELEASE: $ALICE_RELEASE" | |
1068 | echo " inputList: $inputList" | |
1069 | echo " configPath: $configPath" | |
1070 | echo " commonOutputPath: $commonOutputPath" | |
1071 | echo " defaultOCDB: $defaultOCDB" | |
1072 | echo " autoOCDB: $autoOCDB" | |
1073 | echo " recoTriggerOptions: $recoTriggerOptions" | |
1074 | echo " runs:" | |
1075 | echo " ${listOfRuns[*]}" | |
1076 | echo "" | |
1077 | echo "THE TRAIN WILL RUN:" | |
fc7e1e45 | 1078 | |
5dfac970 | 1079 | if [ $runCPass0reco -eq 1 ]; then |
1080 | echo " Pass0 - Recontruction" | |
1081 | fi | |
fc7e1e45 | 1082 | |
5dfac970 | 1083 | if [ $runCPass0MergeMakeOCDB -eq 1 ]; then |
1084 | echo " Pass0 - merging and OCDB export" | |
1085 | fi | |
fc7e1e45 | 1086 | |
5dfac970 | 1087 | if [ $runCPass1reco -eq 1 ]; then |
1088 | echo " Pass1 - Recontruction" | |
1089 | fi | |
1090 | if [ $runCPass1MergeMakeOCDB -eq 1 ]; then | |
1091 | echo " Pass1 - merging and OCDB export" | |
1092 | fi | |
fc7e1e45 | 1093 | |
5dfac970 | 1094 | echo "" |
1095 | echo "LIMITS:" | |
1096 | echo " max. Events/Chunk: $nEvents" | |
1097 | echo " max. Number of Chunks per Run: $nMaxChunks" | |
1098 | echo "" | |
1099 | echo "### END CONFIGURATION ###" | |
1100 | echo "" | |
fc7e1e45 | 1101 | |
36b5de1a | 1102 | |
5dfac970 | 1103 | # check if input file is there |
1104 | if [ ! -f $inputList ]; then | |
1105 | echo "ERROR! Input List '$inputList' not found" >&2 | |
1106 | exit | |
1107 | fi | |
1108 | ||
1109 | # define jobid (for dependent jobs) | |
1110 | JOBID1=p0_${productionID//"/"/_}_${date} | |
1111 | JOBID1wait=w0_${productionID//"/"/_}_${date} | |
1112 | JOBID2=m0_${productionID//"/"/_}_${date} | |
1113 | JOBID2wait=wm0_${productionID//"/"/_}_${date} | |
1114 | JOBID3=op0_${productionID//"/"/_}_${date} | |
1115 | JOBID3wait=wop0_${productionID//"/"/_}_${date} | |
1116 | JOBID4=p1_${productionID//"/"/_}_${date} | |
1117 | JOBID4wait=w1_${productionID//"/"/_}_${date} | |
1118 | JOBID5=m1_${productionID//"/"/_}_${date} | |
1119 | JOBID5wait=wm1_${productionID//"/"/_}_${date} | |
1120 | JOBID6=s1_${productionID//"/"/_}_${date} | |
1121 | JOBID6wait=ws1_${productionID//"/"/_}_${date} | |
1122 | JOBID7=QA_${productionID//"/"/_}_${date} | |
1123 | LASTJOB="" | |
1124 | ||
1125 | #for each run we submit one jobarray: | |
1126 | numberOfSubmittedCPass1MergingJobs=0 | |
fc7e1e45 | 1127 | for runNumber in ${listOfRuns[*]}; do |
1128 | [[ -z $runNumber ]] && continue | |
1129 | [[ ! ${runNumber} =~ ^[0-9]*[0-9]$ ]] && continue | |
5dfac970 | 1130 | oneInputFile=$(egrep -m1 "$runNumber\/" ${inputList}) |
1131 | currentDefaultOCDB=$defaultOCDB | |
1132 | [[ $autoOCDB -ne 0 ]] && currentDefaultOCDB=$(setYear $oneInputFile $defaultOCDB) | |
1133 | echo "submitting run $runNumber with OCDB $currentDefaultOCDB" | |
fc7e1e45 | 1134 | |
5dfac970 | 1135 | ################################################################################ |
1136 | ################################################################################ | |
1137 | # run the CPass0 if requested | |
fc7e1e45 | 1138 | |
5dfac970 | 1139 | if [ $runCPass0reco -eq 1 ]; then |
fc7e1e45 | 1140 | |
fc7e1e45 | 1141 | echo |
5dfac970 | 1142 | echo "starting CPass0... for run $runNumber" |
fc7e1e45 | 1143 | echo |
fc7e1e45 | 1144 | |
5dfac970 | 1145 | # create directory and copy all files that are needed |
1146 | targetDirectory="${commonOutputPath}/000${runNumber}/CPass0" | |
1147 | mkdir -p $targetDirectory | |
1148 | mkdir -p $targetDirectory/logs | |
1149 | ||
1150 | localInputList=$targetDirectory/${inputList##*/} | |
1151 | theScript="$targetDirectory/${self##*/}" | |
1152 | cp -f $self $theScript | |
1153 | chmod u+x $theScript | |
fc7e1e45 | 1154 | |
5dfac970 | 1155 | rm -f $localInputList |
1156 | egrep "\/000$runNumber\/" $inputList >> $localInputList | |
1157 | cp -f $configFile $targetDirectory | |
1158 | [[ -f $configPath/runCPass0.sh ]] && cp -f $configPath/runCPass0.sh $targetDirectory && echo "## using local runCPass0.sh" | |
1159 | [[ -f $configPath/recCPass0.C ]] && cp -f $configPath/recCPass0.C $targetDirectory && echo "## using local recCPass0.C" | |
1160 | [[ -f $configPath/runCalibTrain.C ]] && cp -f $configPath/runCalibTrain.C $targetDirectory && echo "## using local runCalibTrain.C" | |
1161 | [[ -f $configPath/localOCDBaccessConfig.C ]] && cp -f $configPath/localOCDBaccessConfig.C $targetDirectory && echo "## using local localOCDBaccessConfig.C" | |
fc7e1e45 | 1162 | |
5dfac970 | 1163 | [[ -f $configPath/CPass0/runCPass0.sh ]] && cp -f $configPath/CPass0/runCPass0.sh $targetDirectory && echo "## using local runCPass0.sh" |
1164 | [[ -f $configPath/CPass0/recCPass0.C ]] && cp -f $configPath/CPass0/recCPass0.C $targetDirectory && echo "## using local recCPass0.C" | |
1165 | [[ -f $configPath/CPass0/runCalibTrain.C ]] && cp -f $configPath/CPass0/runCalibTrain.C $targetDirectory && echo "## using local runCalibTrain.C" | |
1166 | [[ -f $configPath/CPass0/localOCDBaccessConfig.C ]] && cp -f $configPath/CPass0/localOCDBaccessConfig.C $targetDirectory && echo "## using local localOCDBaccessConfig.C" | |
fc7e1e45 | 1167 | |
5dfac970 | 1168 | echo "... files copied." |
1169 | echo | |
fc7e1e45 | 1170 | |
5dfac970 | 1171 | # limit nFiles to nMaxChunks |
1172 | nFiles=`wc -l < $localInputList` | |
1173 | [[ $nFiles -eq 0 ]] && echo "list contains ZERO files! exiting..." && exit 1 | |
1174 | echo "raw files in list: $nFiles" | |
1175 | if [[ $nMaxChunks -gt 0 && $nMaxChunks -le $nFiles ]]; then | |
1176 | nFiles=$nMaxChunks | |
1177 | fi | |
1178 | echo "raw files to process: $nFiles" | |
1179 | [[ -z "$percentProcessedFilesToContinue" ]] && percentProcessedFilesToContinue=100 | |
1180 | if [[ $percentProcessedFilesToContinue -eq 100 ]]; then | |
1181 | nFilesToWaitFor=$nFiles | |
1182 | else | |
1183 | nFilesToWaitFor=$(( $nFiles-$nFiles/(100/(100-$percentProcessedFilesToContinue)) )) | |
1184 | fi | |
1185 | echo "requested success rate is $percentProcessedFilesToContinue%" | |
1186 | echo "merging will start after $nFilesToWaitFor jobs are done" | |
fc7e1e45 | 1187 | |
5dfac970 | 1188 | submit $JOBID1 1 $nFiles "" "$theScript" "CPass0 $targetDirectory $localInputList $nEvents $currentDefaultOCDB $configFile $runNumber" |
fc7e1e45 | 1189 | |
5dfac970 | 1190 | ## submit a monitoring job that will run until a certain number of jobs are done with reconstruction |
1191 | submit "$JOBID1wait" 1 1 "" "$theScript" "WaitForOutput ${commonOutputPath} 'cpass0.job*.run$runNumber.done' $nFilesToWaitFor $maxSecondsToWait '-maxdepth 1'" | |
1192 | LASTJOB=$JOBID1wait | |
fc7e1e45 | 1193 | |
5dfac970 | 1194 | fi #end running CPass0 |
1195 | ################################################################################ | |
fc7e1e45 | 1196 | |
fc7e1e45 | 1197 | |
5dfac970 | 1198 | ################################################################################ |
1199 | # submit merging of CPass0, depends on the reconstruction | |
fc7e1e45 | 1200 | |
5dfac970 | 1201 | if [ $runCPass0MergeMakeOCDB -eq 1 ]; then |
fc7e1e45 | 1202 | |
5dfac970 | 1203 | echo |
1204 | echo "submit CPass0 merging for run $runNumber" | |
1205 | echo | |
fc7e1e45 | 1206 | |
5dfac970 | 1207 | targetDirectory="${commonOutputPath}/000${runNumber}/CPass0" |
1208 | mkdir -p $targetDirectory | |
fc7e1e45 | 1209 | |
5dfac970 | 1210 | # copy the scripts |
1211 | cp -f $self $targetDirectory | |
1212 | [[ -f $configPath/mergeMakeOCDB.sh ]] && cp -f $configPath/mergeMakeOCDB.sh $targetDirectory && echo "## using local mergeMakeOCDB.sh" | |
1213 | [[ -f $configPath/merge.C ]] && cp -f $configPath/merge.C $targetDirectory && echo "## using local merge.C" | |
1214 | [[ -f $configPath/mergeMakeOCDB.byComponent.sh ]] && cp -f $configPath/mergeMakeOCDB.byComponent.sh $targetDirectory && echo "## using local mergeMakeOCDB.byComponent.sh" | |
1215 | [[ -f $configPath/mergeByComponent.C ]] && cp -f $configPath/mergeByComponent.C $targetDirectory && echo "## using local mergeByComponent.C" | |
1216 | [[ -f $configPath/makeOCDB.C ]] && cp -f $configPath/makeOCDB.C $targetDirectory && echo "## using local makeOCDB.C" | |
fc7e1e45 | 1217 | |
5dfac970 | 1218 | theScript="$targetDirectory/${self##*/}" |
1219 | chmod u+x $theScript | |
fc7e1e45 | 1220 | |
5dfac970 | 1221 | echo submit $JOBID2 1 1 "$LASTJOB" "$theScript" "MergeCPass0 $targetDirectory $currentDefaultOCDB $configFile $runNumber" |
1222 | submit $JOBID2 1 1 "$LASTJOB" "$theScript" "MergeCPass0 $targetDirectory $currentDefaultOCDB $configFile $runNumber" | |
1223 | LASTJOB=$JOBID2 | |
fc7e1e45 | 1224 | |
5dfac970 | 1225 | cd $configPath |
1226 | echo | |
fc7e1e45 | 1227 | fi |
5dfac970 | 1228 | # end of merging CPass0 |
1229 | ################################################################################ | |
fc7e1e45 | 1230 | |
5dfac970 | 1231 | ################################################################################ |
1232 | ################################################################################ | |
1233 | # run the CPass1 if requested | |
fc7e1e45 | 1234 | |
5dfac970 | 1235 | if [ $runCPass1reco -eq 1 ]; then |
fc7e1e45 | 1236 | |
5dfac970 | 1237 | targetDirectory="${commonOutputPath}/000${runNumber}/CPass1" |
fc7e1e45 | 1238 | |
5dfac970 | 1239 | # safety feature: if we are re-running for any reason we want to delete the previous output first. |
1240 | [[ -d $targetDirectory ]] && rm -rf $targetDirectory/* && echo "removed old output at $targetDirectory/*" | |
fc7e1e45 | 1241 | |
5dfac970 | 1242 | ################################################################################ |
1243 | # for the CPass1, based on the OCDB entries produced, we need to create the script | |
1244 | # to set the specific storages to the output of CPass0 | |
1245 | # submit a job that will execute after merging/OCDB export that will do it. | |
fc7e1e45 | 1246 | |
5dfac970 | 1247 | mkdir -p $targetDirectory |
1248 | mkdir -p $targetDirectory/logs | |
1249 | cp -f $self $targetDirectory | |
1250 | theScript="$targetDirectory/${self##*/}" | |
fc7e1e45 | 1251 | |
5dfac970 | 1252 | echo |
1253 | echo submitting the OCDB specific storage config making script for run $runNumber | |
1254 | echo | |
fc7e1e45 | 1255 | |
5dfac970 | 1256 | [[ -f $configPath/localOCDBaccessConfig.C ]] && cp -f $configPath/localOCDBaccessConfig.C $targetDirectory && echo "## using local localOCDBaccessConfig.C" |
fc7e1e45 | 1257 | |
5dfac970 | 1258 | submit $JOBID3 1 1 "$LASTJOB" "$theScript" "ConfOCDB ${commonOutputPath}/000${runNumber}/CPass0/OCDB ${targetDirectory} " |
1259 | LASTJOB=$JOBID3 | |
1260 | ################################################################################ | |
fc7e1e45 | 1261 | |
5dfac970 | 1262 | echo |
1263 | echo "starting CPass1... for run $runNumber" | |
1264 | echo | |
1265 | ||
1266 | # create directory and copy all files that are needed | |
1267 | mkdir -p $targetDirectory | |
1268 | mkdir -p $targetDirectory/logs | |
1269 | ||
1270 | localInputList=$targetDirectory/${inputList##*/} | |
1271 | theScript="$targetDirectory/${self##*/}" | |
1272 | cp -f $self $theScript | |
1273 | chmod u+x $theScript | |
1274 | ||
1275 | rm -f $localInputList | |
1276 | egrep "\/000$runNumber\/" $inputList >> $localInputList | |
1277 | cp -f $configFile $targetDirectory | |
1278 | [[ -f $configPath/runCPass1.sh ]] && cp -f $configPath/runCPass1.sh $targetDirectory && echo "## using local runCPass1.sh" | |
1279 | [[ -f $configPath/recCPass1.C ]] && cp -f $configPath/recCPass1.C $targetDirectory && echo "## using local recCPass1.C" | |
1280 | [[ -f $configPath/recCPass1_OuterDet.C ]] && cp -f $configPath/recCPass1_OuterDet.C $targetDirectory && echo "## using local recCPass1_OuterDet.C" | |
1281 | [[ -f $configPath/runCalibTrain.C ]] && cp -f $configPath/runCalibTrain.C $targetDirectory && echo "## using local runCalibTrain.C" | |
1282 | [[ -f $configPath/QAtrain.C ]] && cp -f $configPath/QAtrain.C $targetDirectory && echo "## using local QAtrain.C" | |
1283 | [[ -f $configPath/QAtrain_duo.C ]] && cp -f $configPath/QAtrain_duo.C $targetDirectory && echo "## using local QAtrain_duo.C" | |
1284 | ||
1285 | [[ -f $configPath/CPass1/runCPass1.sh ]] && cp -f $configPath/CPass1/runCPass1.sh $targetDirectory && echo "## using local runCPass1.sh" | |
1286 | [[ -f $configPath/CPass1/recCPass1.C ]] && cp -f $configPath/CPass1/recCPass1.C $targetDirectory && echo "## using local recCPass1.C" | |
1287 | [[ -f $configPath/CPass1/recCPass1_OuterDet.C ]] && cp -f $configPath/CPass1/recCPass1_OuterDet.C $targetDirectory && echo "## using local recCPass1_OuterDet.C" | |
1288 | [[ -f $configPath/CPass1/runCalibTrain.C ]] && cp -f $configPath/CPass1/runCalibTrain.C $targetDirectory && echo "## using local runCalibTrain.C" | |
1289 | [[ -f $configPath/CPass1/QAtrain.C ]] && cp -f $configPath/CPass1/QAtrain.C $targetDirectory && echo "## using local QAtrain.C" | |
1290 | [[ -f $configPath/CPass1/QAtrain_duo.C ]] && cp -f $configPath/CPass1/QAtrain_duo.C $targetDirectory && echo "## using local QAtrain_duo.C" | |
1291 | ||
1292 | echo "... files copied." | |
1293 | echo | |
1294 | ||
1295 | # limit nFiles to nMaxChunks | |
1296 | nFiles=`wc -l < $localInputList` | |
1297 | [[ $nFiles -eq 0 ]] && echo "list contains ZERO files! exiting..." && exit 1 | |
1298 | echo "raw files in list: $nFiles" | |
1299 | if [[ $nMaxChunks -gt 0 && $nMaxChunks -le $nFiles ]]; then | |
1300 | nFiles=$nMaxChunks | |
1301 | fi | |
1302 | echo "raw files to process: $nFiles" | |
1303 | [[ -z "$percentProcessedFilesToContinue" ]] && percentProcessedFilesToContinue=100 | |
1304 | if [[ $percentProcessedFilesToContinue -eq 100 ]]; then | |
1305 | nFilesToWaitFor=$nFiles | |
fc7e1e45 | 1306 | else |
5dfac970 | 1307 | nFilesToWaitFor=$(( $nFiles-$nFiles/(100/(100-$percentProcessedFilesToContinue)) )) |
fc7e1e45 | 1308 | fi |
5dfac970 | 1309 | echo "requested success rate is $percentProcessedFilesToContinue%" |
1310 | echo "merging will start after $nFilesToWaitFor jobs are done" | |
fc7e1e45 | 1311 | |
5dfac970 | 1312 | submit $JOBID4 1 $nFiles "$LASTJOB" "$theScript" "CPass1 $targetDirectory $localInputList $nEvents $currentDefaultOCDB $configFile $runNumber" |
fc7e1e45 | 1313 | |
5dfac970 | 1314 | ################################################################################ |
1315 | ## submit a monitoring job that will run until a certain number of jobs are done with reconstruction | |
1316 | submit "$JOBID4wait" 1 1 "$LASTJOB" "$theScript" "WaitForOutput ${commonOutputPath} 'cpass1.job*.run${runNumber}.done' $nFilesToWaitFor $maxSecondsToWait '-maxdepth 1'" | |
1317 | LASTJOB=$JOBID4wait | |
1318 | ################################################################################ | |
fc7e1e45 | 1319 | |
5dfac970 | 1320 | echo |
1321 | fi #end running CPass1 | |
fc7e1e45 | 1322 | |
5dfac970 | 1323 | ################################################################################ |
1324 | # submit merging of CPass1, depends on the reconstruction | |
1325 | if [ $runCPass1MergeMakeOCDB -eq 1 ]; then | |
fc7e1e45 | 1326 | |
5dfac970 | 1327 | echo |
1328 | echo "submit CPass1 merging for run $runNumber" | |
1329 | echo | |
fc7e1e45 | 1330 | |
5dfac970 | 1331 | targetDirectory="${commonOutputPath}/000${runNumber}/CPass1" |
1332 | mkdir -p $targetDirectory | |
fc7e1e45 | 1333 | |
5dfac970 | 1334 | # copy the scripts |
1335 | cp -f $self $targetDirectory | |
1336 | [[ -f $configPath/mergeMakeOCDB.sh ]] && cp -f $configPath/mergeMakeOCDB.sh $targetDirectory && echo "## using local mergeMakeOCDB.sh" | |
1337 | [[ -f $configPath/merge.C ]] && cp -f $configPath/merge.C $targetDirectory && echo "## using local merge.C" | |
1338 | [[ -f $configPath/mergeMakeOCDB.byComponent.sh ]] && cp -f $configPath/mergeMakeOCDB.byComponent.sh $targetDirectory && echo "## using local mergeMakeOCDB.byComponent.sh" | |
1339 | [[ -f $configPath/mergeByComponent.C ]] && cp -f $configPath/mergeByComponent.C $targetDirectory && echo "## using local mergeByComponent.C" | |
1340 | [[ -f $configPath/makeOCDB.C ]] && cp -f $configPath/makeOCDB.C $targetDirectory && echo "## using local makeOCDB.C" | |
fc7e1e45 | 1341 | |
5dfac970 | 1342 | theScript="$targetDirectory/${self##*/}" |
1343 | chmod u+x $theScript | |
1344 | ||
1345 | echo submit "$JOBID5" 1 1 "$LASTJOB" "$theScript" "MergeCPass1 $targetDirectory $currentDefaultOCDB $configFile $runNumber" | |
1346 | submit "$JOBID5" 1 1 "$LASTJOB" "$theScript" "MergeCPass1 $targetDirectory $currentDefaultOCDB $configFile $runNumber" | |
1347 | ((numberOfSubmittedCPass1MergingJobs++)) | |
1348 | LASTJOB=$JOBID5 | |
1349 | echo | |
1350 | fi | |
1351 | ||
1352 | done | |
1353 | ||
1354 | ################################################################################ | |
1355 | ################################################################################ | |
1356 | [[ -z $runMakeSummary ]] && runMakeSummary=0 | |
1357 | if [ $runMakeSummary -eq 1 ]; then | |
fc7e1e45 | 1358 | echo |
5dfac970 | 1359 | echo "submit make a summary" |
1360 | echo | |
1361 | ||
1362 | targetDirectory="${commonOutputPath}" | |
1363 | mkdir -p $targetDirectory | |
1364 | mkdir -p $targetDirectory/logs | |
1365 | [[ ! -f $targetDirectory/${self##*/} ]] && cp -f $self $targetDirectory | |
1366 | theScript="$targetDirectory/${self##*/}" | |
1367 | submit "$JOBID6" 1 1 "$LASTJOB" "$theScript" "MakeSummary $targetDirectory $configFile" | |
1368 | fi | |
1369 | ################################################################################ | |
1370 | ||
1371 | ################################################################################ | |
1372 | ################################################################################ | |
1373 | if [ $runMakeQAplots -eq 1 ]; then | |
1374 | echo | |
1375 | echo "submit make QA plots" | |
1376 | echo | |
1377 | ||
1378 | ## submit a monitoring job that will wait for a specified number of files to appear somewhere | |
1379 | targetDirectory="${commonOutputPath}" | |
1380 | [[ ! -f $targetDirectory/${self##*/} ]] && cp -f $self $targetDirectory | |
1381 | theScript="$targetDirectory/${self##*/}" | |
1382 | echo submit "$JOBID5wait 1 1 $theScript WaitForOutput ${commonOutputPath} 'merge.cpass1.run*.done' $numberOfSubmittedCPass1MergingJobs $maxSecondsToWait '-maxdepth 1'" | |
1383 | submit "$JOBID5wait" 1 1 "" "$theScript" "WaitForOutput ${commonOutputPath} 'merge.cpass1.run*.done' $numberOfSubmittedCPass1MergingJobs $maxSecondsToWait '-maxdepth 1'" | |
1384 | LASTJOB=$JOBID5wait | |
1385 | ||
1386 | targetDirectory="${commonOutputPath}/QAplots/" | |
1387 | [[ -d $targetDirectory ]] && rm -rf $targetDirectory/* | |
1388 | mkdir -p $targetDirectory | |
1389 | mkdir -p $targetDirectory/logs | |
1390 | qaFilesDirectory="${commonOutputPath}" | |
1391 | [[ ! -f $targetDirectory/${self##*/} ]] && cp -f $self $targetDirectory | |
1392 | theScript="$targetDirectory/${self##*/}" | |
1393 | ||
1394 | echo submit "$JOBID7" 1 1 "$LASTJOB" "$theScript" "CreateQAplots QAplots CPass1 $targetDirectory $qaFilesDirectory $qaPlotsScript $configFile" | |
1395 | submit "$JOBID7" 1 1 "$LASTJOB" "$theScript" "CreateQAplots QAplots CPass1 $targetDirectory $qaFilesDirectory $qaPlotsScript $configFile" | |
1396 | LASTJOB=$JOBID7 | |
fc7e1e45 | 1397 | fi |
5dfac970 | 1398 | |
1399 | #restore stdout | |
1400 | exec 1>&7 7>&- | |
1401 | echo "jobs submitted." | |
fc7e1e45 | 1402 | } |
1403 | ||
1404 | main "$@" |