]>
Commit | Line | Data |
---|---|---|
fc7e1e45 | 1 | #!/bin/bash |
2 | # this script runs the CPass0/CPass1 train | |
3 | # produced OCDB updates are local | |
4 | ||
5 | main() | |
6 | { | |
7 | #run in proper mode depending on the selection | |
8 | runMode=$1 | |
9 | if [[ $# -gt 0 ]]; then | |
10 | echo "# $0 $*" | |
11 | fi | |
12 | umask 0002 | |
13 | shift | |
14 | case $runMode in | |
15 | "CPass0") goCPass0 "$@";; | |
16 | "CPass1") goCPass1 "$@";; | |
17 | "ConfOCDB") goConfigureOCDBforCPass1 "$@";; | |
18 | "MergeCPass0") goMergeCPass0 "$@";; | |
19 | "MergeCPass1") goMergeCPass1 "$@";; | |
20 | "CreateQAplots") goCreateQAplots "$@";; | |
21 | "WaitForOutput") goWaitForOutput "$@";; | |
22 | "makeSummary") goMakeSummary "$@";; | |
23 | "submit") goSubmit "$@";; | |
24 | "submitQA") goSubmitQA "$@";; | |
25 | "test") goTest "$@";; | |
26 | "generateMakeflow") goGenerateMakeflow "$@";; | |
27 | "makeflow") goMakeflow "$@";; | |
28 | #and the default: unless sourced print some info | |
29 | *) if [[ ! "$0" =~ "bash" ]]; then | |
30 | echo " batch: $0 \"submit\" inputList productionID [configFile=benchmark.config] [runNumber]" | |
31 | echo " makeflow: $0 \"makeflow\" inputList productionID [configFile=benchmark.config] [runNumber]" | |
32 | fi | |
33 | ;; | |
34 | esac | |
35 | } | |
36 | ||
37 | goCPass0() | |
38 | { | |
39 | umask 0002 | |
40 | ||
41 | configFile=$5 | |
42 | source $configFile | |
36b5de1a | 43 | [[ -f ${setupAliROOTenvInCurrentShell} && -z ${alirootEnv} ]] && source $setupAliROOTenvInCurrentShell |
fc7e1e45 | 44 | |
45 | targetDirectory=$1 | |
46 | inputList=$2 | |
47 | nEvents=$3 | |
48 | ocdbPath=$4 | |
49 | configFile=$5 | |
50 | runNumber=$6 | |
51 | jobindex=$7 | |
52 | ||
53 | #use the jobindex only if set and non-negative | |
54 | if [[ -z $jobindex || $jobindex -lt 0 ]]; then | |
55 | [[ -n "$LSB_JOBINDEX" ]] && jobindex=$LSB_JOBINDEX | |
56 | [[ -n "$SGE_TASK_ID" ]] && jobindex=$SGE_TASK_ID | |
57 | fi | |
58 | ||
59 | [[ ! -f ${inputList} && -z ${pretend} ]] && echo "input file $inputList not found, exiting..." && exit 1 | |
60 | if [[ "${inputList}" =~ \.root$ ]]; then | |
61 | infile=$inputList | |
62 | else | |
63 | infile=`sed -ne "${jobindex}p" $inputList` | |
64 | fi | |
65 | ||
66 | chunkName=${infile##*/} | |
67 | outputDir=${targetDirectory}/${jobindex} | |
68 | mkdir -p $outputDir | |
69 | [[ ! -d $outputDir ]] && echo "cannot make $outputDir" && exit 1 | |
70 | ||
71 | runpath=${PWD}/rundir_cpass0_${runNumber}_${jobindex} | |
72 | [[ -z $commonOutputPath ]] && commonOutputPath=$PWD | |
73 | [[ $reconstructInTemporaryDir -eq 1 && -n $TMPDIR ]] && runpath=$TMPDIR | |
74 | [[ $reconstructInTemporaryDir -eq 1 && -z $TMPDIR ]] && runpath=$(mktemp -d) | |
75 | ||
76 | mkdir -p $runpath | |
77 | [[ ! -d ${runpath} ]] && echo "cannot make runpath ${runpath}" && exit 1 | |
78 | cd $runpath | |
79 | ||
80 | logOutputDir=$runpath | |
81 | [[ -n $logToFinalDestination ]] && logOutputDir=$outputDir | |
5ed7c82c | 82 | [[ -z $dontRedirectStdOutToLog ]] && exec 1> $logOutputDir/stdout |
83 | [[ -z $dontRedirectStdOutToLog ]] && exec 2> $logOutputDir/stderr | |
fc7e1e45 | 84 | echo "$0 $*" |
85 | ||
86 | calibDoneFile="$commonOutputPath/cpass0.job${jobindex}.run${runNumber}.done" | |
87 | ||
88 | echo "#####################" | |
89 | echo CPass0: | |
90 | echo JOB setup | |
91 | echo nEvents $nEvents | |
92 | echo runNumber $runNumber | |
93 | echo ocdbPath $ocdbPath | |
94 | echo infile $infile | |
95 | echo chunkName $chunkName | |
96 | echo jobindex $jobindex | |
97 | echo recoTriggerOptions $recoTriggerOptions | |
98 | echo targetDirectory $targetDirectory | |
99 | echo commonOutputPath $commonOutputPath | |
100 | echo calibDoneFile $calibDoneFile | |
101 | echo runpath $runpath | |
102 | echo outputDir $outputDir | |
103 | echo ALICE_ROOT $ALICE_ROOT | |
104 | echo PWD $PWD | |
105 | echo "########## ###########" | |
106 | ||
107 | alirootInfo > ALICE_ROOT_svn.log | |
108 | ||
109 | filesCPass0=( | |
110 | "$commonOutputPath/runCPass0.sh" | |
111 | "$commonOutputPath/recCPass0.C" | |
112 | "$commonOutputPath/runCalibTrain.C" | |
113 | "$commonOutputPath/localOCDBaccessConfig.C" | |
36b5de1a | 114 | "$commonOutputPath/OCDB.root" |
fc7e1e45 | 115 | "$ALICE_ROOT/PWGPP/CalibMacros/CPass0/runCPass0.sh" |
116 | "$ALICE_ROOT/PWGPP/CalibMacros/CPass0/recCPass0.C" | |
117 | "$ALICE_ROOT/PWGPP/CalibMacros/CPass0/runCalibTrain.C" | |
118 | ) | |
119 | ||
120 | for file in ${filesCPass0[*]}; do | |
121 | [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} . | |
122 | done | |
123 | ||
124 | ln -s $infile $runpath/$chunkName | |
125 | ||
126 | echo "this directory ($PWD) contents:" | |
127 | ls -lh | |
128 | echo | |
129 | chmod u+x runCPass0.sh | |
130 | ||
131 | if [[ -n $postSetUpActionCPass0 ]]; then | |
132 | echo "running $postSetUpActionCPass0" | |
133 | eval $postSetUpActionCPass0 | |
134 | fi | |
135 | ||
136 | #run CPass0 | |
137 | echo "$runpath/runCPass0.sh $infile $nEvents $runNumber $ocdbPath $recoTriggerOptions" | |
138 | if [[ -n $pretend ]]; then | |
139 | touch AliESDfriends_v1.root | |
140 | touch rec.log | |
141 | touch calib.log | |
142 | else | |
143 | ./runCPass0.sh "$infile" "$nEvents" "$runNumber" "$ocdbPath" "$recoTriggerOptions" | |
144 | fi | |
145 | ||
fc7e1e45 | 146 | #move stuff to final destination |
147 | echo "this directory ($PWD) contents:" | |
148 | ls -lh | |
149 | echo | |
150 | ||
151 | echo rm -f ./$chunkName | |
152 | rm -f ./$chunkName | |
153 | echo "cp --recursive $runpath/* ${outputDir}" | |
154 | cp --recursive $runpath/* $outputDir | |
155 | echo | |
5ed7c82c | 156 | |
157 | #validate CPass0 | |
158 | cd ${outputDir} | |
159 | touch ${calibDoneFile} | |
160 | [[ -f AliESDfriends_v1.root ]] && echo "calibfile ${outputDir}/AliESDfriends_v1.root" > ${calibDoneFile} | |
161 | summarizeLogs >> ${calibDoneFile} | |
fc7e1e45 | 162 | |
163 | rm -rf ${runpath} | |
164 | } | |
165 | ||
166 | goCPass1() | |
167 | { | |
168 | umask 0002 | |
169 | ||
170 | configFile=$5 | |
171 | source $configFile | |
36b5de1a | 172 | [[ -f ${setupAliROOTenvInCurrentShell} && -z ${alirootEnv} ]] && source $setupAliROOTenvInCurrentShell |
fc7e1e45 | 173 | |
174 | targetDirectory=$1 | |
175 | inputList=$2 | |
176 | nEvents=$3 | |
177 | ocdbPath=$4 | |
178 | configFile=$5 | |
179 | runNumber=$6 | |
180 | jobindex=$7 | |
181 | ||
182 | #use the jobindex only if set and non-negative | |
183 | if [[ -z $jobindex || $jobindex -lt 0 ]]; then | |
184 | [[ -n "$LSB_JOBINDEX" ]] && jobindex=$LSB_JOBINDEX | |
185 | [[ -n "$SGE_TASK_ID" ]] && jobindex=$SGE_TASK_ID | |
186 | fi | |
187 | ||
188 | [[ ! -f ${inputList} && -z ${pretend} ]] && echo "input file $inputList not found, exiting..." && exit 1 | |
189 | if [[ "${inputList}" =~ \.root$ ]]; then | |
190 | infile=$inputList | |
191 | else | |
192 | infile=`sed -ne "${jobindex}p" $inputList` | |
193 | fi | |
194 | ||
195 | chunkName=${infile##*/} | |
196 | outputDir=${targetDirectory}/${jobindex} | |
197 | mkdir -p $outputDir | |
198 | [[ ! -d $outputDir ]] && echo "cannot make $outputDir" && exit 1 | |
199 | ||
200 | runpath=${PWD}/rundir_cpass1_${runNumber}_${jobindex} | |
201 | [[ -z $commonOutputPath ]] && commonOutputPath=$PWD | |
202 | [[ $reconstructInTemporaryDir -eq 1 && -n $TMPDIR ]] && runpath=$TMPDIR | |
203 | [[ $reconstructInTemporaryDir -eq 1 && -z $TMPDIR ]] && runpath=$(mktemp -d) | |
204 | ||
205 | #init the running path | |
206 | mkdir -p $runpath | |
207 | [[ ! -d ${runpath} ]] && echo "cannot make runpath ${runpath}" && exit 1 | |
208 | cd $runpath | |
209 | ||
210 | ||
211 | calibDoneFile="${commonOutputPath}/cpass1.job${jobindex}.run${runNumber}.done" | |
212 | ||
213 | logOutputDir=$runpath | |
214 | [[ -n $logToFinalDestination ]] && logOutputDir=$outputDir | |
5ed7c82c | 215 | [[ -z $dontRedirectStdOutToLog ]] && exec 1> $logOutputDir/stdout |
216 | [[ -z $dontRedirectStdOutToLog ]] && exec 2> $logOutputDir/stderr | |
fc7e1e45 | 217 | echo "$0 $*" |
218 | ||
219 | echo "#####################" | |
220 | echo CPass1: | |
221 | echo JOB setup | |
222 | echo nEvents $nEvents | |
223 | echo runNumber $runNumber | |
224 | echo ocdbPath $ocdbPath | |
225 | echo infile $infile | |
226 | echo chunkName $chunkName | |
227 | echo jobindex $jobindex | |
228 | echo recoTriggerOptions $recoTriggerOptions | |
229 | echo targetDirectory $targetDirectory | |
230 | echo commonOutputPath $commonOutputPath | |
231 | echo calibDoneFile $calibDoneFile | |
232 | echo runpath $runpath | |
233 | echo outputDir $outputDir | |
234 | echo ALICE_ROOT $ALICE_ROOT | |
235 | echo PWD $PWD | |
236 | echo "########## ###########" | |
237 | ||
238 | alirootInfo > ALICE_ROOT_svn.log | |
239 | ||
240 | filesCPass1=( | |
241 | "$commonOutputPath/runCPass1.sh" | |
242 | "$commonOutputPath/recCPass1.C" | |
243 | "$commonOutputPath/recCPass1_OuterDet.C" | |
244 | "$commonOutputPath/runCalibTrain.C" | |
245 | "$commonOutputPath/QAtrain_duo.C" | |
246 | "$commonOutputPath/localOCDBaccessConfig.C" | |
247 | "$commonOutputPath/cpass0.localOCDB.${runNumber}.tgz" | |
36b5de1a | 248 | "$commonOutputPath/OCDB.root" |
fc7e1e45 | 249 | "$ALICE_ROOT/PWGPP/CalibMacros/CPass1/runCPass1.sh" |
250 | "$ALICE_ROOT/PWGPP/CalibMacros/CPass1/recCPass1.C" | |
251 | "$ALICE_ROOT/PWGPP/CalibMacros/CPass1/recCPass1_OuterDet.C" | |
252 | "$ALICE_ROOT/PWGPP/CalibMacros/CPass1/runCalibTrain.C" | |
253 | "$ALICE_ROOT/ANALYSIS/macros/QAtrain_duo.C" | |
254 | ) | |
255 | ||
256 | for file in ${filesCPass1[*]}; do | |
257 | [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} . | |
258 | done | |
259 | ||
260 | ln -s $infile $runpath/$chunkName | |
261 | ||
262 | echo "this directory ($PWD) contents:" | |
263 | ls -lh | |
264 | echo | |
265 | ||
266 | if [[ -n $postSetUpActionCPass1 ]]; then | |
267 | echo "running $postSetUpActionCPass1" | |
268 | eval $postSetUpActionCPass1 | |
269 | echo | |
270 | fi | |
271 | ||
272 | #configure local OCDB storage from CPass0 (creates the localOCDBaccessConfig.C script) | |
273 | if [[ -f cpass0.localOCDB.${runNumber}.tgz ]]; then | |
274 | echo goConfigureOCDBforCPass1 "cpass0.localOCDB.${runNumber}.tgz" | |
275 | goConfigureOCDBforCPass1 "cpass0.localOCDB.${runNumber}.tgz" | |
276 | else | |
277 | echo "WARNING: file cpass0.localOCDB.${runNumber}.tgz not found!" | |
278 | fi | |
279 | ||
280 | #run CPass1 | |
281 | chmod u+x runCPass1.sh | |
282 | echo "$runpath/runCPass1.sh $infile $nEvents $runNumber $ocdbPath $recoTriggerOptions" | |
283 | if [[ -n $pretend ]]; then | |
284 | touch AliESDfriends_v1.root | |
285 | touch QAresults_Barrel.root | |
286 | touch QAresults_Outer.root | |
287 | touch rec.log | |
288 | touch calib.log | |
289 | touch qa.log | |
290 | else | |
291 | ./runCPass1.sh "$infile" "$nEvents" "$runNumber" "$ocdbPath" "$recoTriggerOptions" | |
292 | fi | |
293 | ||
fc7e1e45 | 294 | #move stuff to final destination |
295 | echo "this directory ($PWD) contents:" | |
296 | ls | |
297 | echo | |
298 | ||
299 | echo rm -f ./$chunkName | |
300 | rm -f ./$chunkName | |
301 | echo "cp --recursive ${runpath}/* ${outputDir}" | |
302 | cp --recursive ${runpath}/* ${outputDir} | |
303 | echo | |
304 | ||
5ed7c82c | 305 | #validate CPass1 |
306 | cd ${outputDir} | |
307 | touch ${calibDoneFile} | |
308 | [[ -f AliESDfriends_v1.root ]] && echo "calibfile ${outputDir}/AliESDfriends_v1.root" > ${calibDoneFile} | |
309 | [[ -f QAresults_Barrel.root ]] && echo "qafile ${outputDir}/QAresults_Barrel.root" >> ${calibDoneFile} | |
310 | [[ -f QAresults_Outer.root ]] && echo "qafile ${outputDir}/QAresults_Outer.root" >> ${calibDoneFile} | |
311 | summarizeLogs >> ${calibDoneFile} | |
312 | ||
fc7e1e45 | 313 | rm -rf ${runpath} |
314 | } | |
315 | ||
316 | ||
317 | goMergeCPass0() | |
318 | { | |
319 | umask 0002 | |
320 | # | |
321 | # find the output files and merge them | |
322 | # | |
323 | ||
324 | outputDir=$1 | |
325 | defaultOCDB=$2 | |
326 | configFile=$3 | |
327 | runNumber=$4 | |
328 | calibrationFilesToMergeExternal=$5 | |
329 | ||
330 | source $configFile | |
36b5de1a | 331 | [[ -f ${setupAliROOTenvInCurrentShell} && -z ${alirootEnv} ]] && source $setupAliROOTenvInCurrentShell |
fc7e1e45 | 332 | |
333 | runpath=${PWD}/rundir_cpass0_Merge_${runNumber} | |
334 | [[ -z $commonOutputPath ]] && commonOutputPath=$PWD | |
335 | [[ $reconstructInTemporaryDir -eq 1 && -n $TMPDIR ]] && runpath=$TMPDIR | |
336 | [[ $reconstructInTemporaryDir -eq 1 && -z $TMPDIR ]] && runpath=$(mktemp -d) | |
337 | ||
338 | mkdir -p $runpath | |
339 | [[ ! -d $runpath ]] && echo "not able to make the runpath $runpath" && exit 1 | |
340 | cd $runpath | |
341 | ||
342 | logOutputDir=$runpath | |
343 | [[ -n $logToFinalDestination ]] && logOutputDir=$outputDir | |
344 | [[ -z $dontRedirectStdOutToLog ]] && exec 2>&1 > $logOutputDir/mergeMakeOCDB.log | |
345 | echo "$0 $*" | |
346 | ||
347 | calibrationFilesToMerge=$calibrationFilesToMergeExternal | |
348 | [[ -z $calibrationFilesToMerge ]] && calibrationFilesToMerge="calibrationFilesToMerge.list" | |
349 | calibrationOutputFileName="AliESDfriends_v1.root" | |
350 | mergingScript="mergeMakeOCDB.byComponent.sh" | |
351 | qaFilesToMerge="qaFilesToMerge.list" | |
352 | qaOutputFileName="QAresults*.root" | |
353 | qaMergedOutputFileName="QAresults_merged.root" | |
354 | ||
355 | echo goMergeCPass0 SETUP: | |
356 | echo runNumber=$runNumber | |
357 | echo outputDir=$outputDir | |
358 | echo defaultOCDB=$defaultOCDB | |
359 | echo calibrationFilesToMerge=$calibrationFilesToMerge | |
360 | echo calibrationOutputFileName=$calibrationOutputFileName | |
361 | echo mergingScript=$mergingScript | |
362 | ||
363 | # copy files in case they are not already there | |
364 | filesMergeCPass0=( | |
36b5de1a | 365 | "$commonOutputPath/${calibrationFilesToMerge}" |
366 | "$commonOutputPath/OCDB.root" | |
367 | "$commonOutputPath/localOCDBaccessConfig.C" | |
fc7e1e45 | 368 | "$ALICE_ROOT/PWGPP/CalibMacros/CPass0/mergeMakeOCDB.byComponent.sh" |
369 | "$ALICE_ROOT/PWGPP/CalibMacros/CPass0/mergeByComponent.C" | |
370 | "$ALICE_ROOT/PWGPP/CalibMacros/CPass0/makeOCDB.C" | |
371 | "$ALICE_ROOT/PWGPP/CalibMacros/CPass0/merge.C" | |
372 | "$ALICE_ROOT/PWGPP/CalibMacros/CPass0/mergeMakeOCDB.sh" | |
fc7e1e45 | 373 | ) |
374 | for file in ${filesMergeCPass0[*]}; do | |
375 | [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} . | |
376 | done | |
377 | ||
378 | alirootInfo > ALICE_ROOT_svn.log | |
379 | ||
380 | # | |
381 | ls -lh | |
382 | ||
383 | #merge calibration | |
384 | chmod u+x $mergingScript | |
385 | mkdir -p ./OCDB | |
386 | if [[ -z ${calibrationFilesToMergeExternal} ]]; then | |
387 | echo "find $outputDir -name $calibrationOutputFileName > $calibrationFilesToMerge" | |
388 | find $outputDir -name $calibrationOutputFileName > $calibrationFilesToMerge | |
389 | fi | |
390 | ||
391 | echo "$mergingScript $calibrationFilesToMerge ${runNumber} local://./OCDB $defaultOCDB" | |
392 | if [[ -n $pretend ]]; then | |
393 | touch CalibObjects.root | |
394 | touch ocdb.log | |
395 | touch merge.log | |
396 | mkdir -p ./OCDB/someDetector/ | |
397 | mkdir -p ./OCDB/otherDetector/ | |
398 | touch ./OCDB/someDetector/someCalibObject_0-999999_cpass0.root | |
399 | touch ./OCDB/otherDetector/otherCalibObject_0-999999_cpass0.root | |
400 | else | |
401 | ./$mergingScript $calibrationFilesToMerge ${runNumber} "local://./OCDB" $defaultOCDB | |
402 | fi | |
403 | ||
404 | ### produce the output | |
405 | #tar the produced OCDB for reuse | |
406 | tar czf $commonOutputPath/cpass0.localOCDB.${runNumber}.tgz ./OCDB | |
407 | ||
5ed7c82c | 408 | ls -ltrh |
409 | ||
410 | #copy all to output dir | |
411 | cp --recursive ${runpath}/* $outputDir | |
412 | ||
fc7e1e45 | 413 | #validate merging cpass0 |
5ed7c82c | 414 | cd ${outputDir} |
fc7e1e45 | 415 | calibDoneFile="${commonOutputPath}/merge.cpass0.run${runNumber}.done" |
416 | touch ${calibDoneFile} | |
417 | [[ -f CalibObjects.root ]] && echo "calibfile $outputDir/CalibObjects.root" > ${calibDoneFile} | |
418 | summarizeLogs >> ${calibDoneFile} | |
419 | ||
fc7e1e45 | 420 | rm -rf ${runpath} |
421 | } | |
422 | ||
423 | goMergeCPass1() | |
424 | { | |
425 | umask 0002 | |
426 | # | |
427 | # find the output files and merge them | |
428 | # | |
429 | ||
430 | outputDir=$1 | |
431 | defaultOCDB=$2 | |
432 | configFile=$3 | |
433 | runNumber=$4 | |
434 | calibrationFilesToMergeExternal=$5 | |
435 | qaFilesToMergeExternal=$6 | |
436 | ||
437 | #clean up first: | |
438 | rm -f $outputDir/*.log | |
439 | rm -f $outputDir/*.root | |
440 | rm -f $outputDir/*done | |
441 | ||
442 | source $configFile | |
36b5de1a | 443 | [[ -f ${setupAliROOTenvInCurrentShell} && -z ${alirootEnv} ]] && source $setupAliROOTenvInCurrentShell |
fc7e1e45 | 444 | |
445 | runpath=${PWD}/rundir_cpass1_Merge_${runNumber} | |
446 | [[ -z $commonOutputPath ]] && commonOutputPath=$PWD | |
447 | [[ $reconstructInTemporaryDir -eq 1 && -n $TMPDIR ]] && runpath=$TMPDIR | |
448 | [[ $reconstructInTemporaryDir -eq 1 && -z $TMPDIR ]] && runpath=$(mktemp -d) | |
449 | ||
450 | mkdir -p $runpath | |
451 | [[ ! -d $runpath ]] && echo "not able to make the runpath $runpath" && exit 1 | |
452 | cd $runpath | |
453 | ||
454 | logOutputDir=$runpath | |
455 | [[ -n $logToFinalDestination ]] && logOutputDir=$outputDir | |
456 | [[ -z $dontRedirectStdOutToLog ]] && exec 2>&1 > $logOutputDir/mergeMakeOCDB.log | |
457 | echo "$0 $*" | |
458 | ||
459 | calibrationFilesToMerge=$calibrationFilesToMergeExternal | |
460 | [[ -z $calibrationFilesToMerge ]] && calibrationFilesToMerge="calibrationFilesToMerge.list" | |
461 | calibrationOutputFileName="AliESDfriends_v1.root" | |
462 | mergingScript="mergeMakeOCDB.byComponent.sh" | |
463 | qaFilesToMerge=$qaFilesToMergeExternal | |
464 | [[ -z $qaFilesToMerge ]] && qaFilesToMerge="qaFilesToMerge.list" | |
465 | qaOutputFileName="QAresults*.root" | |
466 | qaMergedOutputFileName="QAresults_merged.root" | |
467 | ||
468 | echo goMergeCPass1 SETUP: | |
469 | echo runNumber=$runNumber | |
470 | echo outputDir=$outputDir | |
471 | echo defaultOCDB=$defaultOCDB | |
472 | echo calibrationFilesToMerge=filesToMerge.list | |
473 | echo calibrationOutputFileName=$calibrationOutputFileName | |
474 | echo mergingScript=$mergingScript | |
475 | ||
476 | # copy files in case they are not already there | |
477 | filesMergeCPass1=( | |
36b5de1a | 478 | "$commonOutputPath/${calibrationFilesToMerge}" |
479 | "$commonOutputPath/${qaFilesToMerge}" | |
480 | "$commonOutputPath/OCDB.root" | |
481 | "$commonOutputPath/localOCDBaccessConfig.C" | |
fc7e1e45 | 482 | "$ALICE_ROOT/PWGPP/CalibMacros/CPass1/mergeMakeOCDB.byComponent.sh" |
483 | "$ALICE_ROOT/PWGPP/CalibMacros/CPass1/mergeByComponent.C" | |
484 | "$ALICE_ROOT/PWGPP/CalibMacros/CPass1/makeOCDB.C" | |
485 | "$ALICE_ROOT/PWGPP/CalibMacros/CPass1/merge.C" | |
486 | "$ALICE_ROOT/PWGPP/CalibMacros/CPass1/mergeMakeOCDB.sh" | |
fc7e1e45 | 487 | ) |
488 | for file in ${filesMergeCPass1[*]}; do | |
489 | [[ ! -f ${file##*/} && -f ${file} ]] && echo "copying ${file}" && cp -f ${file} . | |
490 | done | |
491 | ||
492 | alirootInfo > ALICE_ROOT_svn.log | |
493 | ||
494 | # | |
495 | ls -lh | |
496 | ||
497 | #merge calibration | |
498 | chmod u+x $mergingScript | |
499 | mkdir -p OCDB | |
500 | if [[ -z ${calibrationFilesToMergeExternal} ]]; then | |
501 | echo "find $outputDir -name $calibrationOutputFileName > $calibrationFilesToMerge" | |
502 | find $outputDir -name $calibrationOutputFileName > $calibrationFilesToMerge | |
503 | fi | |
504 | ||
505 | echo "$mergingScript $calibrationFilesToMerge ${runNumber} local://./OCDB $defaultOCDB" | |
506 | if [[ -n $pretend ]]; then | |
507 | touch CalibObjects.root | |
508 | touch ocdb.log | |
509 | touch merge.log | |
510 | else | |
511 | ./$mergingScript $calibrationFilesToMerge ${runNumber} "local://./OCDB" $defaultOCDB | |
512 | fi | |
513 | ||
514 | tar czf localCPass1_${runNumber}.tgz ./OCDB | |
515 | ||
516 | #merge QA | |
5ed7c82c | 517 | [[ -n ${AliAnalysisTaskFilteredTree_fLowPtTrackDownscaligF} ]] && export AliAnalysisTaskFilteredTree_fLowPtTrackDownscaligF |
518 | [[ -n ${AliAnalysisTaskFilteredTree_fLowPtV0DownscaligF} ]] && export AliAnalysisTaskFilteredTree_fLowPtV0DownscaligF | |
519 | ||
520 | if [[ -z $qaFilesToMergeExternal ]]; then | |
fc7e1e45 | 521 | echo "find $outputDir -name $qaOutputFileName > $qaFilesToMerge" |
522 | find $outputDir -name $qaOutputFileName > $qaFilesToMerge | |
523 | fi | |
524 | ||
525 | echo aliroot -l -b -q "merge.C(\"$qaFilesToMerge\",\"\",kFALSE,\"$qaMergedOutputFileName\")" | |
526 | if [[ -n $pretend ]]; then | |
527 | touch $qaMergedOutputFileName | |
528 | touch merge.log | |
529 | else | |
530 | aliroot -l -b -q "merge.C(\"$qaFilesToMerge\",\"\",kFALSE,\"$qaMergedOutputFileName\")" | |
531 | fi | |
532 | ||
5ed7c82c | 533 | ls -ltrh |
534 | ||
535 | #copy all to output dir | |
536 | cp --recursive ${runpath}/* ${outputDir} | |
537 | ||
fc7e1e45 | 538 | #validate merge cpass1 |
5ed7c82c | 539 | cd ${outputDir} |
fc7e1e45 | 540 | calibDoneFile="${commonOutputPath}/merge.cpass1.run${runNumber}.done" |
541 | touch ${calibDoneFile} | |
542 | [[ -f CalibObjects.root ]] && echo "calibfile $outputDir/CalibObjects.root" > ${calibDoneFile} | |
543 | [[ -f $qaMergedOutputFileName ]] && echo "qafile $outputDir/$qaMergedOutputFileName" >> ${calibDoneFile} | |
544 | summarizeLogs >> ${calibDoneFile} | |
545 | ||
fc7e1e45 | 546 | rm -rf ${runpath} |
547 | } | |
548 | ||
5dfac970 | 549 | goMakeSummary() |
fc7e1e45 | 550 | { |
5dfac970 | 551 | configFile=$1 |
fc7e1e45 | 552 | source $configFile |
5dfac970 | 553 | |
554 | # log filtering, script needs to take the base dir as argument | |
555 | if [[ -x $logFilteringScript ]]; then | |
556 | commonOutputPath=${baseOutputDirectory}/${productionID} | |
557 | ${logFilteringScript} $commonOutputPath | |
fc7e1e45 | 558 | fi |
559 | ||
5dfac970 | 560 | awk 'BEGIN {nFiles=0;} /^calibfile/ {nFiles++;} END {print "cpass0 produced "nFiles" calib files";}' cpass0.job*done |
561 | awk 'BEGIN {nOK=0; nBAD=0;} /rec.*log OK/ {nOK++;} /rec.*log BAD/ {nBAD++;} END {print "cpass0 reco: OK: "nOK" BAD: "nBAD;}' cpass0.job*done | |
562 | awk 'BEGIN {nOK=0; nBAD=0;} /calib.*log OK/ {nOK++;} /calib.*log BAD/ {nBAD++;} END {print "cpass0 calib: OK: "nOK" BAD: "nBAD;}' cpass0.job*done | |
563 | ||
564 | awk 'BEGIN {nOK=0; nBAD=0;} /merge.*log OK/ {nOK++;} /merge.*log BAD/ {nBAD++;} END {print "cpass0 merge: OK: "nOK" BAD: "nBAD;}' merge.cpass0*done | |
565 | awk 'BEGIN {nOK=0; nBAD=0;} /ocdb.*log OK/ {nOK++;} /ocdb.*log BAD/ {nBAD++;} END {print "cpass0 OCDB: OK: "nOK" BAD: "nBAD;}' merge.cpass0*done | |
566 | ||
567 | awk 'BEGIN {nFiles=0;} /^calibfile/ {nFiles++;} END {print "cpass1 produced "nFiles" calib files";}' cpass1.job*done | |
568 | awk 'BEGIN {nOK=0; nBAD=0;} /rec.*log OK/ {nOK++;} /rec.*log BAD/ {nBAD++;} END {print "cpass1 reco: OK: "nOK" BAD: "nBAD;}' cpass1.job*done | |
569 | awk 'BEGIN {nOK=0; nBAD=0;} /calib.*log OK/ {nOK++;} /calib.*log BAD/ {nBAD++;} END {print "cpass1 calib: OK: "nOK" BAD: "nBAD;}' cpass1.job*done | |
fc7e1e45 | 570 | |
5dfac970 | 571 | awk 'BEGIN {nOK=0; nBAD=0;} /merge.*log OK/ {nOK++;} /merge.*log BAD/ {nBAD++;} END {print "cpass1 merge: OK: "nOK" BAD: "nBAD;}' merge.cpass1*done |
572 | awk 'BEGIN {nOK=0; nBAD=0;} /ocdb.*log OK/ {nOK++;} /ocdb.*log BAD/ {nBAD++;} END {print "cpass1 OCDB: OK: "nOK" BAD: "nBAD;}' merge.cpass1*done | |
573 | ||
574 | #if set email the summary | |
575 | [[ -n $mailSummaryTo ]] && cat $log | mail -s "benchmark $productionID done" $mailSummaryTo | |
fc7e1e45 | 576 | |
5dfac970 | 577 | return 0 |
578 | } | |
fc7e1e45 | 579 | |
5dfac970 | 580 | goMakeflow() |
581 | { | |
582 | #generate the makeflow file and run | |
583 | inputFileList=$1 | |
584 | productionID=$2 | |
585 | configFile=$3 | |
586 | runNumber=$4 | |
fc7e1e45 | 587 | |
5dfac970 | 588 | [[ -z ${configFile} ]] && configFile="benchmark.config" |
589 | [[ ! -f ${configFile} ]] && echo "no config file found (${configFile})" && return 1 | |
590 | source $configFile | |
fc7e1e45 | 591 | |
5dfac970 | 592 | source $configFile |
593 | goGenerateMakeflow "$@" > benchmark.makeflow | |
594 | makeflow ${makeflowOptions} benchmark.makeflow | |
595 | } | |
fc7e1e45 | 596 | |
5dfac970 | 597 | goGenerateMakeflow() |
598 | { | |
599 | #generate the makeflow file | |
600 | inputFileList=$1 | |
601 | productionID=$2 | |
602 | configFile=$3 | |
603 | runNumber=$4 | |
fc7e1e45 | 604 | |
5dfac970 | 605 | [[ -z ${configFile} ]] && configFile="benchmark.config" |
606 | [[ ! -f ${configFile} ]] && echo "no config file found (${configFile})" && return 1 | |
607 | source $configFile | |
fc7e1e45 | 608 | |
5dfac970 | 609 | commonOutputPath=${baseOutputDirectory}/${productionID} |
fc7e1e45 | 610 | |
5dfac970 | 611 | #these files will be made a dependency - will be copied to the working dir of the jobs |
612 | declare -a copyFiles | |
613 | inputFiles=( | |
614 | "OCDB.root" | |
615 | "localOCDBaccessConfig.C" | |
616 | ) | |
617 | for file in ${inputFiles[*]}; do | |
618 | [[ -f ${file} ]] && copyFiles+=("${file}") | |
619 | done | |
620 | ||
621 | #create the makeflow file | |
622 | declare -a arr_cpass1_final | |
623 | declare -a arr_cpass1_QA_final | |
624 | listOfRuns=${runNumber} | |
625 | [[ -z ${runNumber} ]] && listOfRuns=($(while read x; do guessRunNumber $x; done < ${inputFileList} | sort | uniq)) | |
626 | runindex=0 | |
fc7e1e45 | 627 | for runNumber in ${listOfRuns[*]}; do |
628 | [[ -z $runNumber ]] && continue | |
629 | [[ ! ${runNumber} =~ ^[0-9]*[0-9]$ ]] && continue | |
5dfac970 | 630 | jobindex=0 |
fc7e1e45 | 631 | |
5dfac970 | 632 | unset arr_cpass0_outputs |
633 | unset arr_cpass1_outputs | |
634 | declare -a arr_cpass0_outputs | |
635 | declare -a arr_cpass1_outputs | |
636 | unset arr_cpass0_outputs | |
fc7e1e45 | 637 | |
5dfac970 | 638 | while read inputFile; do |
639 | currentDefaultOCDB=${defaultOCDB} | |
640 | [[ ${autoOCDB} -ne 0 ]] && currentDefaultOCDB=$(setYear $inputFile $defaultOCDB) | |
fc7e1e45 | 641 | |
5dfac970 | 642 | #CPass0 |
643 | arr_cpass0_outputs[$jobindex]="cpass0.job${jobindex}.run${runNumber}.done" | |
644 | echo "${arr_cpass0_outputs[$jobindex]}: benchmark.sh ${configFile} ${copyFiles[@]}" | |
645 | echo " ${alirootEnv} ./benchmark.sh CPass0 ${commonOutputPath}/cpass0/000${runNumber} $inputFile $nEvents $currentDefaultOCDB $configFile $runNumber $jobindex" | |
fc7e1e45 | 646 | echo |
5dfac970 | 647 | |
648 | #CPass1 | |
649 | arr_cpass1_outputs[$jobindex]="cpass1.job${jobindex}.run${runNumber}.done" | |
650 | echo "${arr_cpass1_outputs[$jobindex]} : benchmark.sh ${configFile} cpass0.localOCDB.${runNumber}.tgz ${copyFiles[@]}" | |
651 | echo " ${alirootEnv} ./benchmark.sh CPass1 ${commonOutputPath}/cpass1/000${runNumber} $inputFile $nEvents $currentDefaultOCDB $configFile $runNumber $jobindex" | |
fc7e1e45 | 652 | echo |
5dfac970 | 653 | ((jobindex++)) |
fc7e1e45 | 654 | |
5dfac970 | 655 | done< <(grep "/000$runNumber/" $inputFileList) |
fc7e1e45 | 656 | |
5dfac970 | 657 | #CPass0 list of Calib files to merge |
658 | echo "cpass0.calib.run${runNumber}.list: ${arr_cpass0_outputs[*]}" | |
659 | echo " awk '/^calibfile / {print "'\$2'"}' ${arr_cpass0_outputs[*]} > cpass0.calib.run${runNumber}.list" | |
660 | echo | |
fc7e1e45 | 661 | |
5dfac970 | 662 | #CPass1 list of Calib/QA files to merge |
663 | echo "cpass1.calib.run${runNumber}.list cpass1.QA.run${runNumber}.list: ${arr_cpass1_outputs[*]}" | |
664 | echo " awk '/^calibfile / {print "'\$'"2}' ${arr_cpass1_outputs[*]} > cpass1.calib.run${runNumber}.list; awk '/^qafile / {print "'\$'"2}' ${arr_cpass1_outputs[*]} > cpass1.QA.run${runNumber}.list" | |
665 | echo | |
fc7e1e45 | 666 | |
5dfac970 | 667 | #CPass0 merging |
668 | arr_cpass0_final[$runindex]="merge.cpass0.run${runNumber}.done" | |
669 | echo "cpass0.localOCDB.${runNumber}.tgz ${arr_cpass0_final[$runindex]}: cpass0.calib.run${runNumber}.list benchmark.sh ${configFile} ${copyFiles[@]}" | |
670 | echo " ${alirootEnv} ./benchmark.sh MergeCPass0 ${commonOutputPath}/cpass0/000${runNumber} $currentDefaultOCDB ${configFile} $runNumber cpass0.run${runNumber}.list" | |
671 | echo | |
fc7e1e45 | 672 | |
5dfac970 | 673 | #CPass1 Calib/QA merging |
674 | arr_cpass1_final[$runindex]="merge.cpass1.run${runNumber}.done" | |
675 | echo "${arr_cpass1_final[$runindex]}: cpass1.calib.run${runNumber}.list cpass1.QA.run${runNumber}.list benchmark.sh ${configFile} ${copyFiles[@]}" | |
676 | echo " ${alirootEnv} ./benchmark.sh MergeCPass1 ${commonOutputPath}/cpass1/000${runNumber} $currentDefaultOCDB ${configFile} $runNumber cpass1.calib.run${runNumber}.list cpass1.QA.run${runNumber}.list" | |
677 | echo | |
678 | ((runindex++)) | |
679 | done | |
fc7e1e45 | 680 | |
5dfac970 | 681 | #CPass1 list of final Calib/QA files |
682 | echo "cpass1.QA.list cpass1.calib.list: ${arr_cpass1_final[*]}" | |
683 | echo " awk '/^calibfile / {print "'\$'"2}' ${arr_cpass1_final[*]} > cpass1.calib.list; awk '/^qafile / {print "'\$'"2}' ${arr_cpass1_final[*]} > cpass1.QA.list" | |
684 | echo | |
fc7e1e45 | 685 | |
5dfac970 | 686 | #Summary |
687 | echo "summary.log : ${arr_cpass0_outputs[*]} ${arr_cpass1_outputs[*]} ${arr_cpass1_final[*]} ${arr_cpass0_final[*]} benchmark.sh ${configFile}" | |
688 | echo " LOCAL ./benchmark.sh makeSummary ${configFile} |tee summary.log" | |
689 | } | |
fc7e1e45 | 690 | |
5dfac970 | 691 | goCreateQAplots() |
692 | { | |
693 | umask 0002 | |
694 | [[ $# -lt 5 ]] && echo "goCreateQAplots productionID pass outputDir qaFilesDirectory qaPlotScript" && exit 1 | |
695 | productionID=$1 | |
696 | pass=$2 | |
697 | outputDir=$3 | |
698 | qaFilesDirectory=$4 | |
699 | qaPlotsScript=$5 | |
700 | configFile=$6 | |
fc7e1e45 | 701 | |
5dfac970 | 702 | source $configFile |
703 | [[ -f ${setupAliROOTenvInCurrentShell} ]] && source $setupAliROOTenvInCurrentShell | |
fc7e1e45 | 704 | |
5dfac970 | 705 | runpath=${PWD}/rundir_cpass0_Merge_${runNumber} |
706 | [[ -z $commonOutputPath ]] && commonOutputPath=$PWD | |
707 | [[ $reconstructInTemporaryDir -eq 1 && -n $TMPDIR ]] && runpath=$TMPDIR | |
708 | [[ $reconstructInTemporaryDir -eq 1 && -z $TMPDIR ]] && runpath=$(mktemp -d) | |
fc7e1e45 | 709 | |
5dfac970 | 710 | mkdir -p $runpath |
711 | [[ ! -d $runpath ]] && echo "not able to make the runpath $runpath" && exit 1 | |
712 | cd $runpath | |
fc7e1e45 | 713 | |
5dfac970 | 714 | [[ -z $logOutputDir ]] && logOutputDir=$runpath |
715 | [[ -z $dontRedirectStdOutToLog ]] && exec 2>&1 > $logOutputDir/makeQAplots.log | |
716 | echo "$0 $*" | |
fc7e1e45 | 717 | |
5dfac970 | 718 | [[ -z "$qaPlotsScript" ]] && echo "qaPlotsScript not defined"&&exit 1 |
719 | ||
720 | mergedQAfileList=$outputDir/mergedQAfiles.list | |
721 | echo "MakeListOfQAresults $qaFilesDirectory QAresults.root | grep CPass1 > $mergedQAfileList" | |
722 | MakeListOfQAresults $qaFilesDirectory QAresults_merged.root | grep CPass1 |tee $mergedQAfileList | |
723 | echo $qaPlotsScript "$productionID" "cpass1" $mergedQAfileList $outputDir | |
724 | $qaPlotsScript "$productionID" "cpass1" $mergedQAfileList $outputDir | |
fc7e1e45 | 725 | |
5dfac970 | 726 | mkdir -p $outputDir |
727 | [[ ! -d $outputDir ]] && echo "cannot make the output dir $outputDir" && exit 1 | |
728 | mv -f $runpath/* $outputDir | |
729 | rm -rf $runpath | |
730 | } | |
fc7e1e45 | 731 | |
5dfac970 | 732 | goWaitForOutput() |
733 | { | |
734 | umask 0002 | |
735 | [[ $# -lt 3 ]] && echo "goWaitForOutput() wrong number of arguments, exiting.." && exit 1 | |
736 | echo searchPath=$1 | |
737 | echo fileName=$2 | |
738 | echo numberOfFiles=$3 | |
739 | echo maxSecondsToWait=$4 | |
740 | searchPath=$1 | |
741 | fileName=$2 | |
742 | numberOfFiles=$3 | |
743 | maxSecondsToWait=$4 | |
744 | extraFindOptions=$5 | |
745 | echo "command to be executed: find $searchPath -name "$fileName" ${extraFindOptions}" | |
746 | [[ -z "$maxSecondsToWait" ]] && maxSecondsToWait=$(( 3600*12 )) | |
747 | while sleep 60; do | |
748 | n=$(find $searchPath -name "$fileName" ${extraFindOptions}| wc -l) | |
749 | [[ $n -gt 0 ]] && echo "found $n X $fileName" | |
750 | [[ $n -ge $numberOfFiles ]] && break | |
751 | [[ $SECONDS -gt $maxSecondsToWait ]] && break | |
752 | done | |
753 | echo "DONE! exiting..." | |
754 | } | |
fc7e1e45 | 755 | |
5dfac970 | 756 | submit() |
757 | { | |
758 | umask 0002 | |
759 | [[ $# -ne 6 ]] && echo "6 args needed, you supplied $#" && exit 1 | |
760 | JobID=$1 | |
761 | startID=$2 | |
762 | endID=$3 | |
763 | waitForJOBID=$4 | |
764 | command=$5 | |
765 | commandArgs=$6 | |
fc7e1e45 | 766 | |
5dfac970 | 767 | newFarm=$(which qsub|grep "^/usr/bin/qsub") |
768 | ||
769 | batchSystem="SGE" | |
fc7e1e45 | 770 | |
5dfac970 | 771 | if [[ -z "$newFarm" ]] |
772 | then | |
773 | #old LSF | |
774 | # submit it (as job array) | |
775 | nFiles=$(( $endID-$startID+1 )) | |
776 | while [ $startID -le $nFiles ] ; do | |
777 | if [ `expr $nFiles - $startID` -gt 999 ] ; then | |
778 | endID=`expr $startID + 999` | |
779 | else | |
780 | endID=$nFiles | |
781 | fi | |
782 | if [[ -z "$waitForJOBID" ]]; then | |
783 | echo $batchCommand -J "$JobID[$startID-$endID]" -e "$targetDirectory/logs/job_%I.err" -o "$targetDirectory/logs/job_%I.out" "$command" | |
784 | $batchCommand -J "$JobID[$startID-$endID]" -e "$targetDirectory/logs/job_%I.err" -o "$targetDirectory/logs/job_%I.out" "$command" | |
785 | else | |
786 | echo $batchCommand -J "$JobID[$startID-$endID]" -w "ended($waitForJOBID)" -e "$targetDirectory/logs/job_%I.err" -o "$targetDirectory/logs/job_%I.out" "$command" | |
787 | $batchCommand -J "$JobID[$startID-$endID]" -w "ended($waitForJOBID)" -e "$targetDirectory/logs/job_%I.err" -o "$targetDirectory/logs/job_%I.out" "$command" | |
788 | fi | |
789 | startID=`expr $endID + 1` | |
790 | done | |
791 | else | |
792 | #new SGE farm | |
793 | if [[ -z "$waitForJOBID" ]]; then | |
794 | echo $batchCommand -wd ${targetDirectory} -b y -V -N "$JobID" -t "$startID-$endID" -e "$targetDirectory/logs/" -o "$targetDirectory/logs/" "$command" $commandArgs | |
795 | $batchCommand -wd ${targetDirectory} -b y -V -N "$JobID" -t "$startID-$endID" -e "$targetDirectory/logs/" -o "$targetDirectory/logs/" "$command" $commandArgs | |
796 | else | |
797 | echo $batchCommand -wd ${targetDirectory} -b y -V -N "$JobID" -t "$startID-$endID" -hold_jid "$waitForJOBID" -e "$targetDirectory/logs/" -o "$targetDirectory/logs/" "$command" $commandArgs | |
798 | $batchCommand -wd ${targetDirectory} -b y -V -N "$JobID" -t "$startID-$endID" -hold_jid "$waitForJOBID" -e "$targetDirectory/logs/" -o "$targetDirectory/logs/" "$command" $commandArgs | |
799 | fi | |
800 | fi | |
801 | } | |
fc7e1e45 | 802 | |
5dfac970 | 803 | goTest() |
804 | { | |
805 | umask 0002 | |
806 | exec 2>&1 | |
807 | exec > >(tee test.log) | |
808 | echo "$@" | |
809 | echo something | |
810 | } | |
fc7e1e45 | 811 | |
5dfac970 | 812 | alirootInfo() |
813 | { | |
814 | ( | |
815 | umask 0002 | |
816 | # save aliroot svn info | |
817 | [[ -z "$ALICE_ROOT" ]] && exit 1 | |
818 | prevdir=$PWD | |
819 | cd $ALICE_ROOT | |
820 | echo "\$ALICE_ROOT=$ALICE_ROOT" | |
821 | echo | |
822 | svn info 2>/dev/null | |
823 | echo "" | |
824 | echo "" | |
825 | svn diff 2>/dev/null | |
826 | cd $prevdir | |
827 | ) | |
828 | } | |
fc7e1e45 | 829 | |
5dfac970 | 830 | MakeListOfQAresults() |
831 | { | |
832 | ( | |
833 | umask 0002 | |
834 | if [[ $# -eq 0 ]]; then | |
835 | echo "make an input file list for the qa script" | |
836 | echo "Usage: $0 path filename" | |
837 | return 0 | |
838 | fi | |
fc7e1e45 | 839 | |
5dfac970 | 840 | path=$1 |
841 | fileName=$2 | |
842 | ||
843 | while read entry; do | |
844 | runNumber=$(guessRunNumber $entry) | |
845 | echo "$runNumber 0 1 10 $entry" | |
846 | done < <(find $path -name $fileName) | |
847 | ) | |
848 | } | |
fc7e1e45 | 849 | |
5dfac970 | 850 | setYear() |
851 | { | |
852 | #set the year | |
853 | # $1 - year to be set | |
854 | # $2 - where to set the year | |
855 | year1=$(guessYear $1) | |
856 | year2=$(guessYear $2) | |
857 | local path=$2 | |
858 | [[ $year1 -ne $year2 && -n $year2 ]] && path=${2/\/$year2\//\/$year1\/} | |
859 | echo $path | |
860 | } | |
fc7e1e45 | 861 | |
5dfac970 | 862 | guessPeriod() |
863 | { | |
864 | #guess the period from the path, pick the rightmost one | |
865 | local IFS="/" | |
866 | declare -a path=( $1 ) | |
867 | local dirDepth=${#path[*]} | |
868 | for ((x=${dirDepth}-1;x>=0;x--)); do | |
869 | local field=${path[${x}]} | |
870 | [[ ${field} =~ ^LHC[0-9][0-9][a-z]$ ]] && period=${field#000} && break | |
871 | done | |
872 | echo $period | |
873 | } | |
fc7e1e45 | 874 | |
5dfac970 | 875 | guessYear() |
876 | { | |
877 | #guess the year from the path, pick the rightmost one | |
878 | local IFS="/" | |
879 | declare -a path=( $1 ) | |
880 | local dirDepth=${#path[*]} | |
881 | for ((x=${dirDepth}-1;x>=0;x--)); do | |
882 | local field=${path[${x}]} | |
883 | [[ ${field} =~ ^20[0-9][0-9]$ ]] && year=${field#000} && break | |
884 | done | |
885 | echo $year | |
886 | } | |
fc7e1e45 | 887 | |
5dfac970 | 888 | guessRunNumber() |
889 | { | |
890 | #guess the run number from the path, pick the rightmost one | |
891 | local IFS="/" | |
892 | declare -a path=( $1 ) | |
893 | local dirDepth=${#path[*]} | |
894 | for ((x=${dirDepth}-1;x>=0;x--)); do | |
895 | local field=${path[${x}]} | |
896 | [[ ${field} =~ ^000[0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && runNumber=${field#000} && break | |
897 | done | |
898 | echo $runNumber | |
899 | } | |
fc7e1e45 | 900 | |
5dfac970 | 901 | summarizeLogs() |
902 | { | |
903 | #print a summary of logs | |
904 | logFiles=( | |
905 | "*.log" | |
906 | ) | |
fc7e1e45 | 907 | |
5dfac970 | 908 | errorConditions=( |
909 | "There was a crash" | |
910 | "floating" | |
911 | "error while loading shared libraries" | |
912 | "std::bad_alloc" | |
913 | "s_err_syswatch_" | |
914 | "Thread [0-9]* (Thread" | |
915 | ) | |
916 | logstatus=0 | |
917 | for log in ${logFiles[*]}; do | |
918 | finallog=${outputDir%/}/${log} | |
919 | [[ ! -f $log ]] && continue | |
920 | errorSummary="" | |
921 | for ((i=0; i<${#errorConditions[@]};i++)); do | |
922 | local tmp=$(grep -m1 -e "${errorConditions[$i]}" $log) | |
923 | [[ -n $tmp ]] && tmp+=" : " | |
924 | errorSummary+=$tmp | |
925 | done | |
926 | if [[ -z $errorSummary ]]; then | |
927 | #in pretend mode randomly report an error in rec.log some cases | |
928 | if [[ -n $pretend && "$log" == "rec.log" ]]; then | |
929 | [[ $(( $RANDOM%2 )) -ge 1 ]] && echo "$finallog BAD random error" || echo "$finallog OK" | |
fc7e1e45 | 930 | else |
5dfac970 | 931 | echo "$finallog OK" |
fc7e1e45 | 932 | fi |
5dfac970 | 933 | else |
934 | local logstatus=1 | |
935 | echo "$finallog BAD $errorSummary" | |
936 | fi | |
937 | done | |
938 | return $logstatus | |
939 | } | |
fc7e1e45 | 940 | |
5dfac970 | 941 | spitOutLocalOCDBaccessConfig() |
942 | { | |
943 | umask 0002 | |
944 | find $1 -name "*root" | \ | |
945 | while read line | |
946 | do | |
947 | local tmp=${line#$1} | |
948 | echo ${tmp%/*} | \ | |
949 | awk -v ocdb=$1 '{print " man->SetSpecificStorage(\""$1"\",\"local://"ocdb"\");"}' | |
950 | done | |
951 | } | |
fc7e1e45 | 952 | |
5dfac970 | 953 | goConfigureOCDBforCPass1() |
954 | { | |
955 | umask 0002 | |
956 | # make a script that sets the specific storages form all the root files produced by CPass0 | |
957 | # in this case the second argument is taken to be the path to the produced OCDB for specific storage | |
958 | local localOCDBpathCPass0=$1 | |
959 | local outputDir=$2 | |
960 | [[ -d $outputDir ]] && cd $outputDir | |
fc7e1e45 | 961 | |
5dfac970 | 962 | if [[ -f $localOCDBpathCPass0 && $localOCDBpathCPass0 =~ \.tgz$ ]]; then |
963 | tar xzf $localOCDBpathCPass0 | |
964 | local localOCDBpathCPass0="./OCDB" | |
965 | fi | |
fc7e1e45 | 966 | |
5dfac970 | 967 | echo |
968 | echo creating the specific storage script | |
969 | echo localOCDBaccessConfig.C | |
970 | echo based on OCDB: $fileName | |
971 | echo | |
fc7e1e45 | 972 | |
5dfac970 | 973 | local tempLocalOCDB="" |
974 | if [[ -f localOCDBaccessConfig.C ]]; then | |
975 | tempLocalOCDB=$(mktemp) | |
976 | echo "egrep "SetSpecificStorage" localOCDBaccessConfig.C > $tempLocalOCDB" | |
977 | egrep "SetSpecificStorage" localOCDBaccessConfig.C > $tempLocalOCDB | |
978 | fi | |
fc7e1e45 | 979 | |
5dfac970 | 980 | echo "localOCDBaccessConfig()" > localOCDBaccessConfig.C |
981 | echo "{" >> localOCDBaccessConfig.C | |
982 | echo " AliCDBManager* man = AliCDBManager::Instance();" >> localOCDBaccessConfig.C | |
983 | spitOutLocalOCDBaccessConfig $localOCDBpathCPass0|sort|uniq >> localOCDBaccessConfig.C | |
984 | [[ -f "$tempLocalOCDB" ]] && cat $tempLocalOCDB >> localOCDBaccessConfig.C | |
985 | echo "}" >> localOCDBaccessConfig.C | |
fc7e1e45 | 986 | |
5dfac970 | 987 | [[ -f "$tempLocalOCDB" ]] && rm -f $tempLocalOCDB |
fc7e1e45 | 988 | |
5dfac970 | 989 | if ! grep SetSpecificStorage localOCDBaccessConfig.C; then |
990 | echo | |
991 | echo "!!!!!!! CPass0 produced no OCDB entries" | |
992 | return 1 | |
993 | fi | |
994 | } | |
fc7e1e45 | 995 | |
5dfac970 | 996 | goSubmit() |
997 | { | |
998 | inputList=$1 | |
999 | productionID=$2 | |
1000 | configFile="benchmark.config" | |
1001 | [[ -n "$3" ]] && configFile=$3 | |
1002 | configFile=$(readlink -f $configFile) | |
1003 | [[ -n $4 ]] && runNumber=$4 | |
fc7e1e45 | 1004 | |
5dfac970 | 1005 | #redirect all output to submit.log |
1006 | echo "redirecting all output to ${PWD}/submit_${productionID//"/"/_}.log" | |
1007 | exec 7>&1 | |
1008 | exec 1>submit_${productionID//"/"/_}.log 2>&1 | |
fc7e1e45 | 1009 | |
5dfac970 | 1010 | umask 0002 |
1011 | echo $0" submit $*" | |
1012 | if [[ -z "$inputList" || -z "$productionID" ]] | |
1013 | then | |
fc7e1e45 | 1014 | echo |
5dfac970 | 1015 | echo " Usage: $0 submit inputList productionID [configFile=benchmark.config]" |
fc7e1e45 | 1016 | echo |
5dfac970 | 1017 | exit |
fc7e1e45 | 1018 | fi |
fc7e1e45 | 1019 | |
5dfac970 | 1020 | date=`date +%Y-%m-%d_%H%M%S` |
fc7e1e45 | 1021 | |
5dfac970 | 1022 | # check if config file is there |
1023 | if [ ! -f $configFile ]; then | |
1024 | echo "ERROR! Config File '$configFile' not found" >&2 | |
1025 | exit | |
1026 | else | |
1027 | echo "Using Config File: '$configFile'" | |
fc7e1e45 | 1028 | fi |
1029 | ||
5dfac970 | 1030 | # source the config file |
1031 | # and print the configuration | |
fc7e1e45 | 1032 | source $configFile |
5dfac970 | 1033 | [[ -f ${setupAliROOTenvInCurrentShell} && -z ${alirootEnv} ]] && source $setupAliROOTenvInCurrentShell |
fc7e1e45 | 1034 | |
5dfac970 | 1035 | self=$(readlink -f "$0") |
1036 | configPath=`dirname $self` | |
1037 | #exporting makes it available in the environment on the nodes - makes the summary output go there | |
1038 | export commonOutputPath=${baseOutputDirectory}/${productionID} | |
fc7e1e45 | 1039 | |
5dfac970 | 1040 | #[[ -z ${ALIROOT_RELEASE} ]] && ALIROOT_RELEASE=${ALICE_ROOT//"/"/"_"} |
1041 | #ALIROOT_RELEASE_mod=${ALIROOT_RELEASE//"/"/"_"} | |
1042 | #ALIROOT_BASEDIR_mod=${ALIROOT_BASEDIR//"/"/"_"} | |
1043 | #productionID=${ALIROOT_BASEDIR_mod}_${ALIROOT_RELEASE_mod}/${productionID} | |
fc7e1e45 | 1044 | |
5dfac970 | 1045 | #convert to absolut pathnames |
1046 | inputList=$(readlink -f "$inputList") | |
1047 | #make list of runs | |
1048 | if [[ -z $runNumber ]]; then | |
1049 | listOfRuns=($(while read x; do guessRunNumber $x; done < ${inputList} | sort | uniq)) | |
1050 | else | |
1051 | listOfRuns=$runNumber | |
1052 | fi | |
fc7e1e45 | 1053 | |
5dfac970 | 1054 | setupAliROOTenvInCurrentShell=$(readlink -f "$setupAliROOTenvInCurrentShell") |
fc7e1e45 | 1055 | |
5dfac970 | 1056 | echo "" |
1057 | echo "### BEGIN CONFIGURATION ###" | |
1058 | echo "" | |
1059 | echo "GENERAL:" | |
1060 | echo "" | |
1061 | echo " productionID: $productionID" | |
1062 | echo " batchCommand: $batchCommand" | |
1063 | echo " setupAliROOTenvInCurrentShell: $setupAliROOTenvInCurrentShell" | |
1064 | echo " ALICE_ROOT: $ALICE_ROOT" | |
1065 | echo " ALIROOT_RELEASE: $ALICE_RELEASE" | |
1066 | echo " inputList: $inputList" | |
1067 | echo " configPath: $configPath" | |
1068 | echo " commonOutputPath: $commonOutputPath" | |
1069 | echo " defaultOCDB: $defaultOCDB" | |
1070 | echo " autoOCDB: $autoOCDB" | |
1071 | echo " recoTriggerOptions: $recoTriggerOptions" | |
1072 | echo " runs:" | |
1073 | echo " ${listOfRuns[*]}" | |
1074 | echo "" | |
1075 | echo "THE TRAIN WILL RUN:" | |
fc7e1e45 | 1076 | |
5dfac970 | 1077 | if [ $runCPass0reco -eq 1 ]; then |
1078 | echo " Pass0 - Recontruction" | |
1079 | fi | |
fc7e1e45 | 1080 | |
5dfac970 | 1081 | if [ $runCPass0MergeMakeOCDB -eq 1 ]; then |
1082 | echo " Pass0 - merging and OCDB export" | |
1083 | fi | |
fc7e1e45 | 1084 | |
5dfac970 | 1085 | if [ $runCPass1reco -eq 1 ]; then |
1086 | echo " Pass1 - Recontruction" | |
1087 | fi | |
1088 | if [ $runCPass1MergeMakeOCDB -eq 1 ]; then | |
1089 | echo " Pass1 - merging and OCDB export" | |
1090 | fi | |
fc7e1e45 | 1091 | |
5dfac970 | 1092 | echo "" |
1093 | echo "LIMITS:" | |
1094 | echo " max. Events/Chunk: $nEvents" | |
1095 | echo " max. Number of Chunks per Run: $nMaxChunks" | |
1096 | echo "" | |
1097 | echo "### END CONFIGURATION ###" | |
1098 | echo "" | |
fc7e1e45 | 1099 | |
36b5de1a | 1100 | |
5dfac970 | 1101 | # check if input file is there |
1102 | if [ ! -f $inputList ]; then | |
1103 | echo "ERROR! Input List '$inputList' not found" >&2 | |
1104 | exit | |
1105 | fi | |
1106 | ||
1107 | # define jobid (for dependent jobs) | |
1108 | JOBID1=p0_${productionID//"/"/_}_${date} | |
1109 | JOBID1wait=w0_${productionID//"/"/_}_${date} | |
1110 | JOBID2=m0_${productionID//"/"/_}_${date} | |
1111 | JOBID2wait=wm0_${productionID//"/"/_}_${date} | |
1112 | JOBID3=op0_${productionID//"/"/_}_${date} | |
1113 | JOBID3wait=wop0_${productionID//"/"/_}_${date} | |
1114 | JOBID4=p1_${productionID//"/"/_}_${date} | |
1115 | JOBID4wait=w1_${productionID//"/"/_}_${date} | |
1116 | JOBID5=m1_${productionID//"/"/_}_${date} | |
1117 | JOBID5wait=wm1_${productionID//"/"/_}_${date} | |
1118 | JOBID6=s1_${productionID//"/"/_}_${date} | |
1119 | JOBID6wait=ws1_${productionID//"/"/_}_${date} | |
1120 | JOBID7=QA_${productionID//"/"/_}_${date} | |
1121 | LASTJOB="" | |
1122 | ||
1123 | #for each run we submit one jobarray: | |
1124 | numberOfSubmittedCPass1MergingJobs=0 | |
fc7e1e45 | 1125 | for runNumber in ${listOfRuns[*]}; do |
1126 | [[ -z $runNumber ]] && continue | |
1127 | [[ ! ${runNumber} =~ ^[0-9]*[0-9]$ ]] && continue | |
5dfac970 | 1128 | oneInputFile=$(egrep -m1 "$runNumber\/" ${inputList}) |
1129 | currentDefaultOCDB=$defaultOCDB | |
1130 | [[ $autoOCDB -ne 0 ]] && currentDefaultOCDB=$(setYear $oneInputFile $defaultOCDB) | |
1131 | echo "submitting run $runNumber with OCDB $currentDefaultOCDB" | |
fc7e1e45 | 1132 | |
5dfac970 | 1133 | ################################################################################ |
1134 | ################################################################################ | |
1135 | # run the CPass0 if requested | |
fc7e1e45 | 1136 | |
5dfac970 | 1137 | if [ $runCPass0reco -eq 1 ]; then |
fc7e1e45 | 1138 | |
fc7e1e45 | 1139 | echo |
5dfac970 | 1140 | echo "starting CPass0... for run $runNumber" |
fc7e1e45 | 1141 | echo |
fc7e1e45 | 1142 | |
5dfac970 | 1143 | # create directory and copy all files that are needed |
1144 | targetDirectory="${commonOutputPath}/000${runNumber}/CPass0" | |
1145 | mkdir -p $targetDirectory | |
1146 | mkdir -p $targetDirectory/logs | |
1147 | ||
1148 | localInputList=$targetDirectory/${inputList##*/} | |
1149 | theScript="$targetDirectory/${self##*/}" | |
1150 | cp -f $self $theScript | |
1151 | chmod u+x $theScript | |
fc7e1e45 | 1152 | |
5dfac970 | 1153 | rm -f $localInputList |
1154 | egrep "\/000$runNumber\/" $inputList >> $localInputList | |
1155 | cp -f $configFile $targetDirectory | |
1156 | [[ -f $configPath/runCPass0.sh ]] && cp -f $configPath/runCPass0.sh $targetDirectory && echo "## using local runCPass0.sh" | |
1157 | [[ -f $configPath/recCPass0.C ]] && cp -f $configPath/recCPass0.C $targetDirectory && echo "## using local recCPass0.C" | |
1158 | [[ -f $configPath/runCalibTrain.C ]] && cp -f $configPath/runCalibTrain.C $targetDirectory && echo "## using local runCalibTrain.C" | |
1159 | [[ -f $configPath/localOCDBaccessConfig.C ]] && cp -f $configPath/localOCDBaccessConfig.C $targetDirectory && echo "## using local localOCDBaccessConfig.C" | |
fc7e1e45 | 1160 | |
5dfac970 | 1161 | [[ -f $configPath/CPass0/runCPass0.sh ]] && cp -f $configPath/CPass0/runCPass0.sh $targetDirectory && echo "## using local runCPass0.sh" |
1162 | [[ -f $configPath/CPass0/recCPass0.C ]] && cp -f $configPath/CPass0/recCPass0.C $targetDirectory && echo "## using local recCPass0.C" | |
1163 | [[ -f $configPath/CPass0/runCalibTrain.C ]] && cp -f $configPath/CPass0/runCalibTrain.C $targetDirectory && echo "## using local runCalibTrain.C" | |
1164 | [[ -f $configPath/CPass0/localOCDBaccessConfig.C ]] && cp -f $configPath/CPass0/localOCDBaccessConfig.C $targetDirectory && echo "## using local localOCDBaccessConfig.C" | |
fc7e1e45 | 1165 | |
5dfac970 | 1166 | echo "... files copied." |
1167 | echo | |
fc7e1e45 | 1168 | |
5dfac970 | 1169 | # limit nFiles to nMaxChunks |
1170 | nFiles=`wc -l < $localInputList` | |
1171 | [[ $nFiles -eq 0 ]] && echo "list contains ZERO files! exiting..." && exit 1 | |
1172 | echo "raw files in list: $nFiles" | |
1173 | if [[ $nMaxChunks -gt 0 && $nMaxChunks -le $nFiles ]]; then | |
1174 | nFiles=$nMaxChunks | |
1175 | fi | |
1176 | echo "raw files to process: $nFiles" | |
1177 | [[ -z "$percentProcessedFilesToContinue" ]] && percentProcessedFilesToContinue=100 | |
1178 | if [[ $percentProcessedFilesToContinue -eq 100 ]]; then | |
1179 | nFilesToWaitFor=$nFiles | |
1180 | else | |
1181 | nFilesToWaitFor=$(( $nFiles-$nFiles/(100/(100-$percentProcessedFilesToContinue)) )) | |
1182 | fi | |
1183 | echo "requested success rate is $percentProcessedFilesToContinue%" | |
1184 | echo "merging will start after $nFilesToWaitFor jobs are done" | |
fc7e1e45 | 1185 | |
5dfac970 | 1186 | submit $JOBID1 1 $nFiles "" "$theScript" "CPass0 $targetDirectory $localInputList $nEvents $currentDefaultOCDB $configFile $runNumber" |
fc7e1e45 | 1187 | |
5dfac970 | 1188 | ## submit a monitoring job that will run until a certain number of jobs are done with reconstruction |
1189 | submit "$JOBID1wait" 1 1 "" "$theScript" "WaitForOutput ${commonOutputPath} 'cpass0.job*.run$runNumber.done' $nFilesToWaitFor $maxSecondsToWait '-maxdepth 1'" | |
1190 | LASTJOB=$JOBID1wait | |
fc7e1e45 | 1191 | |
5dfac970 | 1192 | fi #end running CPass0 |
1193 | ################################################################################ | |
fc7e1e45 | 1194 | |
fc7e1e45 | 1195 | |
5dfac970 | 1196 | ################################################################################ |
1197 | # submit merging of CPass0, depends on the reconstruction | |
fc7e1e45 | 1198 | |
5dfac970 | 1199 | if [ $runCPass0MergeMakeOCDB -eq 1 ]; then |
fc7e1e45 | 1200 | |
5dfac970 | 1201 | echo |
1202 | echo "submit CPass0 merging for run $runNumber" | |
1203 | echo | |
fc7e1e45 | 1204 | |
5dfac970 | 1205 | targetDirectory="${commonOutputPath}/000${runNumber}/CPass0" |
1206 | mkdir -p $targetDirectory | |
fc7e1e45 | 1207 | |
5dfac970 | 1208 | # copy the scripts |
1209 | cp -f $self $targetDirectory | |
1210 | [[ -f $configPath/mergeMakeOCDB.sh ]] && cp -f $configPath/mergeMakeOCDB.sh $targetDirectory && echo "## using local mergeMakeOCDB.sh" | |
1211 | [[ -f $configPath/merge.C ]] && cp -f $configPath/merge.C $targetDirectory && echo "## using local merge.C" | |
1212 | [[ -f $configPath/mergeMakeOCDB.byComponent.sh ]] && cp -f $configPath/mergeMakeOCDB.byComponent.sh $targetDirectory && echo "## using local mergeMakeOCDB.byComponent.sh" | |
1213 | [[ -f $configPath/mergeByComponent.C ]] && cp -f $configPath/mergeByComponent.C $targetDirectory && echo "## using local mergeByComponent.C" | |
1214 | [[ -f $configPath/makeOCDB.C ]] && cp -f $configPath/makeOCDB.C $targetDirectory && echo "## using local makeOCDB.C" | |
fc7e1e45 | 1215 | |
5dfac970 | 1216 | theScript="$targetDirectory/${self##*/}" |
1217 | chmod u+x $theScript | |
fc7e1e45 | 1218 | |
5dfac970 | 1219 | echo submit $JOBID2 1 1 "$LASTJOB" "$theScript" "MergeCPass0 $targetDirectory $currentDefaultOCDB $configFile $runNumber" |
1220 | submit $JOBID2 1 1 "$LASTJOB" "$theScript" "MergeCPass0 $targetDirectory $currentDefaultOCDB $configFile $runNumber" | |
1221 | LASTJOB=$JOBID2 | |
fc7e1e45 | 1222 | |
5dfac970 | 1223 | cd $configPath |
1224 | echo | |
fc7e1e45 | 1225 | fi |
5dfac970 | 1226 | # end of merging CPass0 |
1227 | ################################################################################ | |
fc7e1e45 | 1228 | |
5dfac970 | 1229 | ################################################################################ |
1230 | ################################################################################ | |
1231 | # run the CPass1 if requested | |
fc7e1e45 | 1232 | |
5dfac970 | 1233 | if [ $runCPass1reco -eq 1 ]; then |
fc7e1e45 | 1234 | |
5dfac970 | 1235 | targetDirectory="${commonOutputPath}/000${runNumber}/CPass1" |
fc7e1e45 | 1236 | |
5dfac970 | 1237 | # safety feature: if we are re-running for any reason we want to delete the previous output first. |
1238 | [[ -d $targetDirectory ]] && rm -rf $targetDirectory/* && echo "removed old output at $targetDirectory/*" | |
fc7e1e45 | 1239 | |
5dfac970 | 1240 | ################################################################################ |
1241 | # for the CPass1, based on the OCDB entries produced, we need to create the script | |
1242 | # to set the specific storages to the output of CPass0 | |
1243 | # submit a job that will execute after merging/OCDB export that will do it. | |
fc7e1e45 | 1244 | |
5dfac970 | 1245 | mkdir -p $targetDirectory |
1246 | mkdir -p $targetDirectory/logs | |
1247 | cp -f $self $targetDirectory | |
1248 | theScript="$targetDirectory/${self##*/}" | |
fc7e1e45 | 1249 | |
5dfac970 | 1250 | echo |
1251 | echo submitting the OCDB specific storage config making script for run $runNumber | |
1252 | echo | |
fc7e1e45 | 1253 | |
5dfac970 | 1254 | [[ -f $configPath/localOCDBaccessConfig.C ]] && cp -f $configPath/localOCDBaccessConfig.C $targetDirectory && echo "## using local localOCDBaccessConfig.C" |
fc7e1e45 | 1255 | |
5dfac970 | 1256 | submit $JOBID3 1 1 "$LASTJOB" "$theScript" "ConfOCDB ${commonOutputPath}/000${runNumber}/CPass0/OCDB ${targetDirectory} " |
1257 | LASTJOB=$JOBID3 | |
1258 | ################################################################################ | |
fc7e1e45 | 1259 | |
5dfac970 | 1260 | echo |
1261 | echo "starting CPass1... for run $runNumber" | |
1262 | echo | |
1263 | ||
1264 | # create directory and copy all files that are needed | |
1265 | mkdir -p $targetDirectory | |
1266 | mkdir -p $targetDirectory/logs | |
1267 | ||
1268 | localInputList=$targetDirectory/${inputList##*/} | |
1269 | theScript="$targetDirectory/${self##*/}" | |
1270 | cp -f $self $theScript | |
1271 | chmod u+x $theScript | |
1272 | ||
1273 | rm -f $localInputList | |
1274 | egrep "\/000$runNumber\/" $inputList >> $localInputList | |
1275 | cp -f $configFile $targetDirectory | |
1276 | [[ -f $configPath/runCPass1.sh ]] && cp -f $configPath/runCPass1.sh $targetDirectory && echo "## using local runCPass1.sh" | |
1277 | [[ -f $configPath/recCPass1.C ]] && cp -f $configPath/recCPass1.C $targetDirectory && echo "## using local recCPass1.C" | |
1278 | [[ -f $configPath/recCPass1_OuterDet.C ]] && cp -f $configPath/recCPass1_OuterDet.C $targetDirectory && echo "## using local recCPass1_OuterDet.C" | |
1279 | [[ -f $configPath/runCalibTrain.C ]] && cp -f $configPath/runCalibTrain.C $targetDirectory && echo "## using local runCalibTrain.C" | |
1280 | [[ -f $configPath/QAtrain.C ]] && cp -f $configPath/QAtrain.C $targetDirectory && echo "## using local QAtrain.C" | |
1281 | [[ -f $configPath/QAtrain_duo.C ]] && cp -f $configPath/QAtrain_duo.C $targetDirectory && echo "## using local QAtrain_duo.C" | |
1282 | ||
1283 | [[ -f $configPath/CPass1/runCPass1.sh ]] && cp -f $configPath/CPass1/runCPass1.sh $targetDirectory && echo "## using local runCPass1.sh" | |
1284 | [[ -f $configPath/CPass1/recCPass1.C ]] && cp -f $configPath/CPass1/recCPass1.C $targetDirectory && echo "## using local recCPass1.C" | |
1285 | [[ -f $configPath/CPass1/recCPass1_OuterDet.C ]] && cp -f $configPath/CPass1/recCPass1_OuterDet.C $targetDirectory && echo "## using local recCPass1_OuterDet.C" | |
1286 | [[ -f $configPath/CPass1/runCalibTrain.C ]] && cp -f $configPath/CPass1/runCalibTrain.C $targetDirectory && echo "## using local runCalibTrain.C" | |
1287 | [[ -f $configPath/CPass1/QAtrain.C ]] && cp -f $configPath/CPass1/QAtrain.C $targetDirectory && echo "## using local QAtrain.C" | |
1288 | [[ -f $configPath/CPass1/QAtrain_duo.C ]] && cp -f $configPath/CPass1/QAtrain_duo.C $targetDirectory && echo "## using local QAtrain_duo.C" | |
1289 | ||
1290 | echo "... files copied." | |
1291 | echo | |
1292 | ||
1293 | # limit nFiles to nMaxChunks | |
1294 | nFiles=`wc -l < $localInputList` | |
1295 | [[ $nFiles -eq 0 ]] && echo "list contains ZERO files! exiting..." && exit 1 | |
1296 | echo "raw files in list: $nFiles" | |
1297 | if [[ $nMaxChunks -gt 0 && $nMaxChunks -le $nFiles ]]; then | |
1298 | nFiles=$nMaxChunks | |
1299 | fi | |
1300 | echo "raw files to process: $nFiles" | |
1301 | [[ -z "$percentProcessedFilesToContinue" ]] && percentProcessedFilesToContinue=100 | |
1302 | if [[ $percentProcessedFilesToContinue -eq 100 ]]; then | |
1303 | nFilesToWaitFor=$nFiles | |
fc7e1e45 | 1304 | else |
5dfac970 | 1305 | nFilesToWaitFor=$(( $nFiles-$nFiles/(100/(100-$percentProcessedFilesToContinue)) )) |
fc7e1e45 | 1306 | fi |
5dfac970 | 1307 | echo "requested success rate is $percentProcessedFilesToContinue%" |
1308 | echo "merging will start after $nFilesToWaitFor jobs are done" | |
fc7e1e45 | 1309 | |
5dfac970 | 1310 | submit $JOBID4 1 $nFiles "$LASTJOB" "$theScript" "CPass1 $targetDirectory $localInputList $nEvents $currentDefaultOCDB $configFile $runNumber" |
fc7e1e45 | 1311 | |
5dfac970 | 1312 | ################################################################################ |
1313 | ## submit a monitoring job that will run until a certain number of jobs are done with reconstruction | |
1314 | submit "$JOBID4wait" 1 1 "$LASTJOB" "$theScript" "WaitForOutput ${commonOutputPath} 'cpass1.job*.run${runNumber}.done' $nFilesToWaitFor $maxSecondsToWait '-maxdepth 1'" | |
1315 | LASTJOB=$JOBID4wait | |
1316 | ################################################################################ | |
fc7e1e45 | 1317 | |
5dfac970 | 1318 | echo |
1319 | fi #end running CPass1 | |
fc7e1e45 | 1320 | |
5dfac970 | 1321 | ################################################################################ |
1322 | # submit merging of CPass1, depends on the reconstruction | |
1323 | if [ $runCPass1MergeMakeOCDB -eq 1 ]; then | |
fc7e1e45 | 1324 | |
5dfac970 | 1325 | echo |
1326 | echo "submit CPass1 merging for run $runNumber" | |
1327 | echo | |
fc7e1e45 | 1328 | |
5dfac970 | 1329 | targetDirectory="${commonOutputPath}/000${runNumber}/CPass1" |
1330 | mkdir -p $targetDirectory | |
fc7e1e45 | 1331 | |
5dfac970 | 1332 | # copy the scripts |
1333 | cp -f $self $targetDirectory | |
1334 | [[ -f $configPath/mergeMakeOCDB.sh ]] && cp -f $configPath/mergeMakeOCDB.sh $targetDirectory && echo "## using local mergeMakeOCDB.sh" | |
1335 | [[ -f $configPath/merge.C ]] && cp -f $configPath/merge.C $targetDirectory && echo "## using local merge.C" | |
1336 | [[ -f $configPath/mergeMakeOCDB.byComponent.sh ]] && cp -f $configPath/mergeMakeOCDB.byComponent.sh $targetDirectory && echo "## using local mergeMakeOCDB.byComponent.sh" | |
1337 | [[ -f $configPath/mergeByComponent.C ]] && cp -f $configPath/mergeByComponent.C $targetDirectory && echo "## using local mergeByComponent.C" | |
1338 | [[ -f $configPath/makeOCDB.C ]] && cp -f $configPath/makeOCDB.C $targetDirectory && echo "## using local makeOCDB.C" | |
fc7e1e45 | 1339 | |
5dfac970 | 1340 | theScript="$targetDirectory/${self##*/}" |
1341 | chmod u+x $theScript | |
1342 | ||
1343 | echo submit "$JOBID5" 1 1 "$LASTJOB" "$theScript" "MergeCPass1 $targetDirectory $currentDefaultOCDB $configFile $runNumber" | |
1344 | submit "$JOBID5" 1 1 "$LASTJOB" "$theScript" "MergeCPass1 $targetDirectory $currentDefaultOCDB $configFile $runNumber" | |
1345 | ((numberOfSubmittedCPass1MergingJobs++)) | |
1346 | LASTJOB=$JOBID5 | |
1347 | echo | |
1348 | fi | |
1349 | ||
1350 | done | |
1351 | ||
1352 | ################################################################################ | |
1353 | ################################################################################ | |
1354 | [[ -z $runMakeSummary ]] && runMakeSummary=0 | |
1355 | if [ $runMakeSummary -eq 1 ]; then | |
fc7e1e45 | 1356 | echo |
5dfac970 | 1357 | echo "submit make a summary" |
1358 | echo | |
1359 | ||
1360 | targetDirectory="${commonOutputPath}" | |
1361 | mkdir -p $targetDirectory | |
1362 | mkdir -p $targetDirectory/logs | |
1363 | [[ ! -f $targetDirectory/${self##*/} ]] && cp -f $self $targetDirectory | |
1364 | theScript="$targetDirectory/${self##*/}" | |
1365 | submit "$JOBID6" 1 1 "$LASTJOB" "$theScript" "MakeSummary $targetDirectory $configFile" | |
1366 | fi | |
1367 | ################################################################################ | |
1368 | ||
1369 | ################################################################################ | |
1370 | ################################################################################ | |
1371 | if [ $runMakeQAplots -eq 1 ]; then | |
1372 | echo | |
1373 | echo "submit make QA plots" | |
1374 | echo | |
1375 | ||
1376 | ## submit a monitoring job that will wait for a specified number of files to appear somewhere | |
1377 | targetDirectory="${commonOutputPath}" | |
1378 | [[ ! -f $targetDirectory/${self##*/} ]] && cp -f $self $targetDirectory | |
1379 | theScript="$targetDirectory/${self##*/}" | |
1380 | echo submit "$JOBID5wait 1 1 $theScript WaitForOutput ${commonOutputPath} 'merge.cpass1.run*.done' $numberOfSubmittedCPass1MergingJobs $maxSecondsToWait '-maxdepth 1'" | |
1381 | submit "$JOBID5wait" 1 1 "" "$theScript" "WaitForOutput ${commonOutputPath} 'merge.cpass1.run*.done' $numberOfSubmittedCPass1MergingJobs $maxSecondsToWait '-maxdepth 1'" | |
1382 | LASTJOB=$JOBID5wait | |
1383 | ||
1384 | targetDirectory="${commonOutputPath}/QAplots/" | |
1385 | [[ -d $targetDirectory ]] && rm -rf $targetDirectory/* | |
1386 | mkdir -p $targetDirectory | |
1387 | mkdir -p $targetDirectory/logs | |
1388 | qaFilesDirectory="${commonOutputPath}" | |
1389 | [[ ! -f $targetDirectory/${self##*/} ]] && cp -f $self $targetDirectory | |
1390 | theScript="$targetDirectory/${self##*/}" | |
1391 | ||
1392 | echo submit "$JOBID7" 1 1 "$LASTJOB" "$theScript" "CreateQAplots QAplots CPass1 $targetDirectory $qaFilesDirectory $qaPlotsScript $configFile" | |
1393 | submit "$JOBID7" 1 1 "$LASTJOB" "$theScript" "CreateQAplots QAplots CPass1 $targetDirectory $qaFilesDirectory $qaPlotsScript $configFile" | |
1394 | LASTJOB=$JOBID7 | |
fc7e1e45 | 1395 | fi |
5dfac970 | 1396 | |
1397 | #restore stdout | |
1398 | exec 1>&7 7>&- | |
1399 | echo "jobs submitted." | |
fc7e1e45 | 1400 | } |
1401 | ||
1402 | main "$@" |