]>
Commit | Line | Data |
---|---|---|
f8b7a926 | 1 | #!/bin/bash |
2 | version=5 | |
3 | tag= | |
4 | id= | |
5 | run= | |
6 | stage=0 | |
7 | upload=0 | |
8 | jobs=2 | |
9 | events=1 | |
f8b7a926 | 10 | aliroot="v5-04-Rev-20" |
11 | root="" | |
12 | geant="" | |
13 | minmerge=30 | |
14 | noact=0 | |
15 | ||
16 | # --- Display help message ------------------------------------------- | |
17 | usage() | |
18 | { | |
19 | cat <<EOF | |
20 | Usage: $0 [OPTIONS] | |
21 | ||
22 | Options: | |
23 | -h,--help This help | |
24 | -t|--tag TAG Job tag [****] ($tag) | |
25 | -i|--id NAME Name of production ($id) | |
26 | -R|--run RUN_NO Run number ($run) | |
f8b7a926 | 27 | -c|--copy Copy files to AliEn |
28 | -n|--jobs JOBS Set number of jobs[**] ($jobs) | |
29 | -m|--events EVENTS Set events/job[**] ($events) | |
30 | -s|--stage STAGE Set the stage[***] ($stage) | |
31 | -o|--output DIR Set base output directory ($aout) | |
32 | -d|--data DIR Set data directory ($adir) | |
33 | -b|--bin DIR Set base bin directory ($abin) | |
34 | -a|--aliroot RELEASE Set AliROOT release [*] ($aliroot) | |
35 | -r|--root RELEASE Set ROOT release [*] ($root) | |
36 | -g|--geant RELEASE Set GEANT3 release [*] ($geant) | |
37 | -f|--final NUMBER Run final merging when down to this ($minmerge) | |
38 | ||
39 | [*] Only make sense with option -c | |
40 | [**] Only make sense for stage 0 | |
41 | [***] If stage is set to 6, try to deduce the stage automatically | |
42 | [****] TAG is a short hand for specific id and run | |
43 | EOF | |
44 | } | |
45 | ||
46 | # --- Process a return value ----------------------------------------- | |
47 | log_msg() | |
48 | { | |
49 | local log=$1 ; shift | |
50 | echo -en "$@\e[0m ... " | |
51 | if test "x$log" != "x" ; then | |
52 | echo "=== $@" >> $log | |
53 | fi | |
54 | } | |
55 | # --- Make error ----------------------------------------------------- | |
56 | log_err() | |
57 | { | |
58 | local pre=$1 | |
59 | local post=$2 | |
60 | echo -e "\e[31;1mError\e[0m: ${pre} \e[35m${post}\e[0m" > /dev/stderr | |
61 | } | |
62 | ||
63 | # --- Process a return value ----------------------------------------- | |
64 | log_end() | |
65 | { | |
66 | local log=$1 | |
67 | local ret=$2 | |
68 | local ext=$3 | |
69 | local msg="" | |
70 | local fmt="" | |
71 | if test $ret -eq 0 ; then | |
72 | msg="success" | |
73 | fmt="32" | |
74 | else | |
75 | msg="failure" | |
76 | fmt="31" | |
77 | fi | |
78 | echo -e "\e[${fmt}m${msg}${ext}\e[0m" | |
79 | if test "x$log" != "x" ; then | |
80 | echo "=== $msg$ext" >> $log | |
81 | fi | |
82 | } | |
83 | ||
84 | # --- Copy a file to AliEn ------------------------------------------- | |
85 | copy() | |
86 | { | |
87 | local file=$1 | |
88 | local dest=$2 | |
89 | local del=$3 | |
90 | local base=`basename $file` | |
91 | ||
92 | if test "x$del" != "x" ; then | |
93 | log_msg cp.log "Removing \e[33malien:${dest}/${base}" | |
94 | if test $noact -lt 1 ; then | |
95 | alien_rm ${dest}/${base} >> cp.log 2>&1 | |
96 | fi | |
97 | log_end cp.log $? " (ignore errors)" | |
98 | fi | |
99 | ||
100 | log_msg cp.log "Uploading \e[33m${file}\e[0m to \e[33m${dest}" | |
101 | if test $noact -lt 1 ; then | |
102 | alien_cp -n file:${file} alien:${dest}/${base} >> cp.log 2>&1 | |
103 | fi | |
104 | log_end cp.log $? | |
105 | } | |
106 | ||
107 | # --- Run merging jpb ------------------------------------------------ | |
108 | merge() | |
109 | { | |
110 | local what=$1 | |
111 | local stage=$2 | |
112 | local dir=$3 | |
113 | local out=$4 | |
114 | local tag=$5 | |
115 | local run=$6 | |
116 | local tmpdir=`mktemp -d` | |
a0bdf9a4 | 117 | local pre=$what |
118 | if test "x$what" = "xAOD" ; then | |
119 | pre="aod"; | |
120 | fi | |
121 | ||
f8b7a926 | 122 | local top=${out}/${tag}/${run} |
123 | local bse=${what}_Stage_${stage}.xml | |
124 | local xml=${tmpdir}/${bse} | |
a0bdf9a4 | 125 | local arc=${pre}_archive.zip |
f8b7a926 | 126 | local jdl=Merge.jdl |
127 | local ret=0 | |
128 | ||
129 | rm -f cp.log | |
130 | ||
131 | log_msg cp.log "Creating XML file \e[33m${xml}" | |
132 | if test $stage -eq 1 ; then | |
133 | rm -f ${xml} | |
134 | alien_find -x ${top}/${bse} ${top} */${arc} > ${xml} 2>>cp.log | |
135 | ret=$? | |
136 | else | |
137 | let prev=$stage-1 | |
138 | rm -f ${xml} | |
139 | alien_find -x ${top}/${bse} ${top}/${what}_Stage_${prev} */${arc} \ | |
140 | > ${xml} 2>>cp.log | |
141 | ret=$? | |
142 | fi | |
143 | log_end cp.log $ret | |
144 | if test $ret -ne 0 ; then | |
145 | log_err "Make XML", "Failed to make XML collection $bse" | |
146 | exit 1 | |
147 | fi | |
148 | local n=`grep "<event name" ${xml} | wc -l 2>/dev/null` | |
149 | if test $n -lt $minmerge ; then | |
150 | old=$bse | |
151 | stage=5 | |
152 | jdl=Final.jdl | |
153 | bse=${what}_Stage_${stage}.xml | |
154 | tmp=${tmpdir}/${bse} | |
155 | sed "s,$old,$bse," < $xml > $tmp | |
156 | xml=$tmp | |
157 | fi | |
158 | echo -e "\e[33m$n\e[0m input files for \e[32m${what} stage ${stage}\e[0m" | |
159 | ||
160 | copy ${xml} ${top} del | |
161 | ||
162 | log_msg "" "Submitting merging job \e[33m${jdl}" | |
163 | if test $noact -lt 1 ; then | |
164 | alien_submit alien:${dir}/${jdl} ${run} ${stage} ${tag} ${what} | |
165 | fi | |
166 | log_end "" $? | |
167 | } | |
168 | ||
169 | # --- Determine the next stage --------------------------------------- | |
170 | progress() | |
171 | { | |
172 | local out=$1 | |
173 | ||
174 | log_msg "" "Deduce next stage for \e[33m$out" | |
175 | # First, check for final merge result | |
176 | # echo -e "\nCheck of ${out}/QA_merge_archive.zip" | |
177 | alien_ls ${out}/QA_merge_archive.zip > /dev/null 2>&1 | |
178 | if test $? -eq 0 ; then | |
179 | stage=6 | |
180 | else | |
181 | # Then check for production data | |
182 | # echo -e "\nCheck of ${out}/001" | |
183 | alien_ls ${out}/001 > /dev/null 2>&1 | |
184 | ret=$? | |
185 | # echo "ret=$ret" | |
186 | if test $ret -ne 0 ; then | |
187 | # echo "No output, stage 0 to be done" | |
188 | stage=0 | |
189 | else | |
190 | # Finally, check each merge stage | |
191 | tmp=0 | |
192 | stage=0 | |
193 | for i in 4 3 2 1; do | |
194 | # echo -e "\nCheck of ${out}/QA_Stage_${i}" | |
195 | alien_ls ${out}/QA_Stage_${i} > /dev/null 2>&1 | |
196 | if test $? -ne 0 ; then | |
197 | tmp=$i | |
198 | else | |
199 | break | |
200 | fi | |
201 | done | |
202 | stage=$tmp | |
203 | fi | |
204 | fi | |
205 | log_msg "" "\e[34;m$stage" | |
206 | log_end "" 0 | |
207 | } | |
208 | # --- Upload files --------------------------------------------------- | |
209 | push() | |
210 | { | |
211 | local bin=$1 | |
212 | local data=$2 | |
213 | local out=$3 | |
214 | local tmpdir=`mktemp -d` | |
215 | ||
216 | rm cp.log | |
217 | ||
218 | jdls="Run.jdl Merge.jdl Final.jdl" | |
219 | for i in $jdls ; do | |
220 | log_msg "" "Creating \e[33m${i}" | |
221 | sed -e "s|@out@|${out}|" \ | |
222 | -e "s|@data@|${data}|" \ | |
223 | -e "s|@bin@|${bin}|" \ | |
224 | -e "s|@aliroot@|${aliroot}|" \ | |
225 | -e "s|@root@|${root}|" \ | |
226 | -e "s|@geant@|${geant}|" \ | |
227 | < ${i}.in > ${tmpdir}/${i} | |
228 | log_end "" $? | |
229 | done | |
230 | ||
231 | log_msg cp.log "Removing and re-creating \e[33m${data}" | |
232 | if test $noact -lt 1 ; then | |
233 | alien_rmdir ${data} >> cp.log 2>&1 | |
234 | alien_mkdir -p $data >> cp.log 2>&1 | |
235 | fi | |
236 | log_end cp.log $? | |
237 | ||
238 | local del=1 | |
239 | if test "X$bin" = "X$data" ; then del=0 ; fi | |
240 | copy run.sh $bin $del | |
241 | copy merge.sh $bin $del | |
242 | ||
243 | ||
244 | files="simrun.sh \ | |
245 | GRP.C \ | |
246 | Simulate.C \ | |
247 | Config.C \ | |
421f877c | 248 | BaseConfig.C \ |
66ff3038 | 249 | EGConfig.C \ |
ee275c29 | 250 | DetConfig.C \ |
251 | OCDBConfig.C \ | |
f8b7a926 | 252 | Reconstruct.C \ |
253 | Check.C \ | |
254 | Tag.C \ | |
255 | QA.C \ | |
256 | QAConfig.C \ | |
257 | AOD.C \ | |
258 | AODConfig.C \ | |
259 | ${tmpdir}/Run.jdl \ | |
260 | ${tmpdir}/Merge.jdl \ | |
261 | ${tmpdir}/Final.jdl \ | |
262 | fmd_corrections.root" | |
263 | ||
264 | for i in $files ; do | |
265 | copy $i ${data} | |
266 | done | |
267 | } | |
268 | ||
269 | # --- Get package versions ------------------------------------------- | |
270 | getVersions() | |
271 | { | |
272 | local ali=$1 | |
273 | local roo=$2 | |
274 | local gean=$3 | |
275 | ||
276 | log_msg "" "Checking software packages" | |
277 | if test "x$ali" = "x" ; then | |
278 | log_end "" 1 | |
279 | log_err "Check versions" "No AliROOT Release specified" | |
280 | exit 1 | |
281 | fi | |
282 | if test "x$roo" != "x" && test "x$gean" = "x" ; then | |
283 | log_msg "" "\e[33mAliROOT=$ali ROOT=$roo GEANT=$gean" | |
284 | log_end "" 0 | |
285 | return | |
286 | fi | |
287 | ||
288 | l=`wget -q http://alimonitor.cern.ch/packages/ -O - | \ | |
289 | sed -n -e '/<tr/,/<\/tr>/ p' | \ | |
290 | sed -n "/<a.*VO_ALICE@AliRoot::${aliroot}/,/VO_ALICE@ROOT::/ p" | \ | |
291 | sed -n -e 's/.*VO_ALICE@\(GEAN\|ROO\)T3*::\(v[-0-9a-zA-Z]*\).*/\L\1=\2/gp'|\ | |
292 | tr '\n' ' '` | |
293 | eval $l | |
294 | if test "X$roo" = "X" || test "X$gean" = "X" ; then | |
295 | log_end "" 1 | |
296 | log_err "Check versions", "Failed to extract ROOT/GEANT3 versions" | |
297 | exit 1 | |
298 | fi | |
299 | root=$roo | |
300 | geant=$gean | |
301 | log_msg "" "\e[33mAliROOT=$ali ROOT=$root GEANT=$geant" | |
302 | log_end "" 0 | |
303 | } | |
304 | ||
305 | # --- Create an arcive for upload ------------------------------------ | |
306 | archive() | |
307 | { | |
308 | log_msg "" "Creating archive of files" | |
309 | local name=sim_files${version} | |
310 | mkdir -p ${name} | |
311 | files="\ | |
312 | run.sh \ | |
313 | AOD.C \ | |
314 | Check.C \ | |
315 | Config.C \ | |
316 | doit.sh \ | |
317 | Final.jdl.in \ | |
318 | GRP.C \ | |
319 | Merge.jdl.in \ | |
320 | QA.C \ | |
321 | README.md \ | |
322 | Reconstruct.C \ | |
323 | Run.jdl.in \ | |
324 | simrun.sh \ | |
325 | Simulate.C \ | |
326 | Tag.C \ | |
327 | merge.sh \ | |
328 | fmd_corrections.root" | |
329 | ||
330 | for i in $files ; do | |
331 | cp $i ${name}/$i | |
332 | done | |
333 | tar -czf ${name}.tar.gz ${name} | |
334 | ret=$? | |
335 | rm -rf ${name} | |
336 | log_end "" $ret | |
337 | } | |
338 | ||
339 | # --- Set some variables --------------------------------------------- | |
340 | auid=`alien_whoami | sed 's/^ *//'` | |
341 | ahome=/alice/cern.ch/user/`echo $auid | sed 's/^\(.\).*/\1/'`/$auid | |
342 | adir=${ahome}/mc | |
343 | abin=${ahome}/mc | |
344 | aout=${ahome}/test | |
a0bdf9a4 | 345 | stages="AOD QA" |
f8b7a926 | 346 | |
347 | # --- Proces command line options ------------------------------------ | |
348 | while test $# -gt 0 ; do | |
349 | case $1 in | |
350 | -h|--help) usage; exit 0;; | |
351 | -t|--tag) tag=$2 ; shift ;; | |
352 | -i|--id) id=$2 ; shift ;; | |
353 | -R|--run) run=$2 ; shift ;; | |
f8b7a926 | 354 | -c|--copy) upload=1 ;; |
355 | -n|--jobs) jobs=$2 ; shift ;; | |
356 | -m|--events) events=$2 ; shift ;; | |
357 | -s|--stage) stage=$2 ; shift ;; | |
a0bdf9a4 | 358 | -S|--stages) stages=$2 ; shift ;; |
f8b7a926 | 359 | -b|--bin) abin=$2 ; shift ;; |
360 | -o|--output) aout=$2 ; shift ;; | |
361 | -d|--data) adir=$2 ; shift ;; | |
362 | -a|--aliroot) aliroot=$2 ; shift ;; | |
363 | -r|--root) root=$2 ; shift ;; | |
364 | -g|--geant) geant=$2 ; shift ;; | |
365 | -f|--final) minmerge=$2 ; shift ;; | |
366 | -A|--archive) archive ; exit 0 ;; | |
367 | -x|--dry-run) noact=1 ;; | |
66ff3038 | 368 | --) shift ; break ;; |
f8b7a926 | 369 | *) log_err "Unknown option" "$1" ; exit 1 ;; |
370 | esac | |
371 | shift | |
372 | done | |
373 | abin=$adir | |
374 | ||
375 | ||
376 | ||
377 | # --- May upload only ------------------------------------------------ | |
378 | if test $upload -gt 0 ; then | |
379 | getVersions "$aliroot" "$root" "$geant" | |
380 | push ${abin} ${adir} ${aout} | |
381 | if test $stage -lt 0 ; then | |
382 | exit 0 | |
383 | fi | |
384 | fi | |
385 | ||
386 | # --- Inspect options ------------------------------------------------ | |
387 | case $tag in | |
388 | pp) id=$tag ; run=118506 ;; | |
389 | PbPb) id=$tag ; run=138190 ;; | |
390 | pPb) id=$tag ; run=195483 ;; | |
391 | Pbp) id=$tag ; run=196433 ;; | |
392 | esac | |
393 | if test "x$id" = "x" ; then | |
394 | log_err "" "No job identifier given" | |
395 | log_end "" 1 | |
396 | exit 1 | |
397 | fi | |
398 | if test "x$run" = "x" ; then | |
399 | log_err "" "No run number given" | |
400 | log_end "" 1 | |
401 | exit 1 | |
402 | fi | |
403 | case $stage in | |
404 | 0|1|2|3|4|5) : ;; | |
405 | 6) progress $aout/$id/$run ;; | |
406 | *) log_err "Invalid stage" "$stage" ; exit 1 ;; | |
407 | esac | |
408 | if test $stage -ge 6 ; then | |
409 | log_msg "" "All done" | |
410 | log_end "" 0 | |
411 | exit 0 | |
412 | fi | |
413 | ||
414 | # --- Either run the job or merge ------------------------------------ | |
415 | if test $stage -le 0 ; then | |
416 | log_msg "" "Removing old ${aout}/${id}/${run}" | |
417 | ret=0 | |
418 | if test $noact -lt 1 ; then | |
419 | alien_rmdir ${aout}/${id}/${run} > /dev/null 2>&1 | |
420 | ret=$? | |
421 | fi | |
422 | log_end "" $ret | |
423 | ||
424 | log_msg "" "Submitting \e[33mRun.jdl\e[0m for \e[34m$id run\e[0m (\e[34m$jobs\e[0m jobs w/\e[34m$events)" | |
425 | if test $noact -lt 1 ; then | |
66ff3038 | 426 | echo "alien_submit alien:${adir}/Run.jdl ${run} ${jobs} ${events} ${id} $@" |
427 | alien_submit alien:${adir}/Run.jdl ${run} ${jobs} ${events} ${id} "$@" | |
f8b7a926 | 428 | ret=$? |
429 | fi | |
430 | log_end "" $ret | |
431 | exit $ret | |
432 | fi | |
433 | ||
a0bdf9a4 | 434 | for s in $stages ; do |
435 | merge ${s} ${stage} ${adir} ${aout} ${id} ${run} | |
436 | done | |
f8b7a926 | 437 | |
438 | # | |
439 | # EOF | |
440 | # | |
441 | ||
442 | ||
443 |