]>
Commit | Line | Data |
---|---|---|
f8b7a926 | 1 | #!/bin/bash |
2 | version=5 | |
3 | tag= | |
4 | id= | |
5 | run= | |
6 | stage=0 | |
7 | upload=0 | |
8 | jobs=2 | |
9 | events=1 | |
10 | gen=default | |
11 | aliroot="v5-04-Rev-20" | |
12 | root="" | |
13 | geant="" | |
14 | minmerge=30 | |
15 | noact=0 | |
16 | ||
17 | # --- Display help message ------------------------------------------- | |
18 | usage() | |
19 | { | |
20 | cat <<EOF | |
21 | Usage: $0 [OPTIONS] | |
22 | ||
23 | Options: | |
24 | -h,--help This help | |
25 | -t|--tag TAG Job tag [****] ($tag) | |
26 | -i|--id NAME Name of production ($id) | |
27 | -R|--run RUN_NO Run number ($run) | |
28 | -e|--generator NAME Event generator ($gen) | |
29 | -c|--copy Copy files to AliEn | |
30 | -n|--jobs JOBS Set number of jobs[**] ($jobs) | |
31 | -m|--events EVENTS Set events/job[**] ($events) | |
32 | -s|--stage STAGE Set the stage[***] ($stage) | |
33 | -o|--output DIR Set base output directory ($aout) | |
34 | -d|--data DIR Set data directory ($adir) | |
35 | -b|--bin DIR Set base bin directory ($abin) | |
36 | -a|--aliroot RELEASE Set AliROOT release [*] ($aliroot) | |
37 | -r|--root RELEASE Set ROOT release [*] ($root) | |
38 | -g|--geant RELEASE Set GEANT3 release [*] ($geant) | |
39 | -f|--final NUMBER Run final merging when down to this ($minmerge) | |
40 | ||
41 | [*] Only make sense with option -c | |
42 | [**] Only make sense for stage 0 | |
43 | [***] If stage is set to 6, try to deduce the stage automatically | |
44 | [****] TAG is a short hand for specific id and run | |
45 | EOF | |
46 | } | |
47 | ||
48 | # --- Process a return value ----------------------------------------- | |
49 | log_msg() | |
50 | { | |
51 | local log=$1 ; shift | |
52 | echo -en "$@\e[0m ... " | |
53 | if test "x$log" != "x" ; then | |
54 | echo "=== $@" >> $log | |
55 | fi | |
56 | } | |
57 | # --- Make error ----------------------------------------------------- | |
58 | log_err() | |
59 | { | |
60 | local pre=$1 | |
61 | local post=$2 | |
62 | echo -e "\e[31;1mError\e[0m: ${pre} \e[35m${post}\e[0m" > /dev/stderr | |
63 | } | |
64 | ||
65 | # --- Process a return value ----------------------------------------- | |
66 | log_end() | |
67 | { | |
68 | local log=$1 | |
69 | local ret=$2 | |
70 | local ext=$3 | |
71 | local msg="" | |
72 | local fmt="" | |
73 | if test $ret -eq 0 ; then | |
74 | msg="success" | |
75 | fmt="32" | |
76 | else | |
77 | msg="failure" | |
78 | fmt="31" | |
79 | fi | |
80 | echo -e "\e[${fmt}m${msg}${ext}\e[0m" | |
81 | if test "x$log" != "x" ; then | |
82 | echo "=== $msg$ext" >> $log | |
83 | fi | |
84 | } | |
85 | ||
86 | # --- Copy a file to AliEn ------------------------------------------- | |
87 | copy() | |
88 | { | |
89 | local file=$1 | |
90 | local dest=$2 | |
91 | local del=$3 | |
92 | local base=`basename $file` | |
93 | ||
94 | if test "x$del" != "x" ; then | |
95 | log_msg cp.log "Removing \e[33malien:${dest}/${base}" | |
96 | if test $noact -lt 1 ; then | |
97 | alien_rm ${dest}/${base} >> cp.log 2>&1 | |
98 | fi | |
99 | log_end cp.log $? " (ignore errors)" | |
100 | fi | |
101 | ||
102 | log_msg cp.log "Uploading \e[33m${file}\e[0m to \e[33m${dest}" | |
103 | if test $noact -lt 1 ; then | |
104 | alien_cp -n file:${file} alien:${dest}/${base} >> cp.log 2>&1 | |
105 | fi | |
106 | log_end cp.log $? | |
107 | } | |
108 | ||
109 | # --- Run merging jpb ------------------------------------------------ | |
110 | merge() | |
111 | { | |
112 | local what=$1 | |
113 | local stage=$2 | |
114 | local dir=$3 | |
115 | local out=$4 | |
116 | local tag=$5 | |
117 | local run=$6 | |
118 | local tmpdir=`mktemp -d` | |
119 | ||
120 | local top=${out}/${tag}/${run} | |
121 | local bse=${what}_Stage_${stage}.xml | |
122 | local xml=${tmpdir}/${bse} | |
123 | local arc=${what}_archive.zip | |
124 | local jdl=Merge.jdl | |
125 | local ret=0 | |
126 | ||
127 | rm -f cp.log | |
128 | ||
129 | log_msg cp.log "Creating XML file \e[33m${xml}" | |
130 | if test $stage -eq 1 ; then | |
131 | rm -f ${xml} | |
132 | alien_find -x ${top}/${bse} ${top} */${arc} > ${xml} 2>>cp.log | |
133 | ret=$? | |
134 | else | |
135 | let prev=$stage-1 | |
136 | rm -f ${xml} | |
137 | alien_find -x ${top}/${bse} ${top}/${what}_Stage_${prev} */${arc} \ | |
138 | > ${xml} 2>>cp.log | |
139 | ret=$? | |
140 | fi | |
141 | log_end cp.log $ret | |
142 | if test $ret -ne 0 ; then | |
143 | log_err "Make XML", "Failed to make XML collection $bse" | |
144 | exit 1 | |
145 | fi | |
146 | local n=`grep "<event name" ${xml} | wc -l 2>/dev/null` | |
147 | if test $n -lt $minmerge ; then | |
148 | old=$bse | |
149 | stage=5 | |
150 | jdl=Final.jdl | |
151 | bse=${what}_Stage_${stage}.xml | |
152 | tmp=${tmpdir}/${bse} | |
153 | sed "s,$old,$bse," < $xml > $tmp | |
154 | xml=$tmp | |
155 | fi | |
156 | echo -e "\e[33m$n\e[0m input files for \e[32m${what} stage ${stage}\e[0m" | |
157 | ||
158 | copy ${xml} ${top} del | |
159 | ||
160 | log_msg "" "Submitting merging job \e[33m${jdl}" | |
161 | if test $noact -lt 1 ; then | |
162 | alien_submit alien:${dir}/${jdl} ${run} ${stage} ${tag} ${what} | |
163 | fi | |
164 | log_end "" $? | |
165 | } | |
166 | ||
167 | # --- Determine the next stage --------------------------------------- | |
168 | progress() | |
169 | { | |
170 | local out=$1 | |
171 | ||
172 | log_msg "" "Deduce next stage for \e[33m$out" | |
173 | # First, check for final merge result | |
174 | # echo -e "\nCheck of ${out}/QA_merge_archive.zip" | |
175 | alien_ls ${out}/QA_merge_archive.zip > /dev/null 2>&1 | |
176 | if test $? -eq 0 ; then | |
177 | stage=6 | |
178 | else | |
179 | # Then check for production data | |
180 | # echo -e "\nCheck of ${out}/001" | |
181 | alien_ls ${out}/001 > /dev/null 2>&1 | |
182 | ret=$? | |
183 | # echo "ret=$ret" | |
184 | if test $ret -ne 0 ; then | |
185 | # echo "No output, stage 0 to be done" | |
186 | stage=0 | |
187 | else | |
188 | # Finally, check each merge stage | |
189 | tmp=0 | |
190 | stage=0 | |
191 | for i in 4 3 2 1; do | |
192 | # echo -e "\nCheck of ${out}/QA_Stage_${i}" | |
193 | alien_ls ${out}/QA_Stage_${i} > /dev/null 2>&1 | |
194 | if test $? -ne 0 ; then | |
195 | tmp=$i | |
196 | else | |
197 | break | |
198 | fi | |
199 | done | |
200 | stage=$tmp | |
201 | fi | |
202 | fi | |
203 | log_msg "" "\e[34;m$stage" | |
204 | log_end "" 0 | |
205 | } | |
206 | # --- Upload files --------------------------------------------------- | |
207 | push() | |
208 | { | |
209 | local bin=$1 | |
210 | local data=$2 | |
211 | local out=$3 | |
212 | local tmpdir=`mktemp -d` | |
213 | ||
214 | rm cp.log | |
215 | ||
216 | jdls="Run.jdl Merge.jdl Final.jdl" | |
217 | for i in $jdls ; do | |
218 | log_msg "" "Creating \e[33m${i}" | |
219 | sed -e "s|@out@|${out}|" \ | |
220 | -e "s|@data@|${data}|" \ | |
221 | -e "s|@bin@|${bin}|" \ | |
222 | -e "s|@aliroot@|${aliroot}|" \ | |
223 | -e "s|@root@|${root}|" \ | |
224 | -e "s|@geant@|${geant}|" \ | |
225 | < ${i}.in > ${tmpdir}/${i} | |
226 | log_end "" $? | |
227 | done | |
228 | ||
229 | log_msg cp.log "Removing and re-creating \e[33m${data}" | |
230 | if test $noact -lt 1 ; then | |
231 | alien_rmdir ${data} >> cp.log 2>&1 | |
232 | alien_mkdir -p $data >> cp.log 2>&1 | |
233 | fi | |
234 | log_end cp.log $? | |
235 | ||
236 | local del=1 | |
237 | if test "X$bin" = "X$data" ; then del=0 ; fi | |
238 | copy run.sh $bin $del | |
239 | copy merge.sh $bin $del | |
240 | ||
241 | ||
242 | files="simrun.sh \ | |
243 | GRP.C \ | |
244 | Simulate.C \ | |
245 | Config.C \ | |
246 | Reconstruct.C \ | |
247 | Check.C \ | |
248 | Tag.C \ | |
249 | QA.C \ | |
250 | QAConfig.C \ | |
251 | AOD.C \ | |
252 | AODConfig.C \ | |
253 | ${tmpdir}/Run.jdl \ | |
254 | ${tmpdir}/Merge.jdl \ | |
255 | ${tmpdir}/Final.jdl \ | |
256 | fmd_corrections.root" | |
257 | ||
258 | for i in $files ; do | |
259 | copy $i ${data} | |
260 | done | |
261 | } | |
262 | ||
263 | # --- Get package versions ------------------------------------------- | |
264 | getVersions() | |
265 | { | |
266 | local ali=$1 | |
267 | local roo=$2 | |
268 | local gean=$3 | |
269 | ||
270 | log_msg "" "Checking software packages" | |
271 | if test "x$ali" = "x" ; then | |
272 | log_end "" 1 | |
273 | log_err "Check versions" "No AliROOT Release specified" | |
274 | exit 1 | |
275 | fi | |
276 | if test "x$roo" != "x" && test "x$gean" = "x" ; then | |
277 | log_msg "" "\e[33mAliROOT=$ali ROOT=$roo GEANT=$gean" | |
278 | log_end "" 0 | |
279 | return | |
280 | fi | |
281 | ||
282 | l=`wget -q http://alimonitor.cern.ch/packages/ -O - | \ | |
283 | sed -n -e '/<tr/,/<\/tr>/ p' | \ | |
284 | sed -n "/<a.*VO_ALICE@AliRoot::${aliroot}/,/VO_ALICE@ROOT::/ p" | \ | |
285 | sed -n -e 's/.*VO_ALICE@\(GEAN\|ROO\)T3*::\(v[-0-9a-zA-Z]*\).*/\L\1=\2/gp'|\ | |
286 | tr '\n' ' '` | |
287 | eval $l | |
288 | if test "X$roo" = "X" || test "X$gean" = "X" ; then | |
289 | log_end "" 1 | |
290 | log_err "Check versions", "Failed to extract ROOT/GEANT3 versions" | |
291 | exit 1 | |
292 | fi | |
293 | root=$roo | |
294 | geant=$gean | |
295 | log_msg "" "\e[33mAliROOT=$ali ROOT=$root GEANT=$geant" | |
296 | log_end "" 0 | |
297 | } | |
298 | ||
299 | # --- Create an arcive for upload ------------------------------------ | |
300 | archive() | |
301 | { | |
302 | log_msg "" "Creating archive of files" | |
303 | local name=sim_files${version} | |
304 | mkdir -p ${name} | |
305 | files="\ | |
306 | run.sh \ | |
307 | AOD.C \ | |
308 | Check.C \ | |
309 | Config.C \ | |
310 | doit.sh \ | |
311 | Final.jdl.in \ | |
312 | GRP.C \ | |
313 | Merge.jdl.in \ | |
314 | QA.C \ | |
315 | README.md \ | |
316 | Reconstruct.C \ | |
317 | Run.jdl.in \ | |
318 | simrun.sh \ | |
319 | Simulate.C \ | |
320 | Tag.C \ | |
321 | merge.sh \ | |
322 | fmd_corrections.root" | |
323 | ||
324 | for i in $files ; do | |
325 | cp $i ${name}/$i | |
326 | done | |
327 | tar -czf ${name}.tar.gz ${name} | |
328 | ret=$? | |
329 | rm -rf ${name} | |
330 | log_end "" $ret | |
331 | } | |
332 | ||
333 | # --- Set some variables --------------------------------------------- | |
334 | auid=`alien_whoami | sed 's/^ *//'` | |
335 | ahome=/alice/cern.ch/user/`echo $auid | sed 's/^\(.\).*/\1/'`/$auid | |
336 | adir=${ahome}/mc | |
337 | abin=${ahome}/mc | |
338 | aout=${ahome}/test | |
339 | ||
340 | # --- Proces command line options ------------------------------------ | |
341 | while test $# -gt 0 ; do | |
342 | case $1 in | |
343 | -h|--help) usage; exit 0;; | |
344 | -t|--tag) tag=$2 ; shift ;; | |
345 | -i|--id) id=$2 ; shift ;; | |
346 | -R|--run) run=$2 ; shift ;; | |
347 | -e|--generator) gen=$2 ; shift ;; | |
348 | -c|--copy) upload=1 ;; | |
349 | -n|--jobs) jobs=$2 ; shift ;; | |
350 | -m|--events) events=$2 ; shift ;; | |
351 | -s|--stage) stage=$2 ; shift ;; | |
352 | -b|--bin) abin=$2 ; shift ;; | |
353 | -o|--output) aout=$2 ; shift ;; | |
354 | -d|--data) adir=$2 ; shift ;; | |
355 | -a|--aliroot) aliroot=$2 ; shift ;; | |
356 | -r|--root) root=$2 ; shift ;; | |
357 | -g|--geant) geant=$2 ; shift ;; | |
358 | -f|--final) minmerge=$2 ; shift ;; | |
359 | -A|--archive) archive ; exit 0 ;; | |
360 | -x|--dry-run) noact=1 ;; | |
361 | *) log_err "Unknown option" "$1" ; exit 1 ;; | |
362 | esac | |
363 | shift | |
364 | done | |
365 | abin=$adir | |
366 | ||
367 | ||
368 | ||
369 | # --- May upload only ------------------------------------------------ | |
370 | if test $upload -gt 0 ; then | |
371 | getVersions "$aliroot" "$root" "$geant" | |
372 | push ${abin} ${adir} ${aout} | |
373 | if test $stage -lt 0 ; then | |
374 | exit 0 | |
375 | fi | |
376 | fi | |
377 | ||
378 | # --- Inspect options ------------------------------------------------ | |
379 | case $tag in | |
380 | pp) id=$tag ; run=118506 ;; | |
381 | PbPb) id=$tag ; run=138190 ;; | |
382 | pPb) id=$tag ; run=195483 ;; | |
383 | Pbp) id=$tag ; run=196433 ;; | |
384 | esac | |
385 | if test "x$id" = "x" ; then | |
386 | log_err "" "No job identifier given" | |
387 | log_end "" 1 | |
388 | exit 1 | |
389 | fi | |
390 | if test "x$run" = "x" ; then | |
391 | log_err "" "No run number given" | |
392 | log_end "" 1 | |
393 | exit 1 | |
394 | fi | |
395 | case $stage in | |
396 | 0|1|2|3|4|5) : ;; | |
397 | 6) progress $aout/$id/$run ;; | |
398 | *) log_err "Invalid stage" "$stage" ; exit 1 ;; | |
399 | esac | |
400 | if test $stage -ge 6 ; then | |
401 | log_msg "" "All done" | |
402 | log_end "" 0 | |
403 | exit 0 | |
404 | fi | |
405 | ||
406 | # --- Either run the job or merge ------------------------------------ | |
407 | if test $stage -le 0 ; then | |
408 | log_msg "" "Removing old ${aout}/${id}/${run}" | |
409 | ret=0 | |
410 | if test $noact -lt 1 ; then | |
411 | alien_rmdir ${aout}/${id}/${run} > /dev/null 2>&1 | |
412 | ret=$? | |
413 | fi | |
414 | log_end "" $ret | |
415 | ||
416 | log_msg "" "Submitting \e[33mRun.jdl\e[0m for \e[34m$id run\e[0m (\e[34m$jobs\e[0m jobs w/\e[34m$events)" | |
417 | if test $noact -lt 1 ; then | |
418 | alien_submit alien:${adir}/Run.jdl ${run} ${jobs} ${events} ${id} ${gen} | |
419 | ret=$? | |
420 | fi | |
421 | log_end "" $ret | |
422 | exit $ret | |
423 | fi | |
424 | ||
425 | merge QA ${stage} ${adir} ${aout} ${id} ${run} | |
426 | ||
427 | # | |
428 | # EOF | |
429 | # | |
430 | ||
431 | ||
432 |