3 # This script runs the Forward QA for the specified production number
5 # The scripts downloads and runs the single run QA in parallel
8 # --- Some aux files -------------------------------------------------
9 style=$ALICE_ROOT/PWGLF/FORWARD/analysis2/qa/style.css
10 favicon=$ALICE_ROOT/PWGLF/FORWARD/analysis2/qa/fmd_favicon.png
11 logo=$ALICE_ROOT/PWGLF/FORWARD/analysis2/qa/fmd_logo.png
13 # --- Help output ----------------------------------------------
17 Usage: $0 [OPTIONS] -j [JOBID]
18 $0 [OPTIONS] -p PRODUCTION -P PASS
22 -j,--jobid NUMBER The master job id of the production [$jobid]
23 -v,--verbose Increase verbosity [$verb]
24 -m,--max-files NUMBER Max number of files to get [$maxf]
25 -M,--max-jobs NUMBER Max number of consequtive jobs [$maxjobs]
26 -t,--top DIRECTORY Output directory [$top]
27 -R,--also-results Also get QAresults.root for each run
28 -Q,--qa-number NUMBER Custom QA id [$qanumber]
29 -p,--production IDENTIFIER Production identifier [$prodfull]
30 -P,--pass IDENTIFIER Pass identifier [$passfull]
31 -l,--log-file Log file output [$redir]
32 -b,--barrel MODE Fetch barrel data [$barrel]
33 -f,--force Force re-run analysis [$force]
34 -V,--variance Errors=variance (not min/max) [$variance]
35 -L,--local Local trending_<X>.root files [$from_local]
36 -d,--directory DIR Search custom AliEn directory [$path]
38 Note the option -j and the options -p and -P are mutually exclusive,
39 The option -Q is only used if the options -p and -P are given.
40 Production identifiers are of the form LHC11h, LHC11h3, or LHC11h_2.
41 Pass identifers are of the form pass2, pass1_HLT, or cpass1.
42 If barrel mode>0, then do not assume ESD directory.
43 If barrel mode>1, then get trending_barrel.root and QAresults_barrel.root
44 Option -d is for hand-made QA passes.
45 If optiond -d is not specified then official QA passes are assumed.
49 # --- Check AliEn token ----------------------------------------------
53 genv_file=/tmp/gclient_env_${uid}
55 if test ! -f ${genv_file} ; then
56 echo "No such file: ${genv_file}, please do alien-token-init" \
61 alien-token-info | grep -q "Token is still valid"
62 if test $? -ne 0 ; then
63 echo "Token not valid, please re-new" > /dev/stderr
68 # --- Diagnostics output ---------------------------------------------
72 if test $1 -gt $verb ; then return ; fi
77 # --- Handling of exit -----------------------------------------------
81 if test "x$lock" = "x" ; then return ; fi
82 if test "x$store" != "x" && test "x${top}" != "x" ; then
83 chmod -R g+rwX ${top}/${proddir} >> ${redir} 2>&1
84 chmod -R g+rwX ${top}/$store >> ${redir} 2>&1
90 # --- Handling of errors ---------------------------------------------
106 # --- Check the lock -------------------------------------------------
112 if test -f $lock ; then
113 echo "Another QA process is already running:" > /dev/stderr
114 echo "Content of ${lock}:" > /dev/stderr
115 cat $lock > /dev/stderr
127 # --- Parse production information -----------------------------------
130 prodyear=`echo $prodfull | sed 's/LHC\(..\).*/\1/'`
131 prodletter=`echo $prodfull | sed "s/LHC${prodyear}\(.\).*/\1/"`
132 prodpost=`echo $prodfull | sed "s/LHC${prodyear}${prodletter}//"`
137 passno=`echo $passfull | sed 's/.*pass\([0-9]*\).*/\1/'`
138 passpost=`echo $passfull | sed -n "s/.*pass${passno}_//p"`
139 passpre=`echo $passfull | sed -n "s/pass.*//p"`
142 # --- Extract parts from the found path ------------------------------
158 mess 1 "Parsing information from job $@"
161 local lrunn=$1 ; shift
162 local ltype=$1 ; shift
167 mess 10 "prodfull=$prodfull"
168 mess 10 "lrunn=$lrunn"
169 mess 10 "ltype=$ltype"
170 mess 10 "passfull=$passfull"
171 mess 10 "remainder=$remainder"
173 if test "x$passfull" = "x" ; then
179 *) remainder=$passfull
187 xQA*) qanumber=`echo $remainder | sed 's/QA//'` ;;
192 # --- Get the index for information ----------------------------------
194 jobu="http://alimonitor.cern.ch/prod/jobs.jsp?t="
198 mess 1 "Getting the job information"
199 wget -q ${jobu}${jobid} -O job.html
202 p=`grep "/catalogue/index.jsp?path" job.html | head -n $lskip | tail -n 1 | sed -e 's,.*/alice/\(data\|sim\)/\([^<]*\)<.*,\2,' | tr '/' ' '`
207 # --- Append path element --------------------------------------------
212 if test "x$tmp" != "x" ; then tmp="${tmp}/" ; fi
216 # --- Get a list of files to get -------------------------------------
225 mess 3 "Getting file list"
229 if test ${barrel} -gt 0 ; then
232 if test ${barrel} -gt 1 ; then
233 file=trending_barrel.root
234 other=QAresults_barrel.root
239 *) mess 3 "Assuming simulation output"
246 if echo "$passno" | grep -q -E '^[0-9]*[.]?[0-9]*$' ; then
247 if test "x$passfull" != "x" && test $passno -gt 0 ; then
255 local post=${passpost}
259 *) post="_${post}" ;;
263 if test "x$path" = "x" ; then
264 # Assume official productions
265 path=/alice/${datd}/${year}/${prodfull}/
266 search="${esdd}${passpre}${paid}${post}"
271 if test $qanumber -gt 0 ; then
272 qapost=`printf "QA%02d" $qanumber`
273 search=`append_to_path "$search" $qapost`
276 search=`append_to_path "$search" $file`
282 if test $from_local -lt 1 ; then
284 mess 1 "Getting list of files from AliEn - can take minutes - be patient"
285 mess 2 "alien_find ${path} ${search}"
286 files=`alien_find ${path} ${search} | grep -v "\(files found\|AND THE\)" 2>> ${redir}`
288 files=`ls ${top}/${store}/trending_*.root | sed 's,${top}/${store}/,,g'`
293 mess 1 -n "Total of $numf files ... "
294 if test $maxf -lt 0 ; then
297 mess 1 "using $maxf first of these"
301 # --- Change permissions on files ------------------------------------
304 # if test ! -f $1 ; then return ; fi
305 chmod g+rwX $1 >> /dev/null 2>&1
306 chmod o+rX $1 >> /dev/null 2>&1
308 # chmod g+rwX $1 >> ${redir} 2>&1
309 # chmod o+rX $1 >> ${redir} 2>&1
312 # --- Check if a file is OK ------------------------------------------
316 if test $docheck -lt 1 ; then return 0; fi
317 root -l -b <<EOF >> ${redir} 2>&1
318 .L $ALICE_ROOT/PWGLF/FORWARD/analysis2/qa/CheckQAFile.C
323 mess 2 "Check of $1 -> $ret"
328 # --- Analyse a file -------------------------------------------------
331 local dir=`dirname $1`
332 local inp=`basename $1`
333 local out=`echo $inp | sed 's/trending_/tree_/'`
335 mess 2 -n "Analysing $inp -> $out ... "
337 if test -f $dir/$out ; then
338 if test $force -lt 1 ; then
348 RunFileQA("$inp", "$out", $prodyear, "$prodletter");
358 # --- Download a single file -----------------------------------------
362 local source=$1 ; shift
363 local store=$1 ; shift
365 local o=${store}/`basename $file .root`_${r}.root
367 mess 2 -n "$source -> $o ... "
368 if test -f $o && test $force -lt 2; then
373 mess 2 -n "copying ... "
374 alien_cp alien:${source} file:${o} >> ${redir} 2>&1
378 if test ! -f $o ; then return 1 ; fi
380 if test $also_results -gt 0 ; then
381 local s=`dirname ${source}`/${other}
382 local q=${store}/`basename $other .root`_${r}.root
384 mess 2 -n "$s -> $q ... "
385 if test -f $q && test $force -lt 2 ; then
389 mess 2 -n "copying ... "
390 alien_cp alien:${s} file:${q} >> ${redir} 2>&1
401 1|3|4|5|6) return 2 ;;
409 # --- Submit run analysis to background ------------------------------
418 mess 5 "Submitting $maxjobs jobs from $sta/$maxf"
420 let cur=$sta+$counter
423 if test $from_local -lt 1 ; then
424 local b=`echo $i | sed -e "s,${path},,"`
425 r=`echo $b | sed -e "s,/.*,,"`
427 r=`basename $i .root | sed 's/trending_//'`
430 printf "%3d/%3d: %s\n" $cur $max $r
433 let counter=$counter+1
435 analyse_run $i $out $r &
436 j=`jobs %% | sed -e 's/^[^0-9]*//' -e 's/[^0-9]*$//'`
437 joblist="$joblist $j"
441 mess 5 "will wait for jobs $joblist"
442 for i in $joblist ; do
443 mess 5 "waiting for $i of $joblist"
445 mess 5 "Analysing ${runs[$counter]} returned $?"
446 let counter=$counter+1
450 # --- Analyse each run in turn ---------------------------------------
451 maxjobs=`grep "processor" /proc/cpuinfo | wc -l`
461 while test $# -gt 0 ; do
462 if test $counter -ge $max ; then
469 let counter=$counter+1
471 if test $queued -eq 1 ; then
475 if test $queued -ge $maxjobs ; then
476 mess 1 "Submitting $queued jobs from $start/$max"
477 submit_runs $out $start $max $list
482 if test $queued -gt 0 ; then
483 mess 1 "Submitting $queued jobs from $start/$max"
484 submit_runs $out $start $max $list
488 # --- Copy style -----------------------------------------------------
492 if test ! -f $file ; then return ; fi
493 base=`basename $file`
500 copy_aliroot_file $style
503 # --- Run the final trending -----------------------------------------
509 mess 1 -n "Analysing for trend $dir ... "
517 RunFinalQA(".", $prodyear, "$prodletter", $variance);
521 # mess 3 -n "root -l -b -q ${scr}.C "
522 # root -l -b -q ${scr}.C > /dev/null 2>&1
528 local idx=`ls trend_*_*.html 2> /dev/null`
530 mess 1 "Making index.html point to $i"
531 sed -e 's,index.html,../index.html,' \
532 -e "s,!--JOBID--,a target='_blank' href='${jobu}${jobid}'>Job</a," \
537 if test ! -f index.html ; then
538 echo "No index file found"
542 fix_perm . > /dev/null 2>&1
546 copy_aliroot_file $favicon
547 copy_aliroot_file $logo
552 # --- Make index file ------------------------------------------------
556 local title=$1 ; shift
557 local desc=$1 ; shift
558 mess 1 "Making index in $dir ($title)"
565 cat <<EOF > index.html
569 <title>$title</title>
570 <link rel='stylesheet' href='style.css'>
571 <link rel='shortcut icon' href='fmd_favicon.png' type='image/x-png'>
574 <h1><img style='width: 200px;' src='fmd_logo.png'> $title</h1>
576 if test ! "x$desc" = "x" ; then
577 echo "<p>$desc</p>" >> index.html
579 echo " <ul>" >> index.html
581 if test ! -d $i ; then continue ; fi
582 echo " <li><a href='$i'>$i</a></li>" >> index.html
584 echo " </ul>" >> index.html
585 if test "x$desc" = "x" ; then
586 echo " <div class='back'><a href='../'>Back</a></div>" \
589 cat <<EOF >> index.html
590 <div class='change'>Last update: $date</div>
595 copy_aliroot_file $favicon
596 copy_aliroot_file $logo
598 fix_perm . > /dev/null 2>&1
603 # --- Pass command line options --------------------------------------
607 while test $# -gt 0 ; do
609 -h|--help) usage ; exit 0 ;;
610 -v|--verbose) let verb=$verb+1 ;;
611 -j|--jobid) jobid=$2 ; shift ;;
612 -m|--max-files) maxf=$2 ; shift ;;
613 -M|--max-jobs) maxjobs=$2 ; shift ;;
614 -t|--top) top=$2 ; shift ;;
615 -R|--also-results) also_results=1 ;;
616 -Q|--qa-number) qanumber=$2 ; shift ;;
617 -l|--log-file) redir= ;;
618 -L|--local) from_local=1 ;;
619 -V|--variance) variance=1 ;;
620 -b|--barrel) barrel=$2 ; shift ;;
621 -f|--force) let force=$force+1 ;;
623 prodfull=$2; shift; parse_prod ; year=20${prodyear} ;;
625 passfull=$2; shift; parse_pass ;;
626 -d|--directory) path=$2 ; shift ;;
627 *) echo "$0: Unknown argument: $1" > /dev/stderr ; exit 1 ;;
631 # --- Initial setup --------------------------------------------------
632 # First, check we have a valid AliEn token, then retrieve the job
633 # information to parse out the location of the files we need, and
634 # finally make our output directory and check the lock
637 if test ! "x$jobid" = x ; then
638 if test ! "x$prodfull" = "x" || test ! "x$passfull" = "x" ; then
639 cat <<EOF > /dev/stderr
640 Option -j ${jobid} and options -p and -P are mutually exclusive
646 if test "x$prodfull" = "x" || test "x$passfull" = "x" ; then
647 cat<<EOF > /dev/stderr
648 When specifying prodcution and/or pass both options -p and -P _must_
652 elif test ! "x$jobid" = "x" ; then
653 cat <<EOF > /dev/stderr
654 Option -j and options -p ${prodfull} and -P ${passfull} are mutually exclusive
660 proddir=LHC${prodyear}${prodletter}
662 if test ! "x$passno" = "x" ; then
663 if test "x${passpre}" = "xv" || test "x${passpre}" = "xc"; then
664 store=${store}/${passpre}pass${passno}
666 store=${store}/pass${passno}
668 elif test ! "x$prodpost" = "x" ; then
669 proddir=${proddir}${prodpost}
671 elif test ! "x$remainder" = "x" ; then
672 store=${store}/${remainder}
674 if test ! "x$qanumber" = "x" && test $qanumber -gt 0 ; then
675 store=${store}_QA${qanumber}
677 mkdir -p ${top}/$store
678 fix_perm ${top}/${proddir}
679 fix_perm ${top}/$store
681 if test "x$redir" = "x" ; then
682 redir=${top}/$store/qa.log
687 check_lock ${top}/$store
689 # --- Some friendly information --------------------------------------
692 Production: $prodfull
702 Output directory: ${store}
706 Use variance: ${variance}
707 Use pre-downloaded: ${from_local}
709 # --- Do a search to find our files ----------------------------------
712 if test $maxf -gt 0 && test $maxf -lt $numf ; then
716 # --- Copy scripts to target and compile -----------------------------
717 for i in QABase.h QAPlotter.C QARing.h QAStructs.h QATrender.C \
718 RunFileQA.C RunFinalQA.C ; do
719 cp $ALICE_ROOT/PWGLF/FORWARD/analysis2/qa/$i ${store}/${i}
720 rm -f ${store}/`echo $i | tr '.' '_'`.{so,d}
721 fix_perm ${store}/${i}
723 mess 1 "Compiling QATrender.C"
724 (cd $store && root -l -b <<EOF
725 gROOT->LoadMacro("QABase.h++g");
726 gROOT->LoadMacro("QATrender.C++g");
730 mess 1 "Compiling QAPlotter.C"
731 (cd $store && root -l -b <<EOF
732 gROOT->LoadMacro("QABase.h++g");
733 gROOT->LoadMacro("QAPlotter.C++g");
737 (cd ${store} && for i in *.so *.d ; do fix_perm $i ; done)
739 # --- Now get and analyse each run -----------------------------------
740 analyse_runs ${top}/$store $numf $files
742 # --- Now analyse all runs -------------------------------------------
743 make_trend ${top}/$store
745 # --- Make index files -----------------------------------------------
746 make_index ${top}/${proddir} ${proddir}
747 make_index ${top} "QA for the FMD" \
748 "For more information see <a href='https://twiki.cern.ch/twiki/bin/viewauth/ALICE/FMDQA'>TWiki pages</a>."
750 chmod -R g+rwX ${top}/${proddir} >> ${redir} 2>&1