3 # This script runs the Forward QA for the specified production number
5 # The scripts downloads and runs the single run QA in parallel
8 # --- Some aux files -------------------------------------------------
9 style=$ALICE_ROOT/PWGLF/FORWARD/analysis2/qa/style.css
10 favicon=$ALICE_ROOT/PWGLF/FORWARD/analysis2/qa/fmd_favicon.png
11 logo=$ALICE_ROOT/PWGLF/FORWARD/analysis2/qa/fmd_logo.png
12 script=$ALICE_ROOT/PWGLF/FORWARD/analysis2/qa/script.js
13 topmk=$ALICE_ROOT/PWGLF/FORWARD/analysis2/qa/makeIndex.sh
15 # --- Help output ----------------------------------------------
19 Usage: $0 [OPTIONS] -j [JOBID]
20 $0 [OPTIONS] -p PRODUCTION -P PASS
24 -j,--jobid NUMBER The master job id of the production [$jobid]
25 -v,--verbose Increase verbosity [$verb]
26 -m,--max-files NUMBER Max number of files to get [$maxf]
27 -M,--max-jobs NUMBER Max number of consequtive jobs [$maxjobs]
28 -t,--top DIRECTORY Output directory [$top]
29 -R,--also-results Also get QAresults.root for each run
30 -Q,--qa-number NUMBER Custom QA id [$qanumber]
31 -p,--production IDENTIFIER Production identifier [$prodfull]
32 -P,--pass IDENTIFIER Pass identifier [$passfull]
33 -l,--log-file Log file output [$redir]
34 -b,--barrel MODE Fetch barrel data [$barrel]
35 -f,--force Force re-run analysis [$force]
36 -V,--variance Errors=variance (not min/max) [$variance]
37 -L,--local Local trending_<X>.root files [$from_local]
38 -d,--directory DIR Search custom AliEn directory [$path]
40 Note the option -j and the options -p and -P are mutually exclusive,
41 The option -Q is only used if the options -p and -P are given.
42 Production identifiers are of the form LHC11h, LHC11h3, or LHC11h_2.
43 Pass identifers are of the form pass2, pass1_HLT, or cpass1.
44 If barrel mode>0, then do not assume ESD directory.
45 If barrel mode>1, then get trending_barrel.root and QAresults_barrel.root
46 Option -d is for hand-made QA passes.
47 If optiond -d is not specified then official QA passes are assumed.
51 # --- Check AliEn token ----------------------------------------------
55 genv_file=/tmp/gclient_env_${uid}
57 if test ! -f ${genv_file} ; then
58 echo "No such file: ${genv_file}, please do alien-token-init" \
63 alien-token-info | grep -q "Token is still valid"
64 if test $? -ne 0 ; then
65 echo "Token not valid, please re-new" > /dev/stderr
70 # --- Diagnostics output ---------------------------------------------
74 if test $1 -gt $verb ; then return ; fi
79 # --- Handling of exit -----------------------------------------------
83 if test "x$lock" = "x" ; then return ; fi
84 if test "x$store" != "x" && test "x${top}" != "x" ; then
85 chmod -R g+rwX ${top}/${proddir} >> ${redir} 2>&1
86 chmod -R g+rwX ${top}/$store >> ${redir} 2>&1
92 # --- Handling of errors ---------------------------------------------
108 # --- Check the lock -------------------------------------------------
114 if test -f $lock ; then
115 echo "Another QA process is already running:" > /dev/stderr
116 echo "Content of ${lock}:" > /dev/stderr
117 cat $lock > /dev/stderr
129 # --- Parse production information -----------------------------------
132 prodyear=`echo $prodfull | sed 's/LHC\(..\).*/\1/'`
133 prodletter=`echo $prodfull | sed "s/LHC${prodyear}\(.\).*/\1/"`
134 prodpost=`echo $prodfull | sed "s/LHC${prodyear}${prodletter}//"`
139 passno=`echo $passfull | sed 's/.*pass\([0-9]*\).*/\1/'`
140 passpost=`echo $passfull | sed -n "s/.*pass${passno}_//p"`
141 passpre=`echo $passfull | sed -n "s/pass.*//p"`
144 # --- Extract parts from the found path ------------------------------
160 mess 1 "Parsing information from job $@"
163 local lrunn=$1 ; shift
164 local ltype=$1 ; shift
169 mess 10 "prodfull=$prodfull"
170 mess 10 "lrunn=$lrunn"
171 mess 10 "ltype=$ltype"
172 mess 10 "passfull=$passfull"
173 mess 10 "remainder=$remainder"
175 if test "x$passfull" = "x" ; then
181 *) remainder=$passfull
189 xQA*) qanumber=`echo $remainder | sed 's/QA//'` ;;
194 # --- Get the index for information ----------------------------------
196 jobu="http://alimonitor.cern.ch/prod/jobs.jsp?t="
200 mess 1 "Getting the job information"
201 wget -q ${jobu}${jobid} -O job.html
204 p=`grep "/catalogue/index.jsp?path" job.html | head -n $lskip | tail -n 1 | sed -e 's,.*/alice/\(data\|sim\)/\([^<]*\)<.*,\2,' | tr '/' ' '`
209 # --- Append path element --------------------------------------------
214 if test "x$tmp" != "x" ; then tmp="${tmp}/" ; fi
218 # --- Get a list of files to get -------------------------------------
227 mess 3 "Getting file list"
231 if test ${barrel} -gt 0 ; then
234 if test ${barrel} -gt 1 ; then
235 file=trending_barrel.root
236 other=QAresults_barrel.root
241 *) mess 3 "Assuming simulation output"
248 if echo "$passno" | grep -q -E '^[0-9]*[.]?[0-9]*$' ; then
249 if test "x$passfull" != "x" && test $passno -gt 0 ; then
257 local post=${passpost}
261 *) post="_${post}" ;;
265 if test "x$path" = "x" ; then
266 # Assume official productions
267 path=/alice/${datd}/${year}/${prodfull}/
268 search="${esdd}${passpre}${paid}${post}"
273 if test $qanumber -gt 0 ; then
274 qapost=`printf "QA%02d" $qanumber`
275 search=`append_to_path "$search" $qapost`
278 search=`append_to_path "$search" $file`
284 if test $from_local -lt 1 ; then
286 mess 1 "Getting list of files from AliEn - can take minutes - be patient"
287 mess 2 "alien_find ${path} ${search}"
288 files=`alien_find ${path} ${search} | grep -v "\(files found\|AND THE\)" 2>> ${redir}`
290 files=`ls ${top}/${store}/trending_*.root | sed 's,${top}/${store}/,,g'`
295 mess 1 -n "Total of $numf files ... "
296 if test $maxf -lt 0 ; then
299 mess 1 "using $maxf first of these"
303 # --- Change permissions on files ------------------------------------
306 # if test ! -f $1 ; then return ; fi
307 chmod g+rwX $1 >> /dev/null 2>&1
308 chmod o+rX $1 >> /dev/null 2>&1
310 # chmod g+rwX $1 >> ${redir} 2>&1
311 # chmod o+rX $1 >> ${redir} 2>&1
314 # --- Check if a file is OK ------------------------------------------
318 if test $docheck -lt 1 ; then return 0; fi
319 root -l -b <<EOF >> ${redir} 2>&1
320 .L $ALICE_ROOT/PWGLF/FORWARD/analysis2/qa/CheckQAFile.C
325 mess 2 "Check of $1 -> $ret"
330 # --- Analyse a file -------------------------------------------------
333 local dir=`dirname $1`
334 local inp=`basename $1`
335 local out=`echo $inp | sed 's/trending_/tree_/'`
337 mess 2 -n "Analysing $inp -> $out ... "
339 if test -f $dir/$out ; then
340 if test $force -lt 1 ; then
350 RunFileQA("$inp", "$out", $prodyear, "$prodletter");
360 # --- Download a single file -----------------------------------------
364 local source=$1 ; shift
365 local store=$1 ; shift
367 local o=${store}/`basename $file .root`_${r}.root
369 mess 2 -n "$source -> $o ... "
370 if test -f $o && test $force -lt 2; then
375 mess 2 -n "copying ... "
376 alien_cp alien:${source} file:${o} >> ${redir} 2>&1
380 if test ! -f $o ; then return 1 ; fi
382 if test $also_results -gt 0 ; then
383 local s=`dirname ${source}`/${other}
384 local q=${store}/`basename $other .root`_${r}.root
386 mess 2 -n "$s -> $q ... "
387 if test -f $q && test $force -lt 2 ; then
391 mess 2 -n "copying ... "
392 alien_cp alien:${s} file:${q} >> ${redir} 2>&1
403 1|3|4|5|6) return 2 ;;
411 # --- Submit run analysis to background ------------------------------
420 mess 5 "Submitting $maxjobs jobs from $sta/$maxf"
422 let cur=$sta+$counter
425 if test $from_local -lt 1 ; then
426 local b=`echo $i | sed -e "s,${path},,"`
427 r=`echo $b | sed -e "s,/.*,,"`
429 r=`basename $i .root | sed 's/trending_//'`
432 printf "%3d/%3d: %s\n" $cur $max $r
435 let counter=$counter+1
437 analyse_run $i $out $r &
438 j=`jobs %% | sed -e 's/^[^0-9]*//' -e 's/[^0-9]*$//'`
439 joblist="$joblist $j"
443 mess 5 "will wait for jobs $joblist"
444 for i in $joblist ; do
445 mess 5 "waiting for $i of $joblist"
447 mess 5 "Analysing ${runs[$counter]} returned $?"
448 let counter=$counter+1
452 # --- Analyse each run in turn ---------------------------------------
453 maxjobs=`grep "processor" /proc/cpuinfo | wc -l`
463 while test $# -gt 0 ; do
464 if test $counter -ge $max ; then
471 let counter=$counter+1
473 if test $queued -eq 1 ; then
477 if test $queued -ge $maxjobs ; then
478 mess 1 "Submitting $queued jobs from $start/$max"
479 submit_runs $out $start $max $list
484 if test $queued -gt 0 ; then
485 mess 1 "Submitting $queued jobs from $start/$max"
486 submit_runs $out $start $max $list
490 # --- Copy style -----------------------------------------------------
494 if test ! -f $file ; then return ; fi
495 base=`basename $file`
502 copy_aliroot_file $style
505 # --- Run the final trending -----------------------------------------
511 mess 1 -n "Analysing for trend $dir ... "
519 RunFinalQA(".", $prodyear, "$prodletter", $variance);
523 # mess 3 -n "root -l -b -q ${scr}.C "
524 # root -l -b -q ${scr}.C > /dev/null 2>&1
530 local idx=`ls trend_*_*.html 2> /dev/null`
532 mess 1 "Making index.html point to $i"
533 sed -e 's,index.html,../index.html,' \
534 -e "s,!--JOBID--,a target='_blank' href='${jobu}${jobid}'>Job</a," \
539 if test ! -f index.html ; then
540 echo "No index file found"
544 fix_perm . > /dev/null 2>&1
548 copy_aliroot_file $favicon
549 copy_aliroot_file $logo
554 # --- Make index file ------------------------------------------------
558 local title=$1 ; shift
559 local desc=$1 ; shift
560 mess 1 "Making index in $dir ($title)"
567 cat <<EOF > index.html
571 <title>$title</title>
572 <link rel='stylesheet' href='style.css'>
573 <link rel='shortcut icon' href='fmd_favicon.png' type='image/x-png'>
576 <h1><img style='width: 200px;' src='fmd_logo.png'> $title</h1>
578 if test ! "x$desc" = "x" ; then
579 echo "<p>$desc</p>" >> index.html
581 echo " <ul>" >> index.html
583 if test ! -d $i ; then continue ; fi
584 echo " <li><a href='$i'>$i</a></li>" >> index.html
586 echo " </ul>" >> index.html
587 if test "x$desc" = "x" ; then
588 echo " <div class='back'><a href='../'>Back</a></div>" \
591 cat <<EOF >> index.html
592 <div class='change'>Last update: $date</div>
597 copy_aliroot_file $favicon
598 copy_aliroot_file $logo
600 fix_perm . > /dev/null 2>&1
605 # --- Pass command line options --------------------------------------
609 while test $# -gt 0 ; do
611 -h|--help) usage ; exit 0 ;;
612 -v|--verbose) let verb=$verb+1 ;;
613 -j|--jobid) jobid=$2 ; shift ;;
614 -m|--max-files) maxf=$2 ; shift ;;
615 -M|--max-jobs) maxjobs=$2 ; shift ;;
616 -t|--top) top=$2 ; shift ;;
617 -R|--also-results) also_results=1 ;;
618 -Q|--qa-number) qanumber=$2 ; shift ;;
619 -l|--log-file) redir= ;;
620 -L|--local) from_local=1 ;;
621 -V|--variance) variance=1 ;;
622 -b|--barrel) barrel=$2 ; shift ;;
623 -f|--force) let force=$force+1 ;;
625 prodfull=$2; shift; parse_prod ; year=20${prodyear} ;;
627 passfull=$2; shift; parse_pass ;;
628 -d|--directory) path=$2 ; shift ;;
629 *) echo "$0: Unknown argument: $1" > /dev/stderr ; exit 1 ;;
633 # --- Initial setup --------------------------------------------------
634 # First, check we have a valid AliEn token, then retrieve the job
635 # information to parse out the location of the files we need, and
636 # finally make our output directory and check the lock
639 if test ! "x$jobid" = x ; then
640 if test ! "x$prodfull" = "x" || test ! "x$passfull" = "x" ; then
641 cat <<EOF > /dev/stderr
642 Option -j ${jobid} and options -p and -P are mutually exclusive
648 if test "x$prodfull" = "x" || test "x$passfull" = "x" ; then
649 cat<<EOF > /dev/stderr
650 When specifying prodcution and/or pass both options -p and -P _must_
654 elif test ! "x$jobid" = "x" ; then
655 cat <<EOF > /dev/stderr
656 Option -j and options -p ${prodfull} and -P ${passfull} are mutually exclusive
662 proddir=LHC${prodyear}${prodletter}
664 if test ! "x$passno" = "x" ; then
665 if test "x${passpre}" = "xv" || test "x${passpre}" = "xc"; then
666 store=${store}/${passpre}pass${passno}
668 store=${store}/pass${passno}
670 elif test ! "x$prodpost" = "x" ; then
671 proddir=${proddir}${prodpost}
673 elif test ! "x$remainder" = "x" ; then
674 store=${store}/${remainder}
676 if test ! "x$qanumber" = "x" && test $qanumber -gt 0 ; then
677 store=${store}_QA${qanumber}
679 mkdir -p ${top}/$store
680 fix_perm ${top}/${proddir}
681 fix_perm ${top}/$store
683 if test "x$redir" = "x" ; then
684 redir=${top}/$store/qa.log
689 check_lock ${top}/$store
691 # --- Some friendly information --------------------------------------
694 Production: $prodfull
704 Output directory: ${store}
708 Use variance: ${variance}
709 Use pre-downloaded: ${from_local}
711 # --- Do a search to find our files ----------------------------------
714 if test $maxf -gt 0 && test $maxf -lt $numf ; then
718 # --- Copy scripts to target and compile -----------------------------
719 for i in QABase.h QAPlotter.C QARing.h QAStructs.h QATrender.C \
720 RunFileQA.C RunFinalQA.C ; do
721 cp $ALICE_ROOT/PWGLF/FORWARD/analysis2/qa/$i ${store}/${i}
722 rm -f ${store}/`echo $i | tr '.' '_'`.{so,d}
723 fix_perm ${store}/${i}
725 mess 1 "Compiling QATrender.C"
726 (cd $store && root -l -b <<EOF
727 gROOT->LoadMacro("QABase.h++g");
728 gROOT->LoadMacro("QATrender.C++g");
732 mess 1 "Compiling QAPlotter.C"
733 (cd $store && root -l -b <<EOF
734 gROOT->LoadMacro("QABase.h++g");
735 gROOT->LoadMacro("QAPlotter.C++g");
739 (cd ${store} && for i in *.so *.d ; do fix_perm $i ; done)
741 # --- Now get and analyse each run -----------------------------------
742 analyse_runs ${top}/$store $numf $files
744 # --- Now analyse all runs -------------------------------------------
745 make_trend ${top}/$store
747 # --- Make index files -----------------------------------------------
748 make_index ${top}/${proddir} ${proddir}
749 # make_index ${top} "QA for the FMD" \
750 # "For more information see <a href='https://twiki.cern.ch/twiki/bin/viewauth/ALICE/FMDQA'>TWiki pages</a>."
751 $topmk --title "QA for the FMD" \
752 --description "For more information see <a href='https://twiki.cern.ch/twiki/bin/viewauth/ALICE/FMDQA'>TWiki pages</a>." \
757 copy_aliroot_file $script
759 chmod -R g+rwX ${top}/${proddir} >> ${redir} 2>&1