3 # This script runs the Forward QA for the specified production number
5 # The scripts downloads and runs the single run QA in parallel
8 # --- Help output ----------------------------------------------
12 Usage: $0 [OPTIONS] -j [JOBID]
13 $0 [OPTIONS] -p PRODUCTION -P PASS
17 -j,--jobid NUMBER The master job id of the production [$jobid]
18 -v,--verbose Increase verbosity [$verb]
19 -m,--max-files NUMBER Max number of files to get [$maxf]
20 -M,--max-jobs NUMBER Max number of consequtive jobs [$maxjobs]
21 -t,--top DIRECTORY Output directory [$top]
22 -R,--also-results Also get QAresults.root for each run
23 -Q,--qa-number NUMBER Custom QA id [$qanumber]
24 -p,--production IDENTIFIER Production identifier [$prodfull]
25 -P,--pass IDENTIFIER Pass identifier [$passfull]
26 -l,--log-file Log file output [$redir]
27 -b,--barrel MODE Fetch barrel data [$barrel]
28 -f,--force Force re-run analysis [$force]
30 Note the option -j and the options -p and -P are mutually exclusive,
31 The option -Q is only used if the options -p and -P are given.
32 Production identifiers are of the form LHC11h, LHC11h3, or LHC11h_2.
33 Pass identifers are of the form pass2, pass1_HLT, or cpass1. If barrel mode
34 is >0, then do not assume ESD directory. If mode>1, then get
35 trending_barrel.root and QAresults_barrel.root
39 # --- Check AliEn token ----------------------------------------------
43 genv_file=/tmp/gclient_env_${uid}
45 if test ! -f ${genv_file} ; then
46 echo "No such file: ${genv_file}, please do alien-token-init" \
51 alien-token-info | grep -q "Token is still valid"
52 if test $? -ne 0 ; then
53 echo "Token not valid, please re-new" > /dev/stderr
58 # --- Diagnostics output ---------------------------------------------
62 if test $1 -gt $verb ; then return ; fi
67 # --- Handling of exit -----------------------------------------------
71 if test "x$lock" = "x" ; then return ; fi
76 # --- Handling of errors ---------------------------------------------
92 # --- Check the lock -------------------------------------------------
98 if test -f $lock ; then
99 echo "Another QA process is already running:" > /dev/stderr
100 echo "Content of ${lock}:" > /dev/stderr
101 cat $lock > /dev/stderr
113 # --- Parse production information -----------------------------------
116 prodyear=`echo $prodfull | sed 's/LHC\(..\).*/\1/'`
117 prodletter=`echo $prodfull | sed "s/LHC${prodyear}\(.\).*/\1/"`
118 prodpost=`echo $prodfull | sed "s/LHC${prodyear}${prodletter}//"`
123 passno=`echo $passfull | sed 's/.*pass\([0-9]*\).*/\1/'`
124 passpost=`echo $passfull | sed -n "s/.*pass${passno}_//p"`
125 passpre=`echo $passfull | sed -n "s/pass.*//p"`
128 # --- Extract parts from the found path ------------------------------
144 mess 1 "Parsing information from job $@"
147 local lrunn=$1 ; shift
148 local ltype=$1 ; shift
153 mess 10 "prodfull=$prodfull"
154 mess 10 "lrunn=$lrunn"
155 mess 10 "ltype=$ltype"
156 mess 10 "passfull=$passfull"
157 mess 10 "remainder=$remainder"
159 if test "x$passfull" = "x" ; then
165 *) remainder=$passfull
173 xQA*) qanumber=`echo $remainder | sed 's/QA//'` ;;
178 # --- Get the index for information ----------------------------------
180 jobu="http://alimonitor.cern.ch/prod/jobs.jsp?t="
184 mess 1 "Getting the job information"
185 wget -q ${jobu}${jobid} -O job.html
188 p=`grep "/catalogue/index.jsp?path" job.html | head -n $lskip | tail -n 1 | sed -e 's,.*/alice/\(data\|sim\)/\([^<]*\)<.*,\2,' | tr '/' ' '`
193 # --- Append path element --------------------------------------------
198 if test "x$tmp" != "x" ; then tmp="${tmp}/" ; fi
202 # --- Get a list of files to get -------------------------------------
210 mess 3 "Getting file list"
214 if test ${barrel} -gt 0 ; then
217 if test ${barrel} -gt 1 ; then
218 file=trending_barrel.root
219 other=QAresults_barrel.root
224 *) mess 3 "Assuming simulation output"
231 if test "x$passfull" != "x" && test $passno -gt 0 ; then
234 local post=${passpost}
238 *) post="_${post}" ;;
241 path=/alice/${datd}/${year}/${prodfull}/
242 local search="${esdd}${passpre}${paid}${post}"
244 if test $qanumber -gt 0 ; then
245 qapost=`printf "QA%02d" $qanumber`
246 search=`append_to_path "$search" $qapost`
249 search=`append_to_path "$search" $file`
255 mess 1 "Getting list of files from AliEn - can take minutes - be patient"
256 mess 2 "alien_find ${path} ${search}"
257 files=`alien_find ${path} ${search} | grep -v "\(files found\|AND THE\)" 2>> ${redir}`
261 mess 1 -n "Total of $numf files ... "
262 if test $maxf -lt 0 ; then
265 mess 1 "using $maxf first of these"
269 # --- Change permissions on files ------------------------------------
272 if test ! -f $1 ; then return ; fi
277 # --- Check if a file is OK ------------------------------------------
281 if test $docheck -lt 1 ; then return 0; fi
282 root -l -b <<EOF >> ${redir} 2>&1
283 .L $ALICE_ROOT/PWGLF/FORWARD/analysis2/qa/CheckQAFile.C
288 mess 2 "Check of $1 -> $ret"
293 # --- Analyse a file -------------------------------------------------
296 local dir=`dirname $1`
297 local inp=`basename $1`
298 local out=`echo $inp | sed 's/trending_/tree_/'`
300 mess 2 -n "Analysing $inp -> $out ... "
302 if test -f $dir/$out ; then
303 if test $force -lt 0 ; then
313 RunFileQA("$inp", "$out", $prodyear, "$prodletter");
323 # --- Download a single file -----------------------------------------
327 local source=$1 ; shift
328 local store=$1 ; shift
330 local o=${store}/`basename $file .root`_${r}.root
332 mess 2 -n "$source -> $o ... "
337 mess 2 -n "copying ... "
338 alien_cp alien:${source} file:${o} >> ${redir} 2>&1
342 if test ! -f $o ; then return 1 ; fi
344 if test $also_results -gt 0 ; then
345 local s=`dirname ${source}`/${other}
346 local q=${store}/`basename $other .root`_${r}.root
348 mess 2 -n "$s -> $q ... "
352 mess 2 -n "copying ... "
353 alien_cp alien:${s} file:${q} >> ${redir} 2>&1
364 1|3|4|5|6) return 2 ;;
372 # --- Submit run analysis to background ------------------------------
381 mess 5 "Submitting $maxjobs jobs from $sta/$maxf"
383 let cur=$sta+$counter
385 local b=`echo $i | sed -e "s,${path},,"`
386 local r=`echo $b | sed -e "s,/.*,,"`
388 printf "%3d/%3d: %s\n" $cur $max $r
391 let counter=$counter+1
393 analyse_run $i $out $r &
394 j=`jobs %% | sed -e 's/^[^0-9]*//' -e 's/[^0-9]*$//'`
395 joblist="$joblist $j"
399 mess 5 "will wait for jobs $joblist"
400 for i in $joblist ; do
401 mess 5 "waiting for $i of $joblist"
403 mess 5 "Analysing ${runs[$counter]} returned $?"
404 let counter=$counter+1
408 # --- Analyse each run in turn ---------------------------------------
409 maxjobs=`grep "processor" /proc/cpuinfo | wc -l`
419 while test $# -gt 0 ; do
420 if test $counter -ge $max ; then
427 let counter=$counter+1
429 if test $queued -eq 1 ; then
433 if test $queued -ge $maxjobs ; then
434 mess 1 "Submitting $queued jobs from $start/$max"
435 submit_runs $out $start $max $list
440 if test $queued -gt 0 ; then
441 mess 1 "Submitting $queued jobs from $start/$max"
442 submit_runs $out $start $max $list
446 # --- Copy style -----------------------------------------------------
450 if test ! -f $file ; then return ; fi
451 base=`basename $file`
458 copy_aliroot_file $style
461 # --- Run the final trending -----------------------------------------
466 mess 1 -n "Analysing $dir ... "
474 RunFinalQA(".", $prodyear, "$prodletter");
478 # mess 3 -n "root -l -b -q ${scr}.C "
479 # root -l -b -q ${scr}.C > /dev/null 2>&1
485 local idx=`ls trend_*_*.html 2> /dev/null`
487 mess 1 "Making index.html point to $i"
488 sed -e 's,index.html,../index.html,' \
489 -e "s,!--JOBID--,a target='_blank' href='${jobu}${jobid}'>Job</a," \
494 if test ! -f index.html ; then
495 echo "No index file found"
499 fix_perm . > /dev/null 2>&1
503 copy_aliroot_file $favicon
504 copy_aliroot_file $logo
509 # --- Make index file ------------------------------------------------
513 local title=$1 ; shift
514 local desc=$1 ; shift
515 mess 1 "Making index in $dir ($title)"
522 cat <<EOF > index.html
526 <title>$title</title>
527 <link rel='stylesheet' href='style.css'>
528 <link rel='shortcut icon' href='fmd_favicon.png' type='image/x-png'>
531 <h1><img src='fmd_logo.png'>$title</h1>
533 if test ! "x$desc" = "x" ; then
534 echo "<p>$desc</p>" >> index.html
536 echo " <ul>" >> index.html
538 if test ! -d $i ; then continue ; fi
539 echo " <li><a href='$i'>$i</a></li>" >> index.html
541 echo " </ul>" >> index.html
542 if test "x$desc" = "x" ; then
543 echo " <div class='back'><a href='../'>Back</a></div>" \
546 cat <<EOF >> index.html
547 <div class='change'>Last update: $date</div>
552 copy_aliroot_file $favicon
553 copy_aliroot_file $logo
555 fix_perm . > /dev/null 2>&1
560 # --- Pass command line options --------------------------------------
564 while test $# -gt 0 ; do
566 -h|--help) usage ; exit 0 ;;
567 -v|--verbose) let verb=$verb+1 ;;
568 -j|--jobid) jobid=$2 ; shift ;;
569 -m|--max-files) maxf=$2 ; shift ;;
570 -M|--max-jobs) maxjobs=$2 ; shift ;;
571 -t|--top) top=$2 ; shift ;;
572 -R|--also-results) also_results=1 ;;
573 -Q|--qa-number) qanumber=$2 ; shift ;;
585 -l|--log-file) redir= ; shift ;;
586 -b|--barrel) barrel=$2; shift ;;
587 -f|--force) force=1 ;;
588 *) echo "$0: Unknown argument: $1" > /dev/stderr ; exit 1 ;;
592 # --- Initial setup --------------------------------------------------
593 # First, check we have a valid AliEn token, then retrieve the job
594 # information to parse out the location of the files we need, and
595 # finally make our output directory and check the lock
598 if test ! "x$jobid" = x ; then
599 if test ! "x$prodfull" = "x" || test ! "x$passfull" = "x" ; then
600 cat <<EOF > /dev/stderr
601 Option -j ${jobid} and options -p and -P are mutually exclusive
607 if test "x$prodfull" = "x" || test "x$passfull" = "x" ; then
608 cat<<EOF > /dev/stderr
609 When specifying prodcution and/or pass both options -p and -P _must_
613 elif test ! "x$jobid" = "x" ; then
614 cat <<EOF > /dev/stderr
615 Option -j and options -p ${prodfull} and -P ${passfull} are mutually exclusive
621 proddir=LHC${prodyear}${prodletter}
623 if test ! "x$passno" = "x" ; then
624 if test "x${passpre}" = "xv" ; then
625 store=${store}/${passpre}pass${passno}
627 store=${store}/pass${passno}
629 elif test ! "x$prodpost" = "x" ; then
630 proddir=${proddir}${prodpost}
633 if test ! "x$qanumber" = "x" && test $qanumber -gt 0 ; then
634 store=${store}_QA${qanumber}
636 mkdir -p ${top}/$store
637 fix_perm ${top}/${proddir}
638 fix_perm ${top}/$store
640 if test "x$redir" = "x" ; then
641 redir=${top}/$store/qa.log
646 check_lock ${top}/$store
650 Production: $prodfull
660 Output directory: ${store}
665 # --- Copy scripts to target -----------------------------------------
666 for i in QABase.h QAPlotter.C QARing.h QAStructs.h QATrender.C \
667 RunFileQA.C RunFinalQA.C ; do
668 cp $ALICE_ROOT/PWGLF/FORWARD/analysis2/qa/$i ${store}/${i}
669 rm -f ${store}/`echo $i | tr '.' '_'`.{so,d}
671 mess 1 "Compiling QATrender.C"
672 (cd $store && root -l -b <<EOF
673 gROOT->LoadMacro("QABase.h++g");
674 gROOT->LoadMacro("QATrender.C++g");
678 mess 1 "Compiling QAPlotter.C"
679 (cd $store && root -l -b <<EOF
680 gROOT->LoadMacro("QABase.h++g");
681 gROOT->LoadMacro("QAPlotter.C++g");
686 # --- Do a search to find our files ----------------------------------
689 if test $maxf -gt 0 && test $maxf -lt $numf ; then
693 # --- Now get and analyse each run -----------------------------------
694 analyse_runs ${top}/$store $numf $files
696 # --- Now analyse all runs -------------------------------------------
697 style=$ALICE_ROOT/PWGLF/FORWARD/analysis2/qa/style.css
698 favicon=$ALICE_ROOT/PWGLF/FORWARD/analysis2/qa/fmd_favicon.png
699 logo=$ALICE_ROOT/PWGLF/FORWARD/analysis2/qa/fmd_logo.png
700 make_trend ${top}/$store
702 # --- Make index files -----------------------------------------------
703 make_index ${top}/${proddir} ${proddir}
704 make_index ${top} "QA for the FMD" \
705 "For more information see <a href='https://twiki.cern.ch/twiki/bin/viewauth/ALICE/FMDQA'>TWiki pages</a>."