2 # process QA output into plots and trending
3 # run without arguments for examples
4 # origin: Mikolaj Krzewicki, mkrzewic@cern.ch
6 if [ ${BASH_VERSINFO} -lt 4 ]; then
7 echo "bash version >= 4 needed, you have ${BASH_VERSION}, exiting..."
15 echo " ${0##*/} option=value [option=value]"
16 echo " at least inputList should be specified, or configFile containing it:"
17 echo " ${0##*/} inputList=file.list"
18 echo " options override config file (if any), e.g.:"
19 echo " ${0##*/} configFile=runQA.config inputList=file.list outputDirectory=%det"
20 echo "some expert options"
21 echo " inputListHighPtTrees=file.list - external list of filtered trees, requires inputList to be set"
22 echo " includeDetectors=TPC,V0,MU - only process those"
23 echo " excludeDetectors=EVS,TPC - skip processing of those"
24 echo " - see example config file for more"
28 if ! parseConfig "$@"; then
33 [[ -z $ALICE_ROOT ]] && echo "ALICE_ROOT not defined" && return 1
36 if [[ ${ocdbStorage} =~ ${ocdbregex} ]]; then
37 alien-token-init ${alienUserName}
38 #this is a hack! alien-token init seems not enough
39 #but the gclient_env script messes up the LD_LIBRARY_PATH
42 done < <(grep -v "LD_LIBRARY_PATH" /tmp/gclient_env_${UID})
54 #be paranoid and make some full paths
55 [[ ! -f ${inputList} ]] && echo "no input list: ${inputList}" && return 1
56 inputList=$(get_realpath ${inputList})
57 mkdir -p ${workingDirectory}
58 #this is a trick to get the full path of workingDirectory
59 #(on a mac 'readlink -f' does not work...)
60 workingDirectory=$(workingDirectory=${workingDirectory%/}; cd ${workingDirectory%/*}; echo "${PWD}/${workingDirectory##*/}")
61 if [[ ! -d ${workingDirectory} ]]; then
62 echo "working dir $workingDirectory does not exist and cannot be created"
65 cd ${workingDirectory}
68 echo inputList=$inputList
69 echo outputDirectory=$outputDirectory
72 dateString=$(date +%Y-%m-%d-%H-%M-%S-%N)
73 echo "Start time QA process: $dateString"
76 mkdir -p $logDirectory
77 [[ ! -d $logDirectory ]] && echo "no log dir $logDirectory" && return 1
78 logFile="$logDirectory/${0##*/}.${dateString}.log"
80 [[ ! -f ${logFile} ]] && echo "cannot write logfile $logfile" && return 1
81 echo "logFile = $logFile"
84 lockFile=${workingDirectory}/runQA.lock
85 [[ -f ${lockFile} ]] && echo "lock ${lockFile} exists!" | tee ${logFile} && return 1
87 [[ ! -f ${lockFile} ]] && echo "cannot lock $lockFile" | tee ${logFile} && return 1
91 ################################################################
93 for detectorScript in $ALICE_ROOT/PWGPP/QA/detectorQAscripts/*; do
95 echo "##############################################"
98 [[ ! ${detectorScript} =~ .*\.sh$ ]] && continue
99 detector=${detectorScript%.sh}
100 detector=${detector##*/}
101 #by default we expect the container in the QA root file to de named like
103 detectorQAcontainerName=${detector}
106 if [[ "${excludeDetectors}" =~ ${detector} ]]; then
107 echo "${detector} is excluded in config, skipping..."
111 #if includeDetectors set, only process thoe detectors specified there
112 if [[ -n ${includeDetectors} && ! "${includeDetectors}" =~ ${detector} ]]; then
113 echo "${detector} not included in includeDetectors, skipping..."
117 logSummary=${logDirectory}/summary-${detector}-${dateString}.log
118 hostInfo >> ${logSummary}
119 outputDir=$(substituteDetectorName ${detector} ${outputDirectory})
120 tmpDetectorRunDir=${workingDirectory}/tmpQAtmpRunDir${detector}-${dateString}
121 if ! mkdir -p ${tmpDetectorRunDir}; then
122 echo "cannot create the temp dir $tmpDetectorRunDir"
125 cd ${tmpDetectorRunDir}
127 tmpPrefix=${tmpDetectorRunDir}/${outputDir}
128 echo "running QA for ${detector}"
129 echo " outputDir=$outputDir"
130 echo " tmpPrefix=$tmpPrefix"
132 #source the detector script
133 #unset the detector functions from previous iterations (detectors)
135 unset -f runLevelQAouter
136 unset -f periodLevelQA
137 unset -f runLevelEventStatQA
138 unset -f runLevelHighPtTreeQA
139 unset -f periodLevelHighPtTreeQA
140 source ${detectorScript}
142 #################################################################
143 #produce the QA and trending tree for each file (run)
144 unset arrOfTouchedProductions
145 declare -A arrOfTouchedProductions
146 while read inputFile; do
150 #first check if input file exists
151 [[ ! -f ${inputFile%\#*} ]] && echo "file ${inputFile%\#*} not accessible" && continue
153 if ! guessRunData ${inputFile}; then
154 echo "could not guess run data from ${inputFile}"
157 echo "anchorYear for ${originalPeriod} is: ${anchorYear}"
159 tmpProductionDir=${tmpPrefix}/${dataType}/${year}/${period}/${pass}
160 tmpRunDir=${tmpProductionDir}/000${runNumber}
161 mkdir -p ${tmpRunDir}
164 #check what kind of input file we have, default is a zip archive
165 #set the inputs accordingly
170 eventStatFileOuter=""
171 #it is possible we get the highPt trees from somewhere else
172 #search the list of high pt trees for the proper run number
173 if [[ -n ${inputListHighPtTrees} ]]; then
174 highPtTree=$(egrep -m1 ${runNumber} ${inputListHighPtTrees})
175 echo "loaded the highPtTree ${highPtTree} from external file ${inputListHighPtTrees}"
177 #if we are explicit about the input file this takes precedence
178 #over earlier additions
179 [[ "${inputFile}" =~ QAresults.root$ ]] && qaFile=${inputFile}
180 [[ "${inputFile}" =~ QAresults_outer.root$ ]] && qaFileOuter=${inputFile}
181 [[ "${inputFile}" =~ FilterEvents_Trees.root$ ]] && highPtTree=${inputFile}
182 [[ "${inputFile}" =~ event_stat.root$ ]] && eventStatFile=${inputFile}
183 [[ "${inputFile}" =~ event_stat_outer.root$ ]] && eventStatFileOuter=${inputFile}
184 if [[ "${inputFile}" =~ \.zip$ ]]; then
185 [[ -z ${qaFile} ]] && qaFile=${inputFile}
186 [[ -z ${qaFileOuter} ]] && qaFileOuter=${inputFile}
187 [[ -z ${highPtTree} ]] && highPtTree=${inputFile}
188 [[ -z ${eventStatFile} ]] && eventStatFile=${inputFile}
189 [[ -z ${eventStatFileOuter} ]] && eventStatFileOuter=${inputFile}
192 #if we have zip archives in the input, extract the proper file name
193 #from the archive and append in a root-like fashion
194 if [[ "$qaFile" =~ .*.zip$ ]]; then
195 if unzip -l ${qaFile} | egrep "QAresults.root" &>/dev/null; then
196 qaFile+="#QAresults.root"
197 elif unzip -l ${qaFile} | egrep "QAresults_barrel.root" &>/dev/null; then
198 qaFile+="#QAresults_barrel.root"
203 if [[ "$qaFileOuter" =~ .*.zip$ ]]; then
204 if unzip -l ${qaFileOuter} | egrep "QAresults_outer.root" &>/dev/null; then
205 qaFileOuter+="#QAresults_outer.root"
210 if [[ "$highPtTree" =~ .*.zip$ ]]; then
211 if unzip -l ${highPtTree} | egrep "FilterEvents_Trees.root" &>/dev/null; then
212 highPtTree+="#FilterEvents_Trees.root"
217 if [[ "${eventStatFile}" =~ .*.zip$ ]]; then
218 if unzip -l ${eventStatFile} | egrep "event_stat.root" &>/dev/null; then
219 eventStatFile+="#event_stat.root"
220 elif unzip -l ${eventStatFile} | egrep "event_stat_barrel.root" &>/dev/null; then
221 eventStatFile+="#event_stat_barrel.root"
226 if [[ "${eventStatFileOuter}" =~ .*.zip$ ]]; then
227 if unzip -l ${eventStatFileOuter} | egrep "event_stat_outer.root" &>/dev/null; then
228 eventStatFileOuter+="#event_stat.root"
230 eventStatFileOuter=""
235 echo qaFileOuter=$qaFileOuter
236 echo highPtTree=$highPtTree
237 echo eventStatFile=$eventStatFile
238 echo eventStatFileOuter=$eventStatFileOuter
239 echo ocdbStorage=${ocdbStorage}
242 #standard QA based on QAresults.root file (and variants)
243 if [[ -n ${qaFile} && $(type -t runLevelQA) =~ "function" ]]; then
244 echo running ${detector} runLevelQA for run ${runNumber} from ${qaFile}
245 ( runLevelQA "${qaFile}" ) &>> runLevelQA.log
246 #cache the touched production + an example file to guarantee consistent run data parsing
247 arrOfTouchedProductions[${tmpProductionDir}]="${inputFile%\#*}"
249 #standard QA based on QAresults_outer.root file (there in cpass, with different triggers)
250 if [[ -n ${qaFileOuter} && $(type -t runLevelQAouter) =~ "function" ]]; then
251 echo running ${detector} runLevelQAouter for run ${runNumber} from ${qaFileOuter}
252 ( runLevelQAouter "${qaFileOuter}" ) &>> runLevelQA.log
253 #cache the touched production + an example file to guarantee consistent run data parsing
254 arrOfTouchedProductions[${tmpProductionDir}]="${inputFile%\#*}"
256 #expert QA based on high pt trees
257 if [[ -n ${highPtTree} && $(type -t runLevelHighPtTreeQA) =~ "function" ]]; then
258 echo running ${detector} runLevelHighPtTreeQA for run ${runNumber} from ${highPtTree}
259 ( runLevelHighPtTreeQA "${highPtTree}" ) &>> runLevelQA.log
260 #cache the touched production + an example file to guarantee consistent run data parsing
261 arrOfTouchedProductions[${tmpProductionDir}]="${inputFile%\#*}"
263 #event stat QA based on event_stat.root file
264 if [[ -n ${eventStatFile} && $(type -t runLevelEventStatQA) =~ "function" ]]; then
265 echo running ${detector} runLevelEventStatQA for run ${runNumber} from ${eventStatFile}
266 ( runLevelEventStatQA "${eventStatFile}" ) &>> runLevelQA.log
267 #cache the touched production + an example file to guarantee consistent run data parsing
268 arrOfTouchedProductions[${tmpProductionDir}]="${inputFile%\#*}"
270 #event stat QA based on event_stat_outer.root file
271 if [[ -n ${eventStatFileOuter} && $(type -t runLevelEventStatQAouter) =~ "function" ]]; then
272 echo running ${detector} runLevelEventStatQAouter for run ${runNumber} from ${eventStatFileOuter}
273 ( runLevelEventStatQAouter "${eventStatFileOuter}" ) &>> runLevelQA.log
274 #cache the touched production + an example file to guarantee consistent run data parsing
275 arrOfTouchedProductions[${tmpProductionDir}]="${inputFile%\#*}"
278 #perform some default actions:
279 #if trending.root not created, create a default one
280 if [[ ! -f trending.root ]]; then
281 aliroot -b -q -l "$ALICE_ROOT/PWGPP/macros/simpleTrending.C(\"${qaFile}\",${runNumber},\"${detectorQAcontainerName}\",\"trending.root\",\"trending\",\"recreate\")" 2>&1 | tee -a runLevelQA.log
283 if [[ ! -f trending.root ]]; then
284 echo "trending.root not created"
287 cd ${tmpDetectorRunDir}
291 #################################################################
292 #cache which productions were (re)done
293 echo "list of processed productions:"
294 echo " ${!arrOfTouchedProductions[@]}"
297 #################################################################
298 #(re)do the merging/trending
299 for tmpProductionDir in ${!arrOfTouchedProductions[@]}; do
300 cd ${tmpProductionDir}
302 echo "running period level stuff in ${tmpProductionDir}"
305 productionDir=${outputDir}/${tmpProductionDir#${tmpPrefix}}
306 echo productionDir=${outputDir}/${tmpProductionDir#${tmpPrefix}}
308 mkdir -p ${productionDir}
309 if [[ ! -d ${productionDir} ]]; then
310 echo "cannot make productionDir $productionDir" && continue
313 #move runs to final destination
314 for dir in ${tmpProductionDir}/000*; do
316 oldRunDir=${outputDir}/${dir#${tmpPrefix}}
317 if ! guessRunData "${arrOfTouchedProductions[${tmpProductionDir}]}"; then
318 echo "could not guess run data from ${arrOfTouchedProductions[${tmpProductionDir}]}"
322 #before moving - VALIDATE!!!
323 if ! validate ${dir}; then
327 #moving a dir is an atomic operation, no locking necessary
328 if [[ -d ${oldRunDir} ]]; then
329 echo "removing old ${oldRunDir}"
332 echo "moving new ${runNumber} to ${productionDir}"
333 mv -f ${dir} ${productionDir}
336 #go to a temp dir to do the period level stuff in a completely clean dir
337 tmpPeriodLevelQAdir="${tmpProductionDir}/periodLevelQA"
339 echo tmpPeriodLevelQAdir="${tmpPeriodLevelQAdir}"
340 if ! mkdir -p ${tmpPeriodLevelQAdir}; then continue; fi
341 cd ${tmpPeriodLevelQAdir}
343 #link the final list of per-run dirs here, just the dirs
344 #to have a clean working directory
346 declare -a linkedStuff
347 for x in ${productionDir}/000*; do [[ -d $x ]] && ln -s $x && linkedStuff+=(${x##*/}); done
349 #merge trending files if any
350 if /bin/ls 000*/trending.root &>/dev/null; then
351 hadd trending.root 000*/trending.root &> periodLevelQA.log
354 #run the period level trending/QA
355 if [[ -f "trending.root" && $(type -t periodLevelQA) =~ "function" ]]; then
356 echo running ${detector} periodLevelQA for production ${period}/${pass}
357 ( periodLevelQA trending.root ) &>> periodLevelQA.log
359 echo "WARNING: not running ${detector} periodLevelQA for production ${period}/${pass}, no trending.root"
362 if ! validate ${PWD}; then continue; fi
364 #here we are validated so move the produced QA to the final place
365 #clean up linked stuff first
366 [[ -n ${linkedStuff[@]} ]] && rm ${linkedStuff[@]}
367 periodLevelLock=${productionDir}/runQA.lock
368 if [[ ! -f ${periodLevelLock} ]]; then
369 #some of the output could be a directory, so handle that
370 #TODO: maybe use rsync?
371 #lock to avoid conflicts:
372 echo "${HOSTNAME} ${dateString}" > ${periodLevelLock}
373 for x in ${tmpPeriodLevelQAdir}/*; do
374 if [[ -d ${x} ]]; then
375 echo "removing ${productionDir}/${x##*/}"
376 rm -rf ${productionDir}/${x##*/}
377 echo "moving ${x} to ${productionDir}"
378 mv ${x} ${productionDir}
380 if [[ -f ${x} ]]; then
381 echo "moving ${x} to ${productionDir}"
382 mv -f ${x} ${productionDir}
385 rm -f ${periodLevelLock}
387 rm -rf ${tmpPeriodLevelQAdir}
389 echo "ERROR: cannot move to destination" >> ${logSummary}
390 echo "production dir ${productionDir} locked!" >> ${logSummary}
391 echo "check and maybe manually do:" >> ${logSummary}
392 echo " rm ${periodLevelLock}" >> ${logSummary}
393 echo " rsync -av ${tmpPeriodLevelQAdir}/ ${productionDir}/" >> ${logSummary}
399 cd ${workingDirectory}
401 if [[ -z ${planB} ]]; then
403 echo removing ${tmpDetectorRunDir}
404 rm -rf ${tmpDetectorRunDir}
408 done #end of detector loop
417 #in case of emergency
418 #first check if we have the email of the detector expert defined,
419 #if yes, append to the mailing list
420 local mailTo=${MAILTO}
421 local detExpertEmailVar="MAILTO_${detector}"
422 [[ -n "${!detExpertEmailVar}" ]] && mailTo+=" ${!detExpertEmailVar}"
423 if [[ -n ${mailTo} ]]; then
425 echo "trouble detected, sending email to ${mailTo}"
426 cat ${logSummary} | mail -s "${detector} QA in need of assistance" ${mailTo}
433 summarizeLogs ${1} >> ${logSummary}
435 if [[ ${logStatus} -ne 0 ]]; then
436 echo "WARNING not validated: ${1}"
446 [[ ! -d ${dir} ]] && dir=${PWD}
448 #print a summary of logs
457 for log in ${dir}/${logFiles[*]}; do
458 [[ ! -f ${log} ]] && continue
459 errorSummary=$(validateLog ${log})
461 [[ validationStatus -ne 0 ]] && logstatus=1
462 if [[ ${validationStatus} -eq 0 ]]; then
463 #in pretend mode randomly report an error in rec.log some cases
464 if [[ -n ${pretend} && "${log}" == "rec.log" ]]; then
465 [[ $(( ${RANDOM}%2 )) -ge 1 ]] && echo "${log} BAD random error" || echo "${log} OK"
469 elif [[ ${validationStatus} -eq 1 ]]; then
470 echo "${log} BAD ${errorSummary}"
471 elif [[ ${validationStatus} -eq 2 ]]; then
472 echo "${log} OK MWAH ${errorSummary}"
480 gdb --batch --quiet -ex "bt" -ex "quit" aliroot ${x} > stacktrace_${x//\//_}.log
481 done < <(/bin/ls ${PWD}/*/core 2>/dev/null; /bin/ls ${PWD}/core 2>/dev/null)
492 'error while loading shared libraries'
495 'Thread [0-9]* (Thread'
498 '\.C.*error:.*\.h: No such file'
500 'Interpreter error recovered'
508 local errorSummary=""
509 local warningSummary=""
511 for ((i=0; i<${#errorConditions[@]};i++)); do
512 local tmp=$(grep -m1 -e "${errorConditions[${i}]}" ${log})
513 [[ -n ${tmp} ]] && tmp+=" : "
517 for ((i=0; i<${#warningConditions[@]};i++)); do
518 local tmp=$(grep -m1 -e "${warningConditions[${i}]}" ${log})
519 [[ -n ${tmp} ]] && tmp+=" : "
520 warningSummary+=${tmp}
523 if [[ -n ${errorSummary} ]]; then
524 echo "${errorSummary}"
528 if [[ -n ${warningSummary} ]]; then
529 echo "${warningSummary}"
542 #where to search for qa files
545 workingDirectory="${PWD}"
546 #where to place the final qa plots
547 #outputDirectory="/afs/cern.ch/work/a/aliqa%det/www/"
548 outputDirectory="${workingDirectory}/%DET"
549 #filter out detector option
550 excludeDetectors="EXAMPLE"
552 logDirectory=${workingDirectory}/logs
556 #MAILTO="fbellini@cern.ch"
564 #first, check if the config file is configured
565 #is yes - source it so that other options can override it
567 for opt in "${args[@]}"; do
568 if [[ ${opt} =~ configFile=.* ]]; then
570 [[ ! -f ${configFile} ]] && echo "configFile ${configFile} not found, exiting..." && return 1
571 echo "using config file: ${configFile}"
572 source "${configFile}"
577 #then, parse the options as they override the options from file
578 for opt in "${args[@]}"; do
579 if [[ ! "${opt}" =~ .*=.* ]]; then
580 echo "badly formatted option ${var}, should be: option=value, stopping..."
583 local var="${opt%%=*}"
584 local value="${opt#*=}"
585 echo "${var}=${value}"
586 export ${var}="${value}"
593 #guess the period from the path, pick the rightmost one
598 legoTrainRunNumber=""
607 declare -a path=( $1 )
609 local dirDepth=$(( ${#path[*]}-1 ))
611 for ((x=${dirDepth};x>=0;x--)); do
613 [[ $((x-1)) -ge 0 ]] && local fieldPrev=${path[$((x-1))]}
614 local field=${path[${x}]}
615 local fieldNext=${path[$((x+1))]}
617 [[ ${field} =~ ^[0-9]*$ && ${fieldNext} =~ (.*\.zip$|.*\.root$) ]] && legoTrainRunNumber=${field}
618 [[ -n ${legoTrainRunNumber} && -z ${pass} ]] && pass=${fieldPrev}
619 [[ ${field} =~ ^LHC[0-9][0-9][a-z].*$ ]] && period=${field%_*} && originalPeriod=${field}
620 [[ ${field} =~ ^000[0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && runNumber=${field#000}
621 [[ ${field} =~ ^[0-9][0-9][0-9][0-9][0-9][0-9]$ ]] && shortRunNumber=${field}
622 [[ ${field} =~ ^20[0-9][0-9]$ ]] && year=${field}
623 [[ ${field} =~ ^(^sim$|^data$) ]] && dataType=${field}
627 [[ -n ${shortRunNumber} && "${legoTrainRunNumber}" =~ ${shortRunNumber} ]] && legoTrainRunNumber=""
628 [[ -z ${legoTrainRunNumber} ]] && pass=${path[$((dirDepth-1))]}
629 [[ "${dataType}" =~ ^sim$ ]] && pass="passMC" && runNumber=${shortRunNumber} && originalPass="" #for MC not from lego, the runnumber is identified as lego train number, thus needs to be nulled
630 [[ -n ${legoTrainRunNumber} ]] && pass+="_lego${legoTrainRunNumber}"
632 #modify the OCDB: set the year
633 if [[ ${dataType} =~ sim ]]; then
634 anchorYear=$(run2year $runNumber)
635 if [[ -z "${anchorYear}" ]]; then
636 echo "WARNING: anchorYear not available for this production: ${originalPeriod}, runNumber: ${runNumber}. Cannot set the OCDB."
639 ocdbStorage=$(setYear ${anchorYear} ${ocdbStorage})
641 ocdbStorage=$(setYear ${year} ${ocdbStorage})
644 #if [[ -z ${dataType} || -z ${year} || -z ${period} || -z ${runNumber}} || -z ${pass} ]];
645 if [[ -z ${runNumber} ]]
657 #for a given run print the year.
658 #the run-year table is ${runMap} (a string)
659 #defined in the config file
660 #one line per year, format: year runMin runMax
662 [[ -z ${run} ]] && return 1
666 while read year runMin runMax; do
667 [[ -z ${year} || -z ${runMin} || -z ${runMax} ]] && continue
668 [[ ${run} -ge ${runMin} && ${run} -le ${runMax} ]] && echo ${year} && break
669 done < <(echo "${runMap}")
673 substituteDetectorName()
677 [[ ${dir} =~ \%det ]] && det=${det,,} && echo ${dir/\%det/${det}}
678 [[ ${dir} =~ \%DET ]] && det=${det} && echo ${dir/\%DET/${det}}
687 if cd "$(echo "${1%/*}")" &>/dev/null
689 # file *may* not be local
690 # exception is ./file.ext
691 # try 'cd .; cd -;' *works!*
695 # file *must* be local
699 # file *cannot* exist
702 # reassemble realpath
703 echo "$tmppwd"/"${1##*/}"
710 # ${1} - year to be set
711 # ${2} - where to set the year
712 local year1=$(guessYear ${1})
713 local year2=$(guessYear ${2})
715 [[ ${year1} -ne ${year2} && -n ${year2} && -n ${year1} ]] && path=${2/\/${year2}\//\/${year1}\/}
722 #guess the year from the path, pick the rightmost one
724 declare -a pathArray=( ${1} )
727 for field in ${pathArray[@]}; do
728 [[ ${field} =~ ^20[0-9][0-9]$ ]] && year=${field}
736 # Hallo world - Print AliRoot/Root/Alien system info
742 echo --------------------------------------
746 echo HOSTINFO HOSTNAME" "$HOSTNAME
747 echo HOSTINFO DATE" "`date`
748 echo HOSTINFO gccpath" "`which gcc`
749 echo HOSTINFO gcc version" "`gcc --version | grep gcc`
750 echo --------------------------------------
755 echo --------------------------------------
759 echo ROOTINFO ROOT" "`which root`
760 echo ROOTINFO VERSION" "`root-config --version`
762 echo --------------------------------------
768 echo --------------------------------------
772 echo ALIROOTINFO ALIROOT" "`which aliroot`
773 echo ALIROOTINFO VERSION" "`echo $ALICE_LEVEL`
774 echo ALIROOTINFO TARGET" "`echo $ALICE_TARGET`
776 echo --------------------------------------
781 #echo --------------------------------------
784 #for a in `alien --printenv`; do echo ALIENINFO $a; done
786 #echo --------------------------------------