[u/mrichter/AliRoot.git] / PWGLF / FORWARD / analysis2 / scripts / getGridData.sh

#!/bin/bash
#
# This script runs the Forward QA for the specified production number
#
# The scripts downloads and runs the single run QA in parallel 
#
noact=0

# --- Help output ----------------------------------------------
usage()
{
    cat <<EOF
Usage: $0 [OPTIONS] -j [JOBID]
       $0 [OPTIONS] -p PRODUCTION -P PASS 

Options:
	-h,--help		   This help 
	-j,--jobid	NUMBER	   The master job id of the production [$jobid]
	-v,--verbose		   Increase verbosity [$verb]
	-m,--max-files  NUMBER     Max number of files to get [$maxf]
	-M,--max-jobs   NUMBER     Max number of consequtive jobs [$maxjobs]
	-t,--top        DIRECTORY  Output directory [$top]
	-p,--production IDENTIFIER Production identifier [$prodfull]
	-P,--pass       IDENTIFIER Pass identifier [$passfull]
	-l,--log-file              Log file output [$redir]
	-f,--file       FILENAME   Name of files to get [$file]
	-r,--run        NUMBER     Run number [$run]

Note the option -j and the options -p and -P are mutually exclusive,
The option -Q is only used if the options -p and -P are given.
Production identifiers are of the form LHC11h, LHC11h3, or LHC11h_2. 
Pass identifers are of the form pass2, pass1_HLT, or cpass1.
EOF
}

# --- Check AliEn token ----------------------------------------------
check_token()
{
    uid=`id -u`
    genv_file=/tmp/gclient_env_${uid}
    
    if test ! -f ${genv_file} ; then 
	echo "No such file: ${genv_file}, please do alien-token-init" \
	    >/dev/stderr
	exit 1
    fi
    . ${genv_file}
    alien-token-info | grep -q "Token is still valid"
    if test $? -ne 0 ; then 
	echo "Token not valid, please re-new" > /dev/stderr 
	exit 1
    fi
}

# --- Diagnostics output ---------------------------------------------
verb=0
mess()
{
    if test $1 -gt $verb ; then return ; fi 
    shift
    echo $*
}

# --- Handling of exit -----------------------------------------------
lock=
handle_exit()
{
    if test "x$lock" = "x" ; then return ; fi 
    rm -rf $lock 
}
trap handle_exit EXIT

# --- Handling of errors ---------------------------------------------
last=
handle_err()
{
    echo "Error: $last" 
    exit 1
}
enable_trap()
{
    trap handle_err ERR
}
disable_trap()
{
    trap - ERR
}

# --- Check the lock -------------------------------------------------
check_lock()
{
    local loc=$1 
    lock=$loc/.lock

    if test -f $lock ; then 
	echo "Another QA process is already running:" > /dev/stderr 
	echo "Content of ${lock}:" > /dev/stderr 
	cat $lock > /dev/stderr 
	trap - EXIT
    exit 1
    local now=`date` 
    cat <<EOF > $lock
Process: $$
User:    $USER
Start:   $now
EOF
    fi
}

# --- Parse production information -----------------------------------
parse_prod()
{
    prodyear=`echo $prodfull | sed 's/LHC\(..\).*/\1/'` 
    prodletter=`echo $prodfull | sed "s/LHC${prodyear}\(.\).*/\1/"` 
    prodpost=`echo $prodfull | sed "s/LHC${prodyear}${prodletter}//"` 
}

parse_pass()
{
    passno=`echo $passfull | sed 's/.*pass\([0-9]*\).*/\1/'`  
    passpost=`echo $passfull | sed -n "s/.*pass${passno}_//p"` 
    passpre=`echo $passfull | sed -n "s/pass.*//p"` 
}
# --- Extract parts from the found path ------------------------------
year=0
passfull=
passno=0
passpost=
passpre=
prodfull=
prodyear=0
prodletter=
prodpost=
remainder=
get_parts()
{
    mess 1 "Parsing information from job $@" 
    year=$1 ; shift 
    prodfull=$1 ; shift 
    local lrunn=$1 ; shift
    local ltype=$1 ; shift 
    passfull=$1 ; shift 
    remainder=$1 

    mess 10 "year=$year" 
    mess 10 "prodfull=$prodfull" 
    mess 10 "lrunn=$lrunn" 
    mess 10 "ltype=$ltype" 
    mess 10 "passfull=$passfull" 
    mess 10 "remainder=$remainder"

    if test "x$passfull" = "x" ; then 
	remainder=
	passfull=$ltype 
    fi
    case x$passfull in 
	*pass*) ;; 
	*) remainder=$passfull 
	    passfull= 
	    ;;
    esac
    parse_pass 
    parse_prod

    case x$remainder in 
	xQA*) qanumber=`echo $remainder | sed 's/QA//'`  ;; 
	*) ;; 
    esac

}
# --- Append path element --------------------------------------------
append_to_path()
{
    local tmp=$1 ; shift 
    local add=$1
    if test "x$tmp" != "x" ; then tmp="${tmp}/" ; fi 
    echo ${tmp}${add}
}

# --- Get a list of files to get -------------------------------------
file=trending.root
files=
path=
numf=0
mc=0
get_filelist()
{
    mess 3 "Getting file list" 
    
    local datd=data/
    local esdd=ESDs/    
    local yerd=$year/
    case x$prodpost in 
	x_*) ;; 
	x) ;; 
	*)  mess 3 "Assuming simulation output"
	    datd=sim/ 
	    esdd= 
	    if test $year -lt 2013 ; then 
		yerd=
	    fi
	    mc=1
	    passfull=
	    passpost=
	    passno=0
	    passpre=
	    ;; 
    esac
    
    local paid=
    if test "x$passfull" != "x" && \
	test $passno -gt 0 && \
	test $mc -lt 1; then 
	paid=pass${passno}
    fi
    local post=
    if test $mc -lt 1; then 
	post=${passpost}
	case x$post in 
	    x_*) ;; 
	    x) ;; 
	    *) post="_${post}" ;; 
	esac
    fi

    path=/alice/${datd}${yerd}/${prodfull}/
    local search="$file"

    if test "x$run" != "x" ; then 
	case x$datd in 
	    xsim/) rund=$run ;; 
	    *)     rund=`printf %09d $run` ;; 
	esac
	path=$path/$rund/
    fi
    path=${path}${esdd}${passpre}${paid}${post}

    # search=`append_to_path "$search" $file` 

    cat <<EOF
	Path:			$path
	Search:			$search
EOF
    mess 1 "Getting list of files from AliEn - can take minutes - be patient"
    mess 2 "alien_find ${path} ${search}"
    files=`alien_find ${path} ${search} | grep -v "\(does not\|files found\)" 2>> ${redir}` 
    rm -f .list
    for i in $files ; do 
	echo $i >> .list 
	let numf=$numf+1
    done 
    mess 1 -n "Total of $numf files ... "
    if test $maxf -lt 0 ; then 
	mess 1 "using all" 
    else
	mess 1 "using $maxf first of these"
    fi
}

# --- Change permissions on files ------------------------------------
fix_perm()
{
    if test ! -f $1 ; then return ; fi 
    chmod g+rwX $1
    chmod o+rX $1
}
# --- Check if a file is OK ------------------------------------------
docheck=1
check_file()
{
    
    if test $docheck -lt 1 ; then return 0; fi 
    case $1 in 
	*.root) ;;
	*.zip) return 0 ;;
    esac
	    
    root -l -b  <<EOF >> ${redir} 2>&1 
.L $ALICE_ROOT/PWGLF/FORWARD/analysis2/qa/CheckQAFile.C
CheckQAFile("$1");
.q
EOF
    local ret=$? 
    mess 2 "Check of $1 -> $ret"
    rm -f ${scr}.C 
    return $ret
}

# --- Download a single file -----------------------------------------
download_file()
{
    local source=$1 ; shift 
    local store=$1 ; shift 
    local r=$1 ; shift 
    local o=${store}/ 
    case $file in 
	*.root)
	    o=${o}`basename $file .root`_`printf %04d ${r}`.root 
	    ;;
	*.zip)
	    local d=`printf %04d ${r}` 
	    mkdir -p ${o}/${d}
	    o=${o}/${d}/${file}
	    ;;
    esac
    printf "%4d/%4d: %20s -> %20s ..." $cur $max $source $o 

    mess 2 -n "$source -> $o ... "
    if test -f $o ; then 
	printf "exists\n"
	# mess 2 "exists" 
	# sleep 1
    else
	printf "copying\n"
	# mess 2 -n "copying ... " 
	if test $noact -lt 1 ; then 
	    alien_cp alien:${source} file:${o} >> ${redir} 2>&1 
	    fix_perm $o 
	else 
	    sleep 1
	fi
	mess 2 "done"
    fi
    if test $noact -gt 0 ; then return 0 ; fi 
    if test ! -f $o ; then return 1 ; fi 
	

    case $o in 
	*.root) 
	    check_file ${o} 
	    local ret=$? 
	    case $ret in 
		0|2) ;; 
		1|3|4|5|6) return 2 ;; 
	    esac
	    ;;
	*.zip)
	    d=`dirname $o` ;
	    b=`basename $o` ; 
	    mess 3 "Unzipping $b in $d"
	    if ! unzip -n -qq -l $o > /dev/null 2>&1 ; then
		mess 1 "Bad zip file: $o" 
		rm -rf $d 
	    fi
	    # (cd $d && unzip -n -qq $b)
	    ;;
    esac
    # analyse_file ${o}

    return 0
}

# --- Submit run analysis to background ------------------------------
submit_jobs()
{
    local out=$1 ; shift
    local sta=$1 ; shift 
    local max=$1 ; shift
    
    local joblist=
    local counter=0
    mess 5 "Submitting $maxjobs jobs from $sta/$maxf" 
    for i in $@ ; do 
	let cur=$sta+$counter

	local b=`echo $i | sed -e "s,${path},,"` 
	local r=`echo $b | sed -e "s,/.*,,"` 


	let counter=$counter+1

	download_file $i $out $cur &
	j=`jobs %% | sed -e 's/^[^0-9]*//' -e 's/[^0-9]*$//'` 
	joblist="$joblist $j"
    done
    
    counter=0
    mess 5 "will wait for jobs $joblist"
    for i in $joblist ; do 
	mess 5 "waiting for $i of $joblist"
	wait %$i
	let counter=$counter+1
    done
}
    
# --- Analyse each run in turn ---------------------------------------
maxjobs=`grep "processor" /proc/cpuinfo | wc -l`
download_files()
{
    local out=$1 ; shift 
    local max=$1 ; shift 
    
    local queued=0
    local counter=0
    local start=0
    local list=
    while test $# -gt 0 ; do 
	if test $counter -ge $max ; then 
	    break;
	fi

	list="$list $1" 
	shift 
	let queued=$queued+1 
	let counter=$counter+1 

	if test $queued -eq 1 ; then 
	    start=$counter
	fi
	
	if test $queued -ge $maxjobs ; then 
	    mess 1 "Submitting $queued jobs from $start/$max"
	    submit_jobs $out $start $max $list 
	    list=
	    queued=0 
	fi
    done
    if test $queued -gt 0 ; then 
	mess 1 "Submitting $queued jobs from $start/$max"
	submit_jobs $out $start $max $list 
    fi
}

maxjobs=`grep "processor" /proc/cpuinfo | wc -l`
# --- Pass command line options --------------------------------------
redir=/dev/null
maxf=-1
top=.
while test $# -gt 0 ; do 
    case $1 in 
	-h|--help)       usage            ; exit 0 ;; 
	-v|--verbose)    let verb=$verb+1 ;; 
	-j|--jobid)      jobid=$2         ; shift ;; 
	-m|--max-files)  maxf=$2          ; shift ;; 
	-M|--max-jobs)   maxjobs=$2       ; shift ;;
	-t|--top)        top=$2           ; shift ;;
	-l|--log-file)   redir=           ;; 
	-f|--file)       file=$2          ; shift ;;
	-n|--no-action)  noact=1          ;;
	-r|--run)        run=$2           ; shift ;;
	-p|--production) prodfull=$2      ; shift ; parse_prod 
	    year=20${prodyear}            ;; 
	-P|--pass)       passfull=$2      ; shift ; parse_pass    ;;
	*) echo "$0: Unknown argument: $1" > /dev/stderr ; exit 1 ;; 
    esac
    shift
done
# --- Initial setup --------------------------------------------------
# First, check we have a valid AliEn token, then retrieve the job 
# information to parse out the location of the files we need, and 
# finally make our output directory and check the lock 
check_token

if test ! "x$jobid" = x ; then 
    if test ! "x$prodfull" = "x" || test ! "x$passfull" = "x" ; then 
	cat <<EOF > /dev/stderr
Option -j ${jobid} and options -p and -P are mutually exclusive 
EOF
	exit 1
    fi
    get_job
else 
    if test "x$prodfull" = "x" && test "x$passfull" = "x" ; then 
	cat<<EOF > /dev/stderr
When specifying prodcution and/or pass both options -p and -P _must_ 
be specified. 
EOF
	exit 1
    elif test ! "x$jobid"  = "x" ; then 
	cat <<EOF > /dev/stderr
Option -j and options -p ${prodfull} and -P ${passfull} are mutually exclusive 
EOF
	exit 1
    fi
fi	

proddir=LHC${prodyear}${prodletter}
store=${proddir}
if test ! "x$prodpost" = "x" ; then   
    proddir=${proddir}${prodpost}
    store=sim/${proddir}
elif test "x$passfull" != "x" && test $passno -gt 0 ; then 
    store=${store}/pass${passno}
fi
if test "x$run" ; then 
    store=${store}/`printf %06d $run`
fi
mkdir -p ${top}/$store 
fix_perm ${top}/${proddir}
fix_perm ${top}/$store

if test "x$redir" = "x" ; then 
    redir=${top}/$store/download.log 
    rm -f $redir
    fix_perm $redir
fi

check_lock ${top}/$store

cat <<EOF
	Year:			$year
	Production:		$prodfull 
	  Year:			$prodyear
	  Letter:		$prodletter
	  Suffix:		$prodpost
	Pass:			$passfull
	  Number:		$passno
	  Prefix:		$passpre
	  Postfix:		$passpost
	Remainder		$remainder
	  QA number		$qanumber
	Output directory:	${store}
	Lock file:		${lock}
	Log:                    ${redir}
	Run:                    ${run}
EOF
# --- Do a search to find our files ----------------------------------
get_filelist

if test $maxf -ge 0 && test $maxf -lt $numf ; then 
    numf=$maxf 
fi

# --- Now get and analyse each run -----------------------------------
download_files ${top}/$store $numf $files

#
# EOF
#
Commit	Line	Data
99b8fb85	1	#!/bin/bash
	2	#
	3	# This script runs the Forward QA for the specified production number
	4	#
	5	# The scripts downloads and runs the single run QA in parallel
	6	#
	7	noact=0
	8
	9	# --- Help output ----------------------------------------------
	10	usage()
	11	{
	12	cat <<EOF
	13	Usage: $0 [OPTIONS] -j [JOBID]
	14	$0 [OPTIONS] -p PRODUCTION -P PASS
	15
	16	Options:
	17	-h,--help This help
	18	-j,--jobid NUMBER The master job id of the production [$jobid]
	19	-v,--verbose Increase verbosity [$verb]
	20	-m,--max-files NUMBER Max number of files to get [$maxf]
	21	-M,--max-jobs NUMBER Max number of consequtive jobs [$maxjobs]
	22	-t,--top DIRECTORY Output directory [$top]
	23	-p,--production IDENTIFIER Production identifier [$prodfull]
	24	-P,--pass IDENTIFIER Pass identifier [$passfull]
	25	-l,--log-file Log file output [$redir]
	26	-f,--file FILENAME Name of files to get [$file]
	27	-r,--run NUMBER Run number [$run]
	28
	29	Note the option -j and the options -p and -P are mutually exclusive,
	30	The option -Q is only used if the options -p and -P are given.
	31	Production identifiers are of the form LHC11h, LHC11h3, or LHC11h_2.
	32	Pass identifers are of the form pass2, pass1_HLT, or cpass1.
	33	EOF
	34	}
	35
	36	# --- Check AliEn token ----------------------------------------------
	37	check_token()
	38	{
	39	uid=`id -u`
	40	genv_file=/tmp/gclient_env_${uid}
	41
	42	if test ! -f ${genv_file} ; then
	43	echo "No such file: ${genv_file}, please do alien-token-init" \
	44	>/dev/stderr
	45	exit 1
	46	fi
	47	. ${genv_file}
	48	alien-token-info \| grep -q "Token is still valid"
	49	if test $? -ne 0 ; then
	50	echo "Token not valid, please re-new" > /dev/stderr
	51	exit 1
	52	fi
	53	}
	54
	55	# --- Diagnostics output ---------------------------------------------
	56	verb=0
	57	mess()
	58	{
	59	if test $1 -gt $verb ; then return ; fi
	60	shift
	61	echo $*
	62	}
	63
	64	# --- Handling of exit -----------------------------------------------
65	lock=
66	handle_exit()
67	{
68	if test "x$lock" = "x" ; then return ; fi
69	rm -rf $lock
70	}
71	trap handle_exit EXIT
72
73	# --- Handling of errors ---------------------------------------------
74	last=
75	handle_err()
76	{
77	echo "Error: $last"
78	exit 1
79	}
80	enable_trap()
81	{
82	trap handle_err ERR
83	}
84	disable_trap()
85	{
86	trap - ERR
87	}
88
89	# --- Check the lock -------------------------------------------------
90	check_lock()
91	{
92	local loc=$1
93	lock=$loc/.lock
94
95	if test -f $lock ; then
96	echo "Another QA process is already running:" > /dev/stderr
97	echo "Content of ${lock}:" > /dev/stderr
98	cat $lock > /dev/stderr
99	trap - EXIT
100	exit 1
101	local now=`date`
102	cat <<EOF > $lock
103	Process: $$
104	User: $USER
105	Start: $now
106	EOF
107	fi
108	}
109
110	# --- Parse production information -----------------------------------
111	parse_prod()
112	{
113	prodyear=`echo $prodfull \| sed 's/LHC\(..\).*/\1/'`
114	prodletter=`echo $prodfull \| sed "s/LHC${prodyear}\(.\).*/\1/"`
115	prodpost=`echo $prodfull \| sed "s/LHC${prodyear}${prodletter}//"`
116	}
117
118	parse_pass()
119	{
120	passno=`echo $passfull \| sed 's/.pass\([0-9]\).*/\1/'`
121	passpost=`echo $passfull \| sed -n "s/.*pass${passno}_//p"`
122	passpre=`echo $passfull \| sed -n "s/pass.*//p"`
123	}
124	# --- Extract parts from the found path ------------------------------
125	year=0
126	passfull=
127	passno=0
128	passpost=
129	passpre=
130	prodfull=
131	prodyear=0
132	prodletter=
133	prodpost=
134	remainder=
135	get_parts()
136	{
137	mess 1 "Parsing information from job $@"
138	year=$1 ; shift
139	prodfull=$1 ; shift
140	local lrunn=$1 ; shift
141	local ltype=$1 ; shift
142	passfull=$1 ; shift
143	remainder=$1
144
145	mess 10 "year=$year"
146	mess 10 "prodfull=$prodfull"
147	mess 10 "lrunn=$lrunn"
148	mess 10 "ltype=$ltype"
149	mess 10 "passfull=$passfull"
150	mess 10 "remainder=$remainder"
151
152	if test "x$passfull" = "x" ; then
153	remainder=
154	passfull=$ltype
155	fi
156	case x$passfull in
157	pass) ;;
158	*) remainder=$passfull
159	passfull=
160	;;
161	esac
162	parse_pass
163	parse_prod
164
165	case x$remainder in
166	xQA*) qanumber=`echo $remainder \| sed 's/QA//'` ;;
167	*) ;;
168	esac
169
170	}
171	# --- Append path element --------------------------------------------
172	append_to_path()
173	{
174	local tmp=$1 ; shift
175	local add=$1
176	if test "x$tmp" != "x" ; then tmp="${tmp}/" ; fi
177	echo ${tmp}${add}
178	}
179
180	# --- Get a list of files to get -------------------------------------
181	file=trending.root
182	files=
183	path=
184	numf=0
70849dc0	185	mc=0
99b8fb85	186	get_filelist()
	187	{
	188	mess 3 "Getting file list"
	189
bfab35d9	190	local datd=data/
70849dc0	191	local esdd=ESDs/
43ed7920	192	local yerd=$year/
99b8fb85	193	case x$prodpost in
	194	x_*) ;;
	195	x) ;;
	196	*) mess 3 "Assuming simulation output"
bfab35d9	197	datd=sim/
99b8fb85	198	esdd=
70849dc0	199	if test $year -lt 2013 ; then
	200	yerd=
	201	fi
	202	mc=1
	203	passfull=
	204	passpost=
	205	passno=0
	206	passpre=
99b8fb85	207	;;
	208	esac
	209
	210	local paid=
70849dc0	211	if test "x$passfull" != "x" && \
	212	test $passno -gt 0 && \
	213	test $mc -lt 1; then
99b8fb85	214	paid=pass${passno}
99b8fb85	215	fi
70849dc0	216	local post=
	217	if test $mc -lt 1; then
	218	post=${passpost}
	219	case x$post in
	220	x_*) ;;
	221	x) ;;
	222	*) post="_${post}" ;;
	223	esac
	224	fi
99b8fb85	225
43ed7920	226	path=/alice/${datd}${yerd}/${prodfull}/
99b8fb85	227	local search="$file"
	228
	229	if test "x$run" != "x" ; then
	230	case x$datd in
70849dc0	231	xsim/) rund=$run ;;
70849dc0	232	*) rund=`printf %09d $run` ;;
99b8fb85	233	esac
	234	path=$path/$rund/
	235	fi
	236	path=${path}${esdd}${passpre}${paid}${post}
	237
	238	# search=`append_to_path "$search" $file`
	239
	240	cat <<EOF
	241	Path: $path
	242	Search: $search
	243	EOF
	244	mess 1 "Getting list of files from AliEn - can take minutes - be patient"
	245	mess 2 "alien_find ${path} ${search}"
70849dc0	246	files=`alien_find ${path} ${search} \| grep -v "\(does not\\|files found\)" 2>> ${redir}`
43ed7920	247	rm -f .list
99b8fb85	248	for i in $files ; do
43ed7920	249	echo $i >> .list
99b8fb85	250	let numf=$numf+1
	251	done
	252	mess 1 -n "Total of $numf files ... "
	253	if test $maxf -lt 0 ; then
	254	mess 1 "using all"
	255	else
	256	mess 1 "using $maxf first of these"
	257	fi
	258	}
	259
	260	# --- Change permissions on files ------------------------------------
	261	fix_perm()
	262	{
	263	if test ! -f $1 ; then return ; fi
	264	chmod g+rwX $1
	265	chmod o+rX $1
	266	}
	267	# --- Check if a file is OK ------------------------------------------
	268	docheck=1
	269	check_file()
	270	{
43ed7920	271
99b8fb85	272	if test $docheck -lt 1 ; then return 0; fi
43ed7920	273	case $1 in
	274	*.root) ;;
	275	*.zip) return 0 ;;
	276	esac
	277
99b8fb85	278	root -l -b <<EOF >> ${redir} 2>&1
	279	.L $ALICE_ROOT/PWGLF/FORWARD/analysis2/qa/CheckQAFile.C
	280	CheckQAFile("$1");
	281	.q
	282	EOF
	283	local ret=$?
	284	mess 2 "Check of $1 -> $ret"
	285	rm -f ${scr}.C
	286	return $ret
	287	}
	288
	289	# --- Download a single file -----------------------------------------
	290	download_file()
	291	{
	292	local source=$1 ; shift
	293	local store=$1 ; shift
	294	local r=$1 ; shift
	295	local o=${store}/
43ed7920	296	case $file in
	297	*.root)
	298	o=${o}`basename $file .root`_`printf %04d ${r}`.root
	299	;;
	300	*.zip)
	301	local d=`printf %04d ${r}`
	302	mkdir -p ${o}/${d}
	303	o=${o}/${d}/${file}
	304	;;
	305	esac
	306	printf "%4d/%4d: %20s -> %20s ..." $cur $max $source $o
99b8fb85	307
	308	mess 2 -n "$source -> $o ... "
	309	if test -f $o ; then
43ed7920	310	printf "exists\n"
43ed7920	311	# mess 2 "exists"
99b8fb85	312	# sleep 1
99b8fb85	313	else
43ed7920	314	printf "copying\n"
43ed7920	315	# mess 2 -n "copying ... "
99b8fb85	316	if test $noact -lt 1 ; then
	317	alien_cp alien:${source} file:${o} >> ${redir} 2>&1
	318	fix_perm $o
	319	else
	320	sleep 1
	321	fi
	322	mess 2 "done"
	323	fi
	324	if test $noact -gt 0 ; then return 0 ; fi
	325	if test ! -f $o ; then return 1 ; fi
	326
99b8fb85	327
43ed7920	328	case $o in
	329	*.root)
	330	check_file ${o}
	331	local ret=$?
	332	case $ret in
	333	0\|2) ;;
	334	1\|3\|4\|5\|6) return 2 ;;
	335	esac
	336	;;
	337	*.zip)
	338	d=`dirname $o` ;
	339	b=`basename $o` ;
	340	mess 3 "Unzipping $b in $d"
70849dc0	341	if ! unzip -n -qq -l $o > /dev/null 2>&1 ; then
	342	mess 1 "Bad zip file: $o"
	343	rm -rf $d
	344	fi
	345	# (cd $d && unzip -n -qq $b)
43ed7920	346	;;
	347	esac
	348	# analyse_file ${o}
99b8fb85	349
	350	return 0
	351	}
	352
	353	# --- Submit run analysis to background ------------------------------
	354	submit_jobs()
	355	{
	356	local out=$1 ; shift
	357	local sta=$1 ; shift
	358	local max=$1 ; shift
	359
	360	local joblist=
	361	local counter=0
	362	mess 5 "Submitting $maxjobs jobs from $sta/$maxf"
	363	for i in $@ ; do
	364	let cur=$sta+$counter
	365
	366	local b=`echo $i \| sed -e "s,${path},,"`
	367	local r=`echo $b \| sed -e "s,/.*,,"`
	368
99b8fb85	369
	370	let counter=$counter+1
	371
	372	download_file $i $out $cur &
	373	j=`jobs %% \| sed -e 's/^[^0-9]//' -e 's/[^0-9]$//'`
	374	joblist="$joblist $j"
	375	done
	376
	377	counter=0
	378	mess 5 "will wait for jobs $joblist"
	379	for i in $joblist ; do
	380	mess 5 "waiting for $i of $joblist"
	381	wait %$i
	382	let counter=$counter+1
	383	done
	384	}
	385
	386	# --- Analyse each run in turn ---------------------------------------
	387	maxjobs=`grep "processor" /proc/cpuinfo \| wc -l`
	388	download_files()
	389	{
	390	local out=$1 ; shift
	391	local max=$1 ; shift
	392
	393	local queued=0
	394	local counter=0
	395	local start=0
	396	local list=
	397	while test $# -gt 0 ; do
	398	if test $counter -ge $max ; then
	399	break;
	400	fi
	401
	402	list="$list $1"
	403	shift
	404	let queued=$queued+1
	405	let counter=$counter+1
	406
	407	if test $queued -eq 1 ; then
	408	start=$counter
	409	fi
	410
	411	if test $queued -ge $maxjobs ; then
	412	mess 1 "Submitting $queued jobs from $start/$max"
	413	submit_jobs $out $start $max $list
	414	list=
	415	queued=0
	416	fi
	417	done
	418	if test $queued -gt 0 ; then
	419	mess 1 "Submitting $queued jobs from $start/$max"
	420	submit_jobs $out $start $max $list
	421	fi
	422	}
	423
	424	maxjobs=`grep "processor" /proc/cpuinfo \| wc -l`
	425	# --- Pass command line options --------------------------------------
	426	redir=/dev/null
	427	maxf=-1
	428	top=.
	429	while test $# -gt 0 ; do
	430	case $1 in
	431	-h\|--help) usage ; exit 0 ;;
	432	-v\|--verbose) let verb=$verb+1 ;;
433	-j\|--jobid) jobid=$2 ; shift ;;
434	-m\|--max-files) maxf=$2 ; shift ;;
435	-M\|--max-jobs) maxjobs=$2 ; shift ;;
436	-t\|--top) top=$2 ; shift ;;
437	-l\|--log-file) redir= ;;
438	-f\|--file) file=$2 ; shift ;;
439	-n\|--no-action) noact=1 ;;
440	-r\|--run) run=$2 ; shift ;;
441	-p\|--production) prodfull=$2 ; shift ; parse_prod
442	year=20${prodyear} ;;
443	-P\|--pass) passfull=$2 ; shift ; parse_pass ;;
444	*) echo "$0: Unknown argument: $1" > /dev/stderr ; exit 1 ;;
445	esac
446	shift
447	done
448	# --- Initial setup --------------------------------------------------
449	# First, check we have a valid AliEn token, then retrieve the job
450	# information to parse out the location of the files we need, and
451	# finally make our output directory and check the lock
452	check_token
453
454	if test ! "x$jobid" = x ; then
455	if test ! "x$prodfull" = "x" \|\| test ! "x$passfull" = "x" ; then
456	cat <<EOF > /dev/stderr
457	Option -j ${jobid} and options -p and -P are mutually exclusive
458	EOF
459	exit 1
460	fi
461	get_job
462	else
43ed7920	463	if test "x$prodfull" = "x" && test "x$passfull" = "x" ; then
99b8fb85	464	cat<<EOF > /dev/stderr
	465	When specifying prodcution and/or pass both options -p and -P _must_
	466	be specified.
	467	EOF
	468	exit 1
	469	elif test ! "x$jobid" = "x" ; then
	470	cat <<EOF > /dev/stderr
	471	Option -j and options -p ${prodfull} and -P ${passfull} are mutually exclusive
	472	EOF
	473	exit 1
	474	fi
	475	fi
	476
	477	proddir=LHC${prodyear}${prodletter}
	478	store=${proddir}
70849dc0	479	if test ! "x$prodpost" = "x" ; then
99b8fb85	480	proddir=${proddir}${prodpost}
70849dc0	481	store=sim/${proddir}
	482	elif test "x$passfull" != "x" && test $passno -gt 0 ; then
	483	store=${store}/pass${passno}
99b8fb85	484	fi
	485	if test "x$run" ; then
	486	store=${store}/`printf %06d $run`
	487	fi
	488	mkdir -p ${top}/$store
	489	fix_perm ${top}/${proddir}
	490	fix_perm ${top}/$store
	491
	492	if test "x$redir" = "x" ; then
	493	redir=${top}/$store/download.log
	494	rm -f $redir
	495	fix_perm $redir
	496	fi
	497
	498	check_lock ${top}/$store
	499
	500	cat <<EOF
	501	Year: $year
	502	Production: $prodfull
	503	Year: $prodyear
	504	Letter: $prodletter
	505	Suffix: $prodpost
	506	Pass: $passfull
	507	Number: $passno
	508	Prefix: $passpre
	509	Postfix: $passpost
	510	Remainder $remainder
	511	QA number $qanumber
	512	Output directory: ${store}
	513	Lock file: ${lock}
	514	Log: ${redir}
	515	Run: ${run}
	516	EOF
	517	# --- Do a search to find our files ----------------------------------
	518	get_filelist
	519
43ed7920	520	if test $maxf -ge 0 && test $maxf -lt $numf ; then
99b8fb85	521	numf=$maxf
	522	fi
	523
	524	# --- Now get and analyse each run -----------------------------------
	525	download_files ${top}/$store $numf $files
	526
	527	#
	528	# EOF
	529	#