[u/mrichter/AliRoot.git] / TPC / scripts / tpcCalibTrain.sh

#
# parameters:
# 1 - basedir
# 2 - number of chunks processed
# 3 - cosmic or collisions
# Example: 
# /usr/local/grid/AliRoot/HEAD0108/TPC/scripts/tpcCalibTrain.sh `pwd` 20 0 >train.log
# source /lustre/alice/marin/soft64/AliRoot/v4-17-Rev-18/TPC/scripts/tpcCalibTrain.sh `pwd`
#  work directory for test /lustre/alice/marin/rec/testRec1


export balice=/u/miranov/.balice
#export balice=/lustre/alice/marin/soft64/setvar0417rev20.sh 
#export balice=/lustre/alice/marin/soft64/setvartrunk021209.sh 
source $balice
export aliensetup=$HOME/alienSetup.sh
source $aliensetup 

echo $ALICE_ROOT

#
#
# Test setup
#
export workdir=$1
export nChunks=$2
export isCosmic=0
if [ $# -eq 3 ]; then
  isCosmic=$3
fi
echo IsCosmic  $isCosmic 


if [ ! -n length ]; then 
  echo \############################  
  echo Directory was not specified. Exiting
  echo \############################   
  return;
fi;
if [ ! -r $workdir/lists/esd.list  ] ; then
 echo \############################   
 echo File esd list does not exist. Exiting
 echo \############################   
 return;
fi; 
if [ ! -r $workdir/lists/run.list  ] ; then
 echo \############################   
 echo File run list does not exist. Exiting
 echo \############################   
 return;
fi; 

#
# Make directories
#
cd $workdir
#chgrp -R alice $workdir
#chmod -R g+rwx $workdir
#chmod -R o+rx $workdir
mkdirhier  $workdir/calibNoDrift
mkdirhier  $workdir/calibNoRefit
mkdirhier  $workdir/calibQA
#
#modify ConfigOCDB.C
#
# copy predefined Config files 
#

cp   $ALICE_ROOT/TPC/macros/CalibrateTPC.C      calibNoDrift/CalibrateTPC.C
cat  $ALICE_ROOT/TPC/macros/CalibrateTPC.C |    grep -v AddCalibCalib\(task\) > calibNoRefit/CalibrateTPC.C
cp   $ALICE_ROOT/TPC/macros/CalibrateTPC.C      calibQA/CalibrateTPC.C

if [ $isCosmic -eq 1 ] ; then
cat  $ALICE_ROOT/TPC/macros/CalibrateTPC.C|grep -v  calibTimeGain\-\>SetIsCosmic\(kFALSE\)\; > calibNoDrift/CalibrateTPC.C

cat  $ALICE_ROOT/TPC/macros/CalibrateTPC.C |grep -v AddCalibCalib\(task\) | grep -v  calibTimeGain\-\>SetIsCosmic\(kFALSE\)\; > calibNoRefit/CalibrateTPC.C

cat   $ALICE_ROOT/TPC/macros/CalibrateTPC.C |grep -v  calibTimeGain\-\>SetIsCosmic\(kFALSE\)\; >     calibQA/CalibrateTPC.C
fi

cp   $ALICE_ROOT/TPC/macros/ConfigOCDBNoDrift.C calibNoDrift/ConfigOCDB.C
cp   $ALICE_ROOT/TPC/macros/ConfigOCDBNoRefit.C calibNoRefit/ConfigOCDB.C
cp   $ALICE_ROOT/TPC/macros/ConfigOCDBQA.C      calibQA/ConfigOCDB.C
cp   lists/*.list calibNoDrift/
cp   lists/*.list calibNoRefit/
cp   lists/*.list calibQA/
ln -sf $balice          calibNoDrift/balice.sh
ln -sf $balice          calibNoRefit/balice.sh
ln -sf $balice          calibQA/balice.sh
ln -sf $aliensetup      calibNoDrift/alienSetup.sh
ln -sf $aliensetup      calibNoRefit/alienSetup.sh
ln -sf $aliensetup      calibQA/alienSetup.sh
#  make workspaces
#

echo \##################################################   
echo Info Making calibNoDrift workspace and submit jobs
echo \##################################################   


cd $workdir/calibNoDrift
$ALICE_ROOT/TPC/scripts/makeWorkspace.sh run.list 
bgroupNoDrift=/recalib/`pwd | xargs basename`
bgadd $bgroupNoDrift
echo  $bgroupNoDrift
bkill -g $bgroupNoDrift -r 0
$ALICE_ROOT/TPC/scripts/resubmitMissing.sh run.list "alice-t3 -c 3:00 -g $bgroupNoDrift " $nChunks
nJobsNoDriftSub=`bjobs -W | grep submitCalibJob.sh | grep -c calibNoDrift`

echo \############################   
echo Info $nJobsNoDriftSub  submitted
echo \############################   

echo \##################################################   
echo Info Making calibNoRefit workspace and submit jobs
echo \##################################################   

cd $workdir/calibNoRefit
$ALICE_ROOT/TPC/scripts/makeWorkspace.sh run.list 
bgroupNoRefit=/recalib/`pwd | xargs basename`
bgadd $bgroupNoRefit
echo  $bgroupNoRefit
bkill -g $bgroupNoRefit -r 0
$ALICE_ROOT/TPC/scripts/resubmitMissing.sh run.list "alice-t3 -c 3:00 -g $bgroupNoRefit " $nChunks
nJobsNoRefitSub=`bjobs -W | grep submitCalibJob.sh | grep -c calibNoRefit`

echo \############################   
echo Info $nJobsNoRefitSub  submitted
echo \############################   

nJobsNoDriftRun=`bjobs -W | grep submitCalibJob.sh | grep -c calibNoDrift`

nJobsNoRefitRun=`bjobs -W | grep submitCalibJob.sh | grep -c calibNoRefit`


export totalTime=1800
export timeSleep=60
export restTime=$totalTime

while [ $restTime -gt 0 ];do
    nJobsNoDriftRun=`bjobs -g $bgroupNoDrift  -W | grep submitCalibJob.sh | grep -c calibNoDrift`
    let ratioNoDriftRunSub=100*nJobsNoDriftRun/nJobsNoDriftSub
    if [ $nJobsNoDriftSub -eq 0 ]; then
	let ratioNoDriftRunSub=0
    fi
    
    echo \############################   
    echo Info $nJobsNoDriftSub calibNoDrift jobs submitted  $nJobsNoDriftRun  $nJobsGrNoDrift still running $ratioNoDriftRunSub %
    echo \############################   
    
    nJobsNoRefitRun=`bjobs -g $bgroupNoRefit  -W | grep submitCalibJob.sh | grep -c calibNoRefit`
    let ratioNoRefitRunSub=100*nJobsNoRefitRun/nJobsNoRefitSub
    if [ $nJobsNoRefitSub -eq 0 ] ; then
	let ratioNoRefitRunSub=0
    fi


    echo \############################   
    echo Info $nJobsNoRefitSub calibNoRefit jobs  submitted  $nJobsNoRefitRun $nJobsGrNoRefit still running $ratioNoRefitRunSub %
    echo \############################   

    echo Sleeping $timeSleep , time to go $restTime    
    let restTime=restTime-timeSleep
    

   if [ $ratioNoDriftRunSub -le  4 ]; then
	if [ $ratioNoRefitRunSub -le  4 ]; then
	    let restTime=0
	fi
    fi


    sleep $timeSleep ;

done

##############################################
# Submit merging when all calib jobs are done
##############################################


cd $workdir/calibNoDrift
bgroupMgNoDrift=/mergecalib/`pwd | xargs basename`
bgadd $bgroupMgNoDrift
echo $bgroupMgNoDrift
bkill -g $bgroupMgNoDrift -r 0
$ALICE_ROOT/TPC/scripts/submitMerging.sh runMissing.list "alice-t3_8h -c 3:00"  "$bgroupMgNoDrift"

cd $workdir/calibNoRefit
bgroupMgNoRefit=/mergecalib/`pwd | xargs basename`
bgadd $bgroupMgNoRefit
echo  $bgroupMgNoRefit
bkill -g $bgroupMgNoRefit -r 0
$ALICE_ROOT/TPC/scripts/submitMerging.sh runMissing.list "alice-t3_8h -c 3:00"  "$bgroupMgNoRefit"

nJobsNoDriftMergeSub=`bjobs -W | grep -c CalibFileMerger`
nJobsNoDriftMergeRun=`bjobs -W | grep -c CalibFileMerger`


export totalTime=1800
export timeSleep=60
export restTime=$totalTime

while [ $restTime -gt 0 ];do
    nJobsNoDriftMergeRun=`bjobs -W | grep -c CalibFileMerger`
    let ratioNoDriftMergeRunSub=100*nJobsNoDriftMergeRun/nJobsNoDriftMergeSub
    if [ $nJobsNoDriftMergeSub -eq 0 ] ; then
	let ratioNoDriftMergeRunSub=0
    fi


    echo \############################   
    echo Info $nJobsNoDriftMergeSub jobs submitted for merging, $nJobsNoDriftMergeRun running  $ratioNoDriftMergeRunSub % left
    echo \############################   


    echo Sleeping $timeSleep , time to go $restTime    
    let restTime=restTime-timeSleep


    if [ $ratioNoDriftMergeRunSub -le  4 ]; then
	let restTime=0
    fi


     sleep $timeSleep ;
done


echo \############################   
echo Merging  done
echo \############################   

#nJobsNoDriftMergeSub=`bjobs -W | grep -c CalibFileMerger`
#nJobsNoDriftMergeRun=`bjobs -W | grep -c CalibFileMerger`


#################
# Step 7
#################

cd $workdir/calibNoDrift
$ALICE_ROOT/TPC/scripts/mergeCalibRun.sh run.list alice-t3_8h

cd $workdir/calibNoRefit
$ALICE_ROOT/TPC/scripts/mergeCalibRun.sh run.list alice-t3_8h

nJobsNoDriftMergeCalibSub=`bjobs -W | grep -c CalibFileMerger`
nJobsNoDriftMergeCalibRun=`bjobs -W | grep -c CalibFileMerger`

cd $workdir/calibNoDrift
find `pwd`/* | grep err > errRec.log
$ALICE_ROOT/TPC/scripts/filterRecLog.sh errRec.log

cd $workdir/calibNoRefit
find `pwd`/* | grep err > errRec.log
$ALICE_ROOT/TPC/scripts/filterRecLog.sh errRec.log


echo \############################   
echo MakeOCDB 
echo \############################   

cd $workdir/calibNoDrift
test -d OCDB && mv OCDB OCDB.$(date +%y%m%d_%H%M)
mkdir OCDB

runLow=$(cat ../lists/run.list | sort | head -1)
runHig=$(cat ../lists/run.list | sort | tail -1)


runL=$(echo $runLow | sed 's|^0*||')
runH=$(echo $runHig | sed 's|^0*||')

aliroot -x -q $ALICE_ROOT/TPC/CalibMacros/MakeOCDB.C\($runL,$runH,\"mergerun.list/CalibObjectsTrain1.root\"\)
cd $workdir/calibNoRefit
aliroot -x -q $ALICE_ROOT/TPC/CalibMacros/MakeOCDB.C\($runL,$runH,\"mergerun.list/CalibObjectsTrain1.root\"\)


cd $workdir/calibQA
$ALICE_ROOT/TPC/scripts/makeWorkspace.sh run.list 
bgroupQA=/recalib/`pwd | xargs basename`
bgadd $bgroupQA
echo  $bgroupQA
bkill -g $bgroupQA -r 0
$ALICE_ROOT/TPC/scripts/resubmitMissing.sh run.list "alice-t3 -c 3:00 -g $bgroupQA " $nChunks
nJobsQASub=`bjobs -W | grep submitCalibJob.sh | grep -c calibQA`

echo \############################   
echo Info $nJobsQASub  JobsQA submitted
echo \############################   

nJobsQARun=`bjobs -W | grep submitCalibJob.sh | grep -c calibQA`


export totalTime=1800
export timeSleep=60
export restTime=$totalTime

while [ $restTime -gt 0 ];do
    nJobsQARun=`bjobs -g $bgroupQA  -W | grep submitCalibJob.sh | grep -c calibQA`
    let ratioQARunSub=100*nJobsQARun/nJobsQASub
    if [ $nJobsQASub -eq 0 ]; then
	let ratioQARunSub=0
    fi
    
    echo \############################   
    echo Info $nJobsQASub calibQA jobs submitted  $nJobsQARun  still running $ratioQARunSub %
    echo \############################   
    

    echo Sleeping $timeSleep , time to go $restTime    
    let restTime=restTime-timeSleep
    

    if [ $ratioQARunSub -le  4 ]; then
	let restTime=0
    fi


    sleep $timeSleep ;

done

##############################################
# Submit merging when the QA jobs are done
##############################################

echo \############################   
echo Going to Merge individual directories
echo \############################   


cd $workdir/calibQA
bgroupMgQA=/mergecalibQA/`pwd | xargs basename`
bgadd $bgroupMgQA
echo $bgroupMgQA
bkill -g $bgroupQA -r 0
$ALICE_ROOT/TPC/scripts/submitMerging.sh runMissing.list "alice-t3_8h -c 3:00"  "$bgroupMgQA"


echo \####################################   
echo Waiting for calibQA merging to finish 
echo \####################################   


nJobsQAMergeSub=`bjobs -W | grep -c CalibFileMerger`
nJobsQAMergeRun=`bjobs -W | grep -c CalibFileMerger`


export totalTime=1800
export timeSleep=60
export restTime=$totalTime

while [ $restTime -gt 0 ];do
    nJobsQAMergeRun=`bjobs -W | grep -c CalibFileMerger`
    let ratioQAMergeRunSub=100*nJobsQAMergeRun/nJobsQAMergeSub
    if [ $nJobsQAMergeSub -eq 0 ] ; then
	let ratioQAMergeRunSub=0
    fi


    echo \############################   
    echo Info $nJobsQAMergeSub jobs submitted for merging, $nJobsQAMergeRun running  $ratioQAMergeRunSub % left
    echo \############################   


    echo Sleeping $timeSleep , time to go $restTime    
    let restTime=restTime-timeSleep


    if [ $ratioQAMergeRunSub -le  4 ]; then
	let restTime=0
    fi


     sleep $timeSleep ;
done


echo \#####################################   
echo Going to do the last Merge in calibQA 
echo \#####################################   


cd $workdir/calibQA
$ALICE_ROOT/TPC/scripts/mergeCalibRun.sh run.list alice-t3_8h


echo \#####################################   
echo Last Merge in calibQA is done 
echo \#####################################   


echo \##################################   
echo MakeOCDB in the calibQA directory
echo \##################################   
# Step to be verified

cd $workdir/calibQA
test -d OCDB && mv OCDB OCDB.$(date +%y%m%d_%H%M)
mkdir OCDB

runLow=$(cat ../lists/run.list | sort | head -1)
runHig=$(cat ../lists/run.list | sort | tail -1)


runL=$(echo $runLow | sed 's|^0*||')
runH=$(echo $runHig | sed 's|^0*||')

aliroot -x -q $ALICE_ROOT/TPC/CalibMacros/MakeOCDB.C\($runL,$runH,\"mergerun.list/CalibObjectsTrain1.root\"\)


echo \#####################################   
echo Starting validation of calibQA 
echo \#####################################
Commit	Line	Data
8f1b4e07	1	#
	2	# parameters:
	3	# 1 - basedir
	4	# 2 - number of chunks processed
9ad02593	5	# 3 - cosmic or collisions
8f1b4e07	6	# Example:
6b0a14f2	7	# /usr/local/grid/AliRoot/HEAD0108/TPC/scripts/tpcCalibTrain.sh `pwd` 20 0 >train.log
8f1b4e07	8	# source /lustre/alice/marin/soft64/AliRoot/v4-17-Rev-18/TPC/scripts/tpcCalibTrain.sh `pwd`
	9	# work directory for test /lustre/alice/marin/rec/testRec1
	10
	11
9ad02593	12
6b0a14f2	13	export balice=/u/miranov/.balice
6b0a14f2	14	#export balice=/lustre/alice/marin/soft64/setvar0417rev20.sh
9ad02593	15	#export balice=/lustre/alice/marin/soft64/setvartrunk021209.sh
8f1b4e07	16	source $balice
	17	export aliensetup=$HOME/alienSetup.sh
	18	source $aliensetup
	19
8f1b4e07	20	echo $ALICE_ROOT
	21
	22	#
	23	#
	24	# Test setup
	25	#
	26	export workdir=$1
	27	export nChunks=$2
9ad02593	28	export isCosmic=0
	29	if [ $# -eq 3 ]; then
	30	isCosmic=$3
	31	fi
6b0a14f2	32	echo IsCosmic $isCosmic
9ad02593	33
	34
	35
8f1b4e07	36	if [ ! -n length ]; then
	37	echo \############################
	38	echo Directory was not specified. Exiting
	39	echo \############################
	40	return;
	41	fi;
	42	if [ ! -r $workdir/lists/esd.list ] ; then
	43	echo \############################
	44	echo File esd list does not exist. Exiting
	45	echo \############################
	46	return;
	47	fi;
	48	if [ ! -r $workdir/lists/run.list ] ; then
	49	echo \############################
	50	echo File run list does not exist. Exiting
	51	echo \############################
	52	return;
	53	fi;
	54
	55	#
	56	# Make directories
	57	#
	58	cd $workdir
6b0a14f2	59	#chgrp -R alice $workdir
	60	#chmod -R g+rwx $workdir
	61	#chmod -R o+rx $workdir
8f1b4e07	62	mkdirhier $workdir/calibNoDrift
	63	mkdirhier $workdir/calibNoRefit
	64	mkdirhier $workdir/calibQA
	65	#
	66	#modify ConfigOCDB.C
	67	#
	68	# copy predefined Config files
	69	#
9ad02593	70
8f1b4e07	71	cp $ALICE_ROOT/TPC/macros/CalibrateTPC.C calibNoDrift/CalibrateTPC.C
	72	cat $ALICE_ROOT/TPC/macros/CalibrateTPC.C \| grep -v AddCalibCalib\(task\) > calibNoRefit/CalibrateTPC.C
	73	cp $ALICE_ROOT/TPC/macros/CalibrateTPC.C calibQA/CalibrateTPC.C
9ad02593	74
	75	if [ $isCosmic -eq 1 ] ; then
	76	cat $ALICE_ROOT/TPC/macros/CalibrateTPC.C\|grep -v calibTimeGain\-\>SetIsCosmic\(kFALSE\)\; > calibNoDrift/CalibrateTPC.C
	77
	78	cat $ALICE_ROOT/TPC/macros/CalibrateTPC.C \|grep -v AddCalibCalib\(task\) \| grep -v calibTimeGain\-\>SetIsCosmic\(kFALSE\)\; > calibNoRefit/CalibrateTPC.C
	79
	80	cat $ALICE_ROOT/TPC/macros/CalibrateTPC.C \|grep -v calibTimeGain\-\>SetIsCosmic\(kFALSE\)\; > calibQA/CalibrateTPC.C
	81	fi
	82
8f1b4e07	83	cp $ALICE_ROOT/TPC/macros/ConfigOCDBNoDrift.C calibNoDrift/ConfigOCDB.C
	84	cp $ALICE_ROOT/TPC/macros/ConfigOCDBNoRefit.C calibNoRefit/ConfigOCDB.C
	85	cp $ALICE_ROOT/TPC/macros/ConfigOCDBQA.C calibQA/ConfigOCDB.C
	86	cp lists/*.list calibNoDrift/
	87	cp lists/*.list calibNoRefit/
	88	cp lists/*.list calibQA/
	89	ln -sf $balice calibNoDrift/balice.sh
	90	ln -sf $balice calibNoRefit/balice.sh
	91	ln -sf $balice calibQA/balice.sh
	92	ln -sf $aliensetup calibNoDrift/alienSetup.sh
	93	ln -sf $aliensetup calibNoRefit/alienSetup.sh
	94	ln -sf $aliensetup calibQA/alienSetup.sh
	95	# make workspaces
	96	#
	97
	98	echo \##################################################
	99	echo Info Making calibNoDrift workspace and submit jobs
	100	echo \##################################################
	101
	102
	103	cd $workdir/calibNoDrift
	104	$ALICE_ROOT/TPC/scripts/makeWorkspace.sh run.list
	105	bgroupNoDrift=/recalib/`pwd \| xargs basename`
	106	bgadd $bgroupNoDrift
	107	echo $bgroupNoDrift
6b0a14f2	108	bkill -g $bgroupNoDrift -r 0
8f1b4e07	109	$ALICE_ROOT/TPC/scripts/resubmitMissing.sh run.list "alice-t3 -c 3:00 -g $bgroupNoDrift " $nChunks
	110	nJobsNoDriftSub=`bjobs -W \| grep submitCalibJob.sh \| grep -c calibNoDrift`
	111
	112	echo \############################
	113	echo Info $nJobsNoDriftSub submitted
	114	echo \############################
	115
	116	echo \##################################################
	117	echo Info Making calibNoRefit workspace and submit jobs
	118	echo \##################################################
	119
	120	cd $workdir/calibNoRefit
	121	$ALICE_ROOT/TPC/scripts/makeWorkspace.sh run.list
	122	bgroupNoRefit=/recalib/`pwd \| xargs basename`
	123	bgadd $bgroupNoRefit
	124	echo $bgroupNoRefit
6b0a14f2	125	bkill -g $bgroupNoRefit -r 0
8f1b4e07	126	$ALICE_ROOT/TPC/scripts/resubmitMissing.sh run.list "alice-t3 -c 3:00 -g $bgroupNoRefit " $nChunks
	127	nJobsNoRefitSub=`bjobs -W \| grep submitCalibJob.sh \| grep -c calibNoRefit`
	128
	129	echo \############################
	130	echo Info $nJobsNoRefitSub submitted
	131	echo \############################
	132
	133	nJobsNoDriftRun=`bjobs -W \| grep submitCalibJob.sh \| grep -c calibNoDrift`
	134
	135	nJobsNoRefitRun=`bjobs -W \| grep submitCalibJob.sh \| grep -c calibNoRefit`
	136
	137
	138
6b0a14f2	139	export totalTime=1800
8f1b4e07	140	export timeSleep=60
	141	export restTime=$totalTime
	142
	143	while [ $restTime -gt 0 ];do
	144	nJobsNoDriftRun=`bjobs -g $bgroupNoDrift -W \| grep submitCalibJob.sh \| grep -c calibNoDrift`
	145	let ratioNoDriftRunSub=100*nJobsNoDriftRun/nJobsNoDriftSub
	146	if [ $nJobsNoDriftSub -eq 0 ]; then
	147	let ratioNoDriftRunSub=0
	148	fi
8f1b4e07	149
	150	echo \############################
	151	echo Info $nJobsNoDriftSub calibNoDrift jobs submitted $nJobsNoDriftRun $nJobsGrNoDrift still running $ratioNoDriftRunSub %
	152	echo \############################
	153
	154	nJobsNoRefitRun=`bjobs -g $bgroupNoRefit -W \| grep submitCalibJob.sh \| grep -c calibNoRefit`
	155	let ratioNoRefitRunSub=100*nJobsNoRefitRun/nJobsNoRefitSub
	156	if [ $nJobsNoRefitSub -eq 0 ] ; then
	157	let ratioNoRefitRunSub=0
	158	fi
	159
	160
8f1b4e07	161
	162
	163
	164	echo \############################
	165	echo Info $nJobsNoRefitSub calibNoRefit jobs submitted $nJobsNoRefitRun $nJobsGrNoRefit still running $ratioNoRefitRunSub %
	166	echo \############################
	167
	168	echo Sleeping $timeSleep , time to go $restTime
	169	let restTime=restTime-timeSleep
	170
	171
6b0a14f2	172	if [ $ratioNoDriftRunSub -le 4 ]; then
6b0a14f2	173	if [ $ratioNoRefitRunSub -le 4 ]; then
8f1b4e07	174	let restTime=0
	175	fi
	176	fi
	177
	178
	179	sleep $timeSleep ;
	180
	181	done
	182
	183	##############################################
	184	# Submit merging when all calib jobs are done
	185	##############################################
	186
	187
	188	cd $workdir/calibNoDrift
	189	bgroupMgNoDrift=/mergecalib/`pwd \| xargs basename`
	190	bgadd $bgroupMgNoDrift
	191	echo $bgroupMgNoDrift
6b0a14f2	192	bkill -g $bgroupMgNoDrift -r 0
8f1b4e07	193	$ALICE_ROOT/TPC/scripts/submitMerging.sh runMissing.list "alice-t3_8h -c 3:00" "$bgroupMgNoDrift"
	194
	195	cd $workdir/calibNoRefit
	196	bgroupMgNoRefit=/mergecalib/`pwd \| xargs basename`
	197	bgadd $bgroupMgNoRefit
	198	echo $bgroupMgNoRefit
6b0a14f2	199	bkill -g $bgroupMgNoRefit -r 0
8f1b4e07	200	$ALICE_ROOT/TPC/scripts/submitMerging.sh runMissing.list "alice-t3_8h -c 3:00" "$bgroupMgNoRefit"
	201
	202	nJobsNoDriftMergeSub=`bjobs -W \| grep -c CalibFileMerger`
	203	nJobsNoDriftMergeRun=`bjobs -W \| grep -c CalibFileMerger`
	204
	205
6b0a14f2	206	export totalTime=1800
8f1b4e07	207	export timeSleep=60
	208	export restTime=$totalTime
	209
	210	while [ $restTime -gt 0 ];do
	211	nJobsNoDriftMergeRun=`bjobs -W \| grep -c CalibFileMerger`
	212	let ratioNoDriftMergeRunSub=100*nJobsNoDriftMergeRun/nJobsNoDriftMergeSub
	213	if [ $nJobsNoDriftMergeSub -eq 0 ] ; then
	214	let ratioNoDriftMergeRunSub=0
	215	fi
	216
	217
	218	echo \############################
	219	echo Info $nJobsNoDriftMergeSub jobs submitted for merging, $nJobsNoDriftMergeRun running $ratioNoDriftMergeRunSub % left
	220	echo \############################
	221
	222
	223	echo Sleeping $timeSleep , time to go $restTime
	224	let restTime=restTime-timeSleep
	225
	226
6b0a14f2	227	if [ $ratioNoDriftMergeRunSub -le 4 ]; then
8f1b4e07	228	let restTime=0
	229	fi
	230
	231
	232	sleep $timeSleep ;
	233	done
	234
	235
	236	echo \############################
	237	echo Merging done
	238	echo \############################
	239
	240	#nJobsNoDriftMergeSub=`bjobs -W \| grep -c CalibFileMerger`
	241	#nJobsNoDriftMergeRun=`bjobs -W \| grep -c CalibFileMerger`
	242
	243
8f1b4e07	244
	245	#################
	246	# Step 7
	247	#################
	248
	249	cd $workdir/calibNoDrift
	250	$ALICE_ROOT/TPC/scripts/mergeCalibRun.sh run.list alice-t3_8h
	251
	252	cd $workdir/calibNoRefit
	253	$ALICE_ROOT/TPC/scripts/mergeCalibRun.sh run.list alice-t3_8h
	254
	255	nJobsNoDriftMergeCalibSub=`bjobs -W \| grep -c CalibFileMerger`
	256	nJobsNoDriftMergeCalibRun=`bjobs -W \| grep -c CalibFileMerger`
	257
	258	cd $workdir/calibNoDrift
	259	find `pwd`/* \| grep err > errRec.log
	260	$ALICE_ROOT/TPC/scripts/filterRecLog.sh errRec.log
	261
	262	cd $workdir/calibNoRefit
	263	find `pwd`/* \| grep err > errRec.log
	264	$ALICE_ROOT/TPC/scripts/filterRecLog.sh errRec.log
	265
	266
	267
	268
	269	echo \############################
9ad02593	270	echo MakeOCDB
8f1b4e07	271	echo \############################
8f1b4e07	272
9ad02593	273	cd $workdir/calibNoDrift
	274	test -d OCDB && mv OCDB OCDB.$(date +%y%m%d_%H%M)
	275	mkdir OCDB
	276
	277	runLow=$(cat ../lists/run.list \| sort \| head -1)
	278	runHig=$(cat ../lists/run.list \| sort \| tail -1)
	279
	280
	281	runL=$(echo $runLow \| sed 's\|^0*\|\|')
	282	runH=$(echo $runHig \| sed 's\|^0*\|\|')
	283
	284	aliroot -x -q $ALICE_ROOT/TPC/CalibMacros/MakeOCDB.C\($runL,$runH,\"mergerun.list/CalibObjectsTrain1.root\"\)
6b0a14f2	285	cd $workdir/calibNoRefit
6b0a14f2	286	aliroot -x -q $ALICE_ROOT/TPC/CalibMacros/MakeOCDB.C\($runL,$runH,\"mergerun.list/CalibObjectsTrain1.root\"\)
8f1b4e07	287
	288
	289
9ad02593	290	cd $workdir/calibQA
	291	$ALICE_ROOT/TPC/scripts/makeWorkspace.sh run.list
	292	bgroupQA=/recalib/`pwd \| xargs basename`
	293	bgadd $bgroupQA
	294	echo $bgroupQA
6b0a14f2	295	bkill -g $bgroupQA -r 0
9ad02593	296	$ALICE_ROOT/TPC/scripts/resubmitMissing.sh run.list "alice-t3 -c 3:00 -g $bgroupQA " $nChunks
	297	nJobsQASub=`bjobs -W \| grep submitCalibJob.sh \| grep -c calibQA`
	298
	299	echo \############################
	300	echo Info $nJobsQASub JobsQA submitted
	301	echo \############################
	302
	303	nJobsQARun=`bjobs -W \| grep submitCalibJob.sh \| grep -c calibQA`
	304
	305
	306
6b0a14f2	307	export totalTime=1800
9ad02593	308	export timeSleep=60
	309	export restTime=$totalTime
	310
	311	while [ $restTime -gt 0 ];do
	312	nJobsQARun=`bjobs -g $bgroupQA -W \| grep submitCalibJob.sh \| grep -c calibQA`
	313	let ratioQARunSub=100*nJobsQARun/nJobsQASub
	314	if [ $nJobsQASub -eq 0 ]; then
	315	let ratioQARunSub=0
	316	fi
	317
	318	echo \############################
	319	echo Info $nJobsQASub calibQA jobs submitted $nJobsQARun still running $ratioQARunSub %
	320	echo \############################
	321
	322
	323
	324	echo Sleeping $timeSleep , time to go $restTime
	325	let restTime=restTime-timeSleep
	326
	327
6b0a14f2	328	if [ $ratioQARunSub -le 4 ]; then
9ad02593	329	let restTime=0
	330	fi
	331
	332
	333	sleep $timeSleep ;
	334
	335	done
	336
	337	##############################################
	338	# Submit merging when the QA jobs are done
	339	##############################################
	340
	341	echo \############################
	342	echo Going to Merge individual directories
	343	echo \############################
	344
	345
	346	cd $workdir/calibQA
	347	bgroupMgQA=/mergecalibQA/`pwd \| xargs basename`
	348	bgadd $bgroupMgQA
	349	echo $bgroupMgQA
6b0a14f2	350	bkill -g $bgroupQA -r 0
9ad02593	351	$ALICE_ROOT/TPC/scripts/submitMerging.sh runMissing.list "alice-t3_8h -c 3:00" "$bgroupMgQA"
	352
	353
	354	echo \####################################
	355	echo Waiting for calibQA merging to finish
	356	echo \####################################
	357
	358
	359	nJobsQAMergeSub=`bjobs -W \| grep -c CalibFileMerger`
	360	nJobsQAMergeRun=`bjobs -W \| grep -c CalibFileMerger`
	361
	362
6b0a14f2	363	export totalTime=1800
9ad02593	364	export timeSleep=60
	365	export restTime=$totalTime
	366
	367	while [ $restTime -gt 0 ];do
	368	nJobsQAMergeRun=`bjobs -W \| grep -c CalibFileMerger`
	369	let ratioQAMergeRunSub=100*nJobsQAMergeRun/nJobsQAMergeSub
	370	if [ $nJobsQAMergeSub -eq 0 ] ; then
	371	let ratioQAMergeRunSub=0
	372	fi
	373
	374
	375	echo \############################
	376	echo Info $nJobsQAMergeSub jobs submitted for merging, $nJobsQAMergeRun running $ratioQAMergeRunSub % left
	377	echo \############################
	378
	379
	380	echo Sleeping $timeSleep , time to go $restTime
	381	let restTime=restTime-timeSleep
	382
	383
6b0a14f2	384	if [ $ratioQAMergeRunSub -le 4 ]; then
9ad02593	385	let restTime=0
	386	fi
	387
	388
	389	sleep $timeSleep ;
	390	done
	391
	392
	393
	394
	395	echo \#####################################
	396	echo Going to do the last Merge in calibQA
	397	echo \#####################################
	398
	399
	400	cd $workdir/calibQA
	401	$ALICE_ROOT/TPC/scripts/mergeCalibRun.sh run.list alice-t3_8h
	402
	403
	404
	405
	406
	407	echo \#####################################
	408	echo Last Merge in calibQA is done
	409	echo \#####################################
	410
	411
	412
	413
	414
9ad02593	415	echo \##################################
6b0a14f2	416	echo MakeOCDB in the calibQA directory
9ad02593	417	echo \##################################
9ad02593	418	# Step to be verified
8f1b4e07	419
6b0a14f2	420	cd $workdir/calibQA
	421	test -d OCDB && mv OCDB OCDB.$(date +%y%m%d_%H%M)
	422	mkdir OCDB
9ad02593	423
6b0a14f2	424	runLow=$(cat ../lists/run.list \| sort \| head -1)
	425	runHig=$(cat ../lists/run.list \| sort \| tail -1)
	426
	427
	428	runL=$(echo $runLow \| sed 's\|^0*\|\|')
	429	runH=$(echo $runHig \| sed 's\|^0*\|\|')
9ad02593	430
6b0a14f2	431	aliroot -x -q $ALICE_ROOT/TPC/CalibMacros/MakeOCDB.C\($runL,$runH,\"mergerun.list/CalibObjectsTrain1.root\"\)
9ad02593	432
9ad02593	433
6b0a14f2	434
	435
	436	echo \#####################################
	437	echo Starting validation of calibQA
	438	echo \#####################################