From 3b9b13eeb0a30a27a3ad2f34acac59f7b1619407 Mon Sep 17 00:00:00 2001 From: marian Date: Wed, 2 Dec 2009 16:47:20 +0000 Subject: [PATCH] M submitMerging.sh - Using groups for jobs submitting M filterRecLog.sh - print fixes M README - submitting using lsf groups example M resubmitMissing.sh - using quotes in order to use custom batch command M makeCalibTree.sh - adding halloWorld call M startGUItime.C - temporay disable the loading of reference data M guiTime - calling guiAlias.C M runTrainBatch.sh - use script to merge result --- TPC/scripts/README | 19 ++++--- TPC/scripts/filterRecLog.sh | 11 ++-- TPC/scripts/guiTime | 2 +- TPC/scripts/makeCalibTree.sh | 1 + TPC/scripts/resubmitMissing.sh | 2 +- TPC/scripts/runTrainBatch.sh | 100 ++++++++++++++++----------------- TPC/scripts/startGUItime.C | 4 -- TPC/scripts/submitMerging.sh | 15 ++++- 8 files changed, 81 insertions(+), 73 deletions(-) diff --git a/TPC/scripts/README b/TPC/scripts/README index ccd35a0a6ec..da0ebce397e 100644 --- a/TPC/scripts/README +++ b/TPC/scripts/README @@ -1,5 +1,4 @@ - -ABOUT: +cdABOUT: Authors: Jens Wiechula Marian Ivanov (marian.ivanov@cern.ch) @@ -29,10 +28,11 @@ Do the following: 1.a) gui dependend variables export SCRIPTDIR=$ALICE_ROOT/TPC/scripts export PATH=$SCRIPTDIR:$PATH + source $ALICE_ROOT/TPC/scripts/guiEnv.sh 1.b) Modify your guiEnv.sh script in $SCRIPTDIR 1.c) Run: source guiEnv.sh debug 1.d) Check the output of script. All checks has to be OK. - + 2. Run tree creation (Jens Wiechula scripts) Options: 2.a) Run on a list of run numbers with name 'list.txt', one run number per line makeCalibTree -l list.txt @@ -45,16 +45,17 @@ Do the following: 3. Run tree creation for extended statistic. This part can be run only if batchcommand specified. (Marian Ivanov part of scripts) WARNING this is just example usage of expert mode: - - ifirst=70000; - ilast=90000; - step=500; + bgroup=/bgui + ifirst=79000 + ilast=102000 + step=250 runList=`pwd`/run.list + bgadd $bgroup for (( startRun=$ifirst; startRun<$ilast; startRun=startRun+$step )) ; do let endRun=$startRun+$step echo $startRun $endRun - echo $BATCHCOMMAND -oo $GUI_OUTDIR/logs/out$startRun.log -eo $GUI_OUTDIR/logs/err$startRun.out makeCalibTree.sh $runList $startRun $endRun; - $BATCHCOMMAND -oo $GUI_OUTDIR/logs/out$startRun.log -eo $GUI_OUTDIR/logs/err$startRun.out makeCalibTree.sh $runList $startRun $endRun; + echo $BATCHCOMMAND -g $bgroup -oo $GUI_OUTDIR/logs/out$startRun.log -eo $GUI_OUTDIR/logs/err$startRun.log makeCalibTree.sh $runList $startRun $endRun; + $BATCHCOMMAND -g $bgroup -oo $GUI_OUTDIR/logs/out$startRun.log -eo $GUI_OUTDIR/logs/err$startRun.log makeCalibTree.sh $runList $startRun $endRun; done; 4. Start the GUI browser. This should work always (even on your laptop). diff --git a/TPC/scripts/filterRecLog.sh b/TPC/scripts/filterRecLog.sh index 318004864ea..e6704d719b1 100755 --- a/TPC/scripts/filterRecLog.sh +++ b/TPC/scripts/filterRecLog.sh @@ -9,10 +9,12 @@ isOK=0 nonOK=0 # -rm abort.log -rm abortout.log -rm syswatchAbort.log -rm syswatchAll.log +rm -f abort.log +rm -f abortout.log +rm -f syswatchAbort.log +rm -f syswatchAll.log +rm -f seg0.out + echo hname/C:sname/C:id0/I:id1/I:id2/I:first/D:stampSec/D:mi.fMemUsed/D:mi.fSwapUsed/D:cI.fUser/D:cI.fSys/D:pI.fMemResident/D:pI.fMemVirtual/D:pI.fCpuUser/D:pI.fCpuSys/D:stampOldSec/D:miOld.fMemUsed/D:miOld.fSwapUsed/D:cIOld.fUser/D:cIOld.fSys/D:pIOld.fMemResident/D:pIOld.fMemVirtual/D:pIOld.fCpuUser/D:pIOld.fCpuSys/D > syswatchAbort.log echo hname/C:sname/C:id0/I:id1/I:id2/I:first/D:stampSec/D:mi.fMemUsed/D:mi.fSwapUsed/D:cI.fUser/D:cI.fSys/D:pI.fMemResident/D:pI.fMemVirtual/D:pI.fCpuUser/D:pI.fCpuSys/D:stampOldSec/D:miOld.fMemUsed/D:miOld.fSwapUsed/D:cIOld.fUser/D:cIOld.fSys/D:pIOld.fMemResident/D:pIOld.fMemVirtual/D:pIOld.fCpuUser/D:pIOld.fCpuSys/D > syswatchAll.log @@ -64,7 +66,6 @@ echo netOK=$netOK netNonOK=$netNonOK # # filter segmentation fault # -rm seg0.out for a in `cat abort.log |sed s_err_out_ ` ;do cat $a | grep 0x | grep \# >> seg0.out done; diff --git a/TPC/scripts/guiTime b/TPC/scripts/guiTime index 73f9958a600..e7f04c4dd25 100755 --- a/TPC/scripts/guiTime +++ b/TPC/scripts/guiTime @@ -8,4 +8,4 @@ if [ "x$SCRIPTDIR" == "x" ]; then fi source $SCRIPTDIR/guiEnv.sh test -d $GUI_OUTDIR_RUNS || mkdir -p $GUI_OUTDIR_RUNS -root.exe $SCRIPTDIR/loadlibsREC.C $SCRIPTDIR/ConfigOCDB.C $SCRIPTDIR/startGUItime.C\(\"$1\"\) +root.exe $SCRIPTDIR/loadlibsREC.C $SCRIPTDIR/ConfigOCDB.C $SCRIPTDIR/startGUItime.C\(\"$1\"\) $ALICE_ROOT/TPC/CalibMacros/guiAlias.C diff --git a/TPC/scripts/makeCalibTree.sh b/TPC/scripts/makeCalibTree.sh index c6d5b841e71..3f456056672 100755 --- a/TPC/scripts/makeCalibTree.sh +++ b/TPC/scripts/makeCalibTree.sh @@ -15,6 +15,7 @@ backupdir=`pwd`/ mkdirhier $workdir cd $workdir source guiEnv.sh +source halloWorld.sh # aliroot -q -b $SCRIPTDIR/ConfigOCDB.C $ALICE_ROOT/TPC/CalibMacros/CalibEnv.C+\(\"$runList\",$startRun,$endRun\) cp dcsTime.root $GUI_OUTDIR/time/calibTreeTime_$startRun_$endRun.root diff --git a/TPC/scripts/resubmitMissing.sh b/TPC/scripts/resubmitMissing.sh index 546902d1d60..f92923672d1 100644 --- a/TPC/scripts/resubmitMissing.sh +++ b/TPC/scripts/resubmitMissing.sh @@ -44,5 +44,5 @@ $ALICE_ROOT/TPC/scripts/makeWorkspace.sh runMissing.list # # 3. Sumbmit calibration jobs for mssing directories # -$ALICE_ROOT/TPC/scripts/submitCalib.sh runMissing.list $bqueue $nchunks +$ALICE_ROOT/TPC/scripts/submitCalib.sh runMissing.list "$bqueue" $nchunks diff --git a/TPC/scripts/runTrainBatch.sh b/TPC/scripts/runTrainBatch.sh index 744b7078bd9..aa28138f3e3 100644 --- a/TPC/scripts/runTrainBatch.sh +++ b/TPC/scripts/runTrainBatch.sh @@ -1,6 +1,6 @@ ####################################################################### # -# Run train commands at GSI: +# Run train commands locally: # marian.ivanov@cern.ch # # This is just pseudo code. Bellow you can find the sequence of steps @@ -8,16 +8,16 @@ ####################################################################### -#################################################################################### +######################################################################## # Make workspace # This is just example (see $ALICE_ROOT/TPC/scripts/ReadmeTrain.txt) -#################################################################################### +######################################################################## cp $ALICE_ROOT/TPC/macros/CalibrateTPC.C . cp $ALICE_ROOT/TPC/macros/ConfigOCDB.C . #modify ConfigOCDB.C ln -sf ~/.balice64HEAD0108 balice.sh #use your favourite aliroot -ln -sf $ALICE_ROOT/TPC/CalibMacros/alienSetupGSI.sh alienSetup.sh +ln -sf $HOME/alienSetup.sh alienSetup.sh #use your alien setup cp $ALICE_ROOT/TPC/scripts/submitCalibJob.sh . cp ../lists/run.list . @@ -30,14 +30,15 @@ cp ../lists/esd.list . # To be in workspace dir: # 0.0 run.list # 0.1 esd.list -$ALICE_ROOT/TPC/scripts/makeWorkspace.sh run.list +$ALICE_ROOT/TPC/scripts/makeWorkspace.sh run.list -#################################################################################### +########################################################################### # 1. Get list of missing # +## $ALICE_ROOT/TPC/scripts/filterMissing.sh -#################################################################################### +########################################################################### # 2. Filter corrupted # $ALICE_ROOT/TPC/scripts/filterCorrupted.sh alice-t3_8h @@ -45,83 +46,82 @@ cat */esd*.txt.Bad > esdBad.txt cat */esd*.txt.Good > esdGood.txt -#################################################################################### +########################################################################### # 3. Run calibration: # You have to wait until the lists are filtered # Only Afterwards you should process with calibration submission # Submitting calibration -# -$ALICE_ROOT/TPC/scripts/submitCalib.sh run.list alice-t3 50 +################################################################## +################################################################## +bgroup=/recalib/`pwd | xargs basename` +bgadd $bgroup +$ALICE_ROOT/TPC/scripts/submitCalib.sh run.list "alice-t3 -g $bgroup" 20 #$ALICE_ROOT/TPC/scripts/submitCalib.sh run85034.list alice-t3 2 #$ALICE_ROOT/TPC/scripts/submitCalib.sh run85034.list alice-t3 5 #$ALICE_ROOT/TPC/scripts/submitCalib.sh run90000.list alice-t3 10 - - +#$ALICE_ROOT/TPC/scripts/submitCalib.sh runMag05.list alice-t3 5 +#$ALICE_ROOT/TPC/scripts/submitCalib.sh runMag02.list alice-t3 5 ################################################################### # # 4. Check the error and out log # find `pwd`/*/err* > errRec.log -find `pwd`/*/out* > outRec.log +#find `pwd`/*/out* > outRec.log $ALICE_ROOT/TPC/scripts/filterRecLog.sh ################################################################### # # 5. Submitting merging # -$ALICE_ROOT/TPC/scripts/submitMerging.sh run.list alice-t3_8h +################################################################## +################################################################## +bgroup=/merge/`pwd | xargs basename` +bgadd $bgroup +$ALICE_ROOT/TPC/scripts/submitMerging.sh run.list "alice-t3_8h -c 0:10" $bgroup +#$ALICE_ROOT/TPC/scripts/submitMerging.sh runMissing.list "alice-t3_8h -c 0:10" $bgroup ################################################################### # # 6. resubmit missing if neccessary # e.g if the lists were updated -$ALICE_ROOT/TPC/scripts/resubmitMissing.sh run.list alice-t3 10 +# submit in groups +# time restriction 3 hours +################################################################## +bgroup=/recalib/`pwd | xargs basename` +bgadd $bgroup +$ALICE_ROOT/TPC/scripts/resubmitMissing.sh run.list "alice-t3 -c 3:00 -g $bgroup" 10 # # #$ALICE_ROOT/TPC/scripts/resubmitMissing.sh runLaser.list alice-t3 5 +################################################################### +# +# 7. Merge separatelly sub run list +################################################################## +$ALICE_ROOT/TPC/scripts/mergeCalibRun.sh runMag05.list +$ALICE_ROOT/TPC/scripts/mergeCalibRun.sh runMag02.list +$ALICE_ROOT/TPC/scripts/mergeCalibRun.sh runMag0.list + +ls -d mergerunMag0*.list > runMagAll.list +$ALICE_ROOT/TPC/scripts/mergeCalibRun.sh runMagAll.list +rm runMagN0.list +echo mergerunMag02.list >runMagN0.list +echo mergerunMag05.list >>runMagN0.list +$ALICE_ROOT/TPC/scripts/mergeCalibRun.sh runMagN0.list - - - - -# -# Merge mag field data # -mydir=`pwd` -mkdir mergeMag -cd mergeMag -rm mergeTrain1.txt -rm mergeTrain2.txt -touch mergeTrain1.txt -touch mergeTrain2.txt -for adir in `cat ../../lists/runMag*s.list`; do -ls $mydir/$adir/CalibObjectsTrain1.root >> mergeTrain1.txt; -ls $mydir/$adir/CalibObjectsTrain2.root >> mergeTrain2.txt; -done; -aliroot $ALICE_ROOT/TPC/macros/CalibFileMerger.C+\(\"CalibObjectsTrain1.root\",10000,\"mergeTrain1.txt\"\) -aliroot $ALICE_ROOT/TPC/macros/CalibFileMerger.C+\(\"CalibObjectsTrain2.root\",10000,\"mergeTrain2.txt\"\) -cd .. +# filter debug streamers # +rlist=runMag02.list +rm debug$rlist +for a in `cat $rlist`; do + ls `pwd`/$a/*/*.root >> debug$rlist +done # -# Merge mag 0 # -mydir=`pwd` -mkdir mergeMag0 -cd mergeMag0 -rm mergeTrain1.txt -rm mergeTrain2.txt -touch mergeTrain1.txt -touch mergeTrain2.txt -for adir in `cat ../../lists/runMag0.list`; do -ls $mydir/$adir/CalibObjectsTrain1.root >> mergeTrain1.txt; -ls $mydir/$adir/CalibObjectsTrain2.root >> mergeTrain2.txt; -done; -aliroot $ALICE_ROOT/TPC/macros/CalibFileMerger.C+\(\"CalibObjectsTrain1.root\",10000,\"mergeTrain1.txt\"\) -aliroot $ALICE_ROOT/TPC/macros/CalibFileMerger.C+\(\"CalibObjectsTrain2.root\",10000,\"mergeTrain2.txt\"\) -cd .. # +ls | grep Run| sed s_Run__| sed s/_/\ /| gawk ' { print $1} ' diff --git a/TPC/scripts/startGUItime.C b/TPC/scripts/startGUItime.C index 2ea2d9c87fc..68c5579dc22 100644 --- a/TPC/scripts/startGUItime.C +++ b/TPC/scripts/startGUItime.C @@ -18,8 +18,4 @@ void startGUItime(const char* file=""){ TString cacheDir=gSystem->ExpandPathName("$GUI_OUTDIR_RUNS"); if (cacheDir=="$GUI_OUTDIR_RUNS") cacheDir="/tmp"; guiTime->SetCacheDir(cacheDir.Data()); - TString refDataTree=gSystem->ExpandPathName("$REF_DATA_TREE"); - if (!refDataTree.IsNull()&&refDataTree!="$REF_DATA_TREE"){ - guiTime->AddReferenceTree(refDataTree.Data(),"Ref"); - } } diff --git a/TPC/scripts/submitMerging.sh b/TPC/scripts/submitMerging.sh index 14ffb0ace5d..10880fee76a 100755 --- a/TPC/scripts/submitMerging.sh +++ b/TPC/scripts/submitMerging.sh @@ -1,10 +1,19 @@ # submit merging of the calibration train # argument 1 - runlist # argument 2 - batch queues +# argument 3 - group runlist=$1 -bqueues=$2 +bqueues=$2 +bgroup=$3 + +if [ -z $bgroup ] ; then + bgroup=/merge + bgadd $bgroup +fi; + mydir=`pwd` +echo bqueues $bqueues for adir in `cat $runlist`; do myvar=0; cd $mydir @@ -17,8 +26,8 @@ for adir in `cat $runlist`; do find `pwd`/*_* | grep CalibObjectsTrain2.root > mergelistTrain2.txt nfiles=`cat mergelistTrain1.txt | grep -c .root` if [ $nfiles -gt 0 ] ; then - bsub -q $bqueues -oo outm1_$myvar.log aliroot $ALICE_ROOT/TPC/macros/CalibFileMerger.C+\(\"CalibObjectsTrain1.root\",10000,\"mergelistTrain1.txt\"\) - bsub -q $bqueues -oo outm2_$myvar.log aliroot $ALICE_ROOT/TPC/macros/CalibFileMerger.C+\(\"CalibObjectsTrain2.root\",10000,\"mergelistTrain2.txt\"\) + bsub -q $bqueues -g $bgroup -oo outm1_$myvar.log aliroot $ALICE_ROOT/TPC/macros/CalibFileMerger.C+\(\"CalibObjectsTrain1.root\",10000,\"mergelistTrain1.txt\"\) + bsub -q $bqueues -g $bgroup -oo outm2_$myvar.log aliroot $ALICE_ROOT/TPC/macros/CalibFileMerger.C+\(\"CalibObjectsTrain2.root\",10000,\"mergelistTrain2.txt\"\) echo Run $adir Nfiles=$nfiles else echo Run $adir Calib Missing -- 2.43.0