M submitMerging.sh - Using groups for jobs submitting

M      filterRecLog.sh        - print fixes
M      README                 - submitting using lsf groups example
M      resubmitMissing.sh     - using quotes in order to use custom batch command
M      makeCalibTree.sh       - adding halloWorld call
M      startGUItime.C         - temporay disable the loading of reference data
M      guiTime                - calling guiAlias.C
M      runTrainBatch.sh       - use script to merge result

diff --git a/TPC/scripts/README b/TPC/scripts/README
index ccd35a0a6ec4af4795c698ff505331bde14629ea..da0ebce397eb07a29ed660e1bad55c7b9cb531b0 100644 (file)
--- a/TPC/scripts/README
+++ b/TPC/scripts/README
@@ -1,5 +1,4 @@
-
-ABOUT:
+cdABOUT:
 Authors: Jens Wiechula 
          Marian Ivanov (marian.ivanov@cern.ch)
  
@@ -29,10 +28,11 @@ Do the following:
    1.a) gui dependend variables
         export SCRIPTDIR=$ALICE_ROOT/TPC/scripts
         export PATH=$SCRIPTDIR:$PATH
+       source $ALICE_ROOT/TPC/scripts/guiEnv.sh
    1.b) Modify your guiEnv.sh script in $SCRIPTDIR
    1.c) Run: source guiEnv.sh debug    
    1.d) Check the output of script. All checks has to be OK.
-   
+     
 2. Run tree creation (Jens Wiechula scripts) Options:
    2.a)  Run on a list of run numbers with name 'list.txt', one run number per line
          makeCalibTree -l list.txt
@@ -45,16 +45,17 @@ Do the following:
 
 3. Run tree creation for extended statistic. This part can be run only if batchcommand specified.  (Marian Ivanov part of scripts)
    WARNING  this is just example usage of expert mode:
-
-   ifirst=70000;       
-   ilast=90000;
-   step=500;
+   bgroup=/bgui
+   ifirst=79000        
+   ilast=102000
+   step=250
    runList=`pwd`/run.list
+   bgadd $bgroup
    for ((  startRun=$ifirst; startRun<$ilast; startRun=startRun+$step )) ; do 
       let endRun=$startRun+$step 
       echo $startRun $endRun   
-       echo $BATCHCOMMAND -oo $GUI_OUTDIR/logs/out$startRun.log -eo $GUI_OUTDIR/logs/err$startRun.out makeCalibTree.sh $runList $startRun $endRun;
-    $BATCHCOMMAND -oo $GUI_OUTDIR/logs/out$startRun.log -eo $GUI_OUTDIR/logs/err$startRun.out makeCalibTree.sh $runList $startRun $endRun;    
+       echo $BATCHCOMMAND -g $bgroup -oo $GUI_OUTDIR/logs/out$startRun.log -eo $GUI_OUTDIR/logs/err$startRun.log makeCalibTree.sh $runList $startRun $endRun;
+    $BATCHCOMMAND -g $bgroup -oo $GUI_OUTDIR/logs/out$startRun.log -eo $GUI_OUTDIR/logs/err$startRun.log  makeCalibTree.sh $runList $startRun $endRun;    
    done;
 
 4. Start the GUI browser. This should work always (even on your laptop). 

diff --git a/TPC/scripts/filterRecLog.sh b/TPC/scripts/filterRecLog.sh
index 318004864ea221a4db912630030e0bd7e6ae83a1..e6704d719b1d7693bb738676592d0eb7ec55bdbd 100755 (executable)
--- a/TPC/scripts/filterRecLog.sh
+++ b/TPC/scripts/filterRecLog.sh
@@ -9,10 +9,12 @@
 isOK=0 
 nonOK=0
 #
-rm abort.log
-rm abortout.log
-rm syswatchAbort.log
-rm syswatchAll.log
+rm -f abort.log
+rm -f abortout.log
+rm -f syswatchAbort.log
+rm -f syswatchAll.log
+rm -f seg0.out
+
 echo hname/C:sname/C:id0/I:id1/I:id2/I:first/D:stampSec/D:mi.fMemUsed/D:mi.fSwapUsed/D:cI.fUser/D:cI.fSys/D:pI.fMemResident/D:pI.fMemVirtual/D:pI.fCpuUser/D:pI.fCpuSys/D:stampOldSec/D:miOld.fMemUsed/D:miOld.fSwapUsed/D:cIOld.fUser/D:cIOld.fSys/D:pIOld.fMemResident/D:pIOld.fMemVirtual/D:pIOld.fCpuUser/D:pIOld.fCpuSys/D > syswatchAbort.log
 echo hname/C:sname/C:id0/I:id1/I:id2/I:first/D:stampSec/D:mi.fMemUsed/D:mi.fSwapUsed/D:cI.fUser/D:cI.fSys/D:pI.fMemResident/D:pI.fMemVirtual/D:pI.fCpuUser/D:pI.fCpuSys/D:stampOldSec/D:miOld.fMemUsed/D:miOld.fSwapUsed/D:cIOld.fUser/D:cIOld.fSys/D:pIOld.fMemResident/D:pIOld.fMemVirtual/D:pIOld.fCpuUser/D:pIOld.fCpuSys/D > syswatchAll.log
 
@@ -64,7 +66,6 @@ echo netOK=$netOK netNonOK=$netNonOK
 #
 # filter segmentation fault
 #
-rm seg0.out
 for a in `cat  abort.log |sed s_err_out_ ` ;do
     cat $a | grep 0x | grep \# >> seg0.out
 done;

diff --git a/TPC/scripts/guiTime b/TPC/scripts/guiTime
index 73f9958a60052065115b0d20eb933325c44e5844..e7f04c4dd254288d64381584307d1669127ca9a4 100755 (executable)
--- a/TPC/scripts/guiTime
+++ b/TPC/scripts/guiTime
@@ -8,4 +8,4 @@ if [ "x$SCRIPTDIR" == "x" ]; then
 fi
 source $SCRIPTDIR/guiEnv.sh
 test -d $GUI_OUTDIR_RUNS || mkdir -p $GUI_OUTDIR_RUNS
-root.exe $SCRIPTDIR/loadlibsREC.C $SCRIPTDIR/ConfigOCDB.C $SCRIPTDIR/startGUItime.C\(\"$1\"\)
+root.exe $SCRIPTDIR/loadlibsREC.C $SCRIPTDIR/ConfigOCDB.C $SCRIPTDIR/startGUItime.C\(\"$1\"\) $ALICE_ROOT/TPC/CalibMacros/guiAlias.C

diff --git a/TPC/scripts/makeCalibTree.sh b/TPC/scripts/makeCalibTree.sh
index c6d5b841e7123747ad2c907ecf8429e97486f910..3f4560566727146a0d07abc8602d12f94a6fa401 100755 (executable)
--- a/TPC/scripts/makeCalibTree.sh
+++ b/TPC/scripts/makeCalibTree.sh
@@ -15,6 +15,7 @@ backupdir=`pwd`/
 mkdirhier $workdir
 cd $workdir
 source guiEnv.sh
+source halloWorld.sh
 #
 aliroot -q -b $SCRIPTDIR/ConfigOCDB.C  $ALICE_ROOT/TPC/CalibMacros/CalibEnv.C+\(\"$runList\",$startRun,$endRun\)
 cp dcsTime.root $GUI_OUTDIR/time/calibTreeTime_$startRun_$endRun.root

diff --git a/TPC/scripts/resubmitMissing.sh b/TPC/scripts/resubmitMissing.sh
index 546902d1d6015a352906dd1378790a1e1a05951e..f92923672d1ad6f67952a325361b63fc876fba3e 100644 (file)
--- a/TPC/scripts/resubmitMissing.sh
+++ b/TPC/scripts/resubmitMissing.sh
@@ -44,5 +44,5 @@ $ALICE_ROOT/TPC/scripts/makeWorkspace.sh runMissing.list
 #
 # 3. Sumbmit calibration jobs for mssing directories
 #
-$ALICE_ROOT/TPC/scripts/submitCalib.sh runMissing.list $bqueue $nchunks
+$ALICE_ROOT/TPC/scripts/submitCalib.sh runMissing.list "$bqueue"  $nchunks
 

diff --git a/TPC/scripts/runTrainBatch.sh b/TPC/scripts/runTrainBatch.sh
index 744b7078bd9917dabc1d7de3726267b5d745eeba..aa28138f3e3fd2ca05e839deacc93b4516ed15c9 100644 (file)
--- a/TPC/scripts/runTrainBatch.sh
+++ b/TPC/scripts/runTrainBatch.sh
@@ -1,6 +1,6 @@
 #######################################################################
 #
-# Run train commands at GSI: 
+# Run train commands locally: 
 # marian.ivanov@cern.ch
 #
 # This is just pseudo code. Bellow you can find the sequence of steps
@@ -8,16 +8,16 @@
 #######################################################################
 
 
-####################################################################################
+########################################################################
 # Make workspace
 # This is just example (see $ALICE_ROOT/TPC/scripts/ReadmeTrain.txt)
-####################################################################################
+########################################################################
 cp $ALICE_ROOT/TPC/macros/CalibrateTPC.C .
 cp $ALICE_ROOT/TPC/macros/ConfigOCDB.C .
 #modify ConfigOCDB.C
 ln -sf ~/.balice64HEAD0108 balice.sh
 #use your favourite aliroot
-ln -sf $ALICE_ROOT/TPC/CalibMacros/alienSetupGSI.sh alienSetup.sh
+ln -sf $HOME/alienSetup.sh alienSetup.sh
 #use your alien setup 
 cp $ALICE_ROOT/TPC/scripts/submitCalibJob.sh .
 cp ../lists/run.list .
@@ -30,14 +30,15 @@ cp ../lists/esd.list .
 #    To be in workspace dir:    
 #      0.0 run.list 
 #      0.1 esd.list
-$ALICE_ROOT/TPC/scripts/makeWorkspace.sh run.list
+$ALICE_ROOT/TPC/scripts/makeWorkspace.sh run.list 
 
 
-####################################################################################
+###########################################################################
 # 1. Get list of missing 
 #
+##
 $ALICE_ROOT/TPC/scripts/filterMissing.sh
-####################################################################################
+###########################################################################
 # 2. Filter corrupted 
 #
 $ALICE_ROOT/TPC/scripts/filterCorrupted.sh alice-t3_8h
@@ -45,83 +46,82 @@ cat  */esd*.txt.Bad > esdBad.txt
 cat  */esd*.txt.Good > esdGood.txt 
 
 
-####################################################################################
+###########################################################################
 # 3. Run calibration: 
 #    You have to wait until the lists are filtered
 #    Only Afterwards you should process with calibration submission
 #     Submitting calibration
-#
-$ALICE_ROOT/TPC/scripts/submitCalib.sh run.list alice-t3 50
+################################################################## 
+################################################################## 
+bgroup=/recalib/`pwd | xargs basename`
+bgadd $bgroup
+$ALICE_ROOT/TPC/scripts/submitCalib.sh run.list "alice-t3 -g $bgroup"  20
 #$ALICE_ROOT/TPC/scripts/submitCalib.sh run85034.list alice-t3 2
 #$ALICE_ROOT/TPC/scripts/submitCalib.sh run85034.list alice-t3 5
 #$ALICE_ROOT/TPC/scripts/submitCalib.sh run90000.list alice-t3 10
-
-
+#$ALICE_ROOT/TPC/scripts/submitCalib.sh runMag05.list alice-t3 5
+#$ALICE_ROOT/TPC/scripts/submitCalib.sh runMag02.list alice-t3 5
 ###################################################################
 #
 # 4. Check the error and out log
 # 
 find `pwd`/*/err*  > errRec.log
-find `pwd`/*/out*  > outRec.log
+#find `pwd`/*/out*  > outRec.log
 $ALICE_ROOT/TPC/scripts/filterRecLog.sh
 
 ###################################################################
 #
 # 5. Submitting merging
 #
-$ALICE_ROOT/TPC/scripts/submitMerging.sh run.list alice-t3_8h
+################################################################## 
+################################################################## 
+bgroup=/merge/`pwd | xargs basename`
+bgadd $bgroup
+$ALICE_ROOT/TPC/scripts/submitMerging.sh run.list "alice-t3_8h -c 0:10" $bgroup
+#$ALICE_ROOT/TPC/scripts/submitMerging.sh runMissing.list "alice-t3_8h -c 0:10" $bgroup
 
 ###################################################################
 # 
 # 6. resubmit missing if neccessary
 # e.g if the lists were updated
-$ALICE_ROOT/TPC/scripts/resubmitMissing.sh  run.list alice-t3 10
+# submit in groups
+# time restriction 3 hours
+################################################################## 
+bgroup=/recalib/`pwd | xargs basename`
+bgadd $bgroup
+$ALICE_ROOT/TPC/scripts/resubmitMissing.sh  run.list "alice-t3 -c 3:00  -g $bgroup"  10
 #
 #
 #$ALICE_ROOT/TPC/scripts/resubmitMissing.sh  runLaser.list alice-t3 5
 
 
+###################################################################
+# 
+# 7. Merge separatelly sub run list
+################################################################## 
 
+$ALICE_ROOT/TPC/scripts/mergeCalibRun.sh runMag05.list 
+$ALICE_ROOT/TPC/scripts/mergeCalibRun.sh runMag02.list
+$ALICE_ROOT/TPC/scripts/mergeCalibRun.sh runMag0.list
+   
 
+ls -d mergerunMag0*.list > runMagAll.list
+$ALICE_ROOT/TPC/scripts/mergeCalibRun.sh runMagAll.list
+rm runMagN0.list
+echo mergerunMag02.list >runMagN0.list
+echo mergerunMag05.list >>runMagN0.list 
+$ALICE_ROOT/TPC/scripts/mergeCalibRun.sh runMagN0.list
 
 
-
-
-
-
-#
-# Merge mag field data
 #
-mydir=`pwd`
-mkdir mergeMag
-cd  mergeMag
-rm mergeTrain1.txt
-rm mergeTrain2.txt
-touch mergeTrain1.txt
-touch mergeTrain2.txt
-for adir in `cat ../../lists/runMag*s.list`; do
-ls $mydir/$adir/CalibObjectsTrain1.root >> mergeTrain1.txt;
-ls $mydir/$adir/CalibObjectsTrain2.root >> mergeTrain2.txt;
-done;
-aliroot $ALICE_ROOT/TPC/macros/CalibFileMerger.C+\(\"CalibObjectsTrain1.root\",10000,\"mergeTrain1.txt\"\)
-aliroot $ALICE_ROOT/TPC/macros/CalibFileMerger.C+\(\"CalibObjectsTrain2.root\",10000,\"mergeTrain2.txt\"\)
-cd ..
+# filter debug streamers
 #
+rlist=runMag02.list
+rm debug$rlist
+for a in `cat $rlist`; do 
+    ls `pwd`/$a/*/*.root  >> debug$rlist
+done 
 #
-# Merge mag 0
 #
-mydir=`pwd`
-mkdir mergeMag0
-cd  mergeMag0
-rm mergeTrain1.txt
-rm mergeTrain2.txt
-touch mergeTrain1.txt
-touch mergeTrain2.txt
-for adir in `cat ../../lists/runMag0.list`; do
-ls $mydir/$adir/CalibObjectsTrain1.root >> mergeTrain1.txt;
-ls $mydir/$adir/CalibObjectsTrain2.root >> mergeTrain2.txt;
-done;
-aliroot $ALICE_ROOT/TPC/macros/CalibFileMerger.C+\(\"CalibObjectsTrain1.root\",10000,\"mergeTrain1.txt\"\) 
-aliroot $ALICE_ROOT/TPC/macros/CalibFileMerger.C+\(\"CalibObjectsTrain2.root\",10000,\"mergeTrain2.txt\"\)
-cd ..
 #
+ls | grep Run|  sed s_Run__| sed s/_/\ /| gawk ' { print $1} '   

diff --git a/TPC/scripts/startGUItime.C b/TPC/scripts/startGUItime.C
index 2ea2d9c87fcc5d62a119738f5702220acd4a42c2..68c5579dc22b43db0c082e42458ecba878e3cc9b 100644 (file)
--- a/TPC/scripts/startGUItime.C
+++ b/TPC/scripts/startGUItime.C
@@ -18,8 +18,4 @@ void startGUItime(const char* file=""){
   TString cacheDir=gSystem->ExpandPathName("$GUI_OUTDIR_RUNS");
   if (cacheDir=="$GUI_OUTDIR_RUNS") cacheDir="/tmp";
   guiTime->SetCacheDir(cacheDir.Data());
-  TString refDataTree=gSystem->ExpandPathName("$REF_DATA_TREE");
-  if (!refDataTree.IsNull()&&refDataTree!="$REF_DATA_TREE"){
-    guiTime->AddReferenceTree(refDataTree.Data(),"Ref");
-  }
 }

diff --git a/TPC/scripts/submitMerging.sh b/TPC/scripts/submitMerging.sh
index 14ffb0ace5d54da50dacc5490c14d55e07925750..10880fee76a9639014c89dc3c70bf3f3b9e5dc71 100755 (executable)
--- a/TPC/scripts/submitMerging.sh
+++ b/TPC/scripts/submitMerging.sh
@@ -1,10 +1,19 @@
 # submit merging of the calibration train
 # argument 1 - runlist
 # argument 2 - batch queues
+# argument 3 - group
 
 runlist=$1
-bqueues=$2
+bqueues=$2 
+bgroup=$3
+
+if [ -z $bgroup ] ; then
+  bgroup=/merge
+  bgadd $bgroup 
+fi;
+
 mydir=`pwd`
+echo bqueues $bqueues
 for adir in `cat $runlist`; do
     myvar=0;
     cd $mydir
@@ -17,8 +26,8 @@ for adir in `cat $runlist`; do
        find  `pwd`/*_*  | grep CalibObjectsTrain2.root > mergelistTrain2.txt
        nfiles=`cat mergelistTrain1.txt  | grep -c .root`
        if [ $nfiles -gt 0 ] ; then
-           bsub -q $bqueues -oo outm1_$myvar.log aliroot $ALICE_ROOT/TPC/macros/CalibFileMerger.C+\(\"CalibObjectsTrain1.root\",10000,\"mergelistTrain1.txt\"\)
-           bsub -q $bqueues  -oo outm2_$myvar.log aliroot $ALICE_ROOT/TPC/macros/CalibFileMerger.C+\(\"CalibObjectsTrain2.root\",10000,\"mergelistTrain2.txt\"\)
+           bsub -q $bqueues -g $bgroup -oo outm1_$myvar.log aliroot $ALICE_ROOT/TPC/macros/CalibFileMerger.C+\(\"CalibObjectsTrain1.root\",10000,\"mergelistTrain1.txt\"\)
+           bsub -q $bqueues -g $bgroup -oo outm2_$myvar.log aliroot $ALICE_ROOT/TPC/macros/CalibFileMerger.C+\(\"CalibObjectsTrain2.root\",10000,\"mergelistTrain2.txt\"\)
            echo Run $adir  Nfiles=$nfiles
         else
             echo Run $adir Calib Missing