[u/mrichter/AliRoot.git] / PWGPP / CalibMacros / CPass1 / mergeMakeOCDB.byComponent.sh

#!/bin/bash

# Script to merge objects coming out of the calibration train and export of the OCDB:
# Arguments for local use:
#    1 - directory on which to look for the files to be merged 
#    2 - runNumber
#    3 - OCDB output path
#    optional:
#    4 - input default OCDB, default="raw://"
#    5 - option: "alien_cp" will use alien_cp to copy files from alien
#                "tfilecp" will copy the files first using TFile::Cp()
#                "nocopy" will access files over the network directly TFile::Open()
#                default is tfilecp, use 0
#    6 - number of files to download per iteration, default 50
#    7 - maximum number of chunks to be merged for TPC, default 3000"
#
# example local use (files will be downloaded from alien)
#./mergeMakeOCDB.byComponent.sh /alice/data/2012/LHC12g/000188362/cpass0/ 188362 local://./OCDB local:///cvmfs/alice.gsi.de/alice/data/2011/OCDB/ 0 30
#
# on ALIEN just do:
# $1 = directory where to perform the find 
# $2 = runNumber
# $3 = OCDB path

if [[ $# -eq 0 ]]; then
  echo arguments:
  echo  "  1 - directory on which to look for the files to be merged or local file list"
  echo  "  2 - runNumber"
  echo  "  3 - OCDB output path"
  echo  "  optional:"
  echo  "  4 - input default OCDB, default=\"raw://\""
  echo  "  5 - option: \"alien_cp\" will use alien_cp to copy files from alien"
  echo  "              \"tfilecp\" will copy the files first using TFile::Cp()"
  echo  "              \"nocopy\" will access files over the network directly TFile::Open()"
  echo  "              default is tfilecp (use 0)"
  echo  "  6 - number of files to download/merge per iteration, default 50"
  echo  "  7 - maximum number of chunks to be merged for TPC, default 3000"
  echo
  echo "example:"
  echo  " ./mergeMakeOCDB.byComponent.sh /alice/data/2012/LHC12g/000188362/cpass1/ 188362 local://./OCDB raw:// alien_cp 10"
  echo

  exit 0
fi

# init
path=$1
run=$(echo "$2" | sed 's/^0*//')
ocdb=$3
defaultOCDB="raw://"
[[ -n $4 ]] && defaultOCDB=$4
fileAccessMethod=$5
[[ "$fileAccessMethod" != "alien_cp" && "$fileAccessMethod" != "tfilecp" && "$fileAccessMethod" != "nocopy" ]] && fileAccessMethod="tfilecp"
numberOfFilesInAbunch=50
[[ $6 =~ ^[0-9]+$ ]] && numberOfFilesInAbunch=$6

isLocal=0
[[ -f $path ]] && isLocal=1
cleanup=1
[[ $isLocal -eq 1 ]] && cleanup=0
[[ $isLocal -eq 1 ]] && fileAccessMethod="nocopy"

maxChunksTPC=3000
[[ -n $7 ]] && maxChunksTPC=$7

maxTimeToLive=53000
[[ -n ${ALIEN_JDL_TTL} ]] && maxTimeToLive=$(( ${ALIEN_JDL_TTL}-2000 ))

# setup components to be merged
components="TOF MeanVertex T0 SDD TRD TPCCalib TPCCluster TPCAlign"
#components="TOF MeanVertex T0 SDD TRD TPCCalib"

#################################################################
echo "" | tee -a merge.log
echo $0" $*" | tee -a merge.log
echo "" | tee -a merge.log
echo "***********************" | tee -a merge.log
echo mergeMakeOCDB.byComponent.sh started | tee -a merge.log
echo path = $path | tee -a merge.log
echo run  = $run | tee -a merge.log
echo ocdb = $ocdb | tee -a merge.log
echo defaultOCDB=$defaultOCDB | tee -a merge.log
echo isLocal = $isLocal | tee -a merge.log
echo cleanup = $cleanup | tee -a merge.log
echo fileAccessMethod=$fileAccessMethod | tee -a merge.log
echo numberOfFilesInAbunch = $numberOfFilesInAbunch | tee -a merge.log
echo "***********************" | tee -a merge.log

alienFileList="alien.list"
localFileList="local.list"
filesProcessedTPClist="filesProcessedTPC.log"
rm -f $filesProcessedTPClist
touch $filesProcessedTPClist
partialLocalFileListPrefix=${localFileList}_
partialAlienFileListPrefix=${alienFileList}_
runningMergeByComponentLockFile="runningMergeByComponent.lock"

mergeByComponent()
{
  # process by component
  # first argument is the file list to process

  #lock
  touch $runningMergeByComponentLockFile

  # run inside a dedicated running directory
  # whic means copy the file list to process and prefic each line with ../
  # since the file names have no absolute paths!
  runningDirectory="${runningMergeByComponentLockFile}.${1}.dir"
  fileList="$1"
  cleanup=$2
  
  #sanity checks
  if [[ ! -f ${fileList} ]]; then
    echo "${fileList} does not exist"
    rm -f $runningMergeByComponentLockFile
    return 1
  fi
  mkdir -p $runningDirectory
  if [[ ! -d $runningDirectory ]]; then
    echo "cannot create the running directory $runningDirectory"
    echo removing lock $runningMergeByComponentLockFile
    rm -f $runningMergeByComponentLockFile
    return 1
  fi
  
  #move the to be merged files to the running directory and make a list of available files
  #handle the case of archives (x.zip#y.root) as well
  nFiles=0
  while read entry; do
    if [[ $entry =~ ^.*\.root$ ]]; then
      file=${entry%#*}
      fileContent=${entry##*#}
      [[ "${fileContent}" == "${file}" ]] && fileContent=""
      [[ -n ${fileContent} ]] && fileContent="#${fileContent}"
      if [[ -f ${file} ]]; then
        ((nFiles++))
        if [[ -f "./${file}" ]]; then
          echo "../${file}${fileContent}" >> "${runningDirectory}/${fileList}"
        else
          echo "${file}${fileContent}" >> "${runningDirectory}/${fileList}"
        fi
      fi
    fi
  done < ${fileList}
  if [[ $nFiles -lt 1 ]]; then
    echo "no new files in ${fileList}"
    echo rm -rf $runningDirectory 
    rm -rf $runningDirectory 
    echo removing lock $runningMergeByComponentLockFile
    rm -f $runningMergeByComponentLockFile
    return 1
  fi

  #copy the macro to the running directory
  [[ -f mergeByComponent.C ]] && cp mergeByComponent.C $runningDirectory

  #go to running directory
  cd $runningDirectory
  
  numberOfChunksTPC=$(cat ../$filesProcessedTPClist 2>/dev/null | wc -l )

  for det in $components; do
  
    # merge
    echo "***********************" 
    echo merging ${det} data
    echo "***********************"

    #limit the number of chunks processed by TPC
    if [[ "${det}" =~ TPC && $numberOfChunksTPC -ge $maxChunksTPC ]]; then
      echo "Not merging TPC anymore, max number of chunks processed ($maxChunksTPC)"
      continue
    fi

    #add the results of previous iteration to the BEGINNING of the list of files to be merged
    [[ -f ../CalibObjects_${det}.root ]] && echo "../CalibObjects_${det}.root" > ${fileList}_${det}
    cat ${fileList} >> ${fileList}_${det}

    echo "processing following files from ${fileList}_${det}:"
    cat ${fileList}_${det}

    echo aliroot -b -q "mergeByComponent.C(\"${det}\", \"${fileList}_${det}\")"
    aliroot -b -q "mergeByComponent.C(\"${det}\", \"${fileList}_${det}\")" 2>&1 | tee -a merge_${det}.log
    if validateMerging ${det}; then
      echo "### merge OK: mv CalibObjects.root ../CalibObjects_${det}.root"
      mv -f CalibObjects.root ../CalibObjects_${det}.root
      [[ "${det}" =~ TPCCalib ]] && cat ${fileList} >> ../$filesProcessedTPClist
    else 
      echo "### merging not validated"
      rm -f CalibObjects.root
    fi
    mv syswatch.log syswatch_merge_${det}.log

  done

  
  #move stuff back to the parent dir and clean up
  #merge the syswatch logs
  for x in syswatch*log; do
    [[ ! -f $x ]] && continue
    if [[ -f ../$x ]]
    then 
      echo "sed '1d' $x  >> ../$x"
      sed '1d' $x >> ../$x
      rm -f $x
    else 
      echo mv -f $x ..
      mv -f $x ..
    fi
  done

  #merge the other logs
  for x in *.log; do
    [[ ! -f $x ]] && continue
    if [[ -f ../$x ]]
    then
      echo "cat $x >> ../$x"
      cat $x >> ../$x
    else
      echo "mv -f $x .."
      mv -f $x ..
    fi
  done
  
  #final cleanup.
  #go to parent dir first to use the original fileList
  #without ../CalibObjects.root
  cd ..
  if [[ $cleanup -eq 1 ]]; then
    echo "cleaning up merged files..."
    while read file; do
      echo rm -rf $file
      rm -rf $file
    done<$fileList
  fi
  rm -rf $runningDirectory

  #unlock
  echo removing lock $runningMergeByComponentLockFile
  rm -f $runningMergeByComponentLockFile
  echo "***mergeByComponent() DONE"
  echo
  echo "numberOfChunksTPC=$numberOfChunksTPC"
  return 0
}

waitIfLocked()
{
  while [ 1 ]; do
    [[ ! -f $1 ]] && break
    sleep 1
  done
  return 0
}

validateMerging()
{
  det=$1
  retCode=0
  [[ ! -f CalibObjects.root ]] && ((retCode++)) && echo "### no CalibObjects.root..."
  [[ ! -f ${det}_merge_done ]] && ((retCode++)) && echo "### no ${det}_merge_done, job finished abnormally..."
  error=$(grep -e "was a crash" *.log)
  [[ -n $error ]] && ((retCode++)) && echo "### error: $error"
  
  return $retCode
}

copyScripts()
{
  [[ ! -f mergeByComponent.C ]] && \
    cp -f $ALICE_ROOT/PWGPP/CalibMacros/CPass0/mergeByComponent.C $PWD && \
    echo "taking the default scripts from $ALICE_ROOT"
  [[ ! -f makeOCDB.C ]] && \
    cp -f $ALICE_ROOT/PWGPP/CalibMacros/CPass0/makeOCDB.C $PWD && \
    echo "taking the default scripts from $ALICE_ROOT"
}

#first, make sure we have the scripts
copyScripts | tee -a copy.log
ls
#with alien files copy them first to local
echo "***********************" 2>&1 | tee -a copy.log
echo copying files for run $run 2>&1 | tee -a copy.log
echo from $path 2>&1 | tee -a copy.log
echo "***********************" 2>&1 | tee -a copy.log
if [[ $isLocal -eq 0 ]]; then
  if [[ "$fileAccessMethod" == "alien_cp" ]]; then
    echo "alien_find $path AliESDfriends_v1.root | egrep ^/ > $alienFileList" 2>&1 | tee -a copy.log
    alien_find $path "AliESDfriends_v1.root" | egrep "^/" >  $alienFileList
    echo "alien_find done"
    echo
  else 
    echo aliroot -b -q "mergeByComponent.C(\"MAKEALIENLIST\",\"$alienFileList\", \"$path\", \"AliESDfriends_v1.root\")" 2>&1 | tee -a copy.log
    aliroot -b -q "mergeByComponent.C(\"MAKEALIENLIST\",\"$alienFileList\", \"$path\", \"AliESDfriends_v1.root\")" 2>&1 | tee -a copy.log
    echo "MAKEALIENLIST done"
    echo
  fi
else
  cp $path $alienFileList
fi
#randomize the list
#keep the first line intact (it is the largest file of the entire collection)
sed -n '1p' ${alienFileList} > ${alienFileList}.tmp
sed '1d' ${alienFileList} | while read x; do echo "${RANDOM} ${x}"; done|sort -n|cut -d" " -f2- >> ${alienFileList}.tmp
mv -f ${alienFileList}.tmp ${alienFileList}

#split into sublists, each to be processed separately
echo split --numeric-suffixes --suffix-length=6 --lines=$numberOfFilesInAbunch ${alienFileList} ${partialAlienFileListPrefix} | tee -a copy.log
split --numeric-suffixes --suffix-length=6 --lines=$numberOfFilesInAbunch ${alienFileList} ${partialAlienFileListPrefix}
rm -f $runningMergeByComponentLockFile

for partialAlienFileList in ${partialAlienFileListPrefix}*
do

  #if it takes too long, break
  #[[ ${SECONDS} -gt ${maxTimeToLive} ]] && break

  partialAlienFileListPostfix=${partialAlienFileList#$partialAlienFileListPrefix}
  partialLocalFileList=${partialLocalFileListPrefix}${partialAlienFileListPostfix}

  #copy the files if appropriate and make a list of files to use
  rm -f $partialLocalFileList
  if [[ "$fileAccessMethod" == "alien_cp" ]]; then
    while read x; do
      [[ $x != /* ]] && continue
      localName=${x//"/"/_}
      echo "alien_cp "alien://$x" $localName" 2>&1 | tee -a copy.log
      alien_cp "alien://$x" $localName
      echo $localName>>$partialLocalFileList
    done<$partialAlienFileList
  elif [[ "$fileAccessMethod" == "tfilecp" ]]; then
    echo aliroot -b -q "mergeByComponent.C(\"COPY\",\"$partialAlienFileList\",\"noPath\",\"noPattern\",10,\"$partialLocalFileList\")" 2>&1 | tee -a copy.log
    aliroot -b -q "mergeByComponent.C(\"COPY\",\"$partialAlienFileList\",\"noPath\",\"noPattern\",10,\"$partialLocalFileList\")" 2>&1 | tee -a copy.log
  elif [[ "$fileAccessMethod" == "nocopy" ]]; then
    while read x; do
      [[ $isLocal -eq 0 ]] && echo "alien://$x" >> $partialLocalFileList
      [[ $isLocal -eq 1 ]] && echo "$x" >> $partialLocalFileList
    done<$partialAlienFileList
    cleanup=0
  fi

  #handle syswatch
  if [[ -f syswatch_copy.log ]]
  then
    sed '1d' syswatch.log >> syswatch_copy.log
    rm -f syswatch.log
  else 
    [[ -f syswatch.log ]] && mv syswatch.log syswatch_copy.log
  fi

  #merge in parallel, use a simple lockfile
  #echo "waitIfLocked $runningMergeByComponentLockFile" | tee -a merge.log
  waitIfLocked $runningMergeByComponentLockFile
  if [[ $isLocal -eq 1 ]]; then
    #locally we don't need to run in parallel since we dont download,
    #besides it causes problems on some filesystems (like lustre) with file sync
    mergeByComponent $partialLocalFileList $cleanup 2>&1 | tee -a merge.log
  else
    mergeByComponent $partialLocalFileList $cleanup 2>&1 | tee -a merge.log &
  fi
done

#merge all the subfiles into one, wait for the last one to complete
echo "waitIfLocked $runningMergeByComponentLockFile" | tee -a merge.log
waitIfLocked $runningMergeByComponentLockFile
if [[ "$components" =~ ALL && -f CalibObjects_ALL.root ]]; then
  mv -f CalibObjects_ALL.root CalibObjects.root
else
  echo "***********************"
  echo merging ALL data
  echo "***********************"
  finalCalibObjectsList="finalObjects.list"
  ls -1 CalibObjects_*.root > $finalCalibObjectsList
  echo aliroot -b -q "mergeByComponent.C(\"ALL\", \"$finalCalibObjectsList\")" | tee -a merge.log
  aliroot -b -q "mergeByComponent.C(\"ALL\", \"$finalCalibObjectsList\")" 2>&1 | tee -a merge.log
  mv -f syswatch.log syswatch_merge_ALL.log
fi

if ! validateMerging "ALL"; then
  echo final merging not validatet, exiting...
  exit 1
fi
rm -f CalibObjects_*.root

#cleanup
rm -f ${partialAlienFileListPrefix}*
rm -f ${partialLocalFileListPrefix}*
rm -f $alienFileList
rm -f $localFileList

# make OCDB
echo "***********************" 2>&1 | tee -a ocdb.log
echo making ${det} OCDB 2>&1 | tee -a ocdb.log
echo "***********************" 2>&1 | tee -a ocdb.log
echo aliroot -b -q "makeOCDB.C($run, \"$ocdb\", \"$defaultOCDB\")" 2>&1 | tee -a ocdb.log
aliroot -b -q "makeOCDB.C($run, \"$ocdb\", \"$defaultOCDB\")" 2>&1 | tee -a ocdb.log
mv syswatch.log syswatch_makeOCDB.log

# summary
echo "***********************" 2>&1 | tee -a ocdb.log
echo SUMMARY 2>&1 | tee -a ocdb.log
echo "***********************" 2>&1 | tee -a ocdb.log
ls -altr CalibObjects.root *done 2>&1 | tee -a ocdb.log
Commit	Line	Data
e9129651	1	#!/bin/bash
807998f3	2
9e160db9	3	# Script to merge objects coming out of the calibration train and export of the OCDB:
	4	# Arguments for local use:
	5	# 1 - directory on which to look for the files to be merged
	6	# 2 - runNumber
	7	# 3 - OCDB output path
e9129651	8	# optional:
	9	# 4 - input default OCDB, default="raw://"
	10	# 5 - option: "alien_cp" will use alien_cp to copy files from alien
	11	# "tfilecp" will copy the files first using TFile::Cp()
	12	# "nocopy" will access files over the network directly TFile::Open()
	13	# default is tfilecp, use 0
695f9fb2	14	# 6 - number of files to download per iteration, default 50
695f9fb2	15	# 7 - maximum number of chunks to be merged for TPC, default 3000"
e9129651	16	#
	17	# example local use (files will be downloaded from alien)
	18	#./mergeMakeOCDB.byComponent.sh /alice/data/2012/LHC12g/000188362/cpass0/ 188362 local://./OCDB local:///cvmfs/alice.gsi.de/alice/data/2011/OCDB/ 0 30
	19	#
	20	# on ALIEN just do:
9e160db9	21	# $1 = directory where to perform the find
	22	# $2 = runNumber
	23	# $3 = OCDB path
	24
e9129651	25	if [[ $# -eq 0 ]]; then
	26	echo arguments:
	27	echo " 1 - directory on which to look for the files to be merged or local file list"
	28	echo " 2 - runNumber"
	29	echo " 3 - OCDB output path"
	30	echo " optional:"
	31	echo " 4 - input default OCDB, default=\"raw://\""
	32	echo " 5 - option: \"alien_cp\" will use alien_cp to copy files from alien"
	33	echo " \"tfilecp\" will copy the files first using TFile::Cp()"
	34	echo " \"nocopy\" will access files over the network directly TFile::Open()"
	35	echo " default is tfilecp (use 0)"
695f9fb2	36	echo " 6 - number of files to download/merge per iteration, default 50"
695f9fb2	37	echo " 7 - maximum number of chunks to be merged for TPC, default 3000"
e9129651	38	echo
	39	echo "example:"
	40	echo " ./mergeMakeOCDB.byComponent.sh /alice/data/2012/LHC12g/000188362/cpass1/ 188362 local://./OCDB raw:// alien_cp 10"
	41	echo
	42
	43	exit 0
	44	fi
9e160db9	45
807998f3	46	# init
807998f3	47	path=$1
d1c49c14	48	run=$(echo "$2" \| sed 's/^0*//')
807998f3	49	ocdb=$3
e9129651	50	defaultOCDB="raw://"
	51	[[ -n $4 ]] && defaultOCDB=$4
	52	fileAccessMethod=$5
	53	[[ "$fileAccessMethod" != "alien_cp" && "$fileAccessMethod" != "tfilecp" && "$fileAccessMethod" != "nocopy" ]] && fileAccessMethod="tfilecp"
	54	numberOfFilesInAbunch=50
	55	[[ $6 =~ ^[0-9]+$ ]] && numberOfFilesInAbunch=$6
	56
9e160db9	57	isLocal=0
9e160db9	58	[[ -f $path ]] && isLocal=1
e9129651	59	cleanup=1
	60	[[ $isLocal -eq 1 ]] && cleanup=0
	61	[[ $isLocal -eq 1 ]] && fileAccessMethod="nocopy"
9e160db9	62
695f9fb2	63	maxChunksTPC=3000
	64	[[ -n $7 ]] && maxChunksTPC=$7
	65
	66	maxTimeToLive=53000
	67	[[ -n ${ALIEN_JDL_TTL} ]] && maxTimeToLive=$(( ${ALIEN_JDL_TTL}-2000 ))
	68
e9129651	69	# setup components to be merged
1169eaef	70	components="TOF MeanVertex T0 SDD TRD TPCCalib TPCCluster TPCAlign"
1169eaef	71	#components="TOF MeanVertex T0 SDD TRD TPCCalib"
807998f3	72
e9129651	73	#################################################################
695f9fb2	74	echo "" \| tee -a merge.log
	75	echo $0" $*" \| tee -a merge.log
	76	echo "" \| tee -a merge.log
e9129651	77	echo "***********************" \| tee -a merge.log
	78	echo mergeMakeOCDB.byComponent.sh started \| tee -a merge.log
	79	echo path = $path \| tee -a merge.log
	80	echo run = $run \| tee -a merge.log
	81	echo ocdb = $ocdb \| tee -a merge.log
	82	echo defaultOCDB=$defaultOCDB \| tee -a merge.log
	83	echo isLocal = $isLocal \| tee -a merge.log
	84	echo cleanup = $cleanup \| tee -a merge.log
	85	echo fileAccessMethod=$fileAccessMethod \| tee -a merge.log
	86	echo numberOfFilesInAbunch = $numberOfFilesInAbunch \| tee -a merge.log
	87	echo "***********************" \| tee -a merge.log
807998f3	88
9e160db9	89	alienFileList="alien.list"
9e160db9	90	localFileList="local.list"
695f9fb2	91	filesProcessedTPClist="filesProcessedTPC.log"
	92	rm -f $filesProcessedTPClist
	93	touch $filesProcessedTPClist
9e160db9	94	partialLocalFileListPrefix=${localFileList}_
	95	partialAlienFileListPrefix=${alienFileList}_
	96	runningMergeByComponentLockFile="runningMergeByComponent.lock"
	97
	98	mergeByComponent()
	99	{
	100	# process by component
	101	# first argument is the file list to process
9e160db9	102
	103	#lock
	104	touch $runningMergeByComponentLockFile
	105
40f73007	106	# run inside a dedicated running directory
	107	# whic means copy the file list to process and prefic each line with ../
	108	# since the file names have no absolute paths!
	109	runningDirectory="${runningMergeByComponentLockFile}.${1}.dir"
e9129651	110	fileList="$1"
	111	cleanup=$2
	112
	113	#sanity checks
695f9fb2	114	if [[ ! -f ${fileList} ]]; then
	115	echo "${fileList} does not exist"
	116	rm -f $runningMergeByComponentLockFile
	117	return 1
	118	fi
40f73007	119	mkdir -p $runningDirectory
695f9fb2	120	if [[ ! -d $runningDirectory ]]; then
	121	echo "cannot create the running directory $runningDirectory"
	122	echo removing lock $runningMergeByComponentLockFile
	123	rm -f $runningMergeByComponentLockFile
	124	return 1
9e160db9	125	fi
695f9fb2	126
e9129651	127	#move the to be merged files to the running directory and make a list of available files
	128	#handle the case of archives (x.zip#y.root) as well
	129	nFiles=0
	130	while read entry; do
	131	if [[ $entry =~ ^.*\.root$ ]]; then
	132	file=${entry%#*}
	133	fileContent=${entry##*#}
	134	[[ "${fileContent}" == "${file}" ]] && fileContent=""
	135	[[ -n ${fileContent} ]] && fileContent="#${fileContent}"
	136	if [[ -f ${file} ]]; then
	137	((nFiles++))
	138	if [[ -f "./${file}" ]]; then
695f9fb2	139	echo "../${file}${fileContent}" >> "${runningDirectory}/${fileList}"
e9129651	140	else
695f9fb2	141	echo "${file}${fileContent}" >> "${runningDirectory}/${fileList}"
e9129651	142	fi
	143	fi
	144	fi
695f9fb2	145	done < ${fileList}
	146	if [[ $nFiles -lt 1 ]]; then
	147	echo "no new files in ${fileList}"
	148	echo rm -rf $runningDirectory
	149	rm -rf $runningDirectory
	150	echo removing lock $runningMergeByComponentLockFile
	151	rm -f $runningMergeByComponentLockFile
	152	return 1
	153	fi
e9129651	154
	155	#copy the macro to the running directory
	156	[[ -f mergeByComponent.C ]] && cp mergeByComponent.C $runningDirectory
	157
695f9fb2	158	#go to running directory
e9129651	159	cd $runningDirectory
695f9fb2	160
695f9fb2	161	numberOfChunksTPC=$(cat ../$filesProcessedTPClist 2>/dev/null \| wc -l )
807998f3	162
9e160db9	163	for det in $components; do
695f9fb2	164
807998f3	165	# merge
e9129651	166	echo "***********************"
695f9fb2	167	echo merging ${det} data
e9129651	168	echo "***********************"
695f9fb2	169
	170	#limit the number of chunks processed by TPC
	171	if [[ "${det}" =~ TPC && $numberOfChunksTPC -ge $maxChunksTPC ]]; then
	172	echo "Not merging TPC anymore, max number of chunks processed ($maxChunksTPC)"
	173	continue
	174	fi
	175
	176	#add the results of previous iteration to the BEGINNING of the list of files to be merged
	177	[[ -f ../CalibObjects_${det}.root ]] && echo "../CalibObjects_${det}.root" > ${fileList}_${det}
	178	cat ${fileList} >> ${fileList}_${det}
	179
	180	echo "processing following files from ${fileList}_${det}:"
	181	cat ${fileList}_${det}
	182
1169eaef	183	echo aliroot -b -q "mergeByComponent.C(\"${det}\", \"${fileList}_${det}\")"
1169eaef	184	aliroot -b -q "mergeByComponent.C(\"${det}\", \"${fileList}_${det}\")" 2>&1 \| tee -a merge_${det}.log
e9129651	185	if validateMerging ${det}; then
695f9fb2	186	echo "### merge OK: mv CalibObjects.root ../CalibObjects_${det}.root"
	187	mv -f CalibObjects.root ../CalibObjects_${det}.root
	188	[[ "${det}" =~ TPCCalib ]] && cat ${fileList} >> ../$filesProcessedTPClist
e9129651	189	else
695f9fb2	190	echo "### merging not validated"
e9129651	191	rm -f CalibObjects.root
e9129651	192	fi
695f9fb2	193	mv syswatch.log syswatch_merge_${det}.log
695f9fb2	194
9e160db9	195	done
40f73007	196
e9129651	197
40f73007	198	#move stuff back to the parent dir and clean up
	199	#merge the syswatch logs
	200	for x in syswatch*log; do
e9129651	201	[[ ! -f $x ]] && continue
e9129651	202	if [[ -f ../$x ]]
40f73007	203	then
e9129651	204	echo "sed '1d' $x >> ../$x"
e9129651	205	sed '1d' $x >> ../$x
40f73007	206	rm -f $x
40f73007	207	else
e9129651	208	echo mv -f $x ..
	209	mv -f $x ..
	210	fi
	211	done
	212
	213	#merge the other logs
	214	for x in *.log; do
	215	[[ ! -f $x ]] && continue
	216	if [[ -f ../$x ]]
	217	then
	218	echo "cat $x >> ../$x"
	219	cat $x >> ../$x
	220	else
	221	echo "mv -f $x .."
	222	mv -f $x ..
40f73007	223	fi
40f73007	224	done
e9129651	225
	226	#final cleanup.
	227	#go to parent dir first to use the original fileList
	228	#without ../CalibObjects.root
	229	cd ..
	230	if [[ $cleanup -eq 1 ]]; then
	231	echo "cleaning up merged files..."
	232	while read file; do
	233	echo rm -rf $file
	234	rm -rf $file
	235	done<$fileList
	236	fi
40f73007	237	rm -rf $runningDirectory
807998f3	238
9e160db9	239	#unlock
695f9fb2	240	echo removing lock $runningMergeByComponentLockFile
9e160db9	241	rm -f $runningMergeByComponentLockFile
e9129651	242	echo "***mergeByComponent() DONE"
e9129651	243	echo
695f9fb2	244	echo "numberOfChunksTPC=$numberOfChunksTPC"
e9129651	245	return 0
9e160db9	246	}
807998f3	247
9e160db9	248	waitIfLocked()
	249	{
	250	while [ 1 ]; do
	251	[[ ! -f $1 ]] && break
	252	sleep 1
	253	done
e9129651	254	return 0
9e160db9	255	}
807998f3	256
e9129651	257	validateMerging()
	258	{
	259	det=$1
	260	retCode=0
	261	[[ ! -f CalibObjects.root ]] && ((retCode++)) && echo "### no CalibObjects.root..."
	262	[[ ! -f ${det}_merge_done ]] && ((retCode++)) && echo "### no ${det}_merge_done, job finished abnormally..."
	263	error=$(grep -e "was a crash" *.log)
	264	[[ -n $error ]] && ((retCode++)) && echo "### error: $error"
	265
	266	return $retCode
	267	}
	268
	269	copyScripts()
	270	{
	271	[[ ! -f mergeByComponent.C ]] && \
	272	cp -f $ALICE_ROOT/PWGPP/CalibMacros/CPass0/mergeByComponent.C $PWD && \
	273	echo "taking the default scripts from $ALICE_ROOT"
	274	[[ ! -f makeOCDB.C ]] && \
	275	cp -f $ALICE_ROOT/PWGPP/CalibMacros/CPass0/makeOCDB.C $PWD && \
	276	echo "taking the default scripts from $ALICE_ROOT"
	277	}
9e160db9	278
e9129651	279	#first, make sure we have the scripts
	280	copyScripts \| tee -a copy.log
	281	ls
	282	#with alien files copy them first to local
	283	echo "***********************" 2>&1 \| tee -a copy.log
	284	echo copying files for run $run 2>&1 \| tee -a copy.log
	285	echo from $path 2>&1 \| tee -a copy.log
	286	echo "***********************" 2>&1 \| tee -a copy.log
	287	if [[ $isLocal -eq 0 ]]; then
	288	if [[ "$fileAccessMethod" == "alien_cp" ]]; then
	289	echo "alien_find $path AliESDfriends_v1.root \| egrep ^/ > $alienFileList" 2>&1 \| tee -a copy.log
	290	alien_find $path "AliESDfriends_v1.root" \| egrep "^/" > $alienFileList
	291	echo "alien_find done"
	292	echo
	293	else
1169eaef	294	echo aliroot -b -q "mergeByComponent.C(\"MAKEALIENLIST\",\"$alienFileList\", \"$path\", \"AliESDfriends_v1.root\")" 2>&1 \| tee -a copy.log
1169eaef	295	aliroot -b -q "mergeByComponent.C(\"MAKEALIENLIST\",\"$alienFileList\", \"$path\", \"AliESDfriends_v1.root\")" 2>&1 \| tee -a copy.log
e9129651	296	echo "MAKEALIENLIST done"
	297	echo
	298	fi
9e160db9	299	else
e9129651	300	cp $path $alienFileList
e9129651	301	fi
695f9fb2	302	#randomize the list
	303	#keep the first line intact (it is the largest file of the entire collection)
	304	sed -n '1p' ${alienFileList} > ${alienFileList}.tmp
	305	sed '1d' ${alienFileList} \| while read x; do echo "${RANDOM} ${x}"; done\|sort -n\|cut -d" " -f2- >> ${alienFileList}.tmp
	306	mv -f ${alienFileList}.tmp ${alienFileList}
	307
	308	#split into sublists, each to be processed separately
e9129651	309	echo split --numeric-suffixes --suffix-length=6 --lines=$numberOfFilesInAbunch ${alienFileList} ${partialAlienFileListPrefix} \| tee -a copy.log
	310	split --numeric-suffixes --suffix-length=6 --lines=$numberOfFilesInAbunch ${alienFileList} ${partialAlienFileListPrefix}
	311	rm -f $runningMergeByComponentLockFile
695f9fb2	312
e9129651	313	for partialAlienFileList in ${partialAlienFileListPrefix}*
e9129651	314	do
695f9fb2	315
	316	#if it takes too long, break
	317	#[[ ${SECONDS} -gt ${maxTimeToLive} ]] && break
	318
e9129651	319	partialAlienFileListPostfix=${partialAlienFileList#$partialAlienFileListPrefix}
	320	partialLocalFileList=${partialLocalFileListPrefix}${partialAlienFileListPostfix}
	321
	322	#copy the files if appropriate and make a list of files to use
	323	rm -f $partialLocalFileList
	324	if [[ "$fileAccessMethod" == "alien_cp" ]]; then
	325	while read x; do
	326	[[ $x != /* ]] && continue
	327	localName=${x//"/"/_}
	328	echo "alien_cp "alien://$x" $localName" 2>&1 \| tee -a copy.log
	329	alien_cp "alien://$x" $localName
	330	echo $localName>>$partialLocalFileList
	331	done<$partialAlienFileList
	332	elif [[ "$fileAccessMethod" == "tfilecp" ]]; then
	333	echo aliroot -b -q "mergeByComponent.C(\"COPY\",\"$partialAlienFileList\",\"noPath\",\"noPattern\",10,\"$partialLocalFileList\")" 2>&1 \| tee -a copy.log
	334	aliroot -b -q "mergeByComponent.C(\"COPY\",\"$partialAlienFileList\",\"noPath\",\"noPattern\",10,\"$partialLocalFileList\")" 2>&1 \| tee -a copy.log
	335	elif [[ "$fileAccessMethod" == "nocopy" ]]; then
	336	while read x; do
	337	[[ $isLocal -eq 0 ]] && echo "alien://$x" >> $partialLocalFileList
	338	[[ $isLocal -eq 1 ]] && echo "$x" >> $partialLocalFileList
	339	done<$partialAlienFileList
	340	cleanup=0
	341	fi
	342
	343	#handle syswatch
	344	if [[ -f syswatch_copy.log ]]
	345	then
	346	sed '1d' syswatch.log >> syswatch_copy.log
	347	rm -f syswatch.log
	348	else
	349	[[ -f syswatch.log ]] && mv syswatch.log syswatch_copy.log
	350	fi
	351
	352	#merge in parallel, use a simple lockfile
	353	#echo "waitIfLocked $runningMergeByComponentLockFile" \| tee -a merge.log
	354	waitIfLocked $runningMergeByComponentLockFile
	355	if [[ $isLocal -eq 1 ]]; then
	356	#locally we don't need to run in parallel since we dont download,
	357	#besides it causes problems on some filesystems (like lustre) with file sync
	358	mergeByComponent $partialLocalFileList $cleanup 2>&1 \| tee -a merge.log
	359	else
	360	mergeByComponent $partialLocalFileList $cleanup 2>&1 \| tee -a merge.log &
	361	fi
	362	done
9e160db9	363
695f9fb2	364	#merge all the subfiles into one, wait for the last one to complete
	365	echo "waitIfLocked $runningMergeByComponentLockFile" \| tee -a merge.log
	366	waitIfLocked $runningMergeByComponentLockFile
	367	if [[ "$components" =~ ALL && -f CalibObjects_ALL.root ]]; then
	368	mv -f CalibObjects_ALL.root CalibObjects.root
	369	else
	370	echo "***********************"
	371	echo merging ALL data
	372	echo "***********************"
	373	finalCalibObjectsList="finalObjects.list"
	374	ls -1 CalibObjects_*.root > $finalCalibObjectsList
1169eaef	375	echo aliroot -b -q "mergeByComponent.C(\"ALL\", \"$finalCalibObjectsList\")" \| tee -a merge.log
1169eaef	376	aliroot -b -q "mergeByComponent.C(\"ALL\", \"$finalCalibObjectsList\")" 2>&1 \| tee -a merge.log
695f9fb2	377	mv -f syswatch.log syswatch_merge_ALL.log
	378	fi
	379
	380	if ! validateMerging "ALL"; then
	381	echo final merging not validatet, exiting...
	382	exit 1
	383	fi
	384	rm -f CalibObjects_*.root
	385
9e160db9	386	#cleanup
9e160db9	387	rm -f ${partialAlienFileListPrefix}*
9e160db9	388	rm -f ${partialLocalFileListPrefix}*
e9129651	389	rm -f $alienFileList
695f9fb2	390	rm -f $localFileList
9e160db9	391
	392	# make OCDB
	393	echo "***********************" 2>&1 \| tee -a ocdb.log
695f9fb2	394	echo making ${det} OCDB 2>&1 \| tee -a ocdb.log
9e160db9	395	echo "***********************" 2>&1 \| tee -a ocdb.log
1169eaef	396	echo aliroot -b -q "makeOCDB.C($run, \"$ocdb\", \"$defaultOCDB\")" 2>&1 \| tee -a ocdb.log
1169eaef	397	aliroot -b -q "makeOCDB.C($run, \"$ocdb\", \"$defaultOCDB\")" 2>&1 \| tee -a ocdb.log
9e160db9	398	mv syswatch.log syswatch_makeOCDB.log
807998f3	399
	400	# summary
	401	echo "***********************" 2>&1 \| tee -a ocdb.log
	402	echo SUMMARY 2>&1 \| tee -a ocdb.log
	403	echo "***********************" 2>&1 \| tee -a ocdb.log
695f9fb2	404	ls -altr CalibObjects.root *done 2>&1 \| tee -a ocdb.log