[u/mrichter/AliRoot.git] / PWGPP / CalibMacros / CPass1 / mergeMakeOCDB.byComponent.sh

#!/bin/bash
# example local use (files will be downloaded from alien)
#./mergeMakeOCDB.byComponent.sh /alice/data/2012/LHC12g/000188362/cpass0/ 188362 local://./OCDB defaultOCDB=local:///cvmfs/alice.gsi.de/alice/data/2011/OCDB/ fileAccessMethod=tfilecp numberOfFilesInAbunch=30
#
# on ALIEN just do:
# $1 = directory where to perform the find 
# $2 = runNumber
# $3 = OCDB path

main()
{
  save_args=("$@")
  if [[ $# -eq 0 ]]; then
    echo arguments:
    echo  "  1 - directory on which to look for the files to be merged or local file list"
    echo  "  2 - runNumber"
    echo  "  3 - OCDB output path"
    echo
    echo  "  optionally set any of these"
    echo  "  runParallel={0,1}   (1 to run parallel merging and downloading)"
    echo  "  defaultOCDB=raw://  (or any other valid OCDB storage as input database)"
    echo  "  fileAccessMethod={alien_cp,tfilecp,nocopy}   (by default tfilecp)"
    echo  "                    \"alien_cp\" will use alien_cp to copy files from alien"
    echo  "                    \"tfilecp\" will copy the files first using TFile::Cp()"
    echo  "                    \"nocopy\" will access files over the network directly TFile::Open()"
    echo  "  numberOfFilesInAbunch=50                     (how many files get downloaded in one go)"
    echo  "  maxChunksTPC=3000    (max number of chunks to be merged for TPC calibration)"

    echo
    echo "example:"
    echo  " ./mergeMakeOCDB.byComponent.sh /alice/data/2012/LHC12g/000188362/cpass1/ 188362 local://./OCDB runParallel=0 defaultOCDB=raw:// fileAccessMethod=alien_cp"
    echo

    return 0
  fi

  # init
  path=$1
  run=$(echo "$2" | sed 's/^0*//')
  ocdb=$3

  shift 3

  #default values
  runParallel=0
  defaultOCDB="raw://"
  fileAccessMethod="tfilecp"
  numberOfFilesInAbunch=50
  maxChunksTPC=3000
  [[ -n ${ALIEN_JDL_TTL} ]] && maxTimeToLive=$(( ${ALIEN_JDL_TTL}-2000 ))

  parseConfig "$@"

  #some sanity checks of option values
  [[ ! $numberOfFilesInAbunch =~ ^[0-9]+$ ]] && numberOfFilesInAbunch=50
  [[ "$fileAccessMethod" != "alien_cp" && "$fileAccessMethod" != "tfilecp" && "$fileAccessMethod" != "nocopy" ]] && fileAccessMethod="tfilecp" && echo "using default fileAccessMethod=tfilecp"

  filesAreLocal=0
  [[ -f $path ]] && filesAreLocal=1
  cleanup=1
  [[ $filesAreLocal -eq 1 ]] && cleanup=0
  [[ $filesAreLocal -eq 1 ]] && fileAccessMethod="nocopy"

  # setup components to be merged
  components="TOF MeanVertex T0 SDD TRD TPCCalib TPCCluster TPCAlign"
  #components="TOF MeanVertex T0 SDD TRD TPCCalib"

  # take Data Quality Flags from JDL
  detectorBitsQualityFlag=${ALIEN_JDL_RUNQUALITY--1}

  #################################################################
  echo "" | tee -a merge.log
  echo "$0 ${save_args[*]}" | tee -a merge.log
  echo "" | tee -a merge.log
  echo "***********************" | tee -a merge.log
  echo mergeMakeOCDB.byComponent.sh started | tee -a merge.log
  echo path = $path | tee -a merge.log
  echo run  = $run | tee -a merge.log
  echo ocdb = $ocdb | tee -a merge.log
  echo defaultOCDB = $defaultOCDB | tee -a merge.log
  echo filesAreLocal = $filesAreLocal | tee -a merge.log
  echo cleanup = $cleanup | tee -a merge.log
  echo fileAccessMethod = $fileAccessMethod | tee -a merge.log
  echo numberOfFilesInAbunch = $numberOfFilesInAbunch | tee -a merge.log
  echo runParallel = $runParallel
  echo detectorBitsQualityFlag = $detectorBitsQualityFlag
  echo "***********************" | tee -a merge.log

  alienFileList="alien.list"
  localFileList="local.list"
  filesProcessedTPClist="filesProcessedTPC.log"
  rm -f $filesProcessedTPClist
  touch $filesProcessedTPClist
  partialLocalFileListPrefix=${localFileList}_
  partialAlienFileListPrefix=${alienFileList}_

  #first, make sure we have the scripts
  copyScripts | tee -a copy.log
  ls
  #with alien files copy them first to local
  echo "***********************" 2>&1 | tee -a copy.log
  echo copying files for run $run 2>&1 | tee -a copy.log
  echo from $path 2>&1 | tee -a copy.log
  echo "***********************" 2>&1 | tee -a copy.log
  if [[ $filesAreLocal -eq 0 ]]; then
    if [[ "$fileAccessMethod" == "alien_cp" ]]; then
      echo "alien_find $path AliESDfriends_v1.root | egrep ^/ > $alienFileList" 2>&1 | tee -a copy.log
      alien_find $path "AliESDfriends_v1.root" | egrep "^/" >  $alienFileList
      echo "alien_find done"
      echo
    else 
      echo aliroot -b -q "mergeByComponent.C(\"MAKEALIENLIST\",\"$alienFileList\", \"$path\", \"AliESDfriends_v1.root\")" 2>&1 | tee -a copy.log
      aliroot -b -q "mergeByComponent.C(\"MAKEALIENLIST\",\"$alienFileList\", \"$path\", \"AliESDfriends_v1.root\")" 2>&1 | tee -a copy.log
      echo "MAKEALIENLIST done"
      echo
    fi
  else
    cp $path $alienFileList
  fi
  #randomize the list
  #keep the first line intact (it is the largest file of the entire collection)
  sed -n '1p' ${alienFileList} > ${alienFileList}.tmp
  sed '1d' ${alienFileList} | while read x; do echo "${RANDOM} ${x}"; done|sort -n|cut -d" " -f2- >> ${alienFileList}.tmp
  mv -f ${alienFileList}.tmp ${alienFileList}

  #split into sublists, each to be processed separately
  echo split --numeric-suffixes --suffix-length=6 --lines=$numberOfFilesInAbunch ${alienFileList} ${partialAlienFileListPrefix} | tee -a copy.log
  #split --numeric-suffixes --suffix-length=6 --lines=$numberOfFilesInAbunch ${alienFileList} ${partialAlienFileListPrefix}
  split -a 6 -l $numberOfFilesInAbunch ${alienFileList} ${partialAlienFileListPrefix}

  for partialAlienFileList in ${partialAlienFileListPrefix}*
  do

    #if it takes too long, break
    #[[ ${SECONDS} -gt ${maxTimeToLive} ]] && break

    partialAlienFileListPostfix=${partialAlienFileList#$partialAlienFileListPrefix}
    partialLocalFileList=${partialLocalFileListPrefix}${partialAlienFileListPostfix}

    #copy the files if appropriate and make a list of files to use
    rm -f $partialLocalFileList
    if [[ "$fileAccessMethod" == "alien_cp" ]]; then
      while read x; do
        [[ $x != /* ]] && continue
        localName=${x//"/"/_}
        echo "alien_cp "alien://$x" $localName" 2>&1 | tee -a copy.log
        alien_cp "alien://$x" $localName
        echo $localName>>$partialLocalFileList
      done<$partialAlienFileList
    elif [[ "$fileAccessMethod" == "tfilecp" ]]; then
      echo aliroot -b -q "mergeByComponent.C(\"COPY\",\"$partialAlienFileList\",\"noPath\",\"noPattern\",10,\"$partialLocalFileList\")" 2>&1 | tee -a copy.log
      aliroot -b -q "mergeByComponent.C(\"COPY\",\"$partialAlienFileList\",\"noPath\",\"noPattern\",10,\"$partialLocalFileList\")" 2>&1 | tee -a copy.log
    elif [[ "$fileAccessMethod" == "nocopy" ]]; then
      while read x; do
        [[ $filesAreLocal -eq 0 ]] && echo "alien://$x" >> $partialLocalFileList
        [[ $filesAreLocal -eq 1 ]] && echo "$x" >> $partialLocalFileList
      done<$partialAlienFileList
      cleanup=0
    fi

    #handle syswatch
    if [[ -f syswatch_copy.log ]]
    then
      sed '1d' syswatch.log >> syswatch_copy.log
      rm -f syswatch.log
    else 
      [[ -f syswatch.log ]] && mv syswatch.log syswatch_copy.log
    fi

    echo waiting
    wait $!
    if [[ $runParallel -eq 1 ]]; then
      mergeByComponent $partialLocalFileList $cleanup 2>&1 | tee -a merge.log &
    else
      mergeByComponent $partialLocalFileList $cleanup 2>&1 | tee -a merge.log
    fi
  done

  #merge all the subfiles into one, wait for the last one to complete
  echo waiting
  wait $!
  if [[ "$components" =~ ALL && -f CalibObjects_ALL.root ]]; then
    mv -f CalibObjects_ALL.root CalibObjects.root
  else
    echo "***********************"
    echo merging ALL data
    echo "***********************"
    finalCalibObjectsList="finalObjects.list"
    ls -1 CalibObjects_*.root > $finalCalibObjectsList
    echo aliroot -b -q "mergeByComponent.C(\"ALL\", \"$finalCalibObjectsList\")" | tee -a merge.log
    aliroot -b -q "mergeByComponent.C(\"ALL\", \"$finalCalibObjectsList\")" 2>&1 | tee -a merge.log
    mv -f syswatch.log syswatch_merge_ALL.log
  fi

  if ! validateMerging "ALL"; then
    echo final merging not validatet, exiting...
    return 1
  fi
  rm -f CalibObjects_*.root

  #cleanup
  rm -f ${partialAlienFileListPrefix}*
  rm -f ${partialLocalFileListPrefix}*
  rm -f $alienFileList
  rm -f $localFileList

  # make OCDB
  echo "***********************" 2>&1 | tee -a ocdb.log
  echo making ${det} OCDB 2>&1 | tee -a ocdb.log
  echo "***********************" 2>&1 | tee -a ocdb.log
  echo aliroot -b -q "makeOCDB.C($run, \"$ocdb\", \"$defaultOCDB\", $detectorBitsQualityFlag)" 2>&1 | tee -a ocdb.log
  aliroot -b -q "makeOCDB.C($run, \"$ocdb\", \"$defaultOCDB\", $detectorBitsQualityFlag)" 2>&1 | tee -a ocdb.log
  mv syswatch.log syswatch_makeOCDB.log

  # summary
  echo "***********************" 2>&1 | tee -a ocdb.log
  echo SUMMARY 2>&1 | tee -a ocdb.log
  echo "***********************" 2>&1 | tee -a ocdb.log
  ls -altr CalibObjects.root *done 2>&1 | tee -a ocdb.log
}

mergeByComponent()
{
  # process by component
  # first argument is the file list to process

  # run inside a dedicated running directory
  # whic means copy the file list to process and prefic each line with ../
  # since the file names have no absolute paths!
  runningDirectory="${1}.dir"
  fileList="$1"
  cleanup=$2

  #sanity checks
  if [[ ! -f ${fileList} ]]; then
    echo "${fileList} does not exist"
    return 1
  fi
  mkdir -p $runningDirectory
  if [[ ! -d $runningDirectory ]]; then
    echo "cannot create the running directory $runningDirectory"
    return 1
  fi

  #move the to be merged files to the running directory and make a list of available files
  #handle the case of archives (x.zip#y.root) as well
  nFiles=0
  while read entry; do
    if [[ $entry =~ ^.*\.root$ ]]; then
      file=${entry%#*}
      fileContent=${entry##*#}
      [[ "${fileContent}" == "${file}" ]] && fileContent=""
      [[ -n ${fileContent} ]] && fileContent="#${fileContent}"
      if [[ -f ${file} ]]; then
        ((nFiles++))
        if [[ -f "./${file}" ]]; then
          echo "../${file}${fileContent}" >> "${runningDirectory}/${fileList}"
        else
          echo "${file}${fileContent}" >> "${runningDirectory}/${fileList}"
        fi
      fi
    fi
  done < ${fileList}
  if [[ $nFiles -lt 1 ]]; then
    echo "no new files in ${fileList}"
    echo rm -rf $runningDirectory 
    rm -rf $runningDirectory 
    return 1
  fi

  #copy the macro to the running directory
  [[ -f mergeByComponent.C ]] && cp mergeByComponent.C $runningDirectory

  #go to running directory
  cd $runningDirectory

  numberOfChunksTPC=$(cat ../$filesProcessedTPClist 2>/dev/null | wc -l )

  for det in $components; do

    # merge
    echo "***********************" 
    echo merging ${det} data
    echo "***********************"

    #limit the number of chunks processed by TPC
    if [[ "${det}" =~ TPC && $numberOfChunksTPC -ge $maxChunksTPC ]]; then
      echo "Not merging TPC anymore, max number of chunks processed ($maxChunksTPC)"
      continue
    fi

    #add the results of previous iteration to the BEGINNING of the list of files to be merged
    [[ -f ../CalibObjects_${det}.root ]] && echo "../CalibObjects_${det}.root" > ${fileList}_${det}
    cat ${fileList} >> ${fileList}_${det}

    echo "processing following files from ${fileList}_${det}:"
    cat ${fileList}_${det}

    echo aliroot -b -q "mergeByComponent.C(\"${det}\", \"${fileList}_${det}\")"
    aliroot -b -q "mergeByComponent.C(\"${det}\", \"${fileList}_${det}\")" 2>&1 | tee -a merge_${det}.log
    if validateMerging ${det}; then
      echo "### merge OK: mv CalibObjects.root ../CalibObjects_${det}.root"
      mv -f CalibObjects.root ../CalibObjects_${det}.root
      [[ "${det}" =~ TPCCalib ]] && cat ${fileList} >> ../$filesProcessedTPClist
    else 
      echo "### merging not validated"
      rm -f CalibObjects.root
    fi
    mv syswatch.log syswatch_merge_${det}.log

  done


  #move stuff back to the parent dir and clean up
  #merge the syswatch logs
  for x in syswatch*log; do
    [[ ! -f $x ]] && continue
    if [[ -f ../$x ]]
    then 
      echo "sed '1d' $x  >> ../$x"
      sed '1d' $x >> ../$x
      rm -f $x
    else 
      echo mv -f $x ..
      mv -f $x ..
    fi
  done

  #merge the other logs
  for x in *.log; do
    [[ ! -f $x ]] && continue
    if [[ -f ../$x ]]
    then
      echo "cat $x >> ../$x"
      cat $x >> ../$x
    else
      echo "mv -f $x .."
      mv -f $x ..
    fi
  done

  #final cleanup.
  #go to parent dir first to use the original fileList
  #without ../CalibObjects.root
  cd ..
  if [[ $cleanup -eq 1 ]]; then
    echo "cleaning up merged files..."
    while read file; do
      echo rm -rf $file
      rm -rf $file
    done<$fileList
  fi
  rm -rf $runningDirectory

  echo "***mergeByComponent() DONE"
  echo
  echo "numberOfChunksTPC=$numberOfChunksTPC"
  return 0
}

parseConfig()
{
  args=("$@")
  for opt in "${args[@]}"; do
    [[ -z ${opt} ]] && continue
    [[ -n ${encodedSpaces} ]] && opt="$(decSpaces ${opt})"
    [[ "${opt}" =~ ^[[:space:]]*$ ]] && continue
    if [[ ! "${opt}" =~ .*=.* ]]; then
      echo "badly formatted option \"${opt}\" should be: option=value, stopping..."
      return 1
    fi
    local var="${opt%%=*}"
    local value="${opt#*=}"
    export ${var}="${value}"
  done

  return 0
}

waitIfLocked()
{
  while [ 1 ]; do
    [[ ! -f $1 ]] && break
    sleep 1
  done
  return 0
}

validateMerging()
{
  det=$1
  retCode=0
  [[ ! -f CalibObjects.root ]] && ((retCode++)) && echo "### no CalibObjects.root..."
  [[ ! -f ${det}_merge_done ]] && ((retCode++)) && echo "### no ${det}_merge_done, job finished abnormally..."
  error=$(grep -e "was a crash" *.log)
  [[ -n $error ]] && ((retCode++)) && echo "### error: $error"

  return $retCode
}

copyScripts()
{
  [[ ! -f mergeByComponent.C ]] && \
    cp -f $ALICE_ROOT/PWGPP/CalibMacros/CPass1/mergeByComponent.C $PWD && \
    echo "taking the default scripts from $ALICE_ROOT"
  [[ ! -f makeOCDB.C ]] && \
    cp -f $ALICE_ROOT/PWGPP/CalibMacros/CPass1/makeOCDB.C $PWD && \
    echo "taking the default scripts from $ALICE_ROOT"
}

#these functions encode strings to and from a space-less form
#use when spaces are not well handled (e.g. in arguments to
#commands in makeflow files, etc.
encSpaces()(echo "${1// /@@@@}")
decSpaces()(echo "${1//@@@@/ }")

main "$@"
Commit	Line	Data
e9129651	1	#!/bin/bash
e9129651	2	# example local use (files will be downloaded from alien)
0959f55a	3	#./mergeMakeOCDB.byComponent.sh /alice/data/2012/LHC12g/000188362/cpass0/ 188362 local://./OCDB defaultOCDB=local:///cvmfs/alice.gsi.de/alice/data/2011/OCDB/ fileAccessMethod=tfilecp numberOfFilesInAbunch=30
e9129651	4	#
e9129651	5	# on ALIEN just do:
9e160db9	6	# $1 = directory where to perform the find
	7	# $2 = runNumber
	8	# $3 = OCDB path
	9
421603c5	10	main()
421603c5	11	{
0959f55a	12	save_args=("$@")
421603c5	13	if [[ $# -eq 0 ]]; then
	14	echo arguments:
	15	echo " 1 - directory on which to look for the files to be merged or local file list"
	16	echo " 2 - runNumber"
	17	echo " 3 - OCDB output path"
	18	echo
	19	echo " optionally set any of these"
	20	echo " runParallel={0,1} (1 to run parallel merging and downloading)"
	21	echo " defaultOCDB=raw:// (or any other valid OCDB storage as input database)"
	22	echo " fileAccessMethod={alien_cp,tfilecp,nocopy} (by default tfilecp)"
	23	echo " \"alien_cp\" will use alien_cp to copy files from alien"
	24	echo " \"tfilecp\" will copy the files first using TFile::Cp()"
	25	echo " \"nocopy\" will access files over the network directly TFile::Open()"
	26	echo " numberOfFilesInAbunch=50 (how many files get downloaded in one go)"
	27	echo " maxChunksTPC=3000 (max number of chunks to be merged for TPC calibration)"
	28
	29	echo
	30	echo "example:"
	31	echo " ./mergeMakeOCDB.byComponent.sh /alice/data/2012/LHC12g/000188362/cpass1/ 188362 local://./OCDB runParallel=0 defaultOCDB=raw:// fileAccessMethod=alien_cp"
	32	echo
	33
	34	return 0
	35	fi
	36
	37	# init
	38	path=$1
	39	run=$(echo "$2" \| sed 's/^0*//')
	40	ocdb=$3
	41
	42	shift 3
	43
	44	#default values
e52ef734	45	runParallel=0
421603c5	46	defaultOCDB="raw://"
	47	fileAccessMethod="tfilecp"
	48	numberOfFilesInAbunch=50
	49	maxChunksTPC=3000
	50	[[ -n ${ALIEN_JDL_TTL} ]] && maxTimeToLive=$(( ${ALIEN_JDL_TTL}-2000 ))
	51
	52	parseConfig "$@"
	53
	54	#some sanity checks of option values
	55	[[ ! $numberOfFilesInAbunch =~ ^[0-9]+$ ]] && numberOfFilesInAbunch=50
	56	[[ "$fileAccessMethod" != "alien_cp" && "$fileAccessMethod" != "tfilecp" && "$fileAccessMethod" != "nocopy" ]] && fileAccessMethod="tfilecp" && echo "using default fileAccessMethod=tfilecp"
	57
	58	filesAreLocal=0
	59	[[ -f $path ]] && filesAreLocal=1
	60	cleanup=1
	61	[[ $filesAreLocal -eq 1 ]] && cleanup=0
	62	[[ $filesAreLocal -eq 1 ]] && fileAccessMethod="nocopy"
	63
	64	# setup components to be merged
	65	components="TOF MeanVertex T0 SDD TRD TPCCalib TPCCluster TPCAlign"
	66	#components="TOF MeanVertex T0 SDD TRD TPCCalib"
	67
a4d010b1	68	# take Data Quality Flags from JDL
	69	detectorBitsQualityFlag=${ALIEN_JDL_RUNQUALITY--1}
	70
421603c5	71	#################################################################
421603c5	72	echo "" \| tee -a merge.log
0959f55a	73	echo "$0 ${save_args[*]}" \| tee -a merge.log
421603c5	74	echo "" \| tee -a merge.log
	75	echo "***********************" \| tee -a merge.log
	76	echo mergeMakeOCDB.byComponent.sh started \| tee -a merge.log
	77	echo path = $path \| tee -a merge.log
	78	echo run = $run \| tee -a merge.log
	79	echo ocdb = $ocdb \| tee -a merge.log
a4d010b1	80	echo defaultOCDB = $defaultOCDB \| tee -a merge.log
421603c5	81	echo filesAreLocal = $filesAreLocal \| tee -a merge.log
421603c5	82	echo cleanup = $cleanup \| tee -a merge.log
a4d010b1	83	echo fileAccessMethod = $fileAccessMethod \| tee -a merge.log
421603c5	84	echo numberOfFilesInAbunch = $numberOfFilesInAbunch \| tee -a merge.log
a4d010b1	85	echo runParallel = $runParallel
a4d010b1	86	echo detectorBitsQualityFlag = $detectorBitsQualityFlag
421603c5	87	echo "***********************" \| tee -a merge.log
	88
	89	alienFileList="alien.list"
	90	localFileList="local.list"
	91	filesProcessedTPClist="filesProcessedTPC.log"
	92	rm -f $filesProcessedTPClist
	93	touch $filesProcessedTPClist
	94	partialLocalFileListPrefix=${localFileList}_
	95	partialAlienFileListPrefix=${alienFileList}_
	96
	97	#first, make sure we have the scripts
	98	copyScripts \| tee -a copy.log
	99	ls
	100	#with alien files copy them first to local
	101	echo "***********************" 2>&1 \| tee -a copy.log
	102	echo copying files for run $run 2>&1 \| tee -a copy.log
	103	echo from $path 2>&1 \| tee -a copy.log
	104	echo "***********************" 2>&1 \| tee -a copy.log
	105	if [[ $filesAreLocal -eq 0 ]]; then
	106	if [[ "$fileAccessMethod" == "alien_cp" ]]; then
	107	echo "alien_find $path AliESDfriends_v1.root \| egrep ^/ > $alienFileList" 2>&1 \| tee -a copy.log
	108	alien_find $path "AliESDfriends_v1.root" \| egrep "^/" > $alienFileList
	109	echo "alien_find done"
	110	echo
	111	else
	112	echo aliroot -b -q "mergeByComponent.C(\"MAKEALIENLIST\",\"$alienFileList\", \"$path\", \"AliESDfriends_v1.root\")" 2>&1 \| tee -a copy.log
	113	aliroot -b -q "mergeByComponent.C(\"MAKEALIENLIST\",\"$alienFileList\", \"$path\", \"AliESDfriends_v1.root\")" 2>&1 \| tee -a copy.log
	114	echo "MAKEALIENLIST done"
	115	echo
	116	fi
	117	else
	118	cp $path $alienFileList
	119	fi
	120	#randomize the list
	121	#keep the first line intact (it is the largest file of the entire collection)
	122	sed -n '1p' ${alienFileList} > ${alienFileList}.tmp
	123	sed '1d' ${alienFileList} \| while read x; do echo "${RANDOM} ${x}"; done\|sort -n\|cut -d" " -f2- >> ${alienFileList}.tmp
	124	mv -f ${alienFileList}.tmp ${alienFileList}
	125
	126	#split into sublists, each to be processed separately
	127	echo split --numeric-suffixes --suffix-length=6 --lines=$numberOfFilesInAbunch ${alienFileList} ${partialAlienFileListPrefix} \| tee -a copy.log
	128	#split --numeric-suffixes --suffix-length=6 --lines=$numberOfFilesInAbunch ${alienFileList} ${partialAlienFileListPrefix}
	129	split -a 6 -l $numberOfFilesInAbunch ${alienFileList} ${partialAlienFileListPrefix}
	130
	131	for partialAlienFileList in ${partialAlienFileListPrefix}*
	132	do
	133
	134	#if it takes too long, break
	135	#[[ ${SECONDS} -gt ${maxTimeToLive} ]] && break
	136
	137	partialAlienFileListPostfix=${partialAlienFileList#$partialAlienFileListPrefix}
	138	partialLocalFileList=${partialLocalFileListPrefix}${partialAlienFileListPostfix}
	139
	140	#copy the files if appropriate and make a list of files to use
	141	rm -f $partialLocalFileList
	142	if [[ "$fileAccessMethod" == "alien_cp" ]]; then
	143	while read x; do
	144	[[ $x != /* ]] && continue
	145	localName=${x//"/"/_}
	146	echo "alien_cp "alien://$x" $localName" 2>&1 \| tee -a copy.log
	147	alien_cp "alien://$x" $localName
	148	echo $localName>>$partialLocalFileList
	149	done<$partialAlienFileList
	150	elif [[ "$fileAccessMethod" == "tfilecp" ]]; then
151	echo aliroot -b -q "mergeByComponent.C(\"COPY\",\"$partialAlienFileList\",\"noPath\",\"noPattern\",10,\"$partialLocalFileList\")" 2>&1 \| tee -a copy.log
152	aliroot -b -q "mergeByComponent.C(\"COPY\",\"$partialAlienFileList\",\"noPath\",\"noPattern\",10,\"$partialLocalFileList\")" 2>&1 \| tee -a copy.log
153	elif [[ "$fileAccessMethod" == "nocopy" ]]; then
154	while read x; do
155	[[ $filesAreLocal -eq 0 ]] && echo "alien://$x" >> $partialLocalFileList
156	[[ $filesAreLocal -eq 1 ]] && echo "$x" >> $partialLocalFileList
157	done<$partialAlienFileList
158	cleanup=0
159	fi
160
161	#handle syswatch
162	if [[ -f syswatch_copy.log ]]
163	then
164	sed '1d' syswatch.log >> syswatch_copy.log
165	rm -f syswatch.log
166	else
167	[[ -f syswatch.log ]] && mv syswatch.log syswatch_copy.log
168	fi
e9129651	169
421603c5	170	echo waiting
	171	wait $!
	172	if [[ $runParallel -eq 1 ]]; then
	173	mergeByComponent $partialLocalFileList $cleanup 2>&1 \| tee -a merge.log &
	174	else
	175	mergeByComponent $partialLocalFileList $cleanup 2>&1 \| tee -a merge.log
	176	fi
	177	done
	178
	179	#merge all the subfiles into one, wait for the last one to complete
	180	echo waiting
	181	wait $!
	182	if [[ "$components" =~ ALL && -f CalibObjects_ALL.root ]]; then
	183	mv -f CalibObjects_ALL.root CalibObjects.root
	184	else
	185	echo "***********************"
	186	echo merging ALL data
	187	echo "***********************"
	188	finalCalibObjectsList="finalObjects.list"
	189	ls -1 CalibObjects_*.root > $finalCalibObjectsList
	190	echo aliroot -b -q "mergeByComponent.C(\"ALL\", \"$finalCalibObjectsList\")" \| tee -a merge.log
	191	aliroot -b -q "mergeByComponent.C(\"ALL\", \"$finalCalibObjectsList\")" 2>&1 \| tee -a merge.log
	192	mv -f syswatch.log syswatch_merge_ALL.log
	193	fi
	194
	195	if ! validateMerging "ALL"; then
	196	echo final merging not validatet, exiting...
	197	return 1
	198	fi
	199	rm -f CalibObjects_*.root
	200
	201	#cleanup
	202	rm -f ${partialAlienFileListPrefix}*
	203	rm -f ${partialLocalFileListPrefix}*
	204	rm -f $alienFileList
	205	rm -f $localFileList
	206
	207	# make OCDB
	208	echo "***********************" 2>&1 \| tee -a ocdb.log
	209	echo making ${det} OCDB 2>&1 \| tee -a ocdb.log
	210	echo "***********************" 2>&1 \| tee -a ocdb.log
a4d010b1	211	echo aliroot -b -q "makeOCDB.C($run, \"$ocdb\", \"$defaultOCDB\", $detectorBitsQualityFlag)" 2>&1 \| tee -a ocdb.log
a4d010b1	212	aliroot -b -q "makeOCDB.C($run, \"$ocdb\", \"$defaultOCDB\", $detectorBitsQualityFlag)" 2>&1 \| tee -a ocdb.log
421603c5	213	mv syswatch.log syswatch_makeOCDB.log
	214
	215	# summary
	216	echo "***********************" 2>&1 \| tee -a ocdb.log
	217	echo SUMMARY 2>&1 \| tee -a ocdb.log
	218	echo "***********************" 2>&1 \| tee -a ocdb.log
	219	ls -altr CalibObjects.root *done 2>&1 \| tee -a ocdb.log
	220	}
9e160db9	221
	222	mergeByComponent()
	223	{
	224	# process by component
	225	# first argument is the file list to process
9e160db9	226
40f73007	227	# run inside a dedicated running directory
	228	# whic means copy the file list to process and prefic each line with ../
	229	# since the file names have no absolute paths!
421603c5	230	runningDirectory="${1}.dir"
e9129651	231	fileList="$1"
e9129651	232	cleanup=$2
421603c5	233
e9129651	234	#sanity checks
695f9fb2	235	if [[ ! -f ${fileList} ]]; then
695f9fb2	236	echo "${fileList} does not exist"
695f9fb2	237	return 1
695f9fb2	238	fi
40f73007	239	mkdir -p $runningDirectory
695f9fb2	240	if [[ ! -d $runningDirectory ]]; then
695f9fb2	241	echo "cannot create the running directory $runningDirectory"
695f9fb2	242	return 1
9e160db9	243	fi
421603c5	244
e9129651	245	#move the to be merged files to the running directory and make a list of available files
	246	#handle the case of archives (x.zip#y.root) as well
	247	nFiles=0
	248	while read entry; do
	249	if [[ $entry =~ ^.*\.root$ ]]; then
	250	file=${entry%#*}
	251	fileContent=${entry##*#}
	252	[[ "${fileContent}" == "${file}" ]] && fileContent=""
	253	[[ -n ${fileContent} ]] && fileContent="#${fileContent}"
	254	if [[ -f ${file} ]]; then
	255	((nFiles++))
	256	if [[ -f "./${file}" ]]; then
695f9fb2	257	echo "../${file}${fileContent}" >> "${runningDirectory}/${fileList}"
e9129651	258	else
695f9fb2	259	echo "${file}${fileContent}" >> "${runningDirectory}/${fileList}"
e9129651	260	fi
	261	fi
	262	fi
695f9fb2	263	done < ${fileList}
	264	if [[ $nFiles -lt 1 ]]; then
	265	echo "no new files in ${fileList}"
	266	echo rm -rf $runningDirectory
	267	rm -rf $runningDirectory
695f9fb2	268	return 1
695f9fb2	269	fi
e9129651	270
	271	#copy the macro to the running directory
	272	[[ -f mergeByComponent.C ]] && cp mergeByComponent.C $runningDirectory
	273
695f9fb2	274	#go to running directory
e9129651	275	cd $runningDirectory
421603c5	276
695f9fb2	277	numberOfChunksTPC=$(cat ../$filesProcessedTPClist 2>/dev/null \| wc -l )
807998f3	278
9e160db9	279	for det in $components; do
421603c5	280
807998f3	281	# merge
e9129651	282	echo "***********************"
695f9fb2	283	echo merging ${det} data
e9129651	284	echo "***********************"
695f9fb2	285
	286	#limit the number of chunks processed by TPC
	287	if [[ "${det}" =~ TPC && $numberOfChunksTPC -ge $maxChunksTPC ]]; then
	288	echo "Not merging TPC anymore, max number of chunks processed ($maxChunksTPC)"
	289	continue
	290	fi
	291
	292	#add the results of previous iteration to the BEGINNING of the list of files to be merged
	293	[[ -f ../CalibObjects_${det}.root ]] && echo "../CalibObjects_${det}.root" > ${fileList}_${det}
	294	cat ${fileList} >> ${fileList}_${det}
	295
	296	echo "processing following files from ${fileList}_${det}:"
	297	cat ${fileList}_${det}
	298
421603c5	299	echo aliroot -b -q "mergeByComponent.C(\"${det}\", \"${fileList}_${det}\")"
421603c5	300	aliroot -b -q "mergeByComponent.C(\"${det}\", \"${fileList}_${det}\")" 2>&1 \| tee -a merge_${det}.log
e9129651	301	if validateMerging ${det}; then
695f9fb2	302	echo "### merge OK: mv CalibObjects.root ../CalibObjects_${det}.root"
	303	mv -f CalibObjects.root ../CalibObjects_${det}.root
	304	[[ "${det}" =~ TPCCalib ]] && cat ${fileList} >> ../$filesProcessedTPClist
e9129651	305	else
695f9fb2	306	echo "### merging not validated"
e9129651	307	rm -f CalibObjects.root
e9129651	308	fi
695f9fb2	309	mv syswatch.log syswatch_merge_${det}.log
695f9fb2	310
9e160db9	311	done
40f73007	312
421603c5	313
40f73007	314	#move stuff back to the parent dir and clean up
	315	#merge the syswatch logs
	316	for x in syswatch*log; do
e9129651	317	[[ ! -f $x ]] && continue
e9129651	318	if [[ -f ../$x ]]
40f73007	319	then
e9129651	320	echo "sed '1d' $x >> ../$x"
e9129651	321	sed '1d' $x >> ../$x
40f73007	322	rm -f $x
40f73007	323	else
e9129651	324	echo mv -f $x ..
	325	mv -f $x ..
	326	fi
	327	done
	328
	329	#merge the other logs
	330	for x in *.log; do
	331	[[ ! -f $x ]] && continue
	332	if [[ -f ../$x ]]
	333	then
	334	echo "cat $x >> ../$x"
	335	cat $x >> ../$x
	336	else
	337	echo "mv -f $x .."
	338	mv -f $x ..
40f73007	339	fi
40f73007	340	done
421603c5	341
e9129651	342	#final cleanup.
	343	#go to parent dir first to use the original fileList
	344	#without ../CalibObjects.root
	345	cd ..
	346	if [[ $cleanup -eq 1 ]]; then
	347	echo "cleaning up merged files..."
	348	while read file; do
	349	echo rm -rf $file
	350	rm -rf $file
	351	done<$fileList
	352	fi
40f73007	353	rm -rf $runningDirectory
807998f3	354
e9129651	355	echo "***mergeByComponent() DONE"
e9129651	356	echo
695f9fb2	357	echo "numberOfChunksTPC=$numberOfChunksTPC"
e9129651	358	return 0
9e160db9	359	}
807998f3	360
421603c5	361	parseConfig()
	362	{
	363	args=("$@")
	364	for opt in "${args[@]}"; do
	365	[[ -z ${opt} ]] && continue
	366	[[ -n ${encodedSpaces} ]] && opt="$(decSpaces ${opt})"
	367	[[ "${opt}" =~ ^[[:space:]]*$ ]] && continue
	368	if [[ ! "${opt}" =~ .=. ]]; then
	369	echo "badly formatted option \"${opt}\" should be: option=value, stopping..."
	370	return 1
	371	fi
	372	local var="${opt%%=*}"
	373	local value="${opt#*=}"
	374	export ${var}="${value}"
	375	done
	376
	377	return 0
	378	}
	379
9e160db9	380	waitIfLocked()
	381	{
	382	while [ 1 ]; do
	383	[[ ! -f $1 ]] && break
	384	sleep 1
	385	done
e9129651	386	return 0
9e160db9	387	}
807998f3	388
e9129651	389	validateMerging()
	390	{
	391	det=$1
	392	retCode=0
	393	[[ ! -f CalibObjects.root ]] && ((retCode++)) && echo "### no CalibObjects.root..."
	394	[[ ! -f ${det}_merge_done ]] && ((retCode++)) && echo "### no ${det}_merge_done, job finished abnormally..."
	395	error=$(grep -e "was a crash" *.log)
	396	[[ -n $error ]] && ((retCode++)) && echo "### error: $error"
421603c5	397
e9129651	398	return $retCode
	399	}
	400
	401	copyScripts()
	402	{
	403	[[ ! -f mergeByComponent.C ]] && \
421603c5	404	cp -f $ALICE_ROOT/PWGPP/CalibMacros/CPass1/mergeByComponent.C $PWD && \
e9129651	405	echo "taking the default scripts from $ALICE_ROOT"
e9129651	406	[[ ! -f makeOCDB.C ]] && \
421603c5	407	cp -f $ALICE_ROOT/PWGPP/CalibMacros/CPass1/makeOCDB.C $PWD && \
e9129651	408	echo "taking the default scripts from $ALICE_ROOT"
e9129651	409	}
9e160db9	410
0959f55a	411	#these functions encode strings to and from a space-less form
	412	#use when spaces are not well handled (e.g. in arguments to
	413	#commands in makeflow files, etc.
	414	encSpaces()(echo "${1// /@@@@}")
	415	decSpaces()(echo "${1//@@@@/ }")
	416
421603c5	417	main "$@"