[u/mrichter/AliRoot.git] / PWGPP / CalibMacros / CPass1 / mergeMakeOCDB.byComponent.sh

#!/bin/bash
# example local use (files will be downloaded from alien)
#./mergeMakeOCDB.byComponent.sh /alice/data/2012/LHC12g/000188362/cpass0/ 188362 local://./OCDB defaultOCDB=local:///cvmfs/alice.gsi.de/alice/data/2011/OCDB/ fileAccessMethod=tfilecp numberOfFilesInAbunch=30
#
# on ALIEN just do:
# $1 = directory where to perform the find 
# $2 = runNumber
# $3 = OCDB path

main()
{
  save_args=("$@")
  if [[ $# -eq 0 ]]; then
    echo arguments:
    echo  "  1 - directory on which to look for the files to be merged or local file list"
    echo  "  2 - runNumber"
    echo  "  3 - OCDB output path"
    echo
    echo  "  optionally set any of these"
    echo  "  runParallel={0,1}   (1 to run parallel merging and downloading)"
    echo  "  defaultOCDB=raw://  (or any other valid OCDB storage as input database)"
    echo  "  fileAccessMethod={alien_cp,tfilecp,nocopy}   (by default tfilecp)"
    echo  "                    \"alien_cp\" will use alien_cp to copy files from alien"
    echo  "                    \"tfilecp\" will copy the files first using TFile::Cp()"
    echo  "                    \"nocopy\" will access files over the network directly TFile::Open()"
    echo  "  numberOfFilesInAbunch=50                     (how many files get downloaded in one go)"
    echo  "  maxChunksTPC=3000    (max number of chunks to be merged for TPC calibration)"

    echo
    echo "example:"
    echo  " ./mergeMakeOCDB.byComponent.sh /alice/data/2012/LHC12g/000188362/cpass1/ 188362 local://./OCDB runParallel=0 defaultOCDB=raw:// fileAccessMethod=alien_cp"
    echo

    return 0
  fi

  # init
  path=$1
  run=$(echo "$2" | sed 's/^0*//')
  ocdb=$3

  shift 3

  #default values
  runParallel=0
  defaultOCDB="raw://"
  fileAccessMethod="tfilecp"
  numberOfFilesInAbunch=50
  maxChunksTPC=3000
  [[ -n ${ALIEN_JDL_TTL} ]] && maxTimeToLive=$(( ${ALIEN_JDL_TTL}-2000 ))

  parseConfig "$@"

  #some sanity checks of option values
  [[ ! $numberOfFilesInAbunch =~ ^[0-9]+$ ]] && numberOfFilesInAbunch=50
  [[ "$fileAccessMethod" != "alien_cp" && "$fileAccessMethod" != "tfilecp" && "$fileAccessMethod" != "nocopy" ]] && fileAccessMethod="tfilecp" && echo "using default fileAccessMethod=tfilecp"

  filesAreLocal=0
  [[ -f $path ]] && filesAreLocal=1
  cleanup=1
  [[ $filesAreLocal -eq 1 ]] && cleanup=0
  [[ $filesAreLocal -eq 1 ]] && fileAccessMethod="nocopy"

  # setup components to be merged
  components="TOF MeanVertex T0 SDD TRD TPCCalib TPCCluster TPCAlign"
  #components="TOF MeanVertex T0 SDD TRD TPCCalib"

  #################################################################
  echo "" | tee -a merge.log
  echo "$0 ${save_args[*]}" | tee -a merge.log
  echo "" | tee -a merge.log
  echo "***********************" | tee -a merge.log
  echo mergeMakeOCDB.byComponent.sh started | tee -a merge.log
  echo path = $path | tee -a merge.log
  echo run  = $run | tee -a merge.log
  echo ocdb = $ocdb | tee -a merge.log
  echo defaultOCDB=$defaultOCDB | tee -a merge.log
  echo filesAreLocal = $filesAreLocal | tee -a merge.log
  echo cleanup = $cleanup | tee -a merge.log
  echo fileAccessMethod=$fileAccessMethod | tee -a merge.log
  echo numberOfFilesInAbunch = $numberOfFilesInAbunch | tee -a merge.log
  echo runParallel=$runParallel
  echo "***********************" | tee -a merge.log

  alienFileList="alien.list"
  localFileList="local.list"
  filesProcessedTPClist="filesProcessedTPC.log"
  rm -f $filesProcessedTPClist
  touch $filesProcessedTPClist
  partialLocalFileListPrefix=${localFileList}_
  partialAlienFileListPrefix=${alienFileList}_

  #first, make sure we have the scripts
  copyScripts | tee -a copy.log
  ls
  #with alien files copy them first to local
  echo "***********************" 2>&1 | tee -a copy.log
  echo copying files for run $run 2>&1 | tee -a copy.log
  echo from $path 2>&1 | tee -a copy.log
  echo "***********************" 2>&1 | tee -a copy.log
  if [[ $filesAreLocal -eq 0 ]]; then
    if [[ "$fileAccessMethod" == "alien_cp" ]]; then
      echo "alien_find $path AliESDfriends_v1.root | egrep ^/ > $alienFileList" 2>&1 | tee -a copy.log
      alien_find $path "AliESDfriends_v1.root" | egrep "^/" >  $alienFileList
      echo "alien_find done"
      echo
    else 
      echo aliroot -b -q "mergeByComponent.C(\"MAKEALIENLIST\",\"$alienFileList\", \"$path\", \"AliESDfriends_v1.root\")" 2>&1 | tee -a copy.log
      aliroot -b -q "mergeByComponent.C(\"MAKEALIENLIST\",\"$alienFileList\", \"$path\", \"AliESDfriends_v1.root\")" 2>&1 | tee -a copy.log
      echo "MAKEALIENLIST done"
      echo
    fi
  else
    cp $path $alienFileList
  fi
  #randomize the list
  #keep the first line intact (it is the largest file of the entire collection)
  sed -n '1p' ${alienFileList} > ${alienFileList}.tmp
  sed '1d' ${alienFileList} | while read x; do echo "${RANDOM} ${x}"; done|sort -n|cut -d" " -f2- >> ${alienFileList}.tmp
  mv -f ${alienFileList}.tmp ${alienFileList}

  #split into sublists, each to be processed separately
  echo split --numeric-suffixes --suffix-length=6 --lines=$numberOfFilesInAbunch ${alienFileList} ${partialAlienFileListPrefix} | tee -a copy.log
  #split --numeric-suffixes --suffix-length=6 --lines=$numberOfFilesInAbunch ${alienFileList} ${partialAlienFileListPrefix}
  split -a 6 -l $numberOfFilesInAbunch ${alienFileList} ${partialAlienFileListPrefix}

  for partialAlienFileList in ${partialAlienFileListPrefix}*
  do

    #if it takes too long, break
    #[[ ${SECONDS} -gt ${maxTimeToLive} ]] && break

    partialAlienFileListPostfix=${partialAlienFileList#$partialAlienFileListPrefix}
    partialLocalFileList=${partialLocalFileListPrefix}${partialAlienFileListPostfix}

    #copy the files if appropriate and make a list of files to use
    rm -f $partialLocalFileList
    if [[ "$fileAccessMethod" == "alien_cp" ]]; then
      while read x; do
        [[ $x != /* ]] && continue
        localName=${x//"/"/_}
        echo "alien_cp "alien://$x" $localName" 2>&1 | tee -a copy.log
        alien_cp "alien://$x" $localName
        echo $localName>>$partialLocalFileList
      done<$partialAlienFileList
    elif [[ "$fileAccessMethod" == "tfilecp" ]]; then
      echo aliroot -b -q "mergeByComponent.C(\"COPY\",\"$partialAlienFileList\",\"noPath\",\"noPattern\",10,\"$partialLocalFileList\")" 2>&1 | tee -a copy.log
      aliroot -b -q "mergeByComponent.C(\"COPY\",\"$partialAlienFileList\",\"noPath\",\"noPattern\",10,\"$partialLocalFileList\")" 2>&1 | tee -a copy.log
    elif [[ "$fileAccessMethod" == "nocopy" ]]; then
      while read x; do
        [[ $filesAreLocal -eq 0 ]] && echo "alien://$x" >> $partialLocalFileList
        [[ $filesAreLocal -eq 1 ]] && echo "$x" >> $partialLocalFileList
      done<$partialAlienFileList
      cleanup=0
    fi

    #handle syswatch
    if [[ -f syswatch_copy.log ]]
    then
      sed '1d' syswatch.log >> syswatch_copy.log
      rm -f syswatch.log
    else 
      [[ -f syswatch.log ]] && mv syswatch.log syswatch_copy.log
    fi

    echo waiting
    wait $!
    if [[ $runParallel -eq 1 ]]; then
      mergeByComponent $partialLocalFileList $cleanup 2>&1 | tee -a merge.log &
    else
      mergeByComponent $partialLocalFileList $cleanup 2>&1 | tee -a merge.log
    fi
  done

  #merge all the subfiles into one, wait for the last one to complete
  echo waiting
  wait $!
  if [[ "$components" =~ ALL && -f CalibObjects_ALL.root ]]; then
    mv -f CalibObjects_ALL.root CalibObjects.root
  else
    echo "***********************"
    echo merging ALL data
    echo "***********************"
    finalCalibObjectsList="finalObjects.list"
    ls -1 CalibObjects_*.root > $finalCalibObjectsList
    echo aliroot -b -q "mergeByComponent.C(\"ALL\", \"$finalCalibObjectsList\")" | tee -a merge.log
    aliroot -b -q "mergeByComponent.C(\"ALL\", \"$finalCalibObjectsList\")" 2>&1 | tee -a merge.log
    mv -f syswatch.log syswatch_merge_ALL.log
  fi

  if ! validateMerging "ALL"; then
    echo final merging not validatet, exiting...
    return 1
  fi
  rm -f CalibObjects_*.root

  #cleanup
  rm -f ${partialAlienFileListPrefix}*
  rm -f ${partialLocalFileListPrefix}*
  rm -f $alienFileList
  rm -f $localFileList

  # make OCDB
  echo "***********************" 2>&1 | tee -a ocdb.log
  echo making ${det} OCDB 2>&1 | tee -a ocdb.log
  echo "***********************" 2>&1 | tee -a ocdb.log
  echo aliroot -b -q "makeOCDB.C($run, \"$ocdb\", \"$defaultOCDB\")" 2>&1 | tee -a ocdb.log
  aliroot -b -q "makeOCDB.C($run, \"$ocdb\", \"$defaultOCDB\")" 2>&1 | tee -a ocdb.log
  mv syswatch.log syswatch_makeOCDB.log

  # summary
  echo "***********************" 2>&1 | tee -a ocdb.log
  echo SUMMARY 2>&1 | tee -a ocdb.log
  echo "***********************" 2>&1 | tee -a ocdb.log
  ls -altr CalibObjects.root *done 2>&1 | tee -a ocdb.log
}

mergeByComponent()
{
  # process by component
  # first argument is the file list to process

  # run inside a dedicated running directory
  # whic means copy the file list to process and prefic each line with ../
  # since the file names have no absolute paths!
  runningDirectory="${1}.dir"
  fileList="$1"
  cleanup=$2

  #sanity checks
  if [[ ! -f ${fileList} ]]; then
    echo "${fileList} does not exist"
    return 1
  fi
  mkdir -p $runningDirectory
  if [[ ! -d $runningDirectory ]]; then
    echo "cannot create the running directory $runningDirectory"
    return 1
  fi

  #move the to be merged files to the running directory and make a list of available files
  #handle the case of archives (x.zip#y.root) as well
  nFiles=0
  while read entry; do
    if [[ $entry =~ ^.*\.root$ ]]; then
      file=${entry%#*}
      fileContent=${entry##*#}
      [[ "${fileContent}" == "${file}" ]] && fileContent=""
      [[ -n ${fileContent} ]] && fileContent="#${fileContent}"
      if [[ -f ${file} ]]; then
        ((nFiles++))
        if [[ -f "./${file}" ]]; then
          echo "../${file}${fileContent}" >> "${runningDirectory}/${fileList}"
        else
          echo "${file}${fileContent}" >> "${runningDirectory}/${fileList}"
        fi
      fi
    fi
  done < ${fileList}
  if [[ $nFiles -lt 1 ]]; then
    echo "no new files in ${fileList}"
    echo rm -rf $runningDirectory 
    rm -rf $runningDirectory 
    return 1
  fi

  #copy the macro to the running directory
  [[ -f mergeByComponent.C ]] && cp mergeByComponent.C $runningDirectory

  #go to running directory
  cd $runningDirectory

  numberOfChunksTPC=$(cat ../$filesProcessedTPClist 2>/dev/null | wc -l )

  for det in $components; do

    # merge
    echo "***********************" 
    echo merging ${det} data
    echo "***********************"

    #limit the number of chunks processed by TPC
    if [[ "${det}" =~ TPC && $numberOfChunksTPC -ge $maxChunksTPC ]]; then
      echo "Not merging TPC anymore, max number of chunks processed ($maxChunksTPC)"
      continue
    fi

    #add the results of previous iteration to the BEGINNING of the list of files to be merged
    [[ -f ../CalibObjects_${det}.root ]] && echo "../CalibObjects_${det}.root" > ${fileList}_${det}
    cat ${fileList} >> ${fileList}_${det}

    echo "processing following files from ${fileList}_${det}:"
    cat ${fileList}_${det}

    echo aliroot -b -q "mergeByComponent.C(\"${det}\", \"${fileList}_${det}\")"
    aliroot -b -q "mergeByComponent.C(\"${det}\", \"${fileList}_${det}\")" 2>&1 | tee -a merge_${det}.log
    if validateMerging ${det}; then
      echo "### merge OK: mv CalibObjects.root ../CalibObjects_${det}.root"
      mv -f CalibObjects.root ../CalibObjects_${det}.root
      [[ "${det}" =~ TPCCalib ]] && cat ${fileList} >> ../$filesProcessedTPClist
    else 
      echo "### merging not validated"
      rm -f CalibObjects.root
    fi
    mv syswatch.log syswatch_merge_${det}.log

  done


  #move stuff back to the parent dir and clean up
  #merge the syswatch logs
  for x in syswatch*log; do
    [[ ! -f $x ]] && continue
    if [[ -f ../$x ]]
    then 
      echo "sed '1d' $x  >> ../$x"
      sed '1d' $x >> ../$x
      rm -f $x
    else 
      echo mv -f $x ..
      mv -f $x ..
    fi
  done

  #merge the other logs
  for x in *.log; do
    [[ ! -f $x ]] && continue
    if [[ -f ../$x ]]
    then
      echo "cat $x >> ../$x"
      cat $x >> ../$x
    else
      echo "mv -f $x .."
      mv -f $x ..
    fi
  done

  #final cleanup.
  #go to parent dir first to use the original fileList
  #without ../CalibObjects.root
  cd ..
  if [[ $cleanup -eq 1 ]]; then
    echo "cleaning up merged files..."
    while read file; do
      echo rm -rf $file
      rm -rf $file
    done<$fileList
  fi
  rm -rf $runningDirectory

  echo "***mergeByComponent() DONE"
  echo
  echo "numberOfChunksTPC=$numberOfChunksTPC"
  return 0
}

parseConfig()
{
  args=("$@")
  for opt in "${args[@]}"; do
    [[ -z ${opt} ]] && continue
    [[ -n ${encodedSpaces} ]] && opt="$(decSpaces ${opt})"
    [[ "${opt}" =~ ^[[:space:]]*$ ]] && continue
    if [[ ! "${opt}" =~ .*=.* ]]; then
      echo "badly formatted option \"${opt}\" should be: option=value, stopping..."
      return 1
    fi
    local var="${opt%%=*}"
    local value="${opt#*=}"
    export ${var}="${value}"
  done

  return 0
}

waitIfLocked()
{
  while [ 1 ]; do
    [[ ! -f $1 ]] && break
    sleep 1
  done
  return 0
}

validateMerging()
{
  det=$1
  retCode=0
  [[ ! -f CalibObjects.root ]] && ((retCode++)) && echo "### no CalibObjects.root..."
  [[ ! -f ${det}_merge_done ]] && ((retCode++)) && echo "### no ${det}_merge_done, job finished abnormally..."
  error=$(grep -e "was a crash" *.log)
  [[ -n $error ]] && ((retCode++)) && echo "### error: $error"

  return $retCode
}

copyScripts()
{
  [[ ! -f mergeByComponent.C ]] && \
    cp -f $ALICE_ROOT/PWGPP/CalibMacros/CPass1/mergeByComponent.C $PWD && \
    echo "taking the default scripts from $ALICE_ROOT"
  [[ ! -f makeOCDB.C ]] && \
    cp -f $ALICE_ROOT/PWGPP/CalibMacros/CPass1/makeOCDB.C $PWD && \
    echo "taking the default scripts from $ALICE_ROOT"
}

#these functions encode strings to and from a space-less form
#use when spaces are not well handled (e.g. in arguments to
#commands in makeflow files, etc.
encSpaces()(echo "${1// /@@@@}")
decSpaces()(echo "${1//@@@@/ }")

main "$@"
Commit	Line	Data
e9129651	1	#!/bin/bash
e9129651	2	# example local use (files will be downloaded from alien)
0959f55a	3	#./mergeMakeOCDB.byComponent.sh /alice/data/2012/LHC12g/000188362/cpass0/ 188362 local://./OCDB defaultOCDB=local:///cvmfs/alice.gsi.de/alice/data/2011/OCDB/ fileAccessMethod=tfilecp numberOfFilesInAbunch=30
e9129651	4	#
e9129651	5	# on ALIEN just do:
9e160db9	6	# $1 = directory where to perform the find
	7	# $2 = runNumber
	8	# $3 = OCDB path
	9
421603c5	10	main()
421603c5	11	{
0959f55a	12	save_args=("$@")
421603c5	13	if [[ $# -eq 0 ]]; then
	14	echo arguments:
	15	echo " 1 - directory on which to look for the files to be merged or local file list"
	16	echo " 2 - runNumber"
	17	echo " 3 - OCDB output path"
	18	echo
	19	echo " optionally set any of these"
	20	echo " runParallel={0,1} (1 to run parallel merging and downloading)"
	21	echo " defaultOCDB=raw:// (or any other valid OCDB storage as input database)"
	22	echo " fileAccessMethod={alien_cp,tfilecp,nocopy} (by default tfilecp)"
	23	echo " \"alien_cp\" will use alien_cp to copy files from alien"
	24	echo " \"tfilecp\" will copy the files first using TFile::Cp()"
	25	echo " \"nocopy\" will access files over the network directly TFile::Open()"
	26	echo " numberOfFilesInAbunch=50 (how many files get downloaded in one go)"
	27	echo " maxChunksTPC=3000 (max number of chunks to be merged for TPC calibration)"
	28
	29	echo
	30	echo "example:"
	31	echo " ./mergeMakeOCDB.byComponent.sh /alice/data/2012/LHC12g/000188362/cpass1/ 188362 local://./OCDB runParallel=0 defaultOCDB=raw:// fileAccessMethod=alien_cp"
	32	echo
	33
	34	return 0
	35	fi
	36
	37	# init
	38	path=$1
	39	run=$(echo "$2" \| sed 's/^0*//')
	40	ocdb=$3
	41
	42	shift 3
	43
	44	#default values
e52ef734	45	runParallel=0
421603c5	46	defaultOCDB="raw://"
	47	fileAccessMethod="tfilecp"
	48	numberOfFilesInAbunch=50
	49	maxChunksTPC=3000
	50	[[ -n ${ALIEN_JDL_TTL} ]] && maxTimeToLive=$(( ${ALIEN_JDL_TTL}-2000 ))
	51
	52	parseConfig "$@"
	53
	54	#some sanity checks of option values
	55	[[ ! $numberOfFilesInAbunch =~ ^[0-9]+$ ]] && numberOfFilesInAbunch=50
	56	[[ "$fileAccessMethod" != "alien_cp" && "$fileAccessMethod" != "tfilecp" && "$fileAccessMethod" != "nocopy" ]] && fileAccessMethod="tfilecp" && echo "using default fileAccessMethod=tfilecp"
	57
	58	filesAreLocal=0
	59	[[ -f $path ]] && filesAreLocal=1
	60	cleanup=1
	61	[[ $filesAreLocal -eq 1 ]] && cleanup=0
	62	[[ $filesAreLocal -eq 1 ]] && fileAccessMethod="nocopy"
	63
	64	# setup components to be merged
	65	components="TOF MeanVertex T0 SDD TRD TPCCalib TPCCluster TPCAlign"
	66	#components="TOF MeanVertex T0 SDD TRD TPCCalib"
	67
	68	#################################################################
	69	echo "" \| tee -a merge.log
0959f55a	70	echo "$0 ${save_args[*]}" \| tee -a merge.log
421603c5	71	echo "" \| tee -a merge.log
	72	echo "***********************" \| tee -a merge.log
	73	echo mergeMakeOCDB.byComponent.sh started \| tee -a merge.log
	74	echo path = $path \| tee -a merge.log
	75	echo run = $run \| tee -a merge.log
	76	echo ocdb = $ocdb \| tee -a merge.log
	77	echo defaultOCDB=$defaultOCDB \| tee -a merge.log
	78	echo filesAreLocal = $filesAreLocal \| tee -a merge.log
	79	echo cleanup = $cleanup \| tee -a merge.log
	80	echo fileAccessMethod=$fileAccessMethod \| tee -a merge.log
	81	echo numberOfFilesInAbunch = $numberOfFilesInAbunch \| tee -a merge.log
	82	echo runParallel=$runParallel
	83	echo "***********************" \| tee -a merge.log
	84
	85	alienFileList="alien.list"
	86	localFileList="local.list"
	87	filesProcessedTPClist="filesProcessedTPC.log"
	88	rm -f $filesProcessedTPClist
	89	touch $filesProcessedTPClist
	90	partialLocalFileListPrefix=${localFileList}_
	91	partialAlienFileListPrefix=${alienFileList}_
	92
	93	#first, make sure we have the scripts
	94	copyScripts \| tee -a copy.log
	95	ls
	96	#with alien files copy them first to local
	97	echo "***********************" 2>&1 \| tee -a copy.log
	98	echo copying files for run $run 2>&1 \| tee -a copy.log
	99	echo from $path 2>&1 \| tee -a copy.log
	100	echo "***********************" 2>&1 \| tee -a copy.log
	101	if [[ $filesAreLocal -eq 0 ]]; then
	102	if [[ "$fileAccessMethod" == "alien_cp" ]]; then
	103	echo "alien_find $path AliESDfriends_v1.root \| egrep ^/ > $alienFileList" 2>&1 \| tee -a copy.log
	104	alien_find $path "AliESDfriends_v1.root" \| egrep "^/" > $alienFileList
	105	echo "alien_find done"
	106	echo
	107	else
	108	echo aliroot -b -q "mergeByComponent.C(\"MAKEALIENLIST\",\"$alienFileList\", \"$path\", \"AliESDfriends_v1.root\")" 2>&1 \| tee -a copy.log
	109	aliroot -b -q "mergeByComponent.C(\"MAKEALIENLIST\",\"$alienFileList\", \"$path\", \"AliESDfriends_v1.root\")" 2>&1 \| tee -a copy.log
	110	echo "MAKEALIENLIST done"
	111	echo
	112	fi
	113	else
	114	cp $path $alienFileList
	115	fi
	116	#randomize the list
	117	#keep the first line intact (it is the largest file of the entire collection)
	118	sed -n '1p' ${alienFileList} > ${alienFileList}.tmp
	119	sed '1d' ${alienFileList} \| while read x; do echo "${RANDOM} ${x}"; done\|sort -n\|cut -d" " -f2- >> ${alienFileList}.tmp
	120	mv -f ${alienFileList}.tmp ${alienFileList}
	121
	122	#split into sublists, each to be processed separately
	123	echo split --numeric-suffixes --suffix-length=6 --lines=$numberOfFilesInAbunch ${alienFileList} ${partialAlienFileListPrefix} \| tee -a copy.log
	124	#split --numeric-suffixes --suffix-length=6 --lines=$numberOfFilesInAbunch ${alienFileList} ${partialAlienFileListPrefix}
	125	split -a 6 -l $numberOfFilesInAbunch ${alienFileList} ${partialAlienFileListPrefix}
	126
	127	for partialAlienFileList in ${partialAlienFileListPrefix}*
	128	do
	129
	130	#if it takes too long, break
	131	#[[ ${SECONDS} -gt ${maxTimeToLive} ]] && break
	132
	133	partialAlienFileListPostfix=${partialAlienFileList#$partialAlienFileListPrefix}
	134	partialLocalFileList=${partialLocalFileListPrefix}${partialAlienFileListPostfix}
135
136	#copy the files if appropriate and make a list of files to use
137	rm -f $partialLocalFileList
138	if [[ "$fileAccessMethod" == "alien_cp" ]]; then
139	while read x; do
140	[[ $x != /* ]] && continue
141	localName=${x//"/"/_}
142	echo "alien_cp "alien://$x" $localName" 2>&1 \| tee -a copy.log
143	alien_cp "alien://$x" $localName
144	echo $localName>>$partialLocalFileList
145	done<$partialAlienFileList
146	elif [[ "$fileAccessMethod" == "tfilecp" ]]; then
147	echo aliroot -b -q "mergeByComponent.C(\"COPY\",\"$partialAlienFileList\",\"noPath\",\"noPattern\",10,\"$partialLocalFileList\")" 2>&1 \| tee -a copy.log
148	aliroot -b -q "mergeByComponent.C(\"COPY\",\"$partialAlienFileList\",\"noPath\",\"noPattern\",10,\"$partialLocalFileList\")" 2>&1 \| tee -a copy.log
149	elif [[ "$fileAccessMethod" == "nocopy" ]]; then
150	while read x; do
151	[[ $filesAreLocal -eq 0 ]] && echo "alien://$x" >> $partialLocalFileList
152	[[ $filesAreLocal -eq 1 ]] && echo "$x" >> $partialLocalFileList
153	done<$partialAlienFileList
154	cleanup=0
155	fi
156
157	#handle syswatch
158	if [[ -f syswatch_copy.log ]]
159	then
160	sed '1d' syswatch.log >> syswatch_copy.log
161	rm -f syswatch.log
162	else
163	[[ -f syswatch.log ]] && mv syswatch.log syswatch_copy.log
164	fi
e9129651	165
421603c5	166	echo waiting
	167	wait $!
	168	if [[ $runParallel -eq 1 ]]; then
	169	mergeByComponent $partialLocalFileList $cleanup 2>&1 \| tee -a merge.log &
	170	else
	171	mergeByComponent $partialLocalFileList $cleanup 2>&1 \| tee -a merge.log
	172	fi
	173	done
	174
	175	#merge all the subfiles into one, wait for the last one to complete
	176	echo waiting
	177	wait $!
	178	if [[ "$components" =~ ALL && -f CalibObjects_ALL.root ]]; then
	179	mv -f CalibObjects_ALL.root CalibObjects.root
	180	else
	181	echo "***********************"
	182	echo merging ALL data
	183	echo "***********************"
	184	finalCalibObjectsList="finalObjects.list"
	185	ls -1 CalibObjects_*.root > $finalCalibObjectsList
	186	echo aliroot -b -q "mergeByComponent.C(\"ALL\", \"$finalCalibObjectsList\")" \| tee -a merge.log
	187	aliroot -b -q "mergeByComponent.C(\"ALL\", \"$finalCalibObjectsList\")" 2>&1 \| tee -a merge.log
	188	mv -f syswatch.log syswatch_merge_ALL.log
	189	fi
	190
	191	if ! validateMerging "ALL"; then
	192	echo final merging not validatet, exiting...
	193	return 1
	194	fi
	195	rm -f CalibObjects_*.root
	196
	197	#cleanup
	198	rm -f ${partialAlienFileListPrefix}*
	199	rm -f ${partialLocalFileListPrefix}*
	200	rm -f $alienFileList
	201	rm -f $localFileList
	202
	203	# make OCDB
	204	echo "***********************" 2>&1 \| tee -a ocdb.log
	205	echo making ${det} OCDB 2>&1 \| tee -a ocdb.log
	206	echo "***********************" 2>&1 \| tee -a ocdb.log
	207	echo aliroot -b -q "makeOCDB.C($run, \"$ocdb\", \"$defaultOCDB\")" 2>&1 \| tee -a ocdb.log
	208	aliroot -b -q "makeOCDB.C($run, \"$ocdb\", \"$defaultOCDB\")" 2>&1 \| tee -a ocdb.log
	209	mv syswatch.log syswatch_makeOCDB.log
	210
	211	# summary
	212	echo "***********************" 2>&1 \| tee -a ocdb.log
	213	echo SUMMARY 2>&1 \| tee -a ocdb.log
	214	echo "***********************" 2>&1 \| tee -a ocdb.log
	215	ls -altr CalibObjects.root *done 2>&1 \| tee -a ocdb.log
	216	}
9e160db9	217
	218	mergeByComponent()
	219	{
	220	# process by component
	221	# first argument is the file list to process
9e160db9	222
40f73007	223	# run inside a dedicated running directory
	224	# whic means copy the file list to process and prefic each line with ../
	225	# since the file names have no absolute paths!
421603c5	226	runningDirectory="${1}.dir"
e9129651	227	fileList="$1"
e9129651	228	cleanup=$2
421603c5	229
e9129651	230	#sanity checks
695f9fb2	231	if [[ ! -f ${fileList} ]]; then
695f9fb2	232	echo "${fileList} does not exist"
695f9fb2	233	return 1
695f9fb2	234	fi
40f73007	235	mkdir -p $runningDirectory
695f9fb2	236	if [[ ! -d $runningDirectory ]]; then
695f9fb2	237	echo "cannot create the running directory $runningDirectory"
695f9fb2	238	return 1
9e160db9	239	fi
421603c5	240
e9129651	241	#move the to be merged files to the running directory and make a list of available files
	242	#handle the case of archives (x.zip#y.root) as well
	243	nFiles=0
	244	while read entry; do
	245	if [[ $entry =~ ^.*\.root$ ]]; then
	246	file=${entry%#*}
	247	fileContent=${entry##*#}
	248	[[ "${fileContent}" == "${file}" ]] && fileContent=""
	249	[[ -n ${fileContent} ]] && fileContent="#${fileContent}"
	250	if [[ -f ${file} ]]; then
	251	((nFiles++))
	252	if [[ -f "./${file}" ]]; then
695f9fb2	253	echo "../${file}${fileContent}" >> "${runningDirectory}/${fileList}"
e9129651	254	else
695f9fb2	255	echo "${file}${fileContent}" >> "${runningDirectory}/${fileList}"
e9129651	256	fi
	257	fi
	258	fi
695f9fb2	259	done < ${fileList}
	260	if [[ $nFiles -lt 1 ]]; then
	261	echo "no new files in ${fileList}"
	262	echo rm -rf $runningDirectory
	263	rm -rf $runningDirectory
695f9fb2	264	return 1
695f9fb2	265	fi
e9129651	266
	267	#copy the macro to the running directory
	268	[[ -f mergeByComponent.C ]] && cp mergeByComponent.C $runningDirectory
	269
695f9fb2	270	#go to running directory
e9129651	271	cd $runningDirectory
421603c5	272
695f9fb2	273	numberOfChunksTPC=$(cat ../$filesProcessedTPClist 2>/dev/null \| wc -l )
807998f3	274
9e160db9	275	for det in $components; do
421603c5	276
807998f3	277	# merge
e9129651	278	echo "***********************"
695f9fb2	279	echo merging ${det} data
e9129651	280	echo "***********************"
695f9fb2	281
	282	#limit the number of chunks processed by TPC
	283	if [[ "${det}" =~ TPC && $numberOfChunksTPC -ge $maxChunksTPC ]]; then
	284	echo "Not merging TPC anymore, max number of chunks processed ($maxChunksTPC)"
	285	continue
	286	fi
	287
	288	#add the results of previous iteration to the BEGINNING of the list of files to be merged
	289	[[ -f ../CalibObjects_${det}.root ]] && echo "../CalibObjects_${det}.root" > ${fileList}_${det}
	290	cat ${fileList} >> ${fileList}_${det}
	291
	292	echo "processing following files from ${fileList}_${det}:"
	293	cat ${fileList}_${det}
	294
421603c5	295	echo aliroot -b -q "mergeByComponent.C(\"${det}\", \"${fileList}_${det}\")"
421603c5	296	aliroot -b -q "mergeByComponent.C(\"${det}\", \"${fileList}_${det}\")" 2>&1 \| tee -a merge_${det}.log
e9129651	297	if validateMerging ${det}; then
695f9fb2	298	echo "### merge OK: mv CalibObjects.root ../CalibObjects_${det}.root"
	299	mv -f CalibObjects.root ../CalibObjects_${det}.root
	300	[[ "${det}" =~ TPCCalib ]] && cat ${fileList} >> ../$filesProcessedTPClist
e9129651	301	else
695f9fb2	302	echo "### merging not validated"
e9129651	303	rm -f CalibObjects.root
e9129651	304	fi
695f9fb2	305	mv syswatch.log syswatch_merge_${det}.log
695f9fb2	306
9e160db9	307	done
40f73007	308
421603c5	309
40f73007	310	#move stuff back to the parent dir and clean up
	311	#merge the syswatch logs
	312	for x in syswatch*log; do
e9129651	313	[[ ! -f $x ]] && continue
e9129651	314	if [[ -f ../$x ]]
40f73007	315	then
e9129651	316	echo "sed '1d' $x >> ../$x"
e9129651	317	sed '1d' $x >> ../$x
40f73007	318	rm -f $x
40f73007	319	else
e9129651	320	echo mv -f $x ..
	321	mv -f $x ..
	322	fi
	323	done
	324
	325	#merge the other logs
	326	for x in *.log; do
	327	[[ ! -f $x ]] && continue
	328	if [[ -f ../$x ]]
	329	then
	330	echo "cat $x >> ../$x"
	331	cat $x >> ../$x
	332	else
	333	echo "mv -f $x .."
	334	mv -f $x ..
40f73007	335	fi
40f73007	336	done
421603c5	337
e9129651	338	#final cleanup.
	339	#go to parent dir first to use the original fileList
	340	#without ../CalibObjects.root
	341	cd ..
	342	if [[ $cleanup -eq 1 ]]; then
	343	echo "cleaning up merged files..."
	344	while read file; do
	345	echo rm -rf $file
	346	rm -rf $file
	347	done<$fileList
	348	fi
40f73007	349	rm -rf $runningDirectory
807998f3	350
e9129651	351	echo "***mergeByComponent() DONE"
e9129651	352	echo
695f9fb2	353	echo "numberOfChunksTPC=$numberOfChunksTPC"
e9129651	354	return 0
9e160db9	355	}
807998f3	356
421603c5	357	parseConfig()
	358	{
	359	args=("$@")
	360	for opt in "${args[@]}"; do
	361	[[ -z ${opt} ]] && continue
	362	[[ -n ${encodedSpaces} ]] && opt="$(decSpaces ${opt})"
	363	[[ "${opt}" =~ ^[[:space:]]*$ ]] && continue
	364	if [[ ! "${opt}" =~ .=. ]]; then
	365	echo "badly formatted option \"${opt}\" should be: option=value, stopping..."
	366	return 1
	367	fi
	368	local var="${opt%%=*}"
	369	local value="${opt#*=}"
	370	export ${var}="${value}"
	371	done
	372
	373	return 0
	374	}
	375
9e160db9	376	waitIfLocked()
	377	{
	378	while [ 1 ]; do
	379	[[ ! -f $1 ]] && break
	380	sleep 1
	381	done
e9129651	382	return 0
9e160db9	383	}
807998f3	384
e9129651	385	validateMerging()
	386	{
	387	det=$1
	388	retCode=0
	389	[[ ! -f CalibObjects.root ]] && ((retCode++)) && echo "### no CalibObjects.root..."
	390	[[ ! -f ${det}_merge_done ]] && ((retCode++)) && echo "### no ${det}_merge_done, job finished abnormally..."
	391	error=$(grep -e "was a crash" *.log)
	392	[[ -n $error ]] && ((retCode++)) && echo "### error: $error"
421603c5	393
e9129651	394	return $retCode
	395	}
	396
	397	copyScripts()
	398	{
	399	[[ ! -f mergeByComponent.C ]] && \
421603c5	400	cp -f $ALICE_ROOT/PWGPP/CalibMacros/CPass1/mergeByComponent.C $PWD && \
e9129651	401	echo "taking the default scripts from $ALICE_ROOT"
e9129651	402	[[ ! -f makeOCDB.C ]] && \
421603c5	403	cp -f $ALICE_ROOT/PWGPP/CalibMacros/CPass1/makeOCDB.C $PWD && \
e9129651	404	echo "taking the default scripts from $ALICE_ROOT"
e9129651	405	}
9e160db9	406
0959f55a	407	#these functions encode strings to and from a space-less form
	408	#use when spaces are not well handled (e.g. in arguments to
	409	#commands in makeflow files, etc.
	410	encSpaces()(echo "${1// /@@@@}")
	411	decSpaces()(echo "${1//@@@@/ }")
	412
421603c5	413	main "$@"