[u/mrichter/AliRoot.git] / PWGPP / CalibMacros / CPass1 / mergeMakeOCDB.byComponent.sh

#!/bin/bash
# example local use (files will be downloaded from alien)
#./mergeMakeOCDB.byComponent.sh /alice/data/2012/LHC12g/000188362/cpass0/ 188362 local://./OCDB local:///cvmfs/alice.gsi.de/alice/data/2011/OCDB/ 0 30
#
# on ALIEN just do:
# $1 = directory where to perform the find 
# $2 = runNumber
# $3 = OCDB path

main()
{
  if [[ $# -eq 0 ]]; then
    echo arguments:
    echo  "  1 - directory on which to look for the files to be merged or local file list"
    echo  "  2 - runNumber"
    echo  "  3 - OCDB output path"
    echo
    echo  "  optionally set any of these"
    echo  "  runParallel={0,1}   (1 to run parallel merging and downloading)"
    echo  "  defaultOCDB=raw://  (or any other valid OCDB storage as input database)"
    echo  "  fileAccessMethod={alien_cp,tfilecp,nocopy}   (by default tfilecp)"
    echo  "                    \"alien_cp\" will use alien_cp to copy files from alien"
    echo  "                    \"tfilecp\" will copy the files first using TFile::Cp()"
    echo  "                    \"nocopy\" will access files over the network directly TFile::Open()"
    echo  "  numberOfFilesInAbunch=50                     (how many files get downloaded in one go)"
    echo  "  maxChunksTPC=3000    (max number of chunks to be merged for TPC calibration)"

    echo
    echo "example:"
    echo  " ./mergeMakeOCDB.byComponent.sh /alice/data/2012/LHC12g/000188362/cpass1/ 188362 local://./OCDB runParallel=0 defaultOCDB=raw:// fileAccessMethod=alien_cp"
    echo

    return 0
  fi

  # init
  path=$1
  run=$(echo "$2" | sed 's/^0*//')
  ocdb=$3

  shift 3

  #default values
  runParallel=0
  defaultOCDB="raw://"
  fileAccessMethod="tfilecp"
  numberOfFilesInAbunch=50
  maxChunksTPC=3000
  [[ -n ${ALIEN_JDL_TTL} ]] && maxTimeToLive=$(( ${ALIEN_JDL_TTL}-2000 ))

  parseConfig "$@"

  #some sanity checks of option values
  [[ ! $numberOfFilesInAbunch =~ ^[0-9]+$ ]] && numberOfFilesInAbunch=50
  [[ "$fileAccessMethod" != "alien_cp" && "$fileAccessMethod" != "tfilecp" && "$fileAccessMethod" != "nocopy" ]] && fileAccessMethod="tfilecp" && echo "using default fileAccessMethod=tfilecp"

  filesAreLocal=0
  [[ -f $path ]] && filesAreLocal=1
  cleanup=1
  [[ $filesAreLocal -eq 1 ]] && cleanup=0
  [[ $filesAreLocal -eq 1 ]] && fileAccessMethod="nocopy"

  # setup components to be merged
  components="TOF MeanVertex T0 SDD TRD TPCCalib TPCCluster TPCAlign"
  #components="TOF MeanVertex T0 SDD TRD TPCCalib"

  #################################################################
  echo "" | tee -a merge.log
  echo $0" $*" | tee -a merge.log
  echo "" | tee -a merge.log
  echo "***********************" | tee -a merge.log
  echo mergeMakeOCDB.byComponent.sh started | tee -a merge.log
  echo path = $path | tee -a merge.log
  echo run  = $run | tee -a merge.log
  echo ocdb = $ocdb | tee -a merge.log
  echo defaultOCDB=$defaultOCDB | tee -a merge.log
  echo filesAreLocal = $filesAreLocal | tee -a merge.log
  echo cleanup = $cleanup | tee -a merge.log
  echo fileAccessMethod=$fileAccessMethod | tee -a merge.log
  echo numberOfFilesInAbunch = $numberOfFilesInAbunch | tee -a merge.log
  echo runParallel=$runParallel
  echo "***********************" | tee -a merge.log

  alienFileList="alien.list"
  localFileList="local.list"
  filesProcessedTPClist="filesProcessedTPC.log"
  rm -f $filesProcessedTPClist
  touch $filesProcessedTPClist
  partialLocalFileListPrefix=${localFileList}_
  partialAlienFileListPrefix=${alienFileList}_

  #first, make sure we have the scripts
  copyScripts | tee -a copy.log
  ls
  #with alien files copy them first to local
  echo "***********************" 2>&1 | tee -a copy.log
  echo copying files for run $run 2>&1 | tee -a copy.log
  echo from $path 2>&1 | tee -a copy.log
  echo "***********************" 2>&1 | tee -a copy.log
  if [[ $filesAreLocal -eq 0 ]]; then
    if [[ "$fileAccessMethod" == "alien_cp" ]]; then
      echo "alien_find $path AliESDfriends_v1.root | egrep ^/ > $alienFileList" 2>&1 | tee -a copy.log
      alien_find $path "AliESDfriends_v1.root" | egrep "^/" >  $alienFileList
      echo "alien_find done"
      echo
    else 
      echo aliroot -b -q "mergeByComponent.C(\"MAKEALIENLIST\",\"$alienFileList\", \"$path\", \"AliESDfriends_v1.root\")" 2>&1 | tee -a copy.log
      aliroot -b -q "mergeByComponent.C(\"MAKEALIENLIST\",\"$alienFileList\", \"$path\", \"AliESDfriends_v1.root\")" 2>&1 | tee -a copy.log
      echo "MAKEALIENLIST done"
      echo
    fi
  else
    cp $path $alienFileList
  fi
  #randomize the list
  #keep the first line intact (it is the largest file of the entire collection)
  sed -n '1p' ${alienFileList} > ${alienFileList}.tmp
  sed '1d' ${alienFileList} | while read x; do echo "${RANDOM} ${x}"; done|sort -n|cut -d" " -f2- >> ${alienFileList}.tmp
  mv -f ${alienFileList}.tmp ${alienFileList}

  #split into sublists, each to be processed separately
  echo split --numeric-suffixes --suffix-length=6 --lines=$numberOfFilesInAbunch ${alienFileList} ${partialAlienFileListPrefix} | tee -a copy.log
  #split --numeric-suffixes --suffix-length=6 --lines=$numberOfFilesInAbunch ${alienFileList} ${partialAlienFileListPrefix}
  split -a 6 -l $numberOfFilesInAbunch ${alienFileList} ${partialAlienFileListPrefix}

  for partialAlienFileList in ${partialAlienFileListPrefix}*
  do

    #if it takes too long, break
    #[[ ${SECONDS} -gt ${maxTimeToLive} ]] && break

    partialAlienFileListPostfix=${partialAlienFileList#$partialAlienFileListPrefix}
    partialLocalFileList=${partialLocalFileListPrefix}${partialAlienFileListPostfix}

    #copy the files if appropriate and make a list of files to use
    rm -f $partialLocalFileList
    if [[ "$fileAccessMethod" == "alien_cp" ]]; then
      while read x; do
        [[ $x != /* ]] && continue
        localName=${x//"/"/_}
        echo "alien_cp "alien://$x" $localName" 2>&1 | tee -a copy.log
        alien_cp "alien://$x" $localName
        echo $localName>>$partialLocalFileList
      done<$partialAlienFileList
    elif [[ "$fileAccessMethod" == "tfilecp" ]]; then
      echo aliroot -b -q "mergeByComponent.C(\"COPY\",\"$partialAlienFileList\",\"noPath\",\"noPattern\",10,\"$partialLocalFileList\")" 2>&1 | tee -a copy.log
      aliroot -b -q "mergeByComponent.C(\"COPY\",\"$partialAlienFileList\",\"noPath\",\"noPattern\",10,\"$partialLocalFileList\")" 2>&1 | tee -a copy.log
    elif [[ "$fileAccessMethod" == "nocopy" ]]; then
      while read x; do
        [[ $filesAreLocal -eq 0 ]] && echo "alien://$x" >> $partialLocalFileList
        [[ $filesAreLocal -eq 1 ]] && echo "$x" >> $partialLocalFileList
      done<$partialAlienFileList
      cleanup=0
    fi

    #handle syswatch
    if [[ -f syswatch_copy.log ]]
    then
      sed '1d' syswatch.log >> syswatch_copy.log
      rm -f syswatch.log
    else 
      [[ -f syswatch.log ]] && mv syswatch.log syswatch_copy.log
    fi

    echo waiting
    wait $!
    if [[ $runParallel -eq 1 ]]; then
      mergeByComponent $partialLocalFileList $cleanup 2>&1 | tee -a merge.log &
    else
      mergeByComponent $partialLocalFileList $cleanup 2>&1 | tee -a merge.log
    fi
  done

  #merge all the subfiles into one, wait for the last one to complete
  echo waiting
  wait $!
  if [[ "$components" =~ ALL && -f CalibObjects_ALL.root ]]; then
    mv -f CalibObjects_ALL.root CalibObjects.root
  else
    echo "***********************"
    echo merging ALL data
    echo "***********************"
    finalCalibObjectsList="finalObjects.list"
    ls -1 CalibObjects_*.root > $finalCalibObjectsList
    echo aliroot -b -q "mergeByComponent.C(\"ALL\", \"$finalCalibObjectsList\")" | tee -a merge.log
    aliroot -b -q "mergeByComponent.C(\"ALL\", \"$finalCalibObjectsList\")" 2>&1 | tee -a merge.log
    mv -f syswatch.log syswatch_merge_ALL.log
  fi

  if ! validateMerging "ALL"; then
    echo final merging not validatet, exiting...
    return 1
  fi
  rm -f CalibObjects_*.root

  #cleanup
  rm -f ${partialAlienFileListPrefix}*
  rm -f ${partialLocalFileListPrefix}*
  rm -f $alienFileList
  rm -f $localFileList

  # make OCDB
  echo "***********************" 2>&1 | tee -a ocdb.log
  echo making ${det} OCDB 2>&1 | tee -a ocdb.log
  echo "***********************" 2>&1 | tee -a ocdb.log
  echo aliroot -b -q "makeOCDB.C($run, \"$ocdb\", \"$defaultOCDB\")" 2>&1 | tee -a ocdb.log
  aliroot -b -q "makeOCDB.C($run, \"$ocdb\", \"$defaultOCDB\")" 2>&1 | tee -a ocdb.log
  mv syswatch.log syswatch_makeOCDB.log

  # summary
  echo "***********************" 2>&1 | tee -a ocdb.log
  echo SUMMARY 2>&1 | tee -a ocdb.log
  echo "***********************" 2>&1 | tee -a ocdb.log
  ls -altr CalibObjects.root *done 2>&1 | tee -a ocdb.log
}

mergeByComponent()
{
  # process by component
  # first argument is the file list to process

  # run inside a dedicated running directory
  # whic means copy the file list to process and prefic each line with ../
  # since the file names have no absolute paths!
  runningDirectory="${1}.dir"
  fileList="$1"
  cleanup=$2

  #sanity checks
  if [[ ! -f ${fileList} ]]; then
    echo "${fileList} does not exist"
    return 1
  fi
  mkdir -p $runningDirectory
  if [[ ! -d $runningDirectory ]]; then
    echo "cannot create the running directory $runningDirectory"
    return 1
  fi

  #move the to be merged files to the running directory and make a list of available files
  #handle the case of archives (x.zip#y.root) as well
  nFiles=0
  while read entry; do
    if [[ $entry =~ ^.*\.root$ ]]; then
      file=${entry%#*}
      fileContent=${entry##*#}
      [[ "${fileContent}" == "${file}" ]] && fileContent=""
      [[ -n ${fileContent} ]] && fileContent="#${fileContent}"
      if [[ -f ${file} ]]; then
        ((nFiles++))
        if [[ -f "./${file}" ]]; then
          echo "../${file}${fileContent}" >> "${runningDirectory}/${fileList}"
        else
          echo "${file}${fileContent}" >> "${runningDirectory}/${fileList}"
        fi
      fi
    fi
  done < ${fileList}
  if [[ $nFiles -lt 1 ]]; then
    echo "no new files in ${fileList}"
    echo rm -rf $runningDirectory 
    rm -rf $runningDirectory 
    return 1
  fi

  #copy the macro to the running directory
  [[ -f mergeByComponent.C ]] && cp mergeByComponent.C $runningDirectory

  #go to running directory
  cd $runningDirectory

  numberOfChunksTPC=$(cat ../$filesProcessedTPClist 2>/dev/null | wc -l )

  for det in $components; do

    # merge
    echo "***********************" 
    echo merging ${det} data
    echo "***********************"

    #limit the number of chunks processed by TPC
    if [[ "${det}" =~ TPC && $numberOfChunksTPC -ge $maxChunksTPC ]]; then
      echo "Not merging TPC anymore, max number of chunks processed ($maxChunksTPC)"
      continue
    fi

    #add the results of previous iteration to the BEGINNING of the list of files to be merged
    [[ -f ../CalibObjects_${det}.root ]] && echo "../CalibObjects_${det}.root" > ${fileList}_${det}
    cat ${fileList} >> ${fileList}_${det}

    echo "processing following files from ${fileList}_${det}:"
    cat ${fileList}_${det}

    echo aliroot -b -q "mergeByComponent.C(\"${det}\", \"${fileList}_${det}\")"
    aliroot -b -q "mergeByComponent.C(\"${det}\", \"${fileList}_${det}\")" 2>&1 | tee -a merge_${det}.log
    if validateMerging ${det}; then
      echo "### merge OK: mv CalibObjects.root ../CalibObjects_${det}.root"
      mv -f CalibObjects.root ../CalibObjects_${det}.root
      [[ "${det}" =~ TPCCalib ]] && cat ${fileList} >> ../$filesProcessedTPClist
    else 
      echo "### merging not validated"
      rm -f CalibObjects.root
    fi
    mv syswatch.log syswatch_merge_${det}.log

  done


  #move stuff back to the parent dir and clean up
  #merge the syswatch logs
  for x in syswatch*log; do
    [[ ! -f $x ]] && continue
    if [[ -f ../$x ]]
    then 
      echo "sed '1d' $x  >> ../$x"
      sed '1d' $x >> ../$x
      rm -f $x
    else 
      echo mv -f $x ..
      mv -f $x ..
    fi
  done

  #merge the other logs
  for x in *.log; do
    [[ ! -f $x ]] && continue
    if [[ -f ../$x ]]
    then
      echo "cat $x >> ../$x"
      cat $x >> ../$x
    else
      echo "mv -f $x .."
      mv -f $x ..
    fi
  done

  #final cleanup.
  #go to parent dir first to use the original fileList
  #without ../CalibObjects.root
  cd ..
  if [[ $cleanup -eq 1 ]]; then
    echo "cleaning up merged files..."
    while read file; do
      echo rm -rf $file
      rm -rf $file
    done<$fileList
  fi
  rm -rf $runningDirectory

  echo "***mergeByComponent() DONE"
  echo
  echo "numberOfChunksTPC=$numberOfChunksTPC"
  return 0
}

parseConfig()
{
  args=("$@")
  for opt in "${args[@]}"; do
    [[ -z ${opt} ]] && continue
    [[ -n ${encodedSpaces} ]] && opt="$(decSpaces ${opt})"
    [[ "${opt}" =~ ^[[:space:]]*$ ]] && continue
    if [[ ! "${opt}" =~ .*=.* ]]; then
      echo "badly formatted option \"${opt}\" should be: option=value, stopping..."
      return 1
    fi
    local var="${opt%%=*}"
    local value="${opt#*=}"
    export ${var}="${value}"
  done

  return 0
}

waitIfLocked()
{
  while [ 1 ]; do
    [[ ! -f $1 ]] && break
    sleep 1
  done
  return 0
}

validateMerging()
{
  det=$1
  retCode=0
  [[ ! -f CalibObjects.root ]] && ((retCode++)) && echo "### no CalibObjects.root..."
  [[ ! -f ${det}_merge_done ]] && ((retCode++)) && echo "### no ${det}_merge_done, job finished abnormally..."
  error=$(grep -e "was a crash" *.log)
  [[ -n $error ]] && ((retCode++)) && echo "### error: $error"

  return $retCode
}

copyScripts()
{
  [[ ! -f mergeByComponent.C ]] && \
    cp -f $ALICE_ROOT/PWGPP/CalibMacros/CPass1/mergeByComponent.C $PWD && \
    echo "taking the default scripts from $ALICE_ROOT"
  [[ ! -f makeOCDB.C ]] && \
    cp -f $ALICE_ROOT/PWGPP/CalibMacros/CPass1/makeOCDB.C $PWD && \
    echo "taking the default scripts from $ALICE_ROOT"
}

main "$@"
Commit	Line	Data
e9129651	1	#!/bin/bash
e9129651	2	# example local use (files will be downloaded from alien)
	3	#./mergeMakeOCDB.byComponent.sh /alice/data/2012/LHC12g/000188362/cpass0/ 188362 local://./OCDB local:///cvmfs/alice.gsi.de/alice/data/2011/OCDB/ 0 30
	4	#
	5	# on ALIEN just do:
9e160db9	6	# $1 = directory where to perform the find
	7	# $2 = runNumber
	8	# $3 = OCDB path
	9
421603c5	10	main()
	11	{
	12	if [[ $# -eq 0 ]]; then
	13	echo arguments:
	14	echo " 1 - directory on which to look for the files to be merged or local file list"
	15	echo " 2 - runNumber"
	16	echo " 3 - OCDB output path"
	17	echo
	18	echo " optionally set any of these"
	19	echo " runParallel={0,1} (1 to run parallel merging and downloading)"
	20	echo " defaultOCDB=raw:// (or any other valid OCDB storage as input database)"
	21	echo " fileAccessMethod={alien_cp,tfilecp,nocopy} (by default tfilecp)"
	22	echo " \"alien_cp\" will use alien_cp to copy files from alien"
	23	echo " \"tfilecp\" will copy the files first using TFile::Cp()"
	24	echo " \"nocopy\" will access files over the network directly TFile::Open()"
	25	echo " numberOfFilesInAbunch=50 (how many files get downloaded in one go)"
	26	echo " maxChunksTPC=3000 (max number of chunks to be merged for TPC calibration)"
	27
	28	echo
	29	echo "example:"
	30	echo " ./mergeMakeOCDB.byComponent.sh /alice/data/2012/LHC12g/000188362/cpass1/ 188362 local://./OCDB runParallel=0 defaultOCDB=raw:// fileAccessMethod=alien_cp"
	31	echo
	32
	33	return 0
	34	fi
	35
	36	# init
	37	path=$1
	38	run=$(echo "$2" \| sed 's/^0*//')
	39	ocdb=$3
	40
	41	shift 3
	42
	43	#default values
e52ef734	44	runParallel=0
421603c5	45	defaultOCDB="raw://"
	46	fileAccessMethod="tfilecp"
	47	numberOfFilesInAbunch=50
	48	maxChunksTPC=3000
	49	[[ -n ${ALIEN_JDL_TTL} ]] && maxTimeToLive=$(( ${ALIEN_JDL_TTL}-2000 ))
	50
	51	parseConfig "$@"
	52
	53	#some sanity checks of option values
	54	[[ ! $numberOfFilesInAbunch =~ ^[0-9]+$ ]] && numberOfFilesInAbunch=50
	55	[[ "$fileAccessMethod" != "alien_cp" && "$fileAccessMethod" != "tfilecp" && "$fileAccessMethod" != "nocopy" ]] && fileAccessMethod="tfilecp" && echo "using default fileAccessMethod=tfilecp"
	56
	57	filesAreLocal=0
	58	[[ -f $path ]] && filesAreLocal=1
	59	cleanup=1
	60	[[ $filesAreLocal -eq 1 ]] && cleanup=0
	61	[[ $filesAreLocal -eq 1 ]] && fileAccessMethod="nocopy"
	62
	63	# setup components to be merged
	64	components="TOF MeanVertex T0 SDD TRD TPCCalib TPCCluster TPCAlign"
	65	#components="TOF MeanVertex T0 SDD TRD TPCCalib"
	66
	67	#################################################################
	68	echo "" \| tee -a merge.log
	69	echo $0" $*" \| tee -a merge.log
	70	echo "" \| tee -a merge.log
	71	echo "***********************" \| tee -a merge.log
	72	echo mergeMakeOCDB.byComponent.sh started \| tee -a merge.log
	73	echo path = $path \| tee -a merge.log
	74	echo run = $run \| tee -a merge.log
	75	echo ocdb = $ocdb \| tee -a merge.log
	76	echo defaultOCDB=$defaultOCDB \| tee -a merge.log
	77	echo filesAreLocal = $filesAreLocal \| tee -a merge.log
	78	echo cleanup = $cleanup \| tee -a merge.log
	79	echo fileAccessMethod=$fileAccessMethod \| tee -a merge.log
	80	echo numberOfFilesInAbunch = $numberOfFilesInAbunch \| tee -a merge.log
	81	echo runParallel=$runParallel
	82	echo "***********************" \| tee -a merge.log
	83
	84	alienFileList="alien.list"
	85	localFileList="local.list"
	86	filesProcessedTPClist="filesProcessedTPC.log"
	87	rm -f $filesProcessedTPClist
	88	touch $filesProcessedTPClist
	89	partialLocalFileListPrefix=${localFileList}_
	90	partialAlienFileListPrefix=${alienFileList}_
	91
	92	#first, make sure we have the scripts
	93	copyScripts \| tee -a copy.log
	94	ls
	95	#with alien files copy them first to local
	96	echo "***********************" 2>&1 \| tee -a copy.log
	97	echo copying files for run $run 2>&1 \| tee -a copy.log
	98	echo from $path 2>&1 \| tee -a copy.log
	99	echo "***********************" 2>&1 \| tee -a copy.log
	100	if [[ $filesAreLocal -eq 0 ]]; then
	101	if [[ "$fileAccessMethod" == "alien_cp" ]]; then
	102	echo "alien_find $path AliESDfriends_v1.root \| egrep ^/ > $alienFileList" 2>&1 \| tee -a copy.log
	103	alien_find $path "AliESDfriends_v1.root" \| egrep "^/" > $alienFileList
	104	echo "alien_find done"
	105	echo
	106	else
	107	echo aliroot -b -q "mergeByComponent.C(\"MAKEALIENLIST\",\"$alienFileList\", \"$path\", \"AliESDfriends_v1.root\")" 2>&1 \| tee -a copy.log
	108	aliroot -b -q "mergeByComponent.C(\"MAKEALIENLIST\",\"$alienFileList\", \"$path\", \"AliESDfriends_v1.root\")" 2>&1 \| tee -a copy.log
109	echo "MAKEALIENLIST done"
110	echo
111	fi
112	else
113	cp $path $alienFileList
114	fi
115	#randomize the list
116	#keep the first line intact (it is the largest file of the entire collection)
117	sed -n '1p' ${alienFileList} > ${alienFileList}.tmp
118	sed '1d' ${alienFileList} \| while read x; do echo "${RANDOM} ${x}"; done\|sort -n\|cut -d" " -f2- >> ${alienFileList}.tmp
119	mv -f ${alienFileList}.tmp ${alienFileList}
120
121	#split into sublists, each to be processed separately
122	echo split --numeric-suffixes --suffix-length=6 --lines=$numberOfFilesInAbunch ${alienFileList} ${partialAlienFileListPrefix} \| tee -a copy.log
123	#split --numeric-suffixes --suffix-length=6 --lines=$numberOfFilesInAbunch ${alienFileList} ${partialAlienFileListPrefix}
124	split -a 6 -l $numberOfFilesInAbunch ${alienFileList} ${partialAlienFileListPrefix}
125
126	for partialAlienFileList in ${partialAlienFileListPrefix}*
127	do
128
129	#if it takes too long, break
130	#[[ ${SECONDS} -gt ${maxTimeToLive} ]] && break
131
132	partialAlienFileListPostfix=${partialAlienFileList#$partialAlienFileListPrefix}
133	partialLocalFileList=${partialLocalFileListPrefix}${partialAlienFileListPostfix}
134
135	#copy the files if appropriate and make a list of files to use
136	rm -f $partialLocalFileList
137	if [[ "$fileAccessMethod" == "alien_cp" ]]; then
138	while read x; do
139	[[ $x != /* ]] && continue
140	localName=${x//"/"/_}
141	echo "alien_cp "alien://$x" $localName" 2>&1 \| tee -a copy.log
142	alien_cp "alien://$x" $localName
143	echo $localName>>$partialLocalFileList
144	done<$partialAlienFileList
145	elif [[ "$fileAccessMethod" == "tfilecp" ]]; then
146	echo aliroot -b -q "mergeByComponent.C(\"COPY\",\"$partialAlienFileList\",\"noPath\",\"noPattern\",10,\"$partialLocalFileList\")" 2>&1 \| tee -a copy.log
147	aliroot -b -q "mergeByComponent.C(\"COPY\",\"$partialAlienFileList\",\"noPath\",\"noPattern\",10,\"$partialLocalFileList\")" 2>&1 \| tee -a copy.log
148	elif [[ "$fileAccessMethod" == "nocopy" ]]; then
149	while read x; do
150	[[ $filesAreLocal -eq 0 ]] && echo "alien://$x" >> $partialLocalFileList
151	[[ $filesAreLocal -eq 1 ]] && echo "$x" >> $partialLocalFileList
152	done<$partialAlienFileList
153	cleanup=0
154	fi
155
156	#handle syswatch
157	if [[ -f syswatch_copy.log ]]
158	then
159	sed '1d' syswatch.log >> syswatch_copy.log
160	rm -f syswatch.log
161	else
162	[[ -f syswatch.log ]] && mv syswatch.log syswatch_copy.log
163	fi
e9129651	164
421603c5	165	echo waiting
	166	wait $!
	167	if [[ $runParallel -eq 1 ]]; then
	168	mergeByComponent $partialLocalFileList $cleanup 2>&1 \| tee -a merge.log &
	169	else
	170	mergeByComponent $partialLocalFileList $cleanup 2>&1 \| tee -a merge.log
	171	fi
	172	done
	173
	174	#merge all the subfiles into one, wait for the last one to complete
	175	echo waiting
	176	wait $!
	177	if [[ "$components" =~ ALL && -f CalibObjects_ALL.root ]]; then
	178	mv -f CalibObjects_ALL.root CalibObjects.root
	179	else
	180	echo "***********************"
	181	echo merging ALL data
	182	echo "***********************"
	183	finalCalibObjectsList="finalObjects.list"
	184	ls -1 CalibObjects_*.root > $finalCalibObjectsList
	185	echo aliroot -b -q "mergeByComponent.C(\"ALL\", \"$finalCalibObjectsList\")" \| tee -a merge.log
	186	aliroot -b -q "mergeByComponent.C(\"ALL\", \"$finalCalibObjectsList\")" 2>&1 \| tee -a merge.log
	187	mv -f syswatch.log syswatch_merge_ALL.log
	188	fi
	189
	190	if ! validateMerging "ALL"; then
	191	echo final merging not validatet, exiting...
	192	return 1
	193	fi
	194	rm -f CalibObjects_*.root
	195
	196	#cleanup
	197	rm -f ${partialAlienFileListPrefix}*
	198	rm -f ${partialLocalFileListPrefix}*
	199	rm -f $alienFileList
	200	rm -f $localFileList
	201
	202	# make OCDB
	203	echo "***********************" 2>&1 \| tee -a ocdb.log
	204	echo making ${det} OCDB 2>&1 \| tee -a ocdb.log
	205	echo "***********************" 2>&1 \| tee -a ocdb.log
	206	echo aliroot -b -q "makeOCDB.C($run, \"$ocdb\", \"$defaultOCDB\")" 2>&1 \| tee -a ocdb.log
	207	aliroot -b -q "makeOCDB.C($run, \"$ocdb\", \"$defaultOCDB\")" 2>&1 \| tee -a ocdb.log
	208	mv syswatch.log syswatch_makeOCDB.log
	209
	210	# summary
	211	echo "***********************" 2>&1 \| tee -a ocdb.log
	212	echo SUMMARY 2>&1 \| tee -a ocdb.log
	213	echo "***********************" 2>&1 \| tee -a ocdb.log
	214	ls -altr CalibObjects.root *done 2>&1 \| tee -a ocdb.log
	215	}
9e160db9	216
	217	mergeByComponent()
	218	{
	219	# process by component
	220	# first argument is the file list to process
9e160db9	221
40f73007	222	# run inside a dedicated running directory
	223	# whic means copy the file list to process and prefic each line with ../
	224	# since the file names have no absolute paths!
421603c5	225	runningDirectory="${1}.dir"
e9129651	226	fileList="$1"
e9129651	227	cleanup=$2
421603c5	228
e9129651	229	#sanity checks
695f9fb2	230	if [[ ! -f ${fileList} ]]; then
695f9fb2	231	echo "${fileList} does not exist"
695f9fb2	232	return 1
695f9fb2	233	fi
40f73007	234	mkdir -p $runningDirectory
695f9fb2	235	if [[ ! -d $runningDirectory ]]; then
695f9fb2	236	echo "cannot create the running directory $runningDirectory"
695f9fb2	237	return 1
9e160db9	238	fi
421603c5	239
e9129651	240	#move the to be merged files to the running directory and make a list of available files
	241	#handle the case of archives (x.zip#y.root) as well
	242	nFiles=0
	243	while read entry; do
	244	if [[ $entry =~ ^.*\.root$ ]]; then
	245	file=${entry%#*}
	246	fileContent=${entry##*#}
	247	[[ "${fileContent}" == "${file}" ]] && fileContent=""
	248	[[ -n ${fileContent} ]] && fileContent="#${fileContent}"
	249	if [[ -f ${file} ]]; then
	250	((nFiles++))
	251	if [[ -f "./${file}" ]]; then
695f9fb2	252	echo "../${file}${fileContent}" >> "${runningDirectory}/${fileList}"
e9129651	253	else
695f9fb2	254	echo "${file}${fileContent}" >> "${runningDirectory}/${fileList}"
e9129651	255	fi
	256	fi
	257	fi
695f9fb2	258	done < ${fileList}
	259	if [[ $nFiles -lt 1 ]]; then
	260	echo "no new files in ${fileList}"
	261	echo rm -rf $runningDirectory
	262	rm -rf $runningDirectory
695f9fb2	263	return 1
695f9fb2	264	fi
e9129651	265
	266	#copy the macro to the running directory
	267	[[ -f mergeByComponent.C ]] && cp mergeByComponent.C $runningDirectory
	268
695f9fb2	269	#go to running directory
e9129651	270	cd $runningDirectory
421603c5	271
695f9fb2	272	numberOfChunksTPC=$(cat ../$filesProcessedTPClist 2>/dev/null \| wc -l )
807998f3	273
9e160db9	274	for det in $components; do
421603c5	275
807998f3	276	# merge
e9129651	277	echo "***********************"
695f9fb2	278	echo merging ${det} data
e9129651	279	echo "***********************"
695f9fb2	280
	281	#limit the number of chunks processed by TPC
	282	if [[ "${det}" =~ TPC && $numberOfChunksTPC -ge $maxChunksTPC ]]; then
	283	echo "Not merging TPC anymore, max number of chunks processed ($maxChunksTPC)"
	284	continue
	285	fi
	286
	287	#add the results of previous iteration to the BEGINNING of the list of files to be merged
	288	[[ -f ../CalibObjects_${det}.root ]] && echo "../CalibObjects_${det}.root" > ${fileList}_${det}
	289	cat ${fileList} >> ${fileList}_${det}
	290
	291	echo "processing following files from ${fileList}_${det}:"
	292	cat ${fileList}_${det}
	293
421603c5	294	echo aliroot -b -q "mergeByComponent.C(\"${det}\", \"${fileList}_${det}\")"
421603c5	295	aliroot -b -q "mergeByComponent.C(\"${det}\", \"${fileList}_${det}\")" 2>&1 \| tee -a merge_${det}.log
e9129651	296	if validateMerging ${det}; then
695f9fb2	297	echo "### merge OK: mv CalibObjects.root ../CalibObjects_${det}.root"
	298	mv -f CalibObjects.root ../CalibObjects_${det}.root
	299	[[ "${det}" =~ TPCCalib ]] && cat ${fileList} >> ../$filesProcessedTPClist
e9129651	300	else
695f9fb2	301	echo "### merging not validated"
e9129651	302	rm -f CalibObjects.root
e9129651	303	fi
695f9fb2	304	mv syswatch.log syswatch_merge_${det}.log
695f9fb2	305
9e160db9	306	done
40f73007	307
421603c5	308
40f73007	309	#move stuff back to the parent dir and clean up
	310	#merge the syswatch logs
	311	for x in syswatch*log; do
e9129651	312	[[ ! -f $x ]] && continue
e9129651	313	if [[ -f ../$x ]]
40f73007	314	then
e9129651	315	echo "sed '1d' $x >> ../$x"
e9129651	316	sed '1d' $x >> ../$x
40f73007	317	rm -f $x
40f73007	318	else
e9129651	319	echo mv -f $x ..
	320	mv -f $x ..
	321	fi
	322	done
	323
	324	#merge the other logs
	325	for x in *.log; do
	326	[[ ! -f $x ]] && continue
	327	if [[ -f ../$x ]]
	328	then
	329	echo "cat $x >> ../$x"
	330	cat $x >> ../$x
	331	else
	332	echo "mv -f $x .."
	333	mv -f $x ..
40f73007	334	fi
40f73007	335	done
421603c5	336
e9129651	337	#final cleanup.
	338	#go to parent dir first to use the original fileList
	339	#without ../CalibObjects.root
	340	cd ..
	341	if [[ $cleanup -eq 1 ]]; then
	342	echo "cleaning up merged files..."
	343	while read file; do
	344	echo rm -rf $file
	345	rm -rf $file
	346	done<$fileList
	347	fi
40f73007	348	rm -rf $runningDirectory
807998f3	349
e9129651	350	echo "***mergeByComponent() DONE"
e9129651	351	echo
695f9fb2	352	echo "numberOfChunksTPC=$numberOfChunksTPC"
e9129651	353	return 0
9e160db9	354	}
807998f3	355
421603c5	356	parseConfig()
	357	{
	358	args=("$@")
	359	for opt in "${args[@]}"; do
	360	[[ -z ${opt} ]] && continue
	361	[[ -n ${encodedSpaces} ]] && opt="$(decSpaces ${opt})"
	362	[[ "${opt}" =~ ^[[:space:]]*$ ]] && continue
	363	if [[ ! "${opt}" =~ .=. ]]; then
	364	echo "badly formatted option \"${opt}\" should be: option=value, stopping..."
	365	return 1
	366	fi
	367	local var="${opt%%=*}"
	368	local value="${opt#*=}"
	369	export ${var}="${value}"
	370	done
	371
	372	return 0
	373	}
	374
9e160db9	375	waitIfLocked()
	376	{
	377	while [ 1 ]; do
	378	[[ ! -f $1 ]] && break
	379	sleep 1
	380	done
e9129651	381	return 0
9e160db9	382	}
807998f3	383
e9129651	384	validateMerging()
	385	{
	386	det=$1
	387	retCode=0
	388	[[ ! -f CalibObjects.root ]] && ((retCode++)) && echo "### no CalibObjects.root..."
	389	[[ ! -f ${det}_merge_done ]] && ((retCode++)) && echo "### no ${det}_merge_done, job finished abnormally..."
	390	error=$(grep -e "was a crash" *.log)
	391	[[ -n $error ]] && ((retCode++)) && echo "### error: $error"
421603c5	392
e9129651	393	return $retCode
	394	}
	395
	396	copyScripts()
	397	{
	398	[[ ! -f mergeByComponent.C ]] && \
421603c5	399	cp -f $ALICE_ROOT/PWGPP/CalibMacros/CPass1/mergeByComponent.C $PWD && \
e9129651	400	echo "taking the default scripts from $ALICE_ROOT"
e9129651	401	[[ ! -f makeOCDB.C ]] && \
421603c5	402	cp -f $ALICE_ROOT/PWGPP/CalibMacros/CPass1/makeOCDB.C $PWD && \
e9129651	403	echo "taking the default scripts from $ALICE_ROOT"
e9129651	404	}
9e160db9	405
421603c5	406	main "$@"