# (the local chache location and paths can be manipulated.)
# - needs a configured config file (by default alienSync.config)
# and a working alien environment (token and at least $ALIEN_DIR or $ALIEN_ROOT set)
+# - can be also used without a config file
+#
+# run the script without argument to see the examples
#
# origin: Mikolaj Krzewicki, mikolaj.krzewicki@cern.ch
#
+if [ ${BASH_VERSINFO} -lt 4 ]; then
+ echo "bash version >= 4 needed, you have ${BASH_VERSION}, exiting..."
+ exit 1
+fi
+
main()
{
if [[ $# -lt 1 ]]; then
- echo Usage: $0 configFile
+ echo "Usage: ${0##*/} configFile=/path/to/config"
+ echo "expert: ${0##*/} alienFindCommand=\"alien_find /some/path/ file\" [opt=value]"
+ echo " ${0##*/} alienFindCommand=\"alien_find /some/path/ file\" localPathPrefix=\${PWD}"
+ echo
+ echo "by default files are downloaded to current dir, or \${alienSync_localPathPrefix}, if set."
+ echo "At least specify alienFindCommand, either on command line or in the configFile."
+ echo "the logs go by default to localPathPrefix/alienSyncLogs"
return
fi
+
+ #be nice and allow group members access as well (002 will create dirs with 775 and files with 664)
+ umask 0002
# try to load the config file
- [[ ! -f $1 ]] && echo "config file $1 not found, exiting..." | tee -a $logFile && exit 1
- source $1
-
+ #[[ ! -f $1 ]] && echo "config file $1 not found, exiting..." | tee -a $logFile && exit 1
+ if ! parseConfig "$@"; then return 1; fi
+
+ [[ -z ${alienFindCommand} ]] && echo "alienFindCommand not defined!" && return 1
+
#if not set, use the default group
[[ -z ${alienSyncFilesGroupOwnership} ]] && alienSyncFilesGroupOwnership=$(id -gn)
# do some accounting
- [[ ! -d $logOutputPath ]] && echo "logOutputPath not available, creating..." && sg ${alienSyncFilesGroupOwnership} "mkdir -p $logOutputPath"
+ [[ ! -d $logOutputPath ]] && echo "logOutputPath not available, creating..." && mkdir -p $logOutputPath && chgrp ${alienSyncFilesGroupOwnership} ${logOutputPath}
[[ ! -d $logOutputPath ]] && echo "could not create log dir, exiting..." && exit 1
dateString=$(date +%Y-%m-%d-%H-%M)
logFile=$logOutputPath/alienSync-$dateString.log
echo ""|tee -a $logFile
echo log: $logFile
- #be nice and allow group members access as well (002 will create dirs with 775 and files with 664)
- umask 0002
-
#lock
lockFile=$logOutputPath/runningNow.lock
- [[ -f $lockFile ]] && echo "locked. Another process running? ($lockFile)" | tee -a $logFile && exit 1
+ [[ -f $lockFile && ${allowConcurrent} -ne 1 ]] && echo "locked. Another process running? ($lockFile)" | tee -a $logFile && exit 1
touch $lockFile
[[ ! -f $lockFile ]] && echo "unable to create lock. exiting..." | tee -a $logFile && exit 1
exec 6>&1
exec 1>$logFile 2>&1
fi
-
+
newFilesList=$logOutputPath/"newFiles.list"
rm -f $newFilesList
touch $newFilesList
rm -f $redoneFilesList
touch $redoneFilesList
updatedFilesList="${logOutputPath}/updatedFiles.list"
+ failedDownloadList="${logOutputPath}/failedDownload.list"
+ touch ${failedDownloadList}
+
# check the config
[[ -z $alienFindCommand ]] && echo "alienFindCommand not defined, exiting..." && exitScript 1
destinationdir=${destination%/*}
[[ -n $softLinkName ]] && softlinktodestination=${destinationdir}/${softLinkName}
tmpdestination="${destination}.aliensyncTMP"
+
+ #if we allow concurrent running (DANGEROUS) check if somebody is already trying to process this file
+ if [[ -f ${tmpdestination} && ${allowConcurrent} -eq 1 ]]; then
+ echo "$tmpdestination exists - concurrent donwload? skipping..."
+ continue
+ fi
if [[ -n ${destinationModifyCommand} ]]; then
#find the candidate in the database, in case there are more files trying to go to the same
originalEntryIndex=${candidateDBrecord[0]}
[[ $lineNumber -ne $originalEntryIndex ]] && continue
fi
-
+
redownloading=""
if [[ -f ${destination} ]]; then
#soft link the downloaded file (maybe to provide a consistent link to the latest version)
[[ "$md5local" =~ "." ]] && md5local=""
if [[ $forceLocalMD5recalculation -eq 1 || -z $md5local ]]; then
- tmparrayMD5=($(md5sum ${destination}))
- md5recalculated=${tmparrayMD5[0]}
+ md5recalculated=$(checkMD5sum ${destination})
[[ "$md5local" != "$md5recalculated" ]] && echo "WARNING: local copy change ${destination}"
md5local=${md5recalculated}
fi
[[ -f $tmpdestination ]] && echo "WARNING: stale $tmpdestination, removing" && rm $tmpdestination
- sg ${alienSyncFilesGroupOwnership} "mkdir -p ${destinationdir}"
+ mkdir -p ${destinationdir} && chgrp ${alienSyncFilesGroupOwnership} ${destinationdir}
[[ ! -d $destinationdir ]] && echo cannot access $destinationdir && continue
#check token
# $ALIEN_ROOT/api/bin/alien-token-init $alienUserName
# #source /tmp/gclient_env_$UID
#fi
-
+
export copyMethod
- export copyScript
export copyTimeout
export copyTimeoutHard
echo copyFromAlien "$alienFile" "$tmpdestination"
downloadOK=0
#verify the downloaded md5 if available, validate otherwise...
if [[ -n $md5alien ]]; then
- if (echo "$md5alien $tmpdestination"|md5sum -c --status -); then
+ md5recalculated=$(checkMD5sum ${tmpdestination})
+ if [[ ${md5alien} == ${md5recalculated} ]]; then
echo "OK md5 after download"
downloadOK=1
else
echo ${destination} >> $localFileList
fi
[[ -n ${postCommand} ]] && ( cd ${destinationdir}; eval "${postCommand}" )
+ if grep -q ${alienFile} ${failedDownloadList}; then
+ echo "removing ${alienFile} from ${failedDownloadList}"
+ grep -v ${alienFile} ${failedDownloadList} >tmpUpdatedFailed
+ mv tmpUpdatedFailed ${failedDownloadList}
+ fi
else
echo "download not validated, NOT moving to ${destination}..."
echo "removing $tmpdestination"
rm -f $tmpdestination
+ echo ${alienFile} >> ${failedDownloadList}
continue
fi
+ [[ -f $tmpdestination ]] && echo "WARNING: tmpdestination should not still be here! removing..." && rm -r ${tmpdestination}
+
if [[ $unzipFiles -eq 1 ]]; then
- echo unzip $tmpdestination -d $destinationdir
- unzip $tmpdestination -d $destinationdir
+ echo unzip -o ${destination} -d ${destinationdir}
+ unzip -o ${destination} -d ${destinationdir}
fi
echo
done < ${alienFileListCurrent}
[[ $alienFileCounter -gt 0 ]] && mv -f $alienFileListCurrent $localAlienDatabase
-
- echo ${0##*/} DONE
+
+ echo "${0##*/} DONE"
if [[ $allOutputToLog -eq 1 ]]; then
exec 1>&6 6>&-
fi
cat ${newFilesList} ${redoneFilesList} > ${updatedFilesList}
- eval "${executeEnd}"
-
+
echo alienFindCommand:
echo " $alienFindCommand"
echo
echo "redone files:"
echo
cat $redoneFilesList
+ echo
+ echo
+
+ #output the list of failed files to stdout, so the cronjob can mail it
+ echo '###############################'
+ echo "failed to download from alien:"
+ echo
+ local tmpfailed=$(mktemp)
+ [[ "$(cat ${failedDownloadList} | wc -l)" -gt 0 ]] && sort ${failedDownloadList} | uniq -c | awk 'BEGIN{print "#tries\t file" }{print $1 "\t " $2}' | tee ${tmpfailed}
+
+ [[ -n ${MAILTO} ]] && echo $logFile | mail -s "alienSync ${alienFindCommand} done" ${MAILTO}
- [[ -n $sendMailTo ]] && echo $logFile | mail -s "alienSync $alienFindCommand done" $sendMailTo
+ if [[ -n ${executeEnd} ]]; then
+ echo
+ echo
+ echo '###############################'
+ echo "eval ${executeEnd}"
+ eval "${executeEnd}"
+ fi
exitScript 0
}
alien_find()
{
# like a regular alien_find command
- # output is a list with md5sums and ctimes
+ # output is a list with md5 sums and ctimes
executable="$ALIEN_ROOT/api/bin/gbbox find"
[[ ! -x ${executable% *} ]] && echo "### error, no $executable..." && return 1
[[ -z $logOutputPath ]] && logOutputPath="./"
src="alien://${src}"
dst=$2
if [[ "$copyMethod" == "tfilecp" ]]; then
- echo timeout $copyTimeout root -b -q "$copyScript(\"$src\",\"$dst\")"
- timeout $copyTimeout root -b -q "$copyScript(\"$src\",\"$dst\")"
+ if which timeout &>/dev/null; then
+ echo timeout $copyTimeout "TFile::Cp(\"$src\",\"$dst\")"
+ timeout $copyTimeout root -b <<EOF
+TGrid::Connect("alien://");
+TFile::Cp("${src}","${dst}");
+EOF
+
+ else
+ echo "TFile::Cp(\"$src\",\"$dst\")"
+ root -b <<EOF
+TGrid::Connect("alien://");
+TFile::Cp("${src}","${dst}");
+EOF
+ fi
else
- echo timeout $copyTimeout $ALIEN_ROOT/api/bin/alien_cp $src $dst
- timeout $copyTimeout $ALIEN_ROOT/api/bin/alien_cp $src $dst
+ if which timeout &>/dev/null; then
+ echo timeout $copyTimeout $ALIEN_ROOT/api/bin/alien_cp $src $dst
+ timeout $copyTimeout $ALIEN_ROOT/api/bin/alien_cp $src $dst
+ else
+ echo $ALIEN_ROOT/api/bin/alien_cp $src $dst
+ $ALIEN_ROOT/api/bin/alien_cp $src $dst
+ fi
+ fi
+}
+
+parseConfig()
+{
+ #config file
+ configFile=""
+ alienFindCommand=""
+ secondsToSuicide=$(( 10*3600 ))
+ localPathPrefix="${PWD}"
+ #define alienSync_localPathPrefix in your env to have a default central location
+ [[ -n ${alienSync_localPathPrefix} ]] && localPathPrefix=${alienSync_localPathPrefix}
+ unzipFiles=0
+ allOutputToLog=0
+
+ args=("$@")
+
+ #first, check if the config file is configured
+ #is yes - source it so that other options can override it
+ #if any
+ for opt in "${args[@]}"; do
+ if [[ ${opt} =~ configFile=.* ]]; then
+ eval "${opt}"
+ [[ ! -f ${configFile} ]] && echo "configFile ${configFile} not found, exiting..." && return 1
+ echo "using config file: ${configFile}"
+ source "${configFile}"
+ break
+ fi
+ done
+
+ #then, parse the options as they override the options from file
+ for opt in "${args[@]}"; do
+ if [[ ! "${opt}" =~ .*=.* ]]; then
+ echo "badly formatted option ${var}, should be: option=value, stopping..."
+ return 1
+ fi
+ local var="${opt%%=*}"
+ local value="${opt#*=}"
+ echo "${var} = ${value}"
+ export ${var}="${value}"
+ done
+
+ #things that by default depend on other variables should be set here, after the dependencies
+ [[ -z ${logOutputPath} ]] && logOutputPath="${localPathPrefix}/alienSyncLogs"
+ return 0
+}
+
+checkMD5sum()
+{
+ local file="${1}"
+ local md5=""
+ [[ ! -f ${file} ]] && return 1
+ if which md5sum &>/dev/null; then
+ local tmp=($(md5sum ${file}))
+ md5=${tmp[0]}
+ elif which md5 &>/dev/null; then
+ local tmp=($(md5 ${file}))
+ md5=${tmp[3]}
fi
+ echo ${md5}
}
main "$@"
git://git.uio.no / u / mrichter / AliRoot.git / blobdiff