3 # Script to merge objects coming out of the calibration train and export of the OCDB:
4 # Arguments for local use:
5 # 1 - directory on which to look for the files to be merged
9 # 4 - input default OCDB, default="raw://"
10 # 5 - option: "alien_cp" will use alien_cp to copy files from alien
11 # "tfilecp" will copy the files first using TFile::Cp()
12 # "nocopy" will access files over the network directly TFile::Open()
13 # default is tfilecp, use 0
14 # 6 - number of files to download per iteration, default 50
15 # 7 - maximum number of chunks to be merged for TPC, default 3000"
17 # example local use (files will be downloaded from alien)
18 #./mergeMakeOCDB.byComponent.sh /alice/data/2012/LHC12g/000188362/cpass0/ 188362 local://./OCDB local:///cvmfs/alice.gsi.de/alice/data/2011/OCDB/ 0 30
21 # $1 = directory where to perform the find
25 if [[ $# -eq 0 ]]; then
27 echo " 1 - directory on which to look for the files to be merged or local file list"
29 echo " 3 - OCDB output path"
31 echo " 4 - input default OCDB, default=\"raw://\""
32 echo " 5 - option: \"alien_cp\" will use alien_cp to copy files from alien"
33 echo " \"tfilecp\" will copy the files first using TFile::Cp()"
34 echo " \"nocopy\" will access files over the network directly TFile::Open()"
35 echo " default is tfilecp (use 0)"
36 echo " 6 - number of files to download/merge per iteration, default 50"
37 echo " 7 - maximum number of chunks to be merged for TPC, default 3000"
40 echo " ./mergeMakeOCDB.byComponent.sh /alice/data/2012/LHC12g/000188362/cpass1/ 188362 local://./OCDB raw:// alien_cp 10"
48 run=$(echo "$2" | sed 's/^0*//')
51 [[ -n $4 ]] && defaultOCDB=$4
53 [[ "$fileAccessMethod" != "alien_cp" && "$fileAccessMethod" != "tfilecp" && "$fileAccessMethod" != "nocopy" ]] && fileAccessMethod="tfilecp"
54 numberOfFilesInAbunch=50
55 [[ $6 =~ ^[0-9]+$ ]] && numberOfFilesInAbunch=$6
58 [[ -f $path ]] && isLocal=1
60 [[ $isLocal -eq 1 ]] && cleanup=0
61 [[ $isLocal -eq 1 ]] && fileAccessMethod="nocopy"
64 [[ -n $7 ]] && maxChunksTPC=$7
67 [[ -n ${ALIEN_JDL_TTL} ]] && maxTimeToLive=$(( ${ALIEN_JDL_TTL}-2000 ))
69 # setup components to be merged
70 components="TOF MeanVertex T0 SDD TRD TPCCalib TPCCluster TPCAlign"
71 #components="TOF MeanVertex T0 SDD TRD TPCCalib"
73 #################################################################
74 echo "" | tee -a merge.log
75 echo $0" $*" | tee -a merge.log
76 echo "" | tee -a merge.log
77 echo "***********************" | tee -a merge.log
78 echo mergeMakeOCDB.byComponent.sh started | tee -a merge.log
79 echo path = $path | tee -a merge.log
80 echo run = $run | tee -a merge.log
81 echo ocdb = $ocdb | tee -a merge.log
82 echo defaultOCDB=$defaultOCDB | tee -a merge.log
83 echo isLocal = $isLocal | tee -a merge.log
84 echo cleanup = $cleanup | tee -a merge.log
85 echo fileAccessMethod=$fileAccessMethod | tee -a merge.log
86 echo numberOfFilesInAbunch = $numberOfFilesInAbunch | tee -a merge.log
87 echo "***********************" | tee -a merge.log
89 alienFileList="alien.list"
90 localFileList="local.list"
91 filesProcessedTPClist="filesProcessedTPC.log"
92 rm -f $filesProcessedTPClist
93 touch $filesProcessedTPClist
94 partialLocalFileListPrefix=${localFileList}_
95 partialAlienFileListPrefix=${alienFileList}_
96 runningMergeByComponentLockFile="runningMergeByComponent.lock"
100 # process by component
101 # first argument is the file list to process
104 touch $runningMergeByComponentLockFile
106 # run inside a dedicated running directory
107 # whic means copy the file list to process and prefic each line with ../
108 # since the file names have no absolute paths!
109 runningDirectory="${runningMergeByComponentLockFile}.${1}.dir"
114 if [[ ! -f ${fileList} ]]; then
115 echo "${fileList} does not exist"
116 rm -f $runningMergeByComponentLockFile
119 mkdir -p $runningDirectory
120 if [[ ! -d $runningDirectory ]]; then
121 echo "cannot create the running directory $runningDirectory"
122 echo removing lock $runningMergeByComponentLockFile
123 rm -f $runningMergeByComponentLockFile
127 #move the to be merged files to the running directory and make a list of available files
128 #handle the case of archives (x.zip#y.root) as well
131 if [[ $entry =~ ^.*\.root$ ]]; then
133 fileContent=${entry##*#}
134 [[ "${fileContent}" == "${file}" ]] && fileContent=""
135 [[ -n ${fileContent} ]] && fileContent="#${fileContent}"
136 if [[ -f ${file} ]]; then
138 if [[ -f "./${file}" ]]; then
139 echo "../${file}${fileContent}" >> "${runningDirectory}/${fileList}"
141 echo "${file}${fileContent}" >> "${runningDirectory}/${fileList}"
146 if [[ $nFiles -lt 1 ]]; then
147 echo "no new files in ${fileList}"
148 echo rm -rf $runningDirectory
149 rm -rf $runningDirectory
150 echo removing lock $runningMergeByComponentLockFile
151 rm -f $runningMergeByComponentLockFile
155 #copy the macro to the running directory
156 [[ -f mergeByComponent.C ]] && cp mergeByComponent.C $runningDirectory
158 #go to running directory
161 numberOfChunksTPC=$(cat ../$filesProcessedTPClist 2>/dev/null | wc -l )
163 for det in $components; do
166 echo "***********************"
167 echo merging ${det} data
168 echo "***********************"
170 #limit the number of chunks processed by TPC
171 if [[ "${det}" =~ TPC && $numberOfChunksTPC -ge $maxChunksTPC ]]; then
172 echo "Not merging TPC anymore, max number of chunks processed ($maxChunksTPC)"
176 #add the results of previous iteration to the BEGINNING of the list of files to be merged
177 [[ -f ../CalibObjects_${det}.root ]] && echo "../CalibObjects_${det}.root" > ${fileList}_${det}
178 cat ${fileList} >> ${fileList}_${det}
180 echo "processing following files from ${fileList}_${det}:"
181 cat ${fileList}_${det}
183 echo aliroot -b -q "mergeByComponent.C(\"${det}\",\"${fileList}_${det}\")"
184 aliroot -b -q "mergeByComponent.C(\"${det}\",\"${fileList}_${det}\")" 2>&1 | tee -a merge_${det}.log
185 if validateMerging ${det}; then
186 echo "### merge OK: mv CalibObjects.root ../CalibObjects_${det}.root"
187 mv -f CalibObjects.root ../CalibObjects_${det}.root
188 [[ "${det}" =~ TPCCalib ]] && cat ${fileList} >> ../$filesProcessedTPClist
190 echo "### merging not validated"
191 rm -f CalibObjects.root
193 mv syswatch.log syswatch_merge_${det}.log
198 #move stuff back to the parent dir and clean up
199 #merge the syswatch logs
200 for x in syswatch*log; do
201 [[ ! -f $x ]] && continue
204 echo "sed '1d' $x >> ../$x"
213 #merge the other logs
215 [[ ! -f $x ]] && continue
218 echo "cat $x >> ../$x"
227 #go to parent dir first to use the original fileList
228 #without ../CalibObjects.root
230 if [[ $cleanup -eq 1 ]]; then
231 echo "cleaning up merged files..."
237 rm -rf $runningDirectory
240 echo removing lock $runningMergeByComponentLockFile
241 rm -f $runningMergeByComponentLockFile
242 echo "***mergeByComponent() DONE"
244 echo "numberOfChunksTPC=$numberOfChunksTPC"
251 [[ ! -f $1 ]] && break
261 [[ ! -f CalibObjects.root ]] && ((retCode++)) && echo "### no CalibObjects.root..."
262 [[ ! -f ${det}_merge_done ]] && ((retCode++)) && echo "### no ${det}_merge_done, job finished abnormally..."
263 error=$(grep -e "was a crash" *.log)
264 [[ -n $error ]] && ((retCode++)) && echo "### error: $error"
271 [[ ! -f mergeByComponent.C ]] && \
272 cp -f $ALICE_ROOT/PWGPP/CalibMacros/CPass0/mergeByComponent.C $PWD && \
273 echo "taking the default scripts from $ALICE_ROOT"
274 [[ ! -f makeOCDB.C ]] && \
275 cp -f $ALICE_ROOT/PWGPP/CalibMacros/CPass0/makeOCDB.C $PWD && \
276 echo "taking the default scripts from $ALICE_ROOT"
279 #first, make sure we have the scripts
280 copyScripts | tee -a copy.log
282 #with alien files copy them first to local
283 echo "***********************" 2>&1 | tee -a copy.log
284 echo copying files for run $run 2>&1 | tee -a copy.log
285 echo from $path 2>&1 | tee -a copy.log
286 echo "***********************" 2>&1 | tee -a copy.log
287 if [[ $isLocal -eq 0 ]]; then
288 if [[ "$fileAccessMethod" == "alien_cp" ]]; then
289 echo "alien_find $path AliESDfriends_v1.root | egrep ^/ > $alienFileList" 2>&1 | tee -a copy.log
290 alien_find $path "AliESDfriends_v1.root" | egrep "^/" > $alienFileList
291 echo "alien_find done"
294 echo aliroot -b -q "mergeByComponent.C(\"MAKEALIENLIST\",\"$alienFileList\",\"$path\",\"AliESDfriends_v1.root\")" 2>&1 | tee -a copy.log
295 aliroot -b -q "mergeByComponent.C(\"MAKEALIENLIST\",\"$alienFileList\",\"$path\",\"AliESDfriends_v1.root\")" 2>&1 | tee -a copy.log
296 echo "MAKEALIENLIST done"
300 cp $path $alienFileList
303 #keep the first line intact (it is the largest file of the entire collection)
304 sed -n '1p' ${alienFileList} > ${alienFileList}.tmp
305 sed '1d' ${alienFileList} | while read x; do echo "${RANDOM} ${x}"; done|sort -n|cut -d" " -f2- >> ${alienFileList}.tmp
306 mv -f ${alienFileList}.tmp ${alienFileList}
308 #split into sublists, each to be processed separately
309 echo split --numeric-suffixes --suffix-length=6 --lines=$numberOfFilesInAbunch ${alienFileList} ${partialAlienFileListPrefix} | tee -a copy.log
310 split --numeric-suffixes --suffix-length=6 --lines=$numberOfFilesInAbunch ${alienFileList} ${partialAlienFileListPrefix}
311 rm -f $runningMergeByComponentLockFile
313 for partialAlienFileList in ${partialAlienFileListPrefix}*
316 #if it takes too long, break
317 #[[ ${SECONDS} -gt ${maxTimeToLive} ]] && break
319 partialAlienFileListPostfix=${partialAlienFileList#$partialAlienFileListPrefix}
320 partialLocalFileList=${partialLocalFileListPrefix}${partialAlienFileListPostfix}
322 #copy the files if appropriate and make a list of files to use
323 rm -f $partialLocalFileList
324 if [[ "$fileAccessMethod" == "alien_cp" ]]; then
326 [[ $x != /* ]] && continue
327 localName=${x//"/"/_}
328 echo "alien_cp "alien://$x" $localName" 2>&1 | tee -a copy.log
329 alien_cp "alien://$x" $localName
330 echo $localName>>$partialLocalFileList
331 done<$partialAlienFileList
332 elif [[ "$fileAccessMethod" == "tfilecp" ]]; then
333 echo aliroot -b -q "mergeByComponent.C(\"COPY\",\"$partialAlienFileList\",\"noPath\",\"noPattern\",10,\"$partialLocalFileList\")" 2>&1 | tee -a copy.log
334 aliroot -b -q "mergeByComponent.C(\"COPY\",\"$partialAlienFileList\",\"noPath\",\"noPattern\",10,\"$partialLocalFileList\")" 2>&1 | tee -a copy.log
335 elif [[ "$fileAccessMethod" == "nocopy" ]]; then
337 [[ $isLocal -eq 0 ]] && echo "alien://$x" >> $partialLocalFileList
338 [[ $isLocal -eq 1 ]] && echo "$x" >> $partialLocalFileList
339 done<$partialAlienFileList
344 if [[ -f syswatch_copy.log ]]
346 sed '1d' syswatch.log >> syswatch_copy.log
349 [[ -f syswatch.log ]] && mv syswatch.log syswatch_copy.log
352 #merge in parallel, use a simple lockfile
353 #echo "waitIfLocked $runningMergeByComponentLockFile" | tee -a merge.log
354 #waitIfLocked $runningMergeByComponentLockFile
355 echo "waiting" | tee -a merge.log
357 if [[ $isLocal -eq 1 ]]; then
358 #locally we don't need to run in parallel since we dont download,
359 #besides it causes problems on some filesystems (like lustre) with file sync
360 mergeByComponent $partialLocalFileList $cleanup 2>&1 | tee -a merge.log
362 mergeByComponent $partialLocalFileList $cleanup 2>&1 | tee -a merge.log &
366 #merge all the subfiles into one, wait for the last one to complete
367 #echo "waitIfLocked $runningMergeByComponentLockFile" | tee -a merge.log
368 #waitIfLocked $runningMergeByComponentLockFile
369 echo "waiting" | tee -a merge.log
371 if [[ "$components" =~ ALL && -f CalibObjects_ALL.root ]]; then
372 mv -f CalibObjects_ALL.root CalibObjects.root
374 echo "***********************"
375 echo merging ALL data
376 echo "***********************"
377 finalCalibObjectsList="finalObjects.list"
378 ls -1 CalibObjects_*.root > $finalCalibObjectsList
379 echo aliroot -b -q "mergeByComponent.C(\"ALL\",\"$finalCalibObjectsList\")" | tee -a merge.log
380 aliroot -b -q "mergeByComponent.C(\"ALL\",\"$finalCalibObjectsList\")" 2>&1 | tee -a merge.log
381 mv -f syswatch.log syswatch_merge_ALL.log
384 if ! validateMerging "ALL"; then
385 echo final merging not validatet, exiting...
388 rm -f CalibObjects_*.root
391 rm -f ${partialAlienFileListPrefix}*
392 rm -f ${partialLocalFileListPrefix}*
397 echo "***********************" 2>&1 | tee -a ocdb.log
398 echo making ${det} OCDB 2>&1 | tee -a ocdb.log
399 echo "***********************" 2>&1 | tee -a ocdb.log
400 echo aliroot -b -q "makeOCDB.C($run,\"$ocdb\",\"$defaultOCDB\")" 2>&1 | tee -a ocdb.log
401 aliroot -b -q "makeOCDB.C($run,\"$ocdb\",\"$defaultOCDB\")" 2>&1 | tee -a ocdb.log
402 mv syswatch.log syswatch_makeOCDB.log
405 echo "***********************" 2>&1 | tee -a ocdb.log
406 echo SUMMARY 2>&1 | tee -a ocdb.log
407 echo "***********************" 2>&1 | tee -a ocdb.log
408 ls -altr CalibObjects.root *done 2>&1 | tee -a ocdb.log