]>
Commit | Line | Data |
---|---|---|
e9129651 | 1 | #!/bin/bash |
e9129651 | 2 | # example local use (files will be downloaded from alien) |
3 | #./mergeMakeOCDB.byComponent.sh /alice/data/2012/LHC12g/000188362/cpass0/ 188362 local://./OCDB local:///cvmfs/alice.gsi.de/alice/data/2011/OCDB/ 0 30 | |
4 | # | |
5 | # on ALIEN just do: | |
9e160db9 | 6 | # $1 = directory where to perform the find |
7 | # $2 = runNumber | |
8 | # $3 = OCDB path | |
9 | ||
421603c5 | 10 | main() |
11 | { | |
12 | if [[ $# -eq 0 ]]; then | |
13 | echo arguments: | |
14 | echo " 1 - directory on which to look for the files to be merged or local file list" | |
15 | echo " 2 - runNumber" | |
16 | echo " 3 - OCDB output path" | |
17 | echo | |
18 | echo " optionally set any of these" | |
19 | echo " runParallel={0,1} (1 to run parallel merging and downloading)" | |
20 | echo " defaultOCDB=raw:// (or any other valid OCDB storage as input database)" | |
21 | echo " fileAccessMethod={alien_cp,tfilecp,nocopy} (by default tfilecp)" | |
22 | echo " \"alien_cp\" will use alien_cp to copy files from alien" | |
23 | echo " \"tfilecp\" will copy the files first using TFile::Cp()" | |
24 | echo " \"nocopy\" will access files over the network directly TFile::Open()" | |
25 | echo " numberOfFilesInAbunch=50 (how many files get downloaded in one go)" | |
26 | echo " maxChunksTPC=3000 (max number of chunks to be merged for TPC calibration)" | |
27 | ||
28 | echo | |
29 | echo "example:" | |
30 | echo " ./mergeMakeOCDB.byComponent.sh /alice/data/2012/LHC12g/000188362/cpass1/ 188362 local://./OCDB runParallel=0 defaultOCDB=raw:// fileAccessMethod=alien_cp" | |
31 | echo | |
32 | ||
33 | return 0 | |
34 | fi | |
35 | ||
36 | # init | |
37 | path=$1 | |
38 | run=$(echo "$2" | sed 's/^0*//') | |
39 | ocdb=$3 | |
40 | ||
41 | shift 3 | |
42 | ||
43 | #default values | |
44 | runParallel=1 | |
45 | defaultOCDB="raw://" | |
46 | fileAccessMethod="tfilecp" | |
47 | numberOfFilesInAbunch=50 | |
48 | maxChunksTPC=3000 | |
49 | [[ -n ${ALIEN_JDL_TTL} ]] && maxTimeToLive=$(( ${ALIEN_JDL_TTL}-2000 )) | |
50 | ||
51 | parseConfig "$@" | |
52 | ||
53 | #some sanity checks of option values | |
54 | [[ ! $numberOfFilesInAbunch =~ ^[0-9]+$ ]] && numberOfFilesInAbunch=50 | |
55 | [[ "$fileAccessMethod" != "alien_cp" && "$fileAccessMethod" != "tfilecp" && "$fileAccessMethod" != "nocopy" ]] && fileAccessMethod="tfilecp" && echo "using default fileAccessMethod=tfilecp" | |
56 | ||
57 | filesAreLocal=0 | |
58 | [[ -f $path ]] && filesAreLocal=1 | |
59 | cleanup=1 | |
60 | [[ $filesAreLocal -eq 1 ]] && cleanup=0 | |
61 | [[ $filesAreLocal -eq 1 ]] && fileAccessMethod="nocopy" | |
62 | ||
63 | # setup components to be merged | |
64 | components="TOF MeanVertex T0 SDD TRD TPCCalib TPCCluster TPCAlign" | |
65 | #components="TOF MeanVertex T0 SDD TRD TPCCalib" | |
66 | ||
67 | ################################################################# | |
68 | echo "" | tee -a merge.log | |
69 | echo $0" $*" | tee -a merge.log | |
70 | echo "" | tee -a merge.log | |
71 | echo "***********************" | tee -a merge.log | |
72 | echo mergeMakeOCDB.byComponent.sh started | tee -a merge.log | |
73 | echo path = $path | tee -a merge.log | |
74 | echo run = $run | tee -a merge.log | |
75 | echo ocdb = $ocdb | tee -a merge.log | |
76 | echo defaultOCDB=$defaultOCDB | tee -a merge.log | |
77 | echo filesAreLocal = $filesAreLocal | tee -a merge.log | |
78 | echo cleanup = $cleanup | tee -a merge.log | |
79 | echo fileAccessMethod=$fileAccessMethod | tee -a merge.log | |
80 | echo numberOfFilesInAbunch = $numberOfFilesInAbunch | tee -a merge.log | |
81 | echo runParallel=$runParallel | |
82 | echo "***********************" | tee -a merge.log | |
83 | ||
84 | alienFileList="alien.list" | |
85 | localFileList="local.list" | |
86 | filesProcessedTPClist="filesProcessedTPC.log" | |
87 | rm -f $filesProcessedTPClist | |
88 | touch $filesProcessedTPClist | |
89 | partialLocalFileListPrefix=${localFileList}_ | |
90 | partialAlienFileListPrefix=${alienFileList}_ | |
91 | ||
92 | #first, make sure we have the scripts | |
93 | copyScripts | tee -a copy.log | |
94 | ls | |
95 | #with alien files copy them first to local | |
96 | echo "***********************" 2>&1 | tee -a copy.log | |
97 | echo copying files for run $run 2>&1 | tee -a copy.log | |
98 | echo from $path 2>&1 | tee -a copy.log | |
99 | echo "***********************" 2>&1 | tee -a copy.log | |
100 | if [[ $filesAreLocal -eq 0 ]]; then | |
101 | if [[ "$fileAccessMethod" == "alien_cp" ]]; then | |
102 | echo "alien_find $path AliESDfriends_v1.root | egrep ^/ > $alienFileList" 2>&1 | tee -a copy.log | |
103 | alien_find $path "AliESDfriends_v1.root" | egrep "^/" > $alienFileList | |
104 | echo "alien_find done" | |
105 | echo | |
106 | else | |
107 | echo aliroot -b -q "mergeByComponent.C(\"MAKEALIENLIST\",\"$alienFileList\", \"$path\", \"AliESDfriends_v1.root\")" 2>&1 | tee -a copy.log | |
108 | aliroot -b -q "mergeByComponent.C(\"MAKEALIENLIST\",\"$alienFileList\", \"$path\", \"AliESDfriends_v1.root\")" 2>&1 | tee -a copy.log | |
109 | echo "MAKEALIENLIST done" | |
110 | echo | |
111 | fi | |
112 | else | |
113 | cp $path $alienFileList | |
114 | fi | |
115 | #randomize the list | |
116 | #keep the first line intact (it is the largest file of the entire collection) | |
117 | sed -n '1p' ${alienFileList} > ${alienFileList}.tmp | |
118 | sed '1d' ${alienFileList} | while read x; do echo "${RANDOM} ${x}"; done|sort -n|cut -d" " -f2- >> ${alienFileList}.tmp | |
119 | mv -f ${alienFileList}.tmp ${alienFileList} | |
120 | ||
121 | #split into sublists, each to be processed separately | |
122 | echo split --numeric-suffixes --suffix-length=6 --lines=$numberOfFilesInAbunch ${alienFileList} ${partialAlienFileListPrefix} | tee -a copy.log | |
123 | #split --numeric-suffixes --suffix-length=6 --lines=$numberOfFilesInAbunch ${alienFileList} ${partialAlienFileListPrefix} | |
124 | split -a 6 -l $numberOfFilesInAbunch ${alienFileList} ${partialAlienFileListPrefix} | |
125 | ||
126 | for partialAlienFileList in ${partialAlienFileListPrefix}* | |
127 | do | |
128 | ||
129 | #if it takes too long, break | |
130 | #[[ ${SECONDS} -gt ${maxTimeToLive} ]] && break | |
131 | ||
132 | partialAlienFileListPostfix=${partialAlienFileList#$partialAlienFileListPrefix} | |
133 | partialLocalFileList=${partialLocalFileListPrefix}${partialAlienFileListPostfix} | |
134 | ||
135 | #copy the files if appropriate and make a list of files to use | |
136 | rm -f $partialLocalFileList | |
137 | if [[ "$fileAccessMethod" == "alien_cp" ]]; then | |
138 | while read x; do | |
139 | [[ $x != /* ]] && continue | |
140 | localName=${x//"/"/_} | |
141 | echo "alien_cp "alien://$x" $localName" 2>&1 | tee -a copy.log | |
142 | alien_cp "alien://$x" $localName | |
143 | echo $localName>>$partialLocalFileList | |
144 | done<$partialAlienFileList | |
145 | elif [[ "$fileAccessMethod" == "tfilecp" ]]; then | |
146 | echo aliroot -b -q "mergeByComponent.C(\"COPY\",\"$partialAlienFileList\",\"noPath\",\"noPattern\",10,\"$partialLocalFileList\")" 2>&1 | tee -a copy.log | |
147 | aliroot -b -q "mergeByComponent.C(\"COPY\",\"$partialAlienFileList\",\"noPath\",\"noPattern\",10,\"$partialLocalFileList\")" 2>&1 | tee -a copy.log | |
148 | elif [[ "$fileAccessMethod" == "nocopy" ]]; then | |
149 | while read x; do | |
150 | [[ $filesAreLocal -eq 0 ]] && echo "alien://$x" >> $partialLocalFileList | |
151 | [[ $filesAreLocal -eq 1 ]] && echo "$x" >> $partialLocalFileList | |
152 | done<$partialAlienFileList | |
153 | cleanup=0 | |
154 | fi | |
155 | ||
156 | #handle syswatch | |
157 | if [[ -f syswatch_copy.log ]] | |
158 | then | |
159 | sed '1d' syswatch.log >> syswatch_copy.log | |
160 | rm -f syswatch.log | |
161 | else | |
162 | [[ -f syswatch.log ]] && mv syswatch.log syswatch_copy.log | |
163 | fi | |
e9129651 | 164 | |
421603c5 | 165 | echo waiting |
166 | wait $! | |
167 | if [[ $runParallel -eq 1 ]]; then | |
168 | mergeByComponent $partialLocalFileList $cleanup 2>&1 | tee -a merge.log & | |
169 | else | |
170 | mergeByComponent $partialLocalFileList $cleanup 2>&1 | tee -a merge.log | |
171 | fi | |
172 | done | |
173 | ||
174 | #merge all the subfiles into one, wait for the last one to complete | |
175 | echo waiting | |
176 | wait $! | |
177 | if [[ "$components" =~ ALL && -f CalibObjects_ALL.root ]]; then | |
178 | mv -f CalibObjects_ALL.root CalibObjects.root | |
179 | else | |
180 | echo "***********************" | |
181 | echo merging ALL data | |
182 | echo "***********************" | |
183 | finalCalibObjectsList="finalObjects.list" | |
184 | ls -1 CalibObjects_*.root > $finalCalibObjectsList | |
185 | echo aliroot -b -q "mergeByComponent.C(\"ALL\", \"$finalCalibObjectsList\")" | tee -a merge.log | |
186 | aliroot -b -q "mergeByComponent.C(\"ALL\", \"$finalCalibObjectsList\")" 2>&1 | tee -a merge.log | |
187 | mv -f syswatch.log syswatch_merge_ALL.log | |
188 | fi | |
189 | ||
190 | if ! validateMerging "ALL"; then | |
191 | echo final merging not validatet, exiting... | |
192 | return 1 | |
193 | fi | |
194 | rm -f CalibObjects_*.root | |
195 | ||
196 | #cleanup | |
197 | rm -f ${partialAlienFileListPrefix}* | |
198 | rm -f ${partialLocalFileListPrefix}* | |
199 | rm -f $alienFileList | |
200 | rm -f $localFileList | |
201 | ||
202 | # make OCDB | |
203 | echo "***********************" 2>&1 | tee -a ocdb.log | |
204 | echo making ${det} OCDB 2>&1 | tee -a ocdb.log | |
205 | echo "***********************" 2>&1 | tee -a ocdb.log | |
206 | echo aliroot -b -q "makeOCDB.C($run, \"$ocdb\", \"$defaultOCDB\")" 2>&1 | tee -a ocdb.log | |
207 | aliroot -b -q "makeOCDB.C($run, \"$ocdb\", \"$defaultOCDB\")" 2>&1 | tee -a ocdb.log | |
208 | mv syswatch.log syswatch_makeOCDB.log | |
209 | ||
210 | # summary | |
211 | echo "***********************" 2>&1 | tee -a ocdb.log | |
212 | echo SUMMARY 2>&1 | tee -a ocdb.log | |
213 | echo "***********************" 2>&1 | tee -a ocdb.log | |
214 | ls -altr CalibObjects.root *done 2>&1 | tee -a ocdb.log | |
215 | } | |
9e160db9 | 216 | |
217 | mergeByComponent() | |
218 | { | |
219 | # process by component | |
220 | # first argument is the file list to process | |
9e160db9 | 221 | |
40f73007 | 222 | # run inside a dedicated running directory |
223 | # whic means copy the file list to process and prefic each line with ../ | |
224 | # since the file names have no absolute paths! | |
421603c5 | 225 | runningDirectory="${1}.dir" |
e9129651 | 226 | fileList="$1" |
227 | cleanup=$2 | |
421603c5 | 228 | |
e9129651 | 229 | #sanity checks |
695f9fb2 | 230 | if [[ ! -f ${fileList} ]]; then |
231 | echo "${fileList} does not exist" | |
695f9fb2 | 232 | return 1 |
233 | fi | |
40f73007 | 234 | mkdir -p $runningDirectory |
695f9fb2 | 235 | if [[ ! -d $runningDirectory ]]; then |
236 | echo "cannot create the running directory $runningDirectory" | |
695f9fb2 | 237 | return 1 |
9e160db9 | 238 | fi |
421603c5 | 239 | |
e9129651 | 240 | #move the to be merged files to the running directory and make a list of available files |
241 | #handle the case of archives (x.zip#y.root) as well | |
242 | nFiles=0 | |
243 | while read entry; do | |
244 | if [[ $entry =~ ^.*\.root$ ]]; then | |
245 | file=${entry%#*} | |
246 | fileContent=${entry##*#} | |
247 | [[ "${fileContent}" == "${file}" ]] && fileContent="" | |
248 | [[ -n ${fileContent} ]] && fileContent="#${fileContent}" | |
249 | if [[ -f ${file} ]]; then | |
250 | ((nFiles++)) | |
251 | if [[ -f "./${file}" ]]; then | |
695f9fb2 | 252 | echo "../${file}${fileContent}" >> "${runningDirectory}/${fileList}" |
e9129651 | 253 | else |
695f9fb2 | 254 | echo "${file}${fileContent}" >> "${runningDirectory}/${fileList}" |
e9129651 | 255 | fi |
256 | fi | |
257 | fi | |
695f9fb2 | 258 | done < ${fileList} |
259 | if [[ $nFiles -lt 1 ]]; then | |
260 | echo "no new files in ${fileList}" | |
261 | echo rm -rf $runningDirectory | |
262 | rm -rf $runningDirectory | |
695f9fb2 | 263 | return 1 |
264 | fi | |
e9129651 | 265 | |
266 | #copy the macro to the running directory | |
267 | [[ -f mergeByComponent.C ]] && cp mergeByComponent.C $runningDirectory | |
268 | ||
695f9fb2 | 269 | #go to running directory |
e9129651 | 270 | cd $runningDirectory |
421603c5 | 271 | |
695f9fb2 | 272 | numberOfChunksTPC=$(cat ../$filesProcessedTPClist 2>/dev/null | wc -l ) |
807998f3 | 273 | |
9e160db9 | 274 | for det in $components; do |
421603c5 | 275 | |
807998f3 | 276 | # merge |
e9129651 | 277 | echo "***********************" |
695f9fb2 | 278 | echo merging ${det} data |
e9129651 | 279 | echo "***********************" |
695f9fb2 | 280 | |
281 | #limit the number of chunks processed by TPC | |
282 | if [[ "${det}" =~ TPC && $numberOfChunksTPC -ge $maxChunksTPC ]]; then | |
283 | echo "Not merging TPC anymore, max number of chunks processed ($maxChunksTPC)" | |
284 | continue | |
285 | fi | |
286 | ||
287 | #add the results of previous iteration to the BEGINNING of the list of files to be merged | |
288 | [[ -f ../CalibObjects_${det}.root ]] && echo "../CalibObjects_${det}.root" > ${fileList}_${det} | |
289 | cat ${fileList} >> ${fileList}_${det} | |
290 | ||
291 | echo "processing following files from ${fileList}_${det}:" | |
292 | cat ${fileList}_${det} | |
293 | ||
421603c5 | 294 | echo aliroot -b -q "mergeByComponent.C(\"${det}\", \"${fileList}_${det}\")" |
295 | aliroot -b -q "mergeByComponent.C(\"${det}\", \"${fileList}_${det}\")" 2>&1 | tee -a merge_${det}.log | |
e9129651 | 296 | if validateMerging ${det}; then |
695f9fb2 | 297 | echo "### merge OK: mv CalibObjects.root ../CalibObjects_${det}.root" |
298 | mv -f CalibObjects.root ../CalibObjects_${det}.root | |
299 | [[ "${det}" =~ TPCCalib ]] && cat ${fileList} >> ../$filesProcessedTPClist | |
e9129651 | 300 | else |
695f9fb2 | 301 | echo "### merging not validated" |
e9129651 | 302 | rm -f CalibObjects.root |
303 | fi | |
695f9fb2 | 304 | mv syswatch.log syswatch_merge_${det}.log |
305 | ||
9e160db9 | 306 | done |
40f73007 | 307 | |
421603c5 | 308 | |
40f73007 | 309 | #move stuff back to the parent dir and clean up |
310 | #merge the syswatch logs | |
311 | for x in syswatch*log; do | |
e9129651 | 312 | [[ ! -f $x ]] && continue |
313 | if [[ -f ../$x ]] | |
40f73007 | 314 | then |
e9129651 | 315 | echo "sed '1d' $x >> ../$x" |
316 | sed '1d' $x >> ../$x | |
40f73007 | 317 | rm -f $x |
318 | else | |
e9129651 | 319 | echo mv -f $x .. |
320 | mv -f $x .. | |
321 | fi | |
322 | done | |
323 | ||
324 | #merge the other logs | |
325 | for x in *.log; do | |
326 | [[ ! -f $x ]] && continue | |
327 | if [[ -f ../$x ]] | |
328 | then | |
329 | echo "cat $x >> ../$x" | |
330 | cat $x >> ../$x | |
331 | else | |
332 | echo "mv -f $x .." | |
333 | mv -f $x .. | |
40f73007 | 334 | fi |
335 | done | |
421603c5 | 336 | |
e9129651 | 337 | #final cleanup. |
338 | #go to parent dir first to use the original fileList | |
339 | #without ../CalibObjects.root | |
340 | cd .. | |
341 | if [[ $cleanup -eq 1 ]]; then | |
342 | echo "cleaning up merged files..." | |
343 | while read file; do | |
344 | echo rm -rf $file | |
345 | rm -rf $file | |
346 | done<$fileList | |
347 | fi | |
40f73007 | 348 | rm -rf $runningDirectory |
807998f3 | 349 | |
e9129651 | 350 | echo "***mergeByComponent() DONE" |
351 | echo | |
695f9fb2 | 352 | echo "numberOfChunksTPC=$numberOfChunksTPC" |
e9129651 | 353 | return 0 |
9e160db9 | 354 | } |
807998f3 | 355 | |
421603c5 | 356 | parseConfig() |
357 | { | |
358 | args=("$@") | |
359 | for opt in "${args[@]}"; do | |
360 | [[ -z ${opt} ]] && continue | |
361 | [[ -n ${encodedSpaces} ]] && opt="$(decSpaces ${opt})" | |
362 | [[ "${opt}" =~ ^[[:space:]]*$ ]] && continue | |
363 | if [[ ! "${opt}" =~ .*=.* ]]; then | |
364 | echo "badly formatted option \"${opt}\" should be: option=value, stopping..." | |
365 | return 1 | |
366 | fi | |
367 | local var="${opt%%=*}" | |
368 | local value="${opt#*=}" | |
369 | export ${var}="${value}" | |
370 | done | |
371 | ||
372 | return 0 | |
373 | } | |
374 | ||
9e160db9 | 375 | waitIfLocked() |
376 | { | |
377 | while [ 1 ]; do | |
378 | [[ ! -f $1 ]] && break | |
379 | sleep 1 | |
380 | done | |
e9129651 | 381 | return 0 |
9e160db9 | 382 | } |
807998f3 | 383 | |
e9129651 | 384 | validateMerging() |
385 | { | |
386 | det=$1 | |
387 | retCode=0 | |
388 | [[ ! -f CalibObjects.root ]] && ((retCode++)) && echo "### no CalibObjects.root..." | |
389 | [[ ! -f ${det}_merge_done ]] && ((retCode++)) && echo "### no ${det}_merge_done, job finished abnormally..." | |
390 | error=$(grep -e "was a crash" *.log) | |
391 | [[ -n $error ]] && ((retCode++)) && echo "### error: $error" | |
421603c5 | 392 | |
e9129651 | 393 | return $retCode |
394 | } | |
395 | ||
396 | copyScripts() | |
397 | { | |
398 | [[ ! -f mergeByComponent.C ]] && \ | |
421603c5 | 399 | cp -f $ALICE_ROOT/PWGPP/CalibMacros/CPass1/mergeByComponent.C $PWD && \ |
e9129651 | 400 | echo "taking the default scripts from $ALICE_ROOT" |
401 | [[ ! -f makeOCDB.C ]] && \ | |
421603c5 | 402 | cp -f $ALICE_ROOT/PWGPP/CalibMacros/CPass1/makeOCDB.C $PWD && \ |
e9129651 | 403 | echo "taking the default scripts from $ALICE_ROOT" |
404 | } | |
9e160db9 | 405 | |
421603c5 | 406 | main "$@" |