Adding possibility to disable parallel aliroot sessions + making the script more...
[u/mrichter/AliRoot.git] / PWGPP / CalibMacros / CPass0 / mergeMakeOCDB.byComponent.sh
CommitLineData
e9129651 1#!/bin/bash
e9129651 2# example local use (files will be downloaded from alien)
3#./mergeMakeOCDB.byComponent.sh /alice/data/2012/LHC12g/000188362/cpass0/ 188362 local://./OCDB local:///cvmfs/alice.gsi.de/alice/data/2011/OCDB/ 0 30
4#
5# on ALIEN just do:
d157f353 6# $1 = directory where to perform the find
9e160db9 7# $2 = runNumber
d157f353 8# $3 = OCDB path
9
421603c5 10main()
11{
12 if [[ $# -eq 0 ]]; then
13 echo arguments:
14 echo " 1 - directory on which to look for the files to be merged or local file list"
15 echo " 2 - runNumber"
16 echo " 3 - OCDB output path"
17 echo
18 echo " optionally set any of these"
19 echo " runParallel={0,1} (1 to run parallel merging and downloading)"
20 echo " defaultOCDB=raw:// (or any other valid OCDB storage as input database)"
21 echo " fileAccessMethod={alien_cp,tfilecp,nocopy} (by default tfilecp)"
22 echo " \"alien_cp\" will use alien_cp to copy files from alien"
23 echo " \"tfilecp\" will copy the files first using TFile::Cp()"
24 echo " \"nocopy\" will access files over the network directly TFile::Open()"
25 echo " numberOfFilesInAbunch=50 (how many files get downloaded in one go)"
26 echo " maxChunksTPC=3000 (max number of chunks to be merged for TPC calibration)"
27
28 echo
29 echo "example:"
30 echo " ./mergeMakeOCDB.byComponent.sh /alice/data/2012/LHC12g/000188362/cpass1/ 188362 local://./OCDB runParallel=0 defaultOCDB=raw:// fileAccessMethod=alien_cp"
31 echo
32
33 return 0
34 fi
35
36 # init
37 path=$1
38 run=$(echo "$2" | sed 's/^0*//')
39 ocdb=$3
40
41 shift 3
42
43 #default values
44 runParallel=1
45 defaultOCDB="raw://"
46 fileAccessMethod="tfilecp"
47 numberOfFilesInAbunch=50
48 maxChunksTPC=3000
49 [[ -n ${ALIEN_JDL_TTL} ]] && maxTimeToLive=$(( ${ALIEN_JDL_TTL}-2000 ))
50
51 parseConfig "$@"
52
53 #some sanity checks of option values
54 [[ ! $numberOfFilesInAbunch =~ ^[0-9]+$ ]] && numberOfFilesInAbunch=50
55 [[ "$fileAccessMethod" != "alien_cp" && "$fileAccessMethod" != "tfilecp" && "$fileAccessMethod" != "nocopy" ]] && fileAccessMethod="tfilecp" && echo "using default fileAccessMethod=tfilecp"
56
57 filesAreLocal=0
58 [[ -f $path ]] && filesAreLocal=1
59 cleanup=1
60 [[ $filesAreLocal -eq 1 ]] && cleanup=0
61 [[ $filesAreLocal -eq 1 ]] && fileAccessMethod="nocopy"
62
63 # setup components to be merged
64 components="TOF MeanVertex T0 SDD TRD TPCCalib TPCCluster TPCAlign"
65 #components="TOF MeanVertex T0 SDD TRD TPCCalib"
66
67 #################################################################
68 echo "" | tee -a merge.log
69 echo $0" $*" | tee -a merge.log
70 echo "" | tee -a merge.log
71 echo "***********************" | tee -a merge.log
72 echo mergeMakeOCDB.byComponent.sh started | tee -a merge.log
73 echo path = $path | tee -a merge.log
74 echo run = $run | tee -a merge.log
75 echo ocdb = $ocdb | tee -a merge.log
76 echo defaultOCDB=$defaultOCDB | tee -a merge.log
77 echo filesAreLocal = $filesAreLocal | tee -a merge.log
78 echo cleanup = $cleanup | tee -a merge.log
79 echo fileAccessMethod=$fileAccessMethod | tee -a merge.log
80 echo numberOfFilesInAbunch = $numberOfFilesInAbunch | tee -a merge.log
81 echo runParallel=$runParallel
82 echo "***********************" | tee -a merge.log
83
84 alienFileList="alien.list"
85 localFileList="local.list"
86 filesProcessedTPClist="filesProcessedTPC.log"
87 rm -f $filesProcessedTPClist
88 touch $filesProcessedTPClist
89 partialLocalFileListPrefix=${localFileList}_
90 partialAlienFileListPrefix=${alienFileList}_
91
92 #first, make sure we have the scripts
93 copyScripts | tee -a copy.log
94 ls
95 #with alien files copy them first to local
96 echo "***********************" 2>&1 | tee -a copy.log
97 echo copying files for run $run 2>&1 | tee -a copy.log
98 echo from $path 2>&1 | tee -a copy.log
99 echo "***********************" 2>&1 | tee -a copy.log
100 if [[ $filesAreLocal -eq 0 ]]; then
101 if [[ "$fileAccessMethod" == "alien_cp" ]]; then
102 echo "alien_find $path AliESDfriends_v1.root | egrep ^/ > $alienFileList" 2>&1 | tee -a copy.log
103 alien_find $path "AliESDfriends_v1.root" | egrep "^/" > $alienFileList
104 echo "alien_find done"
105 echo
106 else
107 echo aliroot -b -q "mergeByComponent.C(\"MAKEALIENLIST\",\"$alienFileList\", \"$path\", \"AliESDfriends_v1.root\")" 2>&1 | tee -a copy.log
108 aliroot -b -q "mergeByComponent.C(\"MAKEALIENLIST\",\"$alienFileList\", \"$path\", \"AliESDfriends_v1.root\")" 2>&1 | tee -a copy.log
109 echo "MAKEALIENLIST done"
110 echo
111 fi
112 else
113 cp $path $alienFileList
114 fi
115 #randomize the list
116 #keep the first line intact (it is the largest file of the entire collection)
117 sed -n '1p' ${alienFileList} > ${alienFileList}.tmp
118 sed '1d' ${alienFileList} | while read x; do echo "${RANDOM} ${x}"; done|sort -n|cut -d" " -f2- >> ${alienFileList}.tmp
119 mv -f ${alienFileList}.tmp ${alienFileList}
120
121 #split into sublists, each to be processed separately
122 echo split --numeric-suffixes --suffix-length=6 --lines=$numberOfFilesInAbunch ${alienFileList} ${partialAlienFileListPrefix} | tee -a copy.log
123 #split --numeric-suffixes --suffix-length=6 --lines=$numberOfFilesInAbunch ${alienFileList} ${partialAlienFileListPrefix}
124 split -a 6 -l $numberOfFilesInAbunch ${alienFileList} ${partialAlienFileListPrefix}
125
126 for partialAlienFileList in ${partialAlienFileListPrefix}*
127 do
128
129 #if it takes too long, break
130 #[[ ${SECONDS} -gt ${maxTimeToLive} ]] && break
131
132 partialAlienFileListPostfix=${partialAlienFileList#$partialAlienFileListPrefix}
133 partialLocalFileList=${partialLocalFileListPrefix}${partialAlienFileListPostfix}
134
135 #copy the files if appropriate and make a list of files to use
136 rm -f $partialLocalFileList
137 if [[ "$fileAccessMethod" == "alien_cp" ]]; then
138 while read x; do
139 [[ $x != /* ]] && continue
140 localName=${x//"/"/_}
141 echo "alien_cp "alien://$x" $localName" 2>&1 | tee -a copy.log
142 alien_cp "alien://$x" $localName
143 echo $localName>>$partialLocalFileList
144 done<$partialAlienFileList
145 elif [[ "$fileAccessMethod" == "tfilecp" ]]; then
146 echo aliroot -b -q "mergeByComponent.C(\"COPY\",\"$partialAlienFileList\",\"noPath\",\"noPattern\",10,\"$partialLocalFileList\")" 2>&1 | tee -a copy.log
147 aliroot -b -q "mergeByComponent.C(\"COPY\",\"$partialAlienFileList\",\"noPath\",\"noPattern\",10,\"$partialLocalFileList\")" 2>&1 | tee -a copy.log
148 elif [[ "$fileAccessMethod" == "nocopy" ]]; then
149 while read x; do
150 [[ $filesAreLocal -eq 0 ]] && echo "alien://$x" >> $partialLocalFileList
151 [[ $filesAreLocal -eq 1 ]] && echo "$x" >> $partialLocalFileList
152 done<$partialAlienFileList
153 cleanup=0
154 fi
155
156 #handle syswatch
157 if [[ -f syswatch_copy.log ]]
158 then
159 sed '1d' syswatch.log >> syswatch_copy.log
160 rm -f syswatch.log
161 else
162 [[ -f syswatch.log ]] && mv syswatch.log syswatch_copy.log
163 fi
e9129651 164
421603c5 165 echo waiting
166 wait $!
167 if [[ $runParallel -eq 1 ]]; then
168 mergeByComponent $partialLocalFileList $cleanup 2>&1 | tee -a merge.log &
169 else
170 mergeByComponent $partialLocalFileList $cleanup 2>&1 | tee -a merge.log
171 fi
172 done
173
174 #merge all the subfiles into one, wait for the last one to complete
175 echo waiting
176 wait $!
177 if [[ "$components" =~ ALL && -f CalibObjects_ALL.root ]]; then
178 mv -f CalibObjects_ALL.root CalibObjects.root
179 else
180 echo "***********************"
181 echo merging ALL data
182 echo "***********************"
183 finalCalibObjectsList="finalObjects.list"
184 ls -1 CalibObjects_*.root > $finalCalibObjectsList
185 echo aliroot -b -q "mergeByComponent.C(\"ALL\", \"$finalCalibObjectsList\")" | tee -a merge.log
186 aliroot -b -q "mergeByComponent.C(\"ALL\", \"$finalCalibObjectsList\")" 2>&1 | tee -a merge.log
187 mv -f syswatch.log syswatch_merge_ALL.log
188 fi
189
190 if ! validateMerging "ALL"; then
191 echo final merging not validatet, exiting...
192 return 1
193 fi
194 rm -f CalibObjects_*.root
195
196 #cleanup
197 rm -f ${partialAlienFileListPrefix}*
198 rm -f ${partialLocalFileListPrefix}*
199 rm -f $alienFileList
200 rm -f $localFileList
201
202 # make OCDB
203 echo "***********************" 2>&1 | tee -a ocdb.log
204 echo making ${det} OCDB 2>&1 | tee -a ocdb.log
205 echo "***********************" 2>&1 | tee -a ocdb.log
206 echo aliroot -b -q "makeOCDB.C($run, \"$ocdb\", \"$defaultOCDB\")" 2>&1 | tee -a ocdb.log
207 aliroot -b -q "makeOCDB.C($run, \"$ocdb\", \"$defaultOCDB\")" 2>&1 | tee -a ocdb.log
208 mv syswatch.log syswatch_makeOCDB.log
209
210 # summary
211 echo "***********************" 2>&1 | tee -a ocdb.log
212 echo SUMMARY 2>&1 | tee -a ocdb.log
213 echo "***********************" 2>&1 | tee -a ocdb.log
214 ls -altr CalibObjects.root *done 2>&1 | tee -a ocdb.log
215}
9e160db9 216
217mergeByComponent()
218{
219 # process by component
220 # first argument is the file list to process
9e160db9 221
40f73007 222 # run inside a dedicated running directory
223 # whic means copy the file list to process and prefic each line with ../
224 # since the file names have no absolute paths!
421603c5 225 runningDirectory="${1}.dir"
e9129651 226 fileList="$1"
227 cleanup=$2
421603c5 228
e9129651 229 #sanity checks
695f9fb2 230 if [[ ! -f ${fileList} ]]; then
231 echo "${fileList} does not exist"
695f9fb2 232 return 1
233 fi
40f73007 234 mkdir -p $runningDirectory
695f9fb2 235 if [[ ! -d $runningDirectory ]]; then
236 echo "cannot create the running directory $runningDirectory"
695f9fb2 237 return 1
e9129651 238 fi
421603c5 239
e9129651 240 #move the to be merged files to the running directory and make a list of available files
241 #handle the case of archives (x.zip#y.root) as well
242 nFiles=0
243 while read entry; do
244 if [[ $entry =~ ^.*\.root$ ]]; then
245 file=${entry%#*}
246 fileContent=${entry##*#}
247 [[ "${fileContent}" == "${file}" ]] && fileContent=""
248 [[ -n ${fileContent} ]] && fileContent="#${fileContent}"
249 if [[ -f ${file} ]]; then
250 ((nFiles++))
251 if [[ -f "./${file}" ]]; then
695f9fb2 252 echo "../${file}${fileContent}" >> "${runningDirectory}/${fileList}"
e9129651 253 else
695f9fb2 254 echo "${file}${fileContent}" >> "${runningDirectory}/${fileList}"
e9129651 255 fi
256 fi
257 fi
695f9fb2 258 done < ${fileList}
259 if [[ $nFiles -lt 1 ]]; then
260 echo "no new files in ${fileList}"
261 echo rm -rf $runningDirectory
262 rm -rf $runningDirectory
695f9fb2 263 return 1
264 fi
40f73007 265
e9129651 266 #copy the macro to the running directory
267 [[ -f mergeByComponent.C ]] && cp mergeByComponent.C $runningDirectory
268
695f9fb2 269 #go to running directory
e9129651 270 cd $runningDirectory
421603c5 271
695f9fb2 272 numberOfChunksTPC=$(cat ../$filesProcessedTPClist 2>/dev/null | wc -l )
27eb9bff 273
9e160db9 274 for det in $components; do
421603c5 275
27eb9bff 276 # merge
40f73007 277 echo "***********************"
695f9fb2 278 echo merging ${det} data
40f73007 279 echo "***********************"
695f9fb2 280
281 #limit the number of chunks processed by TPC
282 if [[ "${det}" =~ TPC && $numberOfChunksTPC -ge $maxChunksTPC ]]; then
283 echo "Not merging TPC anymore, max number of chunks processed ($maxChunksTPC)"
284 continue
285 fi
286
287 #add the results of previous iteration to the BEGINNING of the list of files to be merged
288 [[ -f ../CalibObjects_${det}.root ]] && echo "../CalibObjects_${det}.root" > ${fileList}_${det}
289 cat ${fileList} >> ${fileList}_${det}
290
291 echo "processing following files from ${fileList}_${det}:"
292 cat ${fileList}_${det}
293
421603c5 294 echo aliroot -b -q "mergeByComponent.C(\"${det}\", \"${fileList}_${det}\")"
295 aliroot -b -q "mergeByComponent.C(\"${det}\", \"${fileList}_${det}\")" 2>&1 | tee -a merge_${det}.log
e9129651 296 if validateMerging ${det}; then
695f9fb2 297 echo "### merge OK: mv CalibObjects.root ../CalibObjects_${det}.root"
298 mv -f CalibObjects.root ../CalibObjects_${det}.root
299 [[ "${det}" =~ TPCCalib ]] && cat ${fileList} >> ../$filesProcessedTPClist
e9129651 300 else
695f9fb2 301 echo "### merging not validated"
e9129651 302 rm -f CalibObjects.root
303 fi
695f9fb2 304 mv syswatch.log syswatch_merge_${det}.log
305
9e160db9 306 done
e9129651 307
421603c5 308
40f73007 309 #move stuff back to the parent dir and clean up
310 #merge the syswatch logs
311 for x in syswatch*log; do
e9129651 312 [[ ! -f $x ]] && continue
313 if [[ -f ../$x ]]
40f73007 314 then
e9129651 315 echo "sed '1d' $x >> ../$x"
316 sed '1d' $x >> ../$x
40f73007 317 rm -f $x
318 else
e9129651 319 echo mv -f $x ..
320 mv -f $x ..
40f73007 321 fi
322 done
e9129651 323
40f73007 324 #merge the other logs
325 for x in *.log; do
e9129651 326 [[ ! -f $x ]] && continue
327 if [[ -f ../$x ]]
40f73007 328 then
e9129651 329 echo "cat $x >> ../$x"
330 cat $x >> ../$x
40f73007 331 else
e9129651 332 echo "mv -f $x .."
333 mv -f $x ..
40f73007 334 fi
335 done
421603c5 336
e9129651 337 #final cleanup.
338 #go to parent dir first to use the original fileList
339 #without ../CalibObjects.root
340 cd ..
341 if [[ $cleanup -eq 1 ]]; then
342 echo "cleaning up merged files..."
343 while read file; do
344 echo rm -rf $file
345 rm -rf $file
346 done<$fileList
347 fi
40f73007 348 rm -rf $runningDirectory
349
40f73007 350 echo "***mergeByComponent() DONE"
e9129651 351 echo
695f9fb2 352 echo "numberOfChunksTPC=$numberOfChunksTPC"
e9129651 353 return 0
9e160db9 354}
d157f353 355
421603c5 356parseConfig()
357{
358 args=("$@")
359 for opt in "${args[@]}"; do
360 [[ -z ${opt} ]] && continue
361 [[ -n ${encodedSpaces} ]] && opt="$(decSpaces ${opt})"
362 [[ "${opt}" =~ ^[[:space:]]*$ ]] && continue
363 if [[ ! "${opt}" =~ .*=.* ]]; then
364 echo "badly formatted option \"${opt}\" should be: option=value, stopping..."
365 return 1
366 fi
367 local var="${opt%%=*}"
368 local value="${opt#*=}"
369 export ${var}="${value}"
370 done
371
372 return 0
373}
374
9e160db9 375waitIfLocked()
376{
377 while [ 1 ]; do
378 [[ ! -f $1 ]] && break
379 sleep 1
380 done
e9129651 381 return 0
382}
383
384validateMerging()
385{
386 det=$1
387 retCode=0
388 [[ ! -f CalibObjects.root ]] && ((retCode++)) && echo "### no CalibObjects.root..."
389 [[ ! -f ${det}_merge_done ]] && ((retCode++)) && echo "### no ${det}_merge_done, job finished abnormally..."
390 error=$(grep -e "was a crash" *.log)
391 [[ -n $error ]] && ((retCode++)) && echo "### error: $error"
421603c5 392
e9129651 393 return $retCode
9e160db9 394}
d157f353 395
40f73007 396copyScripts()
397{
e9129651 398 [[ ! -f mergeByComponent.C ]] && \
399 cp -f $ALICE_ROOT/PWGPP/CalibMacros/CPass0/mergeByComponent.C $PWD && \
400 echo "taking the default scripts from $ALICE_ROOT"
401 [[ ! -f makeOCDB.C ]] && \
402 cp -f $ALICE_ROOT/PWGPP/CalibMacros/CPass0/makeOCDB.C $PWD && \
403 echo "taking the default scripts from $ALICE_ROOT"
40f73007 404}
405
421603c5 406main "$@"