bug fixed
[u/mrichter/AliRoot.git] / MUON / checkDeps.py
CommitLineData
ea1f7807 1#!/usr/bin/python
2
3import sys
4import os
5import re
6import string
7import getopt
8
9"""
10Given a directory, will look into lib*.pkg files to produce a dependency graph
11 of libraries (and DA if there are some)
ea1f7807 12"""
13
14__author__ = "L. Aphecetche aphecetc_at_in2p3_dot_fr"
48739cf2 15__version__ = "$Id$"
ea1f7807 16
58e0ac49 17notassociatedfiles = {}
18
ea1f7807 19#_______________________________________________________________________________
20def usage():
21 """Describe usage of script
22 """
58e0ac49 23 print "Usage: %s [-h | --help] [-d | --debug] [--da] directory_to_scan" % sys.argv[0]
ea1f7807 24 sys.exit(1)
25
26#_______________________________________________________________________________
58e0ac49 27def append(list,a):
28 """ append a to list, if a not there yet
ea1f7807 29 """
58e0ac49 30 if not a in list:
31 list.append(a)
32
630711ed 33#_______________________________________________________________________________
34def isempty(line):
35
36 return len(line) < 2
37
38#_______________________________________________________________________________
39def iscomment(line):
40
41 return re.search("^#",line)
42
43#_______________________________________________________________________________
44def iscontinuation(line):
45
46 return line.find('\\') > 0
47
48#_______________________________________________________________________________
49def compactLines(ilines):
50 """ Given an array of lines, remove empty lines, comment lines
51 and concatenate lines that should be one (i.e. removing the \ continuation
52 marks...
53 """
54
55 continuation = False
56
57 olines = []
58
59 currentline = ""
60
61 i = 0
62
63 for line in ilines:
64
65 i = i + 1
66
67 if line.find('\\') > 0:
68 line = re.sub("\t|\n|\r|\\\\","",line)
69 continuation = True
70 else:
71 continuation = False
72 if isempty(currentline):
73 currentline = line
74 else:
75 l = re.sub("\t","",line)
76 currentline = currentline + re.sub(" {1,}"," ",l)
77 if not iscomment(currentline) and not isempty(currentline):
78 olines.append(re.sub(" {1,}"," ",currentline))
79 currentline = ""
80
81 if continuation:
82 currentline = currentline + line
83
84 return olines
85
86#_______________________________________________________________________________
87def tokenizeLines(lines):
88 """
89 Return a dict of keys -> value items.
90 keys are the left part of lines supposed to be of the form :
91 KEY:=VALUE
92 or
93 KEY+=VALUE
94 or
95 KEY=VALUE
96 """
97
98 tokens = {}
99
100 define = ":="
101 plus = "+="
102 equal = "="
103
104 separators = [ define, plus, equal ]
105
106 for l in lines:
107 sep = False
108 for s in separators:
109 if s in l:
110 a = l.split(s,1)
111 key = a[0].strip()
112 value = re.sub("\n|\t|\n","",a[1])
113 if len(a) > 2:
114 print "Something fishy here !"
115 sep = True
116 break
117 if not sep:
118 continue
119 if not key in tokens.keys():
120 tokens[key] = ""
121 if s == plus:
122 tokens[key] += value
123 else:
124 tokens[key] = value
125
126 return tokens
127
128#_______________________________________________________________________________
129def variableSubstitution(tokenname,alltokens):
130 """
131 """
132
133 value = alltokens.get(tokenname,"")
134
135 for k in alltokens.keys():
136 if re.search("\$\(%s\)"%k,value):
137 # found something like $(VARIABLE), so expand that variable,
138 # by calling us again
139 rep = value.replace("$(%s)"%k,variableSubstitution(k,alltokens))
140 return rep
141 if re.search("\$\(%s\:"%k,value):
142 # found something like $(VARIABLE:
143 # we suppose it's then something like $(VARIABLE:.x=.y)
144 # i.e. we replace x by y in VARIABLE's expansion
145 t = variableSubstitution(k,alltokens)
146 i1 = value.index(":")
147 i2 = value.index(")")
148 change = value[i1+1:i2].split("=")
149 return t.replace(change[0],change[1])
150
151 return value
152
153#_______________________________________________________________________________
154def patternSubstitution(value):
155
156 if re.search("\$\(patsubst",value):
157 rv = []
158 # found the Makefile function $(patsubst %.x, %.y, list)
159 i1 = value.index("(")
160 i2 = value.rindex(")")
161 a = value[i1+1:i2].split(",")
162 source = a[0].replace("patsubst ","")
163 destination = a[1].replace("%","")
164 if source != "%":
165 print "Houston, we have a problem : ",value
166 sys.exit(1)
167 for l in a[2].split():
168 rv.append(destination + l)
169 return rv
170
171 return value.split()
172
173#_______________________________________________________________________________
174def getSourceFiles2(lib,rootsys,alice_root):
175 """Extract the list of files from a libXXX.pkg file
176 Return a pair of list (sourceFiles,einclude), where einclude
177 is the list of directories needed to be included compile the files.
178 """
179
180 try:
181 f = open(lib)
182 except:
183 print "Cannot open file ", file
184 sys.exit(1)
185
186 filelines = f.readlines()
187
188 f.close()
189
190 lines = compactLines(filelines)
191
192 tokens = tokenizeLines(lines)
193
194 sourcesfiles = patternSubstitution(variableSubstitution("SRCS",tokens))
195 eincludes = patternSubstitution(variableSubstitution("EINCLUDE",tokens))
196
197 pkg = getLibPackage(lib)
198 dir = os.path.join(alice_root,pkg)
199
200 sourcesfiles = [ os.path.join(dir,x) for x in sourcesfiles ]
201
202 return sourcesfiles,eincludes
ea1f7807 203
58e0ac49 204#_______________________________________________________________________________
205def getSourceFiles(lib,rootsys,alice_root):
206 """Extract the list of files from a libXXX.pkg file
207 Return a pair of list (sourceFiles,einclude), where einclude
208 is the list of directories needed to be included compile the files.
209 """
210
211 # list of possible .pkg variables
212 pkgkeys = [ "SRCS","EINCLUDE","HDRS","FSRCS","DHDR","CSRCS","CHDRS","ELIBS","EDEFINE","PACKFFLAGS","PACKCXXFLAGS","PACKCFLAGS","PACKSOFLAGS","EXPORT","EHDRS" ]
213
214 keySRCS = pkgkeys[0]
215 keyEINCLUDE = pkgkeys[1]
216
ea1f7807 217 sourcefiles = []
58e0ac49 218 pkg = getLibPackage(lib)
219 einclude = [ "%s/include" % rootsys, "%s/STEER" % alice_root, "%s/%s" % (alice_root,pkg) ]
220
221 dir = os.path.dirname(lib)
222
223 try:
224 f = open(lib)
225 except:
226 print "getSourceFiles : could not open package file %s" % lib
227 return sourcefiles, einclude
228
229 src = False
230
ea1f7807 231 for line in f:
232 l = line.strip()
58e0ac49 233 key = False
234 for k in pkgkeys:
235 if re.search(k,l):
236 key = True
237 if key:
238 if re.search("^%s" % keySRCS,l):
239 src = True
240 else:
241 src = False
242 if re.search("^%s" % keyEINCLUDE,l):
243 l = re.sub(keyEINCLUDE,' ',l)
244 l = re.sub(':',' ',l)
245 l = re.sub('=',' ',l)
246 l = re.sub('\+',' ',l)
247 a = l.split()
248 for i in a:
249 append(einclude,os.path.join(alice_root,i))
250
251 if src:
252 if re.search('Ali',l) and ( re.search('.cxx',l) or re.search('.h',l) ):
253 l = re.sub(keySRCS,' ',l)
254 l = re.sub(':',' ',l)
255 l = re.sub('=',' ',l)
256 l = re.sub('\+',' ',l)
257 l = re.sub("\\\\",' ',l)
258 for i in l.split():
259 append(sourcefiles,os.path.join(dir,i))
260
ea1f7807 261 f.close()
58e0ac49 262 return sourcefiles,einclude
ea1f7807 263
264#_______________________________________________________________________________
58e0ac49 265def getIncludeFiles2(srcfile,alice_root,alice_target,rootsys):
266 """Extract the list of included classes from a class, using the dep
267 files generated in $ALICE_ROOT/package/tgt_ALICE_TARGET/*.d files
268 It is much faster than getIncludeFile, as it reuses the output of
269 previously preprocessing part. Drawback is that it will only work
270 on a compiled version of aliroot...
ea1f7807 271 """
272
273 includes = []
274
58e0ac49 275 package = getFilePackage(srcfile,alice_root)
276 file = re.sub("%s/%s" % (alice_root,package)," ",srcfile).strip()
277 if file[0] == '/':
278 file = file[1:]
279 depfile = "%s/%s/tgt_%s/%s" % (alice_root,package,alice_target,file)
280 depfile = re.sub("\.cxx",".d",depfile)
281
ea1f7807 282 try:
58e0ac49 283 f = open(depfile)
ea1f7807 284 except:
58e0ac49 285 print "Could not open file %s" % depfile
286 print "From",srcfile
ea1f7807 287 return includes
288
289 for line in f:
290 line = line.strip()
58e0ac49 291 i = line.find(":")
292 if i > 0:
293 line = line[i+1:]
294 parts = line.strip().split()
295 for p in parts:
296 if re.search(rootsys,p):
297 p = rootsys
298 else:
299 if p[0] != '/':
300 p = "%s/%s" % (alice_root,p)
301 p = re.sub("%s/include" % alice_root,"%s/STEER" % alice_root,p)
302 append(includes,p)
303
ea1f7807 304 f.close()
58e0ac49 305
306 return includes
307
308#_______________________________________________________________________________
309def getIncludeFiles(srcfile,eincludes,rootsys):
310 """Extract the list of included classes from a class, using :
311 gcc -MM srcfile -MG
312 and then parses the output...
313 This version is quite slow as we're (re-)doing the preprocessing of
314 all the files, but the advantage is that it'll work for a fresh checkout
315 of aliroot, i.e. even before compilation
316 """
317
318 includes = []
319
320# try:
321# f = open(srcfile)
322# except:
323# print "Could not open file %s" % srcfile
324# return includes
325#
326# f.close()
327
328
329 incdir = ""
330
331 for i in eincludes:
332 incdir = "%s -I%s" % (incdir,i)
333
334 cmd = "gcc %s -MM %s -MG" % (incdir,srcfile)
335
336 pre = os.popen(cmd)
337
338 for line in pre:
339 line = line.strip()
340 line = re.sub("\\\\"," ",line)
341 i = line.find(":")
342 if i > 0:
343 line = line[i+1:]
344 line = line.strip()
345 if len(line) > 0 and line != srcfile:
346 if line.find('/') < 0:
347 print "Got no path for file",srcfile," line=",line
348 print "cmd was",cmd
349 if re.search(rootsys,line):
350 line = rootsys
351 append(includes,line)
352 pre.close()
353
ea1f7807 354 return includes
355
356#_______________________________________________________________________________
357def unique(list):
358 """Extract a unique list from list
359 """
360 d = {}
361 for l in list:
362 d[l] = 1
363 return d.keys()
364
365#_______________________________________________________________________________
58e0ac49 366def libshorten(libname):
367 """From libYYYxxx.pkg to YYYxxx
ea1f7807 368 """
ea1f7807 369
58e0ac49 370 s = os.path.basename(libname)
371 if re.search("^lib",s):
372 s = re.sub("^lib","",s)
373 s = re.sub("\.pkg","",s)
374
375 return s
376
ea1f7807 377#_______________________________________________________________________________
58e0ac49 378def fileshorten(file,path):
379 """From path/toto/file to toto/file
ea1f7807 380 """
58e0ac49 381
382 s = re.sub(path," ",file).strip()
383 if s[0] == '/':
384 s = s[1:]
385
ea1f7807 386 return s
58e0ac49 387
388#_______________________________________________________________________________
389def getFilePackage(file,alice_root):
390 """ Get the package in which this file is defined
391 """
392
393 f = re.sub(alice_root,"/",file)
394 while f[0] == '/':
395 f = f[1:]
396 p = f.split('/')
397 return p[0]
398
399#_______________________________________________________________________________
400def getLibPackage(libname):
401 """ Get the package in which this library is defined
402 """
403
404 p = libname.split('/')
405 return p[len(p)-2]
406
407#_______________________________________________________________________________
408def tryRecover(f,inc2src,src2lib,alice_root):
409 """ This method should try to recover the "father" of file f (most probably
410 f is an include file
411 The idea would be to find a cxx file that *directly* includes f, and take
412 the lib of that cxx file as the source of f...
413 Would that work ?
414 Is it needed really ?
415 """
416
417 """
418 print "tryRecover:",f
419
420 if not f.find('\.h'):
421 return ""
422
423 p = getFilePackage(f,alice_root)
424
425 cxxfiles = inc2src.get(f,[])
426
427 for file in cxxfiles:
428 libs = src2lib.get(file,[])
429
430 for l in libs:
431 pl = getLibPackage(l)
432 print f,file,p,pl
433 """
434
435 return ""
ea1f7807 436
437#_______________________________________________________________________________
438#_______________________________________________________________________________
439#_______________________________________________________________________________
440def main():
441
58e0ac49 442 # we cannot work w/o those environement variables, so check them...
443 requiredVariables = [ "ROOTSYS", "ALICE_ROOT", "ALICE_TARGET" ]
444
445 for r in requiredVariables:
446 if not r in os.environ:
447 print "%s is not defined. Cannot work." % r
448 sys.exit(1)
449
450 alice_root = os.environ.get("ALICE_ROOT")
451 rootsys = os.environ.get("ROOTSYS")
452 alice_target = os.environ.get("ALICE_TARGET")
453
454 debug = 0
455 noda = True
ea1f7807 456
457 try:
58e0ac49 458 opts, args = getopt.getopt(sys.argv[1:],"hd",["help", "debug","da"])
ea1f7807 459 except getopt.GetoptError:
460 print "Error in options"
461 usage()
462
463 for o, a in opts:
464 if o in ( "-d","--debug" ):
58e0ac49 465 debug = debug + 1
ea1f7807 466 elif o in ( "-h","--help" ):
467 usage()
468 sys.exit()
58e0ac49 469 elif o == "--da":
470 noda = False
ea1f7807 471 else:
472 assert False, "unhandled option"
473
58e0ac49 474 dir = os.path.abspath(args[0])
475 dirs = []
476
477 for sd in os.listdir(dir):
478 ld = os.path.join(dir,sd)
479 if os.path.isdir(ld) and not os.path.islink(ld):
480 dirs.append(ld)
481
482 dirs.append(dir)
483
630711ed 484# requestedPackages = [ "MUON", "STEER", "RAW", "ITS", "TRD", "VZERO", "TPC", "PHOS", "TOF", "ZDC", "EMCAL", "HMPID", "SHUTTLE", "ACORDE", "HLT", "EVE" ];
485 requestedPackages = [ "RAW", "STEER", "MUON", "HLT","EVE" ]
ea1f7807 486
487 # find the libraries defined in this directory (looking for libXXXX.pkg files)
488 libraries = []
489
58e0ac49 490 for d in dirs:
491 for f in os.listdir(d):
492 fulllib = os.path.join(d,f)
493 p = getLibPackage(fulllib)
494 if not p in requestedPackages:
495 continue
496 if re.search('^lib',f) and re.search('.pkg$',f):
497 libraries.append(fulllib)
498 if not noda and re.search('da.cxx',f) and not re.search('.svn',f):
499 # append fake libraries for DAs
500 tmp = re.sub("cxx","pkg",f)
501 tmp = "lib%s" % tmp
502 libraries.append(os.path.join(d,tmp))
503
504 # from list of library files (libXXXyyy.pkg), try to find back the list of
505 # "packages" = XXX
506 packages = {}
507
508 for l in libraries:
509 p = getLibPackage(l)
510 packages[p] = []
511
512 for l in libraries:
513 p = getLibPackage(l)
514 packages[p].append(l)
515
516 # src2inc[file.cxx] -> { all included files of that file }
517 src2inc = {}
518
519 # inc2src[file.h] -> { all files that include that one }
520 inc2src = {}
521
522 # lib2src[libXXX.pkg] -> { list of files of that library }
523 lib2src = {}
524
525 # src2lib[file.cxx] -> { list of libraries including that file }
526 src2lib = {}
527
528 # eincludes[libXXX.pkg] -> list of directories to be included to be able to compile the files
529 eincludes = {}
530
531 # lib2inc[libXXX.pkg] -> { list of all included files of that library }
532 lib2inc = {}
533
534 # inc2lib[file.h] -> { list of libraries that include that file }
535 inc2lib = {}
536
537 for p in packages:
538
539 print "Scanning ",p
540
541 for lib in packages[p]:
542
543 lib2inc[lib] = []
544
545 print " ",libshorten(lib),"..."
546
547 if not re.search("da.pkg",lib):
548 # handle the special case of DAs which are not part of libs, really
630711ed 549 lib2src[lib], eincludes[lib] = getSourceFiles2(lib,rootsys,alice_root)
58e0ac49 550 else:
551 l = lib
552 l = re.sub("lib","",l)
553 l = re.sub("\.pkg","",l)
554 lib2src[lib] = [ "%s.cxx" % l ]
555 eincludes[lib] = []
556
557 files = []
ea1f7807 558
58e0ac49 559 for src in lib2src[lib]:
560# inc = getIncludeFiles(src,eincludes[lib],rootsys)
561 inc = getIncludeFiles2(src,alice_root,alice_target,rootsys)
562 src2inc[src] = inc
563
564 if not src in src2lib:
565 src2lib[src] = []
566
567 append(src2lib[src],lib)
568
569 for i in inc:
570 if not i in inc2src.keys():
571 inc2src[i] = []
572 append(inc2src[i],src)
573 append(lib2inc[lib],i)
574 if not i in inc2lib.keys():
575 inc2lib[i] = []
576 append(inc2lib[i],lib)
577
578 # some debug at this point...
ea1f7807 579
58e0ac49 580 if debug>=1:
581 for lib in libraries:
582 print lib," is made of "
583 for f in lib2src[lib]:
584 print " ",fileshorten(f,alice_root)
585 print " and includes "
586 for h in lib2inc[lib]:
587 print " ",fileshorten(h,alice_root)
588 if len(eincludes[lib]) > 0:
589 print " and needs the following directories to be compiled "
590 for f in eincludes[lib]:
591 print " ",f
592
593 if debug>=2:
594 print "src2lib relationship"
595 for src,lib in src2lib.items():
596 print fileshorten(src,alice_root),"(",
597 for l in lib:
598 print libshorten(l)," ",
599 print ")"
600
601 # now fills the ultimate array, lib2lib
602 # lib2lib[libXXX.pkg] -> { libYYY.pkg }, list of libraries that lib depends on
ea1f7807 603
58e0ac49 604 lib2lib = {}
ea1f7807 605
606 for lib in libraries:
58e0ac49 607
608 lib2lib[lib] = []
609
610 for hfile in lib2inc[lib]:
611
612 l = "external"
613
614 # start simple : is f contains ROOTSYS, it's ROOT.
615 if re.search(rootsys,hfile):
616 l = "ROOT"
617 else:
618 # not that simple, let's try to find out...
619 cxx = re.sub("\.h",".cxx",hfile)
620 dl = src2lib.get(cxx,[])
621 if len(dl)==1:
622 l = dl[0]
623 elif len(dl)>1:
624 print "Got several libs(",len(dl),"for ",hfile,":"
625 print dl
626
627 if l =="external":
628 notassociatedfiles[hfile] = 1
629 else:
630 append(lib2lib[lib],l)
631
632 ###################### Debug parts...
633 if debug>=1:
ea1f7807 634 for lib in libraries:
58e0ac49 635 print libshorten(lib),"depends on"
636 for f in lib2lib[lib]:
637 print " ",libshorten(f)
638
639 if debug>=2:
640 print
641 print "From source files to include files : "
642 for cxxfile, hfile in src2inc.items():
643 print fileshorten(cxxfile,alice_root)
644 for h in hfile:
645 print " ",fileshorten(h,alice_root)
646
647 if debug>=3:
ea1f7807 648 print
58e0ac49 649 print "From include files to source files : "
650 for i,sources in inc2src.items():
651 print fileshorten(i,alice_root), len(sources)
652 for s in sources:
653 print " ",fileshorten(s,alice_root),"(",
654 for l in src2lib[s]:
655 print libshorten(l),
656 print ")"
657 ######################
658
659 if len(notassociatedfiles) > 0:
660 print "The following files could not be associated with any library..."
661 for f in notassociatedfiles.keys():
662 print f,
663 t=tryRecover(f,inc2src,src2lib,alice_root)
664 print t
ea1f7807 665
58e0ac49 666 # output the dot file that will have to be processed by the dot program
667
ea1f7807 668 ofile = "%s.dot" % os.path.splitext(os.path.basename(sys.argv[0]))[0]
669
670 f = open(ofile,"w")
671
58e0ac49 672 f.write("digraph G {\n")
673 f.write("rankdir=BT;\n")
ea1f7807 674
58e0ac49 675 defaultcolor = "lightblue"
676
677 colors = {}
678
679 colors["MUON"] = "lightyellow"
680 colors["STEER"] = "lightgray"
681
682 for l,d in lib2lib.items():
ea1f7807 683 for dl in d:
58e0ac49 684 f.write("%s->%s;\n" %(libshorten(l),libshorten(dl)))
ea1f7807 685
58e0ac49 686 for p in packages:
687 f.write("subgraph cluster_%s {\n" % p.lower())
688 color = colors.get(p,defaultcolor)
689 f.write("style=filled;\n")
690 f.write("color=%s;\n" % color)
691 f.write('label="%s";\n' % p)
692 for lib in packages[p]:
693 f.write("%s\n" % libshorten(lib))
694 f.write("}\n")
695
696 f.write("}\n")
ea1f7807 697
698 print "You should now do :"
699 print "tred %s > %s.bis" % ( ofile, ofile )
58e0ac49 700 print "dot -Tpng %s.bis -o %s.png" % (ofile,ofile)
ea1f7807 701
702if __name__ == "__main__":
703 main()
704