10 Given a directory, will look into lib*.pkg files to produce a dependency graph
11 of libraries (and DA if there are some)
14 __author__ = "L. Aphecetche aphecetc_at_in2p3_dot_fr"
17 notassociatedfiles = {}
19 #_______________________________________________________________________________
21 """Describe usage of script
23 print "Usage: %s [-h | --help] [-d | --debug] [--da] directory_to_scan" % sys.argv[0]
26 #_______________________________________________________________________________
28 """ append a to list, if a not there yet
33 #_______________________________________________________________________________
38 #_______________________________________________________________________________
41 return re.search("^#",line)
43 #_______________________________________________________________________________
44 def iscontinuation(line):
46 return line.find('\\') > 0
48 #_______________________________________________________________________________
49 def compactLines(ilines):
50 """ Given an array of lines, remove empty lines, comment lines
51 and concatenate lines that should be one (i.e. removing the \ continuation
67 if line.find('\\') > 0:
68 line = re.sub("\t|\n|\r|\\\\","",line)
72 if isempty(currentline):
75 l = re.sub("\t","",line)
76 currentline = currentline + re.sub(" {1,}"," ",l)
77 if not iscomment(currentline) and not isempty(currentline):
78 olines.append(re.sub(" {1,}"," ",currentline))
82 currentline = currentline + line
86 #_______________________________________________________________________________
87 def tokenizeLines(lines):
89 Return a dict of keys -> value items.
90 keys are the left part of lines supposed to be of the form :
104 separators = [ define, plus, equal ]
112 value = re.sub("\n|\t|\n","",a[1])
114 print "Something fishy here !"
119 if not key in tokens.keys():
128 #_______________________________________________________________________________
129 def variableSubstitution(tokenname,alltokens):
133 value = alltokens.get(tokenname,"")
135 for k in alltokens.keys():
136 if re.search("\$\(%s\)"%k,value):
137 # found something like $(VARIABLE), so expand that variable,
138 # by calling us again
139 rep = value.replace("$(%s)"%k,variableSubstitution(k,alltokens))
141 if re.search("\$\(%s\:"%k,value):
142 # found something like $(VARIABLE:
143 # we suppose it's then something like $(VARIABLE:.x=.y)
144 # i.e. we replace x by y in VARIABLE's expansion
145 t = variableSubstitution(k,alltokens)
146 i1 = value.index(":")
147 i2 = value.index(")")
148 change = value[i1+1:i2].split("=")
149 return t.replace(change[0],change[1])
153 #_______________________________________________________________________________
154 def patternSubstitution(value):
156 if re.search("\$\(patsubst",value):
158 # found the Makefile function $(patsubst %.x, %.y, list)
159 i1 = value.index("(")
160 i2 = value.rindex(")")
161 a = value[i1+1:i2].split(",")
162 source = a[0].replace("patsubst ","")
163 destination = a[1].replace("%","")
165 print "Houston, we have a problem : ",value
167 for l in a[2].split():
168 rv.append(destination + l)
173 #_______________________________________________________________________________
174 def getSourceFiles2(lib,rootsys,alice_root):
175 """Extract the list of files from a libXXX.pkg file
176 Return a pair of list (sourceFiles,einclude), where einclude
177 is the list of directories needed to be included compile the files.
183 print "Cannot open file ", file
186 filelines = f.readlines()
190 lines = compactLines(filelines)
192 tokens = tokenizeLines(lines)
194 sourcesfiles = patternSubstitution(variableSubstitution("SRCS",tokens))
195 eincludes = patternSubstitution(variableSubstitution("EINCLUDE",tokens))
197 pkg = getLibPackage(lib)
198 dir = os.path.join(alice_root,pkg)
200 sourcesfiles = [ os.path.join(dir,x) for x in sourcesfiles ]
202 return sourcesfiles,eincludes
204 #_______________________________________________________________________________
205 def getSourceFiles(lib,rootsys,alice_root):
206 """Extract the list of files from a libXXX.pkg file
207 Return a pair of list (sourceFiles,einclude), where einclude
208 is the list of directories needed to be included compile the files.
211 # list of possible .pkg variables
212 pkgkeys = [ "SRCS","EINCLUDE","HDRS","FSRCS","DHDR","CSRCS","CHDRS","ELIBS","EDEFINE","PACKFFLAGS","PACKCXXFLAGS","PACKCFLAGS","PACKSOFLAGS","EXPORT","EHDRS" ]
215 keyEINCLUDE = pkgkeys[1]
218 pkg = getLibPackage(lib)
219 einclude = [ "%s/include" % rootsys, "%s/STEER" % alice_root, "%s/%s" % (alice_root,pkg) ]
221 dir = os.path.dirname(lib)
226 print "getSourceFiles : could not open package file %s" % lib
227 return sourcefiles, einclude
238 if re.search("^%s" % keySRCS,l):
242 if re.search("^%s" % keyEINCLUDE,l):
243 l = re.sub(keyEINCLUDE,' ',l)
244 l = re.sub(':',' ',l)
245 l = re.sub('=',' ',l)
246 l = re.sub('\+',' ',l)
249 append(einclude,os.path.join(alice_root,i))
252 if re.search('Ali',l) and ( re.search('.cxx',l) or re.search('.h',l) ):
253 l = re.sub(keySRCS,' ',l)
254 l = re.sub(':',' ',l)
255 l = re.sub('=',' ',l)
256 l = re.sub('\+',' ',l)
257 l = re.sub("\\\\",' ',l)
259 append(sourcefiles,os.path.join(dir,i))
262 return sourcefiles,einclude
264 #_______________________________________________________________________________
265 def getIncludeFiles2(srcfile,alice_root,alice_target,rootsys):
266 """Extract the list of included classes from a class, using the dep
267 files generated in $ALICE_ROOT/package/tgt_ALICE_TARGET/*.d files
268 It is much faster than getIncludeFile, as it reuses the output of
269 previously preprocessing part. Drawback is that it will only work
270 on a compiled version of aliroot...
275 package = getFilePackage(srcfile,alice_root)
276 file = re.sub("%s/%s" % (alice_root,package)," ",srcfile).strip()
279 depfile = "%s/%s/tgt_%s/%s" % (alice_root,package,alice_target,file)
280 depfile = re.sub("\.cxx",".d",depfile)
285 print "Could not open file %s" % depfile
294 parts = line.strip().split()
296 if re.search(rootsys,p):
300 p = "%s/%s" % (alice_root,p)
301 p = re.sub("%s/include" % alice_root,"%s/STEER" % alice_root,p)
308 #_______________________________________________________________________________
309 def getIncludeFiles(srcfile,eincludes,rootsys):
310 """Extract the list of included classes from a class, using :
312 and then parses the output...
313 This version is quite slow as we're (re-)doing the preprocessing of
314 all the files, but the advantage is that it'll work for a fresh checkout
315 of aliroot, i.e. even before compilation
323 # print "Could not open file %s" % srcfile
332 incdir = "%s -I%s" % (incdir,i)
334 cmd = "gcc %s -MM %s -MG" % (incdir,srcfile)
340 line = re.sub("\\\\"," ",line)
345 if len(line) > 0 and line != srcfile:
346 if line.find('/') < 0:
347 print "Got no path for file",srcfile," line=",line
349 if re.search(rootsys,line):
351 append(includes,line)
356 #_______________________________________________________________________________
358 """Extract a unique list from list
365 #_______________________________________________________________________________
366 def libshorten(libname):
367 """From libYYYxxx.pkg to YYYxxx
370 s = os.path.basename(libname)
371 if re.search("^lib",s):
372 s = re.sub("^lib","",s)
373 s = re.sub("\.pkg","",s)
377 #_______________________________________________________________________________
378 def fileshorten(file,path):
379 """From path/toto/file to toto/file
382 s = re.sub(path," ",file).strip()
388 #_______________________________________________________________________________
389 def getFilePackage(file,alice_root):
390 """ Get the package in which this file is defined
393 f = re.sub(alice_root,"/",file)
399 #_______________________________________________________________________________
400 def getLibPackage(libname):
401 """ Get the package in which this library is defined
404 p = libname.split('/')
407 #_______________________________________________________________________________
408 def tryRecover(f,inc2src,src2lib,alice_root):
409 """ This method should try to recover the "father" of file f (most probably
411 The idea would be to find a cxx file that *directly* includes f, and take
412 the lib of that cxx file as the source of f...
414 Is it needed really ?
418 print "tryRecover:",f
420 if not f.find('\.h'):
423 p = getFilePackage(f,alice_root)
425 cxxfiles = inc2src.get(f,[])
427 for file in cxxfiles:
428 libs = src2lib.get(file,[])
431 pl = getLibPackage(l)
437 #_______________________________________________________________________________
438 #_______________________________________________________________________________
439 #_______________________________________________________________________________
442 # we cannot work w/o those environement variables, so check them...
443 requiredVariables = [ "ROOTSYS", "ALICE_ROOT", "ALICE_TARGET" ]
445 for r in requiredVariables:
446 if not r in os.environ:
447 print "%s is not defined. Cannot work." % r
450 alice_root = os.environ.get("ALICE_ROOT")
451 rootsys = os.environ.get("ROOTSYS")
452 alice_target = os.environ.get("ALICE_TARGET")
458 opts, args = getopt.getopt(sys.argv[1:],"hd",["help", "debug","da"])
459 except getopt.GetoptError:
460 print "Error in options"
464 if o in ( "-d","--debug" ):
466 elif o in ( "-h","--help" ):
472 assert False, "unhandled option"
474 dir = os.path.abspath(args[0])
477 for sd in os.listdir(dir):
478 ld = os.path.join(dir,sd)
479 if os.path.isdir(ld) and not os.path.islink(ld):
484 # requestedPackages = [ "MUON", "STEER", "RAW", "ITS", "TRD", "VZERO", "TPC", "PHOS", "TOF", "ZDC", "EMCAL", "HMPID", "SHUTTLE", "ACORDE", "HLT", "EVE" ];
485 requestedPackages = [ "RAW", "STEER", "MUON", "HLT","EVE" ]
487 # find the libraries defined in this directory (looking for libXXXX.pkg files)
491 for f in os.listdir(d):
492 fulllib = os.path.join(d,f)
493 p = getLibPackage(fulllib)
494 if not p in requestedPackages:
496 if re.search('^lib',f) and re.search('.pkg$',f):
497 libraries.append(fulllib)
498 if not noda and re.search('da.cxx',f) and not re.search('.svn',f):
499 # append fake libraries for DAs
500 tmp = re.sub("cxx","pkg",f)
502 libraries.append(os.path.join(d,tmp))
504 # from list of library files (libXXXyyy.pkg), try to find back the list of
514 packages[p].append(l)
516 # src2inc[file.cxx] -> { all included files of that file }
519 # inc2src[file.h] -> { all files that include that one }
522 # lib2src[libXXX.pkg] -> { list of files of that library }
525 # src2lib[file.cxx] -> { list of libraries including that file }
528 # eincludes[libXXX.pkg] -> list of directories to be included to be able to compile the files
531 # lib2inc[libXXX.pkg] -> { list of all included files of that library }
534 # inc2lib[file.h] -> { list of libraries that include that file }
541 for lib in packages[p]:
545 print " ",libshorten(lib),"..."
547 if not re.search("da.pkg",lib):
548 # handle the special case of DAs which are not part of libs, really
549 lib2src[lib], eincludes[lib] = getSourceFiles2(lib,rootsys,alice_root)
552 l = re.sub("lib","",l)
553 l = re.sub("\.pkg","",l)
554 lib2src[lib] = [ "%s.cxx" % l ]
559 for src in lib2src[lib]:
560 # inc = getIncludeFiles(src,eincludes[lib],rootsys)
561 inc = getIncludeFiles2(src,alice_root,alice_target,rootsys)
564 if not src in src2lib:
567 append(src2lib[src],lib)
570 if not i in inc2src.keys():
572 append(inc2src[i],src)
573 append(lib2inc[lib],i)
574 if not i in inc2lib.keys():
576 append(inc2lib[i],lib)
578 # some debug at this point...
581 for lib in libraries:
582 print lib," is made of "
583 for f in lib2src[lib]:
584 print " ",fileshorten(f,alice_root)
585 print " and includes "
586 for h in lib2inc[lib]:
587 print " ",fileshorten(h,alice_root)
588 if len(eincludes[lib]) > 0:
589 print " and needs the following directories to be compiled "
590 for f in eincludes[lib]:
594 print "src2lib relationship"
595 for src,lib in src2lib.items():
596 print fileshorten(src,alice_root),"(",
598 print libshorten(l)," ",
601 # now fills the ultimate array, lib2lib
602 # lib2lib[libXXX.pkg] -> { libYYY.pkg }, list of libraries that lib depends on
606 for lib in libraries:
610 for hfile in lib2inc[lib]:
614 # start simple : is f contains ROOTSYS, it's ROOT.
615 if re.search(rootsys,hfile):
618 # not that simple, let's try to find out...
619 cxx = re.sub("\.h",".cxx",hfile)
620 dl = src2lib.get(cxx,[])
624 print "Got several libs(",len(dl),"for ",hfile,":"
628 notassociatedfiles[hfile] = 1
630 append(lib2lib[lib],l)
632 ###################### Debug parts...
634 for lib in libraries:
635 print libshorten(lib),"depends on"
636 for f in lib2lib[lib]:
637 print " ",libshorten(f)
641 print "From source files to include files : "
642 for cxxfile, hfile in src2inc.items():
643 print fileshorten(cxxfile,alice_root)
645 print " ",fileshorten(h,alice_root)
649 print "From include files to source files : "
650 for i,sources in inc2src.items():
651 print fileshorten(i,alice_root), len(sources)
653 print " ",fileshorten(s,alice_root),"(",
657 ######################
659 if len(notassociatedfiles) > 0:
660 print "The following files could not be associated with any library..."
661 for f in notassociatedfiles.keys():
663 t=tryRecover(f,inc2src,src2lib,alice_root)
666 # output the dot file that will have to be processed by the dot program
668 ofile = "%s.dot" % os.path.splitext(os.path.basename(sys.argv[0]))[0]
672 f.write("digraph G {\n")
673 f.write("rankdir=BT;\n")
675 defaultcolor = "lightblue"
679 colors["MUON"] = "lightyellow"
680 colors["STEER"] = "lightgray"
682 for l,d in lib2lib.items():
684 f.write("%s->%s;\n" %(libshorten(l),libshorten(dl)))
687 f.write("subgraph cluster_%s {\n" % p.lower())
688 color = colors.get(p,defaultcolor)
689 f.write("style=filled;\n")
690 f.write("color=%s;\n" % color)
691 f.write('label="%s";\n' % p)
692 for lib in packages[p]:
693 f.write("%s\n" % libshorten(lib))
698 print "You should now do :"
699 print "tred %s > %s.bis" % ( ofile, ofile )
700 print "dot -Tpng %s.bis -o %s.png" % (ofile,ofile)
702 if __name__ == "__main__":