Linux_Kernel_map4vehicle/srcxray.py

487 lines
14 KiB
Python
Raw Normal View History

2018-07-19 02:46:21 +08:00
#!/usr/bin/python3
#
2018-07-19 04:48:22 +08:00
# srcxray - source code X-ray
2018-07-19 02:46:21 +08:00
#
# Analyzes interconnections between functions and structures in source code.
#
# Uses cscope and git grep --show-function to
# reveal references between identifiers.
#
# 2018 Constantine Shulyupin, const@MakeLinux.com
#
2018-07-19 12:52:33 +08:00
import inspect
import random
import os
import sys
import collections
import subprocess
import re
2018-07-22 05:32:07 +08:00
import networkx as nx
2018-07-22 21:19:45 +08:00
from networkx.drawing.nx_agraph import write_dot
2018-07-19 02:46:21 +08:00
2018-07-19 12:52:33 +08:00
black_list = ['aligned', '__attribute__', 'unlikely', 'typeof', 'u32',
2018-07-19 13:34:54 +08:00
'PVOP_CALLEE0', 'PVOP_VCALLEE0', 'PVOP_VCALLEE1', 'if',
'trace_hardirqs_off']
2018-07-19 02:46:21 +08:00
2018-07-21 03:06:51 +08:00
level_limit = 8
2018-07-19 02:46:21 +08:00
limit = 10000
n = 0
2018-07-19 04:48:22 +08:00
2018-07-19 02:46:21 +08:00
def print_limited(a):
print(a)
global n
2018-07-19 04:48:22 +08:00
n += 1
2018-07-22 04:01:11 +08:00
if n > limit + 1:
print('...')
2018-07-19 02:46:21 +08:00
sys.exit(1)
2018-07-22 04:01:11 +08:00
# raise(Exception('Reached limit'))
2018-07-19 02:46:21 +08:00
2018-07-19 04:48:22 +08:00
2018-07-19 02:46:21 +08:00
def log(*args, **kwargs):
2018-07-19 12:52:33 +08:00
print(inspect.stack()[1][3],
str(*args).rstrip(), file=sys.stderr, **kwargs)
2018-07-19 02:46:21 +08:00
pass
2018-07-19 12:52:33 +08:00
2018-07-19 02:46:21 +08:00
def popen(p):
2018-07-22 04:01:48 +08:00
return [a.decode('utf-8') for a in subprocess.check_output(p, shell=True)
.splitlines()]
2018-07-19 02:46:21 +08:00
2018-07-19 12:52:33 +08:00
2018-07-19 02:46:21 +08:00
def extract_referer(line):
line = re.sub(r'__ro_after_init', '', line)
line = re.sub(r'FNAME\((\w+)\)', r'\1', line)
line = re.sub(r'.*TRACE_EVENT.*', '', line)
m = re.match(r'^[^\s]+=[^,]*\(\*(\b\w+)\)\s*[\(\[=][^;]*$', line)
if not m:
m = re.match(r'^[^\s]+=[^,]*(\b\w+)\s*[\(\[=][^;]*$', line)
if m:
return m.group(1)
2018-07-19 12:52:33 +08:00
2018-07-19 02:46:21 +08:00
def extract_referer_test():
for a in {
"fs=good2()",
2018-07-19 13:29:29 +08:00
"f=static int fastop(struct x86_emulate_ctxt *ctxt, "
+ "void (*fop)(struct fastop *))",
2018-07-19 02:46:21 +08:00
"f=int good(a, bad (*func)(arg))",
"f=EXPORT_SYMBOL_GPL(bad);",
"f=bad (*good)()",
"f=int FNAME(good)(a)",
"f=TRACE_EVENT(a)",
2018-07-19 04:48:22 +08:00
"f: a=in bad()"}:
2018-07-19 02:46:21 +08:00
print(a, '->', extract_referer(a))
2018-07-19 12:52:33 +08:00
2018-07-19 02:46:21 +08:00
def func_referers_git_grep(name):
2018-07-19 17:54:11 +08:00
res = set()
2018-07-19 02:46:21 +08:00
r = None
2018-07-19 13:29:29 +08:00
for line in popen(r'git grep --no-index --word-regexp --show-function '
2018-07-19 17:54:54 +08:00
r'"^\s.*\b%s" '
2018-07-22 04:05:18 +08:00
r'**.\[hc\] **.cpp **.cc **.hh' % (name)):
2018-07-19 13:29:29 +08:00
# Filter out names in comment afer function,
# when comment start from ' *'
2018-07-19 02:46:21 +08:00
# To see the problem try "git grep -p and"
2018-07-19 13:34:54 +08:00
for p in {
2018-07-19 17:54:54 +08:00
r'.*:\s+\* .*%s',
2018-07-19 13:34:54 +08:00
r'.*/\*.*%s',
r'.*//.*%s',
r'.*".*\b%s\b.*"'}:
if re.match(p % (name), line):
r = None
break
2018-07-19 13:29:29 +08:00
if r and r != name and r not in black_list:
2018-07-19 17:54:11 +08:00
res.add(r)
2018-07-19 02:46:21 +08:00
r = None
r = extract_referer(line)
return res
2018-07-19 12:52:33 +08:00
2018-07-19 02:46:21 +08:00
cscope_warned = False
2018-07-19 12:52:33 +08:00
2018-07-19 02:46:21 +08:00
def func_referers_cscope(name):
global cscope_warned
if not os.path.isfile('cscope.out'):
if not cscope_warned:
print("Recommended: cscope -bkR", file=sys.stderr)
cscope_warned = True
return []
2018-07-19 17:54:11 +08:00
res = set([l.split()[1] for l in popen(r'cscope -d -L3 "%s"' %
2018-07-22 04:05:18 +08:00
(name)) if l not in black_list])
2018-07-19 02:46:21 +08:00
if not res:
res = func_referers_git_grep(name)
return res
2018-07-19 12:52:33 +08:00
2018-07-19 02:46:21 +08:00
def func_referers_all(name):
return set(func_referers_git_grep(name) + func_referers_cscope(name))
2018-07-19 12:52:33 +08:00
def referers_tree(name, referer=None, printed=None, level=0):
2018-07-19 02:46:21 +08:00
if not referer:
if os.path.isfile('cscope.out'):
referer = func_referers_cscope
else:
2018-07-19 12:52:33 +08:00
print("Using git grep only, recommended to run: cscope -bkR",
file=sys.stderr)
2018-07-19 02:46:21 +08:00
referer = func_referers_git_grep
if isinstance(referer, str):
referer = eval(referer)
2018-07-19 12:52:33 +08:00
if not printed:
printed = set()
2018-07-19 02:46:21 +08:00
if name in printed:
print_limited(level*'\t' + name + ' ^')
return
else:
print_limited(level*'\t' + name)
printed.add(name)
if level > level_limit - 2:
print_limited((level + 1)*'\t' + '...')
return ''
listed = set()
for a in referer(name):
referers_tree(a, referer, printed, level + 1)
listed.add(a)
return ''
2018-07-19 12:52:33 +08:00
2018-07-19 19:03:30 +08:00
def referers_dep(name, referer=None, printed=None, level=0):
if not referer:
if os.path.isfile('cscope.out'):
referer = func_referers_cscope
else:
print("Using git grep only, recommended to run: cscope -bkR",
file=sys.stderr)
referer = func_referers_git_grep
if isinstance(referer, str):
referer = eval(referer)
if not printed:
printed = set()
if name in printed:
return
if level > level_limit - 2:
return ''
referers = set(referer(name))
if referers:
printed.add(name)
print(name, end=': ')
for a in referers:
print(a, end=' ')
print()
for a in referers:
referers_dep(a, referer, printed, level + 1)
else:
pass
# TODO: print terminal
# print('...')
return ''
2018-07-19 12:52:33 +08:00
def call_tree(node, printed=None, level=0):
2018-07-19 02:46:21 +08:00
if not os.path.isfile('cscope.out'):
2018-07-19 12:52:33 +08:00
print("Please run: cscope -bkR", file=sys.stderr)
return False
2018-07-19 13:29:29 +08:00
if printed is None:
2018-07-19 12:52:33 +08:00
printed = set()
2018-07-19 02:46:21 +08:00
if node in printed:
2018-07-19 04:48:22 +08:00
print_limited(level*'\t' + node + ' ^')
2018-07-19 02:46:21 +08:00
return
else:
print_limited(level*'\t' + node)
printed.add(node)
2018-07-19 04:48:22 +08:00
if level > level_limit - 2:
2018-07-19 02:46:21 +08:00
print_limited((level + 1)*'\t' + '...')
return ''
local_printed = set()
2018-07-19 12:52:33 +08:00
for line in popen('cscope -d -L2 "%s"' % (node)):
2018-07-19 13:29:29 +08:00
a = line.split()[1]
if a in local_printed or a in black_list:
2018-07-19 12:52:33 +08:00
continue
2018-07-19 13:29:29 +08:00
local_printed.add(a)
2018-07-19 16:41:45 +08:00
# try:
call_tree(line.split()[1], printed, level + 1)
# except Exception:
# pass
2018-07-19 02:46:21 +08:00
return ''
2018-07-19 12:52:33 +08:00
2018-07-19 16:45:49 +08:00
def call_dep(node, printed=None, level=0):
if not os.path.isfile('cscope.out'):
print("Please run: cscope -bkR", file=sys.stderr)
return False
if printed is None:
printed = set()
if node in printed:
return
calls = set()
for a in [line.split()[1] for line in
popen('cscope -d -L2 "%s"' % (node))]:
if a in black_list:
continue
calls.add(a)
if calls:
if level < level_limit - 1:
printed.add(node)
print(node, end=': ')
for a in calls:
print(a, end=' ')
print()
for a in calls:
call_dep(a, printed, level + 1)
else:
pass
# TODO: print terminal
# print('...')
return ''
2018-07-22 05:32:07 +08:00
def my_graph(name=None):
g = nx.DiGraph(name=name)
g.graph.update({'node': {'shape': 'none', 'fontsize': 50}})
g.graph.update({'rankdir': 'LR', 'nodesep': 0, })
return g
2018-07-22 22:16:40 +08:00
def reduce_graph(g):
print(type(g))
rm = set()
for e in g:
if not g.out_degree(e):
rm.add(e)
print(rm)
g.remove_nodes_from(rm)
return g
2018-07-26 13:28:37 +08:00
def includes(a):
res = []
# log(a)
for a in popen('man -s 2 %s 2> /dev/null |'
' head -n 20 | grep include || true' % (a),
shell=True):
m = re.match('.*<(.*)>', a)
if m:
res.append(m.group(1))
if not res:
for a in popen('grep -l -r " %s *(" '
'/usr/include --include "*.h" '
'2> /dev/null || true' % (a)):
# log(a)
a = re.sub(r'.*/(bits)', r'\1', a)
a = re.sub(r'.*/(sys)', r'\1', a)
a = re.sub(r'/usr/include/(.*)', r'\1', a)
# log(a)
res.append(a)
res = set(res)
if res and len(res) > 1:
r = set()
for f in res:
# log('grep " %s \+\(" --include "%s" -r /usr/include/'%(a,f))
# log(os.system(
# 'grep -w "%s" --include "%s" -r /usr/include/'%(a,f)))
if 0 != os.system(
'grep " %s *(" --include "%s" -r /usr/include/ -q'
% (a, os.path.basename(f))):
r.add(f)
res = res.difference(r)
log(res)
return ','.join(list(res)) if res else 'unexported'
2018-07-22 05:32:07 +08:00
def syscalls():
sc = my_graph('syscalls')
2018-07-26 13:28:37 +08:00
inc = 'includes.list'
if not os.path.isfile(inc):
os.system('ctags --langmap=c:+.h --c-kinds=+pex -I __THROW '
+ ' -R -u -f- /usr/include/ | cut -f1,2 > '
+ inc)
'''
if False:
includes = {}
with open(inc, 'r') as f:
for s in f:
includes[s.split()[0]] = s.split()[1]
log(includes)
'''
2018-07-22 05:32:07 +08:00
scd = 'SYSCALL_DEFINE.list'
if not os.path.isfile(scd):
os.system("grep SYSCALL_DEFINE -r --include='*.c' > " + scd)
with open(scd, 'r') as f:
2018-07-26 13:31:32 +08:00
v = set('sigsuspend', 'llseek', 'sysfs', 'sync_file_range2', 'ustat', 'bdflush')
2018-07-22 05:32:07 +08:00
for s in f:
2018-07-26 13:31:32 +08:00
if any(x in s.lower() for x in ['compat', 'stub']):
continue
2018-07-22 05:32:07 +08:00
m = re.match(r'(.*?):.*SYSCALL.*\(([\w]+)', s)
if m:
for p in {
'^old',
'^xnew',
r'.*64',
r'.*32$',
r'.*16$',
}:
if re.match(p, m.group(2)):
m = None
break
if m:
syscall = m.group(2)
syscall = re.sub('^new', '', syscall)
path = m.group(1).split('/')
2018-07-26 13:31:32 +08:00
if (m.group(1).startswith('mm/nommu.c')
or m.group(1).startswith('arch/x86/ia32')
or m.group(1).startswith('arch/')
or syscall.startswith('vm86')
2018-07-22 05:32:07 +08:00
and not m.group(1).startswith('arch/x86')):
continue
2018-07-26 13:31:32 +08:00
if syscall in v:
continue
v.add(syscall)
2018-07-22 05:32:07 +08:00
p2 = '/'.join(path[1:])
2018-07-26 13:31:32 +08:00
p2 = m.group(1)
# if log(difflib.get_close_matches(syscall,v) or ''):
# log(syscall)
# log(syscall + ' ' + (includes.get(syscall) or '------'))
# man -s 2 timerfd_settime | head -n 20
# sc.add_edge('syscalls', path[0] + '/')
# sc.add_edge(path[0] + '/', p2)
# sc.add_edge(p2, syscall)
i = includes(syscall)
log(p2 + ' ' + str(i) + ' ' + syscall)
sc.add_edge(i, i+' - '+p2)
sc.add_edge(i+' - '+p2, syscall)
# sc.add_edge(includes(syscall), syscall)
2018-07-22 05:32:07 +08:00
return sc
2018-07-22 21:19:45 +08:00
# DiGraph
# write_dot to_agraph AGraph
# agwrite
# srcxray.py 'write_dot(syscalls(), "syscalls.dot")'
2018-07-22 05:32:07 +08:00
def digraph_print(dg):
def digraph_print_sub(node=None, printed=None, level=0):
outs = [_ for _ in dg.successors(node)]
if node in printed:
2018-07-22 22:16:40 +08:00
print_limited(level*'\t' + str(node) + ' ^')
2018-07-22 05:32:07 +08:00
return
else:
s = ' ...' if level > level_limit - 2 and outs else ''
2018-07-22 22:16:40 +08:00
print_limited(level*'\t' + str(node) + s)
2018-07-22 05:32:07 +08:00
printed.add(node)
if level > level_limit - 2:
return ''
passed = set()
for o in outs:
if o in passed or o in black_list:
continue
passed.add(o)
digraph_print_sub(o, printed, level + 1)
starts = {}
printed = set()
for i in [n for (n, d) in dg.in_degree if not d]:
starts[i] = dg.out_degree(i)
starts = sorted(starts.items(), key=lambda k: k[1], reverse=True)
outs = [a[0] for a in starts]
passed = set()
for o in outs:
if o in passed or o in black_list:
continue
passed.add(o)
digraph_print_sub(o, printed)
2018-07-22 21:33:13 +08:00
def cflow_preprocess(a):
with open(a, 'r') as f:
for s in f:
# treat struct like function
s = re.sub(r"^static const struct (.*)\[\] = ", r"\1()", s)
s = re.sub(r"^static __initdata int \(\*actions\[\]\)\(void\) = ",
"int actions()", s) # treat struct like function
s = re.sub(r"^static ", "", s)
s = re.sub(r"COMPAT_SYSCALL_DEFINE[0-9]\((\w*),",
r"compat_sys_\1(", s)
s = re.sub(r"SYSCALL_DEFINE[0-9]\((\w*),", r"sys_\1(", s)
s = re.sub(r"__setup\(.*,(.*)\)", r"void __setup() {\1();}", s)
s = re.sub(r"early_param\(.*,(.*)\)",
r"void early_param() {\1();}", s)
s = re.sub(r"rootfs_initcall\((.*)\)",
r"void rootfs_initcall() {\1();}", s)
s = re.sub(r"^static ", "", s)
s = re.sub(r"__read_mostly", "", s)
s = re.sub(r"^inline ", "", s)
s = re.sub(r"^const ", "", s)
s = re.sub(r"^struct (.*) =", r"\1()", s)
s = re.sub(r"^struct ", "", s)
# for line in sys.stdin:
sys.stdout.write(s)
def import_cflow():
cf = nx.DiGraph()
stack = list()
nprev = -1
# "--depth=%d " %(level_limit+1) +
cflow = (r"cflow " +
"--preprocess='srcxray.py cflow_preprocess' " +
"--include=_sxt --brief --level-indent='0=\t' " +
" *.[ch] *.cpp *.hh ")
# " $(find -name '*.[ch]' -o -name '*.cpp' -o -name '*.hh') "
for line in popen(cflow):
# --print-level
m = re.match(r'^([\t]*)([^(^ ^<]+)', str(line))
if m:
n = len(m.group(1))
id = str(m.group(2))
else:
raise Exception(line)
if n <= nprev:
stack = stack[:n - nprev - 1]
# print(n, id, stack)
if len(stack):
cf.add_edge(stack[-1], id)
stack.append(id)
nprev = n
return cf
2018-07-19 02:46:21 +08:00
me = os.path.basename(sys.argv[0])
2018-07-19 12:52:33 +08:00
2018-07-19 02:46:21 +08:00
def usage():
2018-07-19 19:03:30 +08:00
for c in ["referers_tree", "call_tree", "referers_dep", "call_dep"]:
2018-07-19 16:45:49 +08:00
print(me, c, "<identifier>")
2018-07-19 02:46:21 +08:00
print("Try this:")
print("cd linux/init")
print(me, "referers_tree nfs_root_data")
print(me, "call_tree start_kernel")
print(me, "Emergency termination: ^Z, kill %1")
2018-07-19 12:52:33 +08:00
2018-07-19 02:46:21 +08:00
def main():
try:
ret = False
if len(sys.argv) == 1:
print('Run', me, 'usage')
else:
2018-07-19 04:48:22 +08:00
if '(' in sys.argv[1]:
2018-07-19 02:46:21 +08:00
ret = eval(sys.argv[1])
else:
2018-07-19 12:52:33 +08:00
ret = eval(sys.argv[1] + '(' + ', '.join("'%s'" % (a)
2018-07-19 13:29:29 +08:00
for a in sys.argv[2:]) + ')')
if isinstance(ret, bool) and ret is False:
2018-07-19 02:46:21 +08:00
sys.exit(os.EX_CONFIG)
2018-07-22 21:33:13 +08:00
if (ret is not None):
print(ret)
2018-07-19 02:46:21 +08:00
except KeyboardInterrupt:
warning("\nInterrupted")
2018-07-19 12:52:33 +08:00
2018-07-19 02:46:21 +08:00
if __name__ == "__main__":
main()