In some cases it may be desirable to parse the string expression "f1(*args)" and return some of the key features of the represented function-like call.
This recipe returns the key features in the form of a namedtuple.
e.g. (for the above)
>>> explain("f1(*args)")
[ Call(func='f1', starargs='args') ]
The recipe will return a list of such namedtuples for "f1(*args)\nf2(*args)"
Note that while the passed string expression must evaluate to valid python syntax,
names needn't be declared in current scope.
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 | import ast
from itertools import cycle, chain, islice
from collections import namedtuple
import operator
def explain(code_string, important=None):
"""Parse a string containing a function-like call,
return a namedtuple containing the results
>>> explain('mymod.nestmod.func("arg1", "arg2",
kw1="kword1", kw2="kword2",
*args, **kws')
[Call( args=['arg1', 'arg2'],
keywords={'kw1': 'kword1', 'kw2': 'kword2'},
starargs='args',
func='mymod.nestmod.func',
kwargs='kws')]
optional 'important' argument is a list of features to parse
from the code_string. Features defined for a Call Node:
args - positional arguments,
keywords - keyword arguments,
starargs - excess positional arguments,
kwargs - excess keyword arguments,
func - chained function attribute lookup.
"""
node = ast.parse(code_string)
visitor = StrNodeVisitor(important)
return visitor.visit(node)
#--------------------------------------------------------------------
def attrgetter(name):
"""Get attribute 'name' from object and return
a string representation of it."""
getname = operator.attrgetter(name)
def str_getattr(self, obj=None):
obj = self if obj is None else obj
return str(getname(obj))
return str_getattr
def strmap(show):
"""Hardcode a particular ast Node to string representation 'show'."""
return lambda self, node=None: show
#--------------------------------------------------------------------
class StrNodeVisitor(ast.NodeVisitor):
"""A class to return string representations of visited ast nodes."""
visit_Name = attrgetter('id')
visit_Num = attrgetter('n')
visit_Str = attrgetter('s')
# hardcoded these Nodes to return string argument when visited.
visit_Add = strmap('+')
visit_Sub = strmap('-')
visit_Mult = strmap('*')
visit_Div = strmap('/')
visit_Mod = strmap('%')
visit_Pow = strmap('**')
visit_LShift = strmap('<<')
visit_RShift = strmap('>>')
visit_FloorDiv = strmap('//')
visit_Not = strmap('not')
visit_And = strmap('and')
visit_Or = strmap('or')
visit_Eq = strmap('==')
visit_NotEq = strmap('!=')
visit_Lt = strmap('<')
visit_LtE = strmap('<=')
visit_Gt = strmap('>')
visit_GtE = strmap('>=')
visit_Is = strmap('is')
visit_IsNot = strmap('not is')
visit_In = strmap('in')
visit_NotIn = strmap('not in')
def __init__(self, interested=None):
"""interested - a sequence of features of a function to
include in returned namedtuple. Allowed features:
func, args, keywords, starargs, kwargs"""
try:
self._interested = set(interested)
except TypeError:
self._interested = interested
def visit_Module(self, node):
visit = self.visit
return [visit(body) for body in node.body]
def visit_Expr(self, node):
return self.visit(node.value)
def visit_Call(self, node):
"""return a NamedTuple that represents a Call:
f(arg, kw=1, *args, **kws).
Call node defines:
func, args, keywords, starargs, kwargs"""
# determine which of the fields we are allowed to handle.
defined = set(node._fields)
try:
interested = self._interested & defined
except TypeError:
interested = defined
fields = {}
for field in interested:
field_contents = getattr(node, field)
if field_contents is None:
# short circuit if the node field is a NoneType.
fields[field] = None
continue
# handle the field using one of the convenience functions.
fields[field] = getattr(self, field)(field_contents)
# return the result as a namedtuple rather than dict.
BaseCallTuple = namedtuple(classname(node), interested)
class MyCallTuple(BaseCallTuple):
"""Enable representation in a nicer string format.
Don't use this MyCallTuple class if 'func' is not a field.
as the string representation relies on it."""
__str__ = CallTuple2Str
if 'func' in interested:
mytup = MyCallTuple(**fields)
else:
mytup = BaseCallTuple(**fields)
return mytup
def visit_List(self, node):
"""return a string representation of list."""
return self._sequence(node, '[%s]')
def visit_Tuple(self, node):
"""return a string representation of tuple."""
return self._sequence(node, '(%s)')
def visit_Dict(self, node):
"""return a string representation of a dict."""
visit = self.visit
keyvals = zip(node.keys, node.values)
contents = ', '.join(['%s: %s' % (visit(key), visit(value))
for key, value in keyvals])
return '{%s}' % contents
def visit_Attribute(self, node):
"""Attribute of form: obj.attr."""
return '%s.%s' % (self.visit(node.value), node.attr)
def visit_BoolOp(self, node):
"""BoolOp of form: op values
e.g. a and b."""
visit = self.visit
op = ' %s ' % visit(node.op)
return op.join([visit(n) for n in node.values])
def visit_UnaryOp(self, node):
"""UnaryOp of form: op operand
e.g. not []."""
return '%(op)s %(operand)s' % dict(
op=self.visit(node.op),
operand=self.visit(node.operand))
def visit_BinOp(self, node):
"""BinOp of form: left op right
e.g. 2 * 3."""
visit = self.visit
return '(%(left)s %(op)s %(right)s)' % dict(
left=visit(node.left),
op=visit(node.op),
right=visit(node.right))
def visit_Subscript(self, node):
"""Subscript of form: value[slice].
e.g. a[1:10:2]."""
visit = self.visit
return '%s[%s]' % (visit(node.value), visit(node.slice))
def visit_Slice(self, node):
"""Slice of form: lower:upper:step.
e.g. 1:10:2."""
visit = self.visit
return '%s:%s:%s' % (visit(node.lower),
visit(node.upper), visit(node.step))
def visit_Compare(self, node):
"""Compare of form: left ops comparators.
e.g. x > y > z -> left=x, ops=['>', '>'], comparators=['y', 'z']
"""
visit = self.visit
rest = ' '.join([visit(r)
for r in roundrobin(node.ops, node.comparators)])
return '%s %s' % (visit(node.left), rest)
# Convenience functions.
def _sequence(self, node, signature):
visit = self.visit
contents = ', '.join([visit(elt) for elt in node.elts])
return signature % contents
def func(self, func):
"""convenience function called from visit_Call."""
return self.visit(func)
def args(self, args):
"""convenience function called from visit_Call."""
visit = self.visit
return [visit(n) for n in args]
def keywords(self, keywords):
"""convenience function called from visit_Call."""
visit = self.visit
return dict((kw.arg, visit(kw.value)) for kw in keywords)
def starargs(self, starargs):
"""convenience function called from visit_Call."""
return self.visit(starargs)
def kwargs(self, kwargs):
"""convenience function called from visit_Call."""
return self.visit(kwargs)
def generic_visit(self, node):
"""Called as a fallback handler if all other visit_* functions failed.
return '<unknown>'. if node is NoneType return ''"""
if node is None:
return ''
return '<unknown: %s>' % classname(node)
#--------------------------------------------------------------------
def classname(obj):
return obj.__class__.__name__
def roundrobin(*iterables):
"roundrobin('ABC', 'D', 'EF') --> A D E B F C"
# Recipe credited to George Sakkis
pending = len(iterables)
nexts = cycle(iter(it).next for it in iterables)
while pending:
try:
for next in nexts:
yield next()
except StopIteration:
pending -= 1
nexts = cycle(islice(nexts, pending))
def CallTuple2Str(self):
"""replacement for CallTuple's __str__ method.
Assumes that func field is present.
The print signature should look like:
func(args, keywords, *starargs, **kwargs)."""
func = self.func
order = ['args', 'keywords', 'starargs', 'kwargs']
# handle args.
arg_values = getattr(self, 'args', [])
args = ', '.join([str(arg) for arg in arg_values])
# handle keywords.
kw_values = getattr(self, 'keywords', {})
keywords = ', '.join(['%s=%s' % (k, v) for k, v in kw_values.items()])
# handle starargs.
star = getattr(self, 'starargs', None)
if star:
starargs = '*%s' % star
else:
starargs = ''
# handle kwargs.
kwargs = getattr(self, 'kwargs', None)
if kwargs:
kwargs = '**%s' % kwargs
else:
kwargs = ''
# put it all together.
arguments = [args, keywords, starargs, kwargs]
signature = ', '.join([arg for arg in arguments if arg != ''])
return '%s(%s)' % (func, signature)
#--------------------------------------------------------------------
if __name__ == '__main__':
tests = dict(
tuple_test = "mod1.f_tuple((1,2), kw1=(1,2))",
list_test = "f_list([1,2], kw1=[1,2])",
dict_test = "f_dict({1:2}, kw1={1:2})",
complex_test = "f_complex(1 + 2j, kw1=1 + 2j)",
fn_test = "f_func(abs(-1), kw1=explain('f1(2, 3)'))",
bool_test = "f_bool(True, False, not [], hello or 'hello')",
slice_test = "f_slice(a[:2], b=b[1:2])",
lambda_test = "f_lambda(lambda x: x)",
compare_test = "f_compare(x > y > z not in [True])",
genexp_test = "f_genexp([a for a in range(2)], b=(b for b in range(2)))")
for name, test in tests.iteritems():
print '%s: %s' % (name, explain(test,
['func', 'keywords', 'args'])[0])
|
Approach
Python's ast module is used to parse the code string and create an ast Node. It then walks through the resultant ast.AST node to find the features using a NodeVisitor subclass.
Limitations
Currently can only parse string, number type or name lookup arguments:
explain("f1(1,2, one="one", one=1)")
explain("f1(defined_name, one=defined_name)")
cannot parse more complicated arguments:
>>> explain("f1([1 for a in range(2)])", ['func', 'args'])
[Call(args=['<unknown>'], func='f1')]
>>> explain("f1(lambda x: x)", ['func', 'args'])
[Call(args=['<unknown>'], func='f1')]
Extensions can be made by adding additional visit_* functions to handle other ast Nodes. e.g.
- visit_Lambda,
- visit_IfExp,
- visit_GeneratorExp,
- visit_ListComp
to name a few.
Currently unhandled nodes are returned as a string '<unknown %s>' % Node.
Examples:
Function from module with multiple keywords
>>> f_kws = "mymodule.f1(kw1=1, kw2=2, **kws)"
>>> explain(f_kws)
[Call(keywords={'kw1': 1, 'kw2': 2}, starargs=None, args=[],
func='mymodule.f1', kwargs='kws')]
>>> result = explain(f_kws)[0]
>>> result.func
'mymodule.f1'
>>> result.kwargs
'kws'
Multiple function-like calls
>>> explain("f1('call1')\nf2('call2')")
[Call(keywords={}, starargs=None, args=['call1'], func='f1', kwargs=None),
Call(keywords={}, starargs=None, args=['call2'], func='f2', kwargs=None)]
Limit feature information
If only interested in func and args (for example), rather than any other features..
>>> explain("f1('ignore kw', kw=1)", ['func', 'args'])
[Call(args=['ignore kw'], func='f1')]
Nested function calls
Each Call Node returns a NamedTuple subclass that overrides the __str__ to return a function call-like string representation (looking similar to original function call rather than the default namedtuple __repr__ output.)
>>> explain("f1(nested('arg1'))")
[Call(args=['nested("arg1")'], func='f1')]