| "Usage: unparse.py <path to source file>" | |
| import sys | |
| import ast | |
| import cStringIO | |
| import os | |
| # Large float and imaginary literals get turned into infinities in the AST. | |
| # We unparse those infinities to INFSTR. | |
| INFSTR = "1e" + repr(sys.float_info.max_10_exp + 1) | |
| def interleave(inter, f, seq): | |
| """Call f on each item in seq, calling inter() in between. | |
| """ | |
| seq = iter(seq) | |
| try: | |
| f(next(seq)) | |
| except StopIteration: | |
| pass | |
| else: | |
| for x in seq: | |
| inter() | |
| f(x) | |
| class Unparser: | |
| """Methods in this class recursively traverse an AST and | |
| output source code for the abstract syntax; original formatting | |
| is disregarded. """ | |
| def __init__(self, tree, file = sys.stdout): | |
| """Unparser(tree, file=sys.stdout) -> None. | |
| Print the source for tree to file.""" | |
| self.f = file | |
| self.future_imports = [] | |
| self._indent = 0 | |
| self.dispatch(tree) | |
| self.f.write("") | |
| self.f.flush() | |
| def fill(self, text = ""): | |
| "Indent a piece of text, according to the current indentation level" | |
| self.f.write("\n"+" "*self._indent + text) | |
| def write(self, text): | |
| "Append a piece of text to the current line." | |
| self.f.write(text) | |
| def enter(self): | |
| "Print ':', and increase the indentation." | |
| self.write(":") | |
| self._indent += 1 | |
| def leave(self): | |
| "Decrease the indentation level." | |
| self._indent -= 1 | |
| def dispatch(self, tree): | |
| "Dispatcher function, dispatching tree type T to method _T." | |
| if isinstance(tree, list): | |
| for t in tree: | |
| self.dispatch(t) | |
| return | |
| meth = getattr(self, "_"+tree.__class__.__name__) | |
| meth(tree) | |
| ############### Unparsing methods ###################### | |
| # There should be one method per concrete grammar type # | |
| # Constructors should be grouped by sum type. Ideally, # | |
| # this would follow the order in the grammar, but # | |
| # currently doesn't. # | |
| ######################################################## | |
| def _Module(self, tree): | |
| for stmt in tree.body: | |
| self.dispatch(stmt) | |
| # stmt | |
| def _Expr(self, tree): | |
| self.fill() | |
| self.dispatch(tree.value) | |
| def _Import(self, t): | |
| self.fill("import ") | |
| interleave(lambda: self.write(", "), self.dispatch, t.names) | |
| def _ImportFrom(self, t): | |
| # A from __future__ import may affect unparsing, so record it. | |
| if t.module and t.module == '__future__': | |
| self.future_imports.extend(n.name for n in t.names) | |
| self.fill("from ") | |
| self.write("." * t.level) | |
| if t.module: | |
| self.write(t.module) | |
| self.write(" import ") | |
| interleave(lambda: self.write(", "), self.dispatch, t.names) | |
| def _Assign(self, t): | |
| self.fill() | |
| for target in t.targets: | |
| self.dispatch(target) | |
| self.write(" = ") | |
| self.dispatch(t.value) | |
| def _AugAssign(self, t): | |
| self.fill() | |
| self.dispatch(t.target) | |
| self.write(" "+self.binop[t.op.__class__.__name__]+"= ") | |
| self.dispatch(t.value) | |
| def _Return(self, t): | |
| self.fill("return") | |
| if t.value: | |
| self.write(" ") | |
| self.dispatch(t.value) | |
| def _Pass(self, t): | |
| self.fill("pass") | |
| def _Break(self, t): | |
| self.fill("break") | |
| def _Continue(self, t): | |
| self.fill("continue") | |
| def _Delete(self, t): | |
| self.fill("del ") | |
| interleave(lambda: self.write(", "), self.dispatch, t.targets) | |
| def _Assert(self, t): | |
| self.fill("assert ") | |
| self.dispatch(t.test) | |
| if t.msg: | |
| self.write(", ") | |
| self.dispatch(t.msg) | |
| def _Exec(self, t): | |
| self.fill("exec ") | |
| self.dispatch(t.body) | |
| if t.globals: | |
| self.write(" in ") | |
| self.dispatch(t.globals) | |
| if t.locals: | |
| self.write(", ") | |
| self.dispatch(t.locals) | |
| def _Print(self, t): | |
| self.fill("print ") | |
| do_comma = False | |
| if t.dest: | |
| self.write(">>") | |
| self.dispatch(t.dest) | |
| do_comma = True | |
| for e in t.values: | |
| if do_comma:self.write(", ") | |
| else:do_comma=True | |
| self.dispatch(e) | |
| if not t.nl: | |
| self.write(",") | |
| def _Global(self, t): | |
| self.fill("global ") | |
| interleave(lambda: self.write(", "), self.write, t.names) | |
| def _Yield(self, t): | |
| self.write("(") | |
| self.write("yield") | |
| if t.value: | |
| self.write(" ") | |
| self.dispatch(t.value) | |
| self.write(")") | |
| def _Raise(self, t): | |
| self.fill('raise ') | |
| if t.type: | |
| self.dispatch(t.type) | |
| if t.inst: | |
| self.write(", ") | |
| self.dispatch(t.inst) | |
| if t.tback: | |
| self.write(", ") | |
| self.dispatch(t.tback) | |
| def _TryExcept(self, t): | |
| self.fill("try") | |
| self.enter() | |
| self.dispatch(t.body) | |
| self.leave() | |
| for ex in t.handlers: | |
| self.dispatch(ex) | |
| if t.orelse: | |
| self.fill("else") | |
| self.enter() | |
| self.dispatch(t.orelse) | |
| self.leave() | |
| def _TryFinally(self, t): | |
| if len(t.body) == 1 and isinstance(t.body[0], ast.TryExcept): | |
| # try-except-finally | |
| self.dispatch(t.body) | |
| else: | |
| self.fill("try") | |
| self.enter() | |
| self.dispatch(t.body) | |
| self.leave() | |
| self.fill("finally") | |
| self.enter() | |
| self.dispatch(t.finalbody) | |
| self.leave() | |
| def _ExceptHandler(self, t): | |
| self.fill("except") | |
| if t.type: | |
| self.write(" ") | |
| self.dispatch(t.type) | |
| if t.name: | |
| self.write(" as ") | |
| self.dispatch(t.name) | |
| self.enter() | |
| self.dispatch(t.body) | |
| self.leave() | |
| def _ClassDef(self, t): | |
| self.write("\n") | |
| for deco in t.decorator_list: | |
| self.fill("@") | |
| self.dispatch(deco) | |
| self.fill("class "+t.name) | |
| if t.bases: | |
| self.write("(") | |
| for a in t.bases: | |
| self.dispatch(a) | |
| self.write(", ") | |
| self.write(")") | |
| self.enter() | |
| self.dispatch(t.body) | |
| self.leave() | |
| def _FunctionDef(self, t): | |
| self.write("\n") | |
| for deco in t.decorator_list: | |
| self.fill("@") | |
| self.dispatch(deco) | |
| self.fill("def "+t.name + "(") | |
| self.dispatch(t.args) | |
| self.write(")") | |
| self.enter() | |
| self.dispatch(t.body) | |
| self.leave() | |
| def _For(self, t): | |
| self.fill("for ") | |
| self.dispatch(t.target) | |
| self.write(" in ") | |
| self.dispatch(t.iter) | |
| self.enter() | |
| self.dispatch(t.body) | |
| self.leave() | |
| if t.orelse: | |
| self.fill("else") | |
| self.enter() | |
| self.dispatch(t.orelse) | |
| self.leave() | |
| def _If(self, t): | |
| self.fill("if ") | |
| self.dispatch(t.test) | |
| self.enter() | |
| self.dispatch(t.body) | |
| self.leave() | |
| # collapse nested ifs into equivalent elifs. | |
| while (t.orelse and len(t.orelse) == 1 and | |
| isinstance(t.orelse[0], ast.If)): | |
| t = t.orelse[0] | |
| self.fill("elif ") | |
| self.dispatch(t.test) | |
| self.enter() | |
| self.dispatch(t.body) | |
| self.leave() | |
| # final else | |
| if t.orelse: | |
| self.fill("else") | |
| self.enter() | |
| self.dispatch(t.orelse) | |
| self.leave() | |
| def _While(self, t): | |
| self.fill("while ") | |
| self.dispatch(t.test) | |
| self.enter() | |
| self.dispatch(t.body) | |
| self.leave() | |
| if t.orelse: | |
| self.fill("else") | |
| self.enter() | |
| self.dispatch(t.orelse) | |
| self.leave() | |
| def _With(self, t): | |
| self.fill("with ") | |
| self.dispatch(t.context_expr) | |
| if t.optional_vars: | |
| self.write(" as ") | |
| self.dispatch(t.optional_vars) | |
| self.enter() | |
| self.dispatch(t.body) | |
| self.leave() | |
| # expr | |
| def _Str(self, tree): | |
| # if from __future__ import unicode_literals is in effect, | |
| # then we want to output string literals using a 'b' prefix | |
| # and unicode literals with no prefix. | |
| if "unicode_literals" not in self.future_imports: | |
| self.write(repr(tree.s)) | |
| elif isinstance(tree.s, str): | |
| self.write("b" + repr(tree.s)) | |
| elif isinstance(tree.s, unicode): | |
| self.write(repr(tree.s).lstrip("u")) | |
| else: | |
| assert False, "shouldn't get here" | |
| def _Name(self, t): | |
| self.write(t.id) | |
| def _Repr(self, t): | |
| self.write("`") | |
| self.dispatch(t.value) | |
| self.write("`") | |
| def _Num(self, t): | |
| repr_n = repr(t.n) | |
| # Parenthesize negative numbers, to avoid turning (-1)**2 into -1**2. | |
| if repr_n.startswith("-"): | |
| self.write("(") | |
| # Substitute overflowing decimal literal for AST infinities. | |
| self.write(repr_n.replace("inf", INFSTR)) | |
| if repr_n.startswith("-"): | |
| self.write(")") | |
| def _List(self, t): | |
| self.write("[") | |
| interleave(lambda: self.write(", "), self.dispatch, t.elts) | |
| self.write("]") | |
| def _ListComp(self, t): | |
| self.write("[") | |
| self.dispatch(t.elt) | |
| for gen in t.generators: | |
| self.dispatch(gen) | |
| self.write("]") | |
| def _GeneratorExp(self, t): | |
| self.write("(") | |
| self.dispatch(t.elt) | |
| for gen in t.generators: | |
| self.dispatch(gen) | |
| self.write(")") | |
| def _SetComp(self, t): | |
| self.write("{") | |
| self.dispatch(t.elt) | |
| for gen in t.generators: | |
| self.dispatch(gen) | |
| self.write("}") | |
| def _DictComp(self, t): | |
| self.write("{") | |
| self.dispatch(t.key) | |
| self.write(": ") | |
| self.dispatch(t.value) | |
| for gen in t.generators: | |
| self.dispatch(gen) | |
| self.write("}") | |
| def _comprehension(self, t): | |
| self.write(" for ") | |
| self.dispatch(t.target) | |
| self.write(" in ") | |
| self.dispatch(t.iter) | |
| for if_clause in t.ifs: | |
| self.write(" if ") | |
| self.dispatch(if_clause) | |
| def _IfExp(self, t): | |
| self.write("(") | |
| self.dispatch(t.body) | |
| self.write(" if ") | |
| self.dispatch(t.test) | |
| self.write(" else ") | |
| self.dispatch(t.orelse) | |
| self.write(")") | |
| def _Set(self, t): | |
| assert(t.elts) # should be at least one element | |
| self.write("{") | |
| interleave(lambda: self.write(", "), self.dispatch, t.elts) | |
| self.write("}") | |
| def _Dict(self, t): | |
| self.write("{") | |
| def write_pair(pair): | |
| (k, v) = pair | |
| self.dispatch(k) | |
| self.write(": ") | |
| self.dispatch(v) | |
| interleave(lambda: self.write(", "), write_pair, zip(t.keys, t.values)) | |
| self.write("}") | |
| def _Tuple(self, t): | |
| self.write("(") | |
| if len(t.elts) == 1: | |
| (elt,) = t.elts | |
| self.dispatch(elt) | |
| self.write(",") | |
| else: | |
| interleave(lambda: self.write(", "), self.dispatch, t.elts) | |
| self.write(")") | |
| unop = {"Invert":"~", "Not": "not", "UAdd":"+", "USub":"-"} | |
| def _UnaryOp(self, t): | |
| self.write("(") | |
| self.write(self.unop[t.op.__class__.__name__]) | |
| self.write(" ") | |
| # If we're applying unary minus to a number, parenthesize the number. | |
| # This is necessary: -2147483648 is different from -(2147483648) on | |
| # a 32-bit machine (the first is an int, the second a long), and | |
| # -7j is different from -(7j). (The first has real part 0.0, the second | |
| # has real part -0.0.) | |
| if isinstance(t.op, ast.USub) and isinstance(t.operand, ast.Num): | |
| self.write("(") | |
| self.dispatch(t.operand) | |
| self.write(")") | |
| else: | |
| self.dispatch(t.operand) | |
| self.write(")") | |
| binop = { "Add":"+", "Sub":"-", "Mult":"*", "Div":"/", "Mod":"%", | |
| "LShift":"<<", "RShift":">>", "BitOr":"|", "BitXor":"^", "BitAnd":"&", | |
| "FloorDiv":"//", "Pow": "**"} | |
| def _BinOp(self, t): | |
| self.write("(") | |
| self.dispatch(t.left) | |
| self.write(" " + self.binop[t.op.__class__.__name__] + " ") | |
| self.dispatch(t.right) | |
| self.write(")") | |
| cmpops = {"Eq":"==", "NotEq":"!=", "Lt":"<", "LtE":"<=", "Gt":">", "GtE":">=", | |
| "Is":"is", "IsNot":"is not", "In":"in", "NotIn":"not in"} | |
| def _Compare(self, t): | |
| self.write("(") | |
| self.dispatch(t.left) | |
| for o, e in zip(t.ops, t.comparators): | |
| self.write(" " + self.cmpops[o.__class__.__name__] + " ") | |
| self.dispatch(e) | |
| self.write(")") | |
| boolops = {ast.And: 'and', ast.Or: 'or'} | |
| def _BoolOp(self, t): | |
| self.write("(") | |
| s = " %s " % self.boolops[t.op.__class__] | |
| interleave(lambda: self.write(s), self.dispatch, t.values) | |
| self.write(")") | |
| def _Attribute(self,t): | |
| self.dispatch(t.value) | |
| # Special case: 3.__abs__() is a syntax error, so if t.value | |
| # is an integer literal then we need to either parenthesize | |
| # it or add an extra space to get 3 .__abs__(). | |
| if isinstance(t.value, ast.Num) and isinstance(t.value.n, int): | |
| self.write(" ") | |
| self.write(".") | |
| self.write(t.attr) | |
| def _Call(self, t): | |
| self.dispatch(t.func) | |
| self.write("(") | |
| comma = False | |
| for e in t.args: | |
| if comma: self.write(", ") | |
| else: comma = True | |
| self.dispatch(e) | |
| for e in t.keywords: | |
| if comma: self.write(", ") | |
| else: comma = True | |
| self.dispatch(e) | |
| if t.starargs: | |
| if comma: self.write(", ") | |
| else: comma = True | |
| self.write("*") | |
| self.dispatch(t.starargs) | |
| if t.kwargs: | |
| if comma: self.write(", ") | |
| else: comma = True | |
| self.write("**") | |
| self.dispatch(t.kwargs) | |
| self.write(")") | |
| def _Subscript(self, t): | |
| self.dispatch(t.value) | |
| self.write("[") | |
| self.dispatch(t.slice) | |
| self.write("]") | |
| # slice | |
| def _Ellipsis(self, t): | |
| self.write("...") | |
| def _Index(self, t): | |
| self.dispatch(t.value) | |
| def _Slice(self, t): | |
| if t.lower: | |
| self.dispatch(t.lower) | |
| self.write(":") | |
| if t.upper: | |
| self.dispatch(t.upper) | |
| if t.step: | |
| self.write(":") | |
| self.dispatch(t.step) | |
| def _ExtSlice(self, t): | |
| interleave(lambda: self.write(', '), self.dispatch, t.dims) | |
| # others | |
| def _arguments(self, t): | |
| first = True | |
| # normal arguments | |
| defaults = [None] * (len(t.args) - len(t.defaults)) + t.defaults | |
| for a,d in zip(t.args, defaults): | |
| if first:first = False | |
| else: self.write(", ") | |
| self.dispatch(a), | |
| if d: | |
| self.write("=") | |
| self.dispatch(d) | |
| # varargs | |
| if t.vararg: | |
| if first:first = False | |
| else: self.write(", ") | |
| self.write("*") | |
| self.write(t.vararg) | |
| # kwargs | |
| if t.kwarg: | |
| if first:first = False | |
| else: self.write(", ") | |
| self.write("**"+t.kwarg) | |
| def _keyword(self, t): | |
| self.write(t.arg) | |
| self.write("=") | |
| self.dispatch(t.value) | |
| def _Lambda(self, t): | |
| self.write("(") | |
| self.write("lambda ") | |
| self.dispatch(t.args) | |
| self.write(": ") | |
| self.dispatch(t.body) | |
| self.write(")") | |
| def _alias(self, t): | |
| self.write(t.name) | |
| if t.asname: | |
| self.write(" as "+t.asname) | |
| def roundtrip(filename, output=sys.stdout): | |
| with open(filename, "r") as pyfile: | |
| source = pyfile.read() | |
| tree = compile(source, filename, "exec", ast.PyCF_ONLY_AST) | |
| Unparser(tree, output) | |
| def testdir(a): | |
| try: | |
| names = [n for n in os.listdir(a) if n.endswith('.py')] | |
| except OSError: | |
| sys.stderr.write("Directory not readable: %s" % a) | |
| else: | |
| for n in names: | |
| fullname = os.path.join(a, n) | |
| if os.path.isfile(fullname): | |
| output = cStringIO.StringIO() | |
| print 'Testing %s' % fullname | |
| try: | |
| roundtrip(fullname, output) | |
| except Exception as e: | |
| print ' Failed to compile, exception is %s' % repr(e) | |
| elif os.path.isdir(fullname): | |
| testdir(fullname) | |
| def main(args): | |
| if args[0] == '--testdir': | |
| for a in args[1:]: | |
| testdir(a) | |
| else: | |
| for a in args: | |
| roundtrip(a) | |
| if __name__=='__main__': | |
| main(sys.argv[1:]) |