Welcome, guest | Sign In | My Account | Store | Cart

This code converts variables which is made with upper-leters only, into constants at compile time, if it is possible to be replaced. and generate .pyc file.

This recipe pay attensions to the compile time evaluation, the feature of Constants.

Python, 231 lines
  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
####
# compile.py

#!/usr/bin/env python

import re
import os
import sys
import new
import imp
import time
import struct
import marshal
import compiler
from compiler.ast import Const, AssName, AssTuple

__author__ = 'Shimomura Ikkei'
__date__ = '2005-06-23'
__all__ = ['ConstantCompiler']


# Check the string is valid constant name,
isConstName = re.compile('^[A-Z][A-Z_]+$').match

def ispyfile(filename):
    "ispyfile(filename) ... The file is python source file."
    assert isinstance(filename, str) and filename
    return filename.endswith('.py') and os.path.isfile(filename)

def change_extension(name, ext='.pyc'):
    "change_extension(name, ext) ... Rename exstension."

    assert isinstance(name, str) and name
    assert isinstance(ext, str) and ext
    assert ext.startswith('.'), 'File extension must starts with dot.'

    return os.path.splitext(name)[0] + ext


class ConstantVisitor:
    def __init__(self, constants):
        self.constants = constants

    def __registerConstant(self, node, assign, const):
        assert isinstance(assign, AssName)
        if isConstName(assign.name):
            if self.constants.has_key(assign.name):
                print "Warning: %s at line %d: '%s' is already defined." % \
                  (node.filename, node.lineno, assign.name)
            else:
                if isinstance(const, Const):
                    self.constants[assign.name] = const.value
                else:
                    self.constants[assign.name] = None # dummy data

    def visitAssign(self, node):
        nodes = node.getChildren()

        if isinstance(nodes[0], AssName):
            name, const = nodes
            self.__registerConstant(node, name, const)

        elif isinstance(nodes[0], AssTuple):
            names, consts = nodes
            names = names.getChildren()
            consts = consts.getChildren()
            assert len(names) == len(consts)
            for name, const in zip(names, consts):
                self.__registerConstant(node, name, const)

    def visitName(self, node):
        assert isinstance(node, compiler.ast.Name)
        if isConstName(node.name) and self.constants.has_key(node.name):
            value = self.constants.get(node.name)
            # If the value can be constant(int, long, float, str, ...)
            if [True for type in (int, long, float, str) if isinstance(value, type)]:
                node.__class__ = Const
                node.value = value
                del node.name


class ConstantCompiler:

    def __init__(self, filename=None):
        self.constants = {}
        if os.path.isfile(filename) and filename.endswith('.py'):
            self.__load_constants(filename)

    def __load_constants(self, filename):
        assert isinstance(filename, str) and filename.endswith('.py')
        assert os.path.isfile(filename) and os.access(filename, os.R_OK)

        try:
            fh, filename, opts = imp.find_module(os.path.splitext(filename)[0])
            mod = imp.load_module("", fh, filename, opts)
            for k,v in ((x,getattr(mod,x)) for x in dir(mod) if isConstName(x)):
                self.constants[k] = v
        except ImportError:
            print "Failed to import module '%s'" % filename

    def __walk_ast(self, ast):
        compiler.walk(ast, ConstantVisitor(self.constants))

    def compile(self, filename):
        assert isinstance(filename, str) and filename
        assert os.path.isfile(filename) and filename.endswith('.py')

        # Parse python source -> AST(Abstract Syntax Tree)
        src = open(filename, 'r')
        ast = compiler.parse(src.read())
        src.close()

        # Syntax Macro (Expand constant values before compile)
        compiler.misc.set_filename(filename, ast)
        compiler.syntax.check(ast)
        self.__walk_ast(ast)

        # Compile AST -> code object.
        code = compiler.pycodegen.ModuleCodeGenerator(ast).getCode()

        return CodeWrapper(filename, code)


class CodeWrapper:
    """An utility class to save code object as .pyc file."""

    def __init__(self, src_filename, code):
        "CodeWrapper(code) This class only wrap an object for method chain."

        assert isinstance(src_filename, str) and src_filename
        assert os.path.isfile(src_filename) and src_filename.endswith('.py')
        assert isinstance(code, new.code)

        self.src_filename = src_filename
        self.__code = code

    def getCode(self):
        "getCode() ... Returns code object."
        assert isinstance(self.__code, new.code)
        return self.__code

    def __timestamp(self, pyc_filename):
        "__get_timestamp(pyc_filename) Gets timestamp stored in .pyc file."

        assert isinstance(pyc_filename, str) and pyc_filename
        assert pyc_filename.endswith('.pyc')
        assert os.path.isfile(pyc_filename)
        assert os.access(pyc_filename, os.R_OK)

        try:
            pyc = open(pyc_filename, 'rb')
            # The first 4 bytes is a magic number.for pyc file.
            # this checks the python's version.
            if pyc.read(4) == imp.get_magic():
                # The next 4 bytes is the timestamp stored as long,
                # we need this value.
                return struct.unpack("<l", pyc.read(4))[0]
            else:
                # Not .pyc file or wrong version of python.
                # It should be always updated.
                return -1
        finally:
            pyc.close()

    def __modified(self, src, pyc):
        "__modified(src_filename, pyc_filename) Returns True if src updated."

        assert isinstance(src, str) and src and src.endswith('.py')
        assert isinstance(pyc, str) and pyc and pyc.endswith('.pyc')
        assert os.path.isfile(src)

        # If not exists .pyc file then always True.
        if not os.path.isfile(pyc):
            return True

        # Is source's modified time newer than .pyc's timestamp ?
        return os.stat(src)[9] > self.__timestamp(pyc)

    def save_as(self, pyc_filename):
        "save_as(pyc_filename) ... Save current code object to .pyc file."

        assert isinstance(self.__code, new.code)
        assert isinstance(pyc_filename, str) and pyc_filename
        assert pyc_filename.endswith('.pyc')

        # Skip if the file was already updated.
        if self.__modified(self.src_filename, pyc_filename):

            # Output dump the code object to .pyc file.
            pyc = open(pyc_filename, 'wb')
            pyc.write(imp.get_magic())
            pyc.write(struct.pack('<l', time.time()))
            marshal.dump(self.__code, pyc)
            pyc.close()

        assert os.path.isfile(pyc_filename)
        assert os.path.getsize(pyc_filename) > 0


def main(const_file, *argv):
    pyc = ConstantCompiler(const_file)

    for filename in filter(os.path.exists, argv):
        pyc.compile(filename).save_as(change_extension(filename, ext='.pyc'))

if __name__ == '__main__':
    main(*sys.argv[1:])


####
# define_constants.py

import math
PI = math.atan(1) * 4.0

DEBUG = 1

####
# test_constants.py

print PI

def foo(num):
  if DEBUG:
    print "debug foo(%d)" num
  print num

for i in range(20): foo(i)
####
# how to run
# python compile.py define_constants.py test_constants.py

At the first, you know Python has some simple optimization for constants. If the condition of if-statement is a constant then can be evaluated at compile time, etc.

For example, assume the function foo() is called often. You will notice the "if DEBUG: ..." statement is evaluated each time when the function is called. but, since the DEBUG is constant value, its enough to be evaluated only once at the compile time. This recipe can handle it well.

Known issus: The abstract of this concept is run/compile-time pre-evaluation. Write a pre-processor, AOP's pointcut, decorator, eval also can solve the same task, though. I think implementing constants is the simplest solution.


Known problems

  • some constants this code can not handle are: -- constants in moduled imported at run-time. -- constants which created by eval, or stored through locals(), globals() -- another variables which evaluated at run-time

Since I just wrote this code experimental, there are a lot rest of problems.


  • References

pyc (python compiler written in compiler module) has more optimization tips. http://students.ceid.upatras.gr/~sxanth/pyc/

Created by Ikkei Shimomura on Wed, 29 Jun 2005 (PSF)
Python recipes (4591)
Ikkei Shimomura's recipes (4)

Required Modules

  • (none specified)

Other Information and Tasks