Size of Python objects. (Python recipe) by Jean Brouwers

ActiveState Code (http://code.activestate.com/recipes/544288/)

The recipe below is obsolete.  Please use this one instead

<http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/546530>


#!/usr/bin/env python

 # <http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/544288>

 # Copyright, license and disclaimer are at the end of this file.

'''This module exposes function  asizeof() which calculates the
   (approximate) size of Python objects in bytes.

   The size of an object is defined as the sum of the basic size
   of the object type, the item size times the number of items
   and the size of any referenced objects.

   The basic and item sizes are obtained from the __basicsize__
   resp. __itemsize__ attributes of the type of the object.  Any
   zero item size (for sequence objects) is replaced by the size
   of the C typedef of the item.

   Referenced objects are visited recursively up to a given depth.
   The size of any objects referenced multiple times is included
   only once.

   Multi-precision int (aka long) objects and over-allocation of
   mutable sequence objects as lists are taken into account.

   The (byte)code size of objects as classes, functions, methods,
   modules, etc. can be included, optionally.

   Sizes can be aligned to a given alignment (power of 2).

   To prevent excessive sizes, several object types are ignored,
   e.g. builtins, referenced modules and function globals.  But
   module objects will be sized if passed as arguments.

   In addition, many __...__ attributes of callable objects are
   ignored, except crucial ones, e.g. class attributes __dict__,
   __doc__, __name__.  For more details, check the type-specific
   functions returning referenced objects.

   These and possibly other assumptions are rather arbitrary and
   may need corrections or adjustments.

   Tested with Python 2.2.3, 2.3.4, 2.4.3, 2.5.1 or 3.0a2 on RHEL
   3u7, CentOS 4.6, SuSE 9.3, MacOS X Tiger (Intel) and Panther
   (PPC), Solaris 10 and Windows XP.

   Function  asizeof() is not thread-safe.
'''

__version__ = '2.12 (Feb 21, 2008)'

__all__ = ['asizeof',]

from inspect import isbuiltin, isclass, iscode, isframe, isfunction, ismethod, ismodule
from math    import log
from struct  import pack
from sys     import getrecursionlimit

 # type-specific functions
def _dir(obj, prefix):
    '''Return list of matching attributes, except globals.
    '''
    g = prefix + 'globals'  # sized in modules
    return [a for a in dir(obj) if a.startswith(prefix) and a != g]

def _refs(obj, *ats):
    '''Return list of specified attribute objects.
    '''
    return [getattr(obj, a) for a in ats if hasattr(obj, a)]

def _class(obj):
    '''Class objects.
    '''
    return _refs(obj, '__class__', '__dict__', '__doc__', '__name__', '__slots__', '__weakref__')

def _code(obj):
    '''Code objects.
    '''
    return _refs(obj, *_dir(obj, 'co_'))

def _dict(obj):
    '''Dict keys and value.
    '''
    return list(obj.keys()) + list(obj.values())

def _frame(obj):
    '''Frame objects.
    '''
    return _refs(obj, *_dir(obj, 'f_'))

def _function(obj):
    '''Function or lambda objects.
    '''
    return _refs(obj, '__doc__', '__name__', *_dir(obj, 'func_'))

def _instance(obj):
    '''Class instance objects.
    '''
    return _refs(obj, '__dict__', '__slots__')

def _method(obj):
    '''Method objects.
    '''
    return _refs(obj, '__doc__', '__name__', *_dir(obj, 'im_'))

def _module(obj):
    '''Module objects.
    '''
     # essentially a dict
    return dict([(a, getattr(obj, a)) for a in dir(obj)])

def _seq(obj):
    '''Frozen/set, list, tuple and xrange objects.
    '''
    return list(obj)

def _type(obj):
    '''Type objects.
    '''
    return _refs(obj, '__mro__', '__name__')

 # sizes of some C types
_Csizeof_int      =  len(pack('i', 0))  # sizeof(int)
_Csizeof_long     =  len(pack('l', 0))  # sizeof(long)
_Csizeof_ssize    =  len(pack('P', 0))  # sizeof(ssize_t)
_Csizeof_voidp    =  len(pack('P', 0))  # sizeof(void*)
try:  # multi-precision int (or long)
   _Csizeof_digit =  long.__itemsize__
except NameError:  # no long in Python 3.0
   _Csizeof_digit =  int.__itemsize__
_Csizeof_head     = _Csizeof_voidp + _Csizeof_ssize  # sizeof(PyObject_HEAD)
_Csizeof_var_head = _Csizeof_head  + _Csizeof_ssize  # sizeof(PyObject_VAR_HEAD)

 # length functions
def _len(obj):
    '''Safe len().
    '''
    try:
        return len(obj)
    except:  # TypeError
        return 0

def _len1(obj):
    '''Length of str and unicode.
    '''
    return len(obj) + 1  # XXX sentinel

def _len8(obj):
    '''Length of mutable sequences.
    '''
    n = len(obj)
     # estimate over-allocation
    if n < 9:
        n += 4
    else:
        n += 6 + (n >> 3)
    return n

_digit2p2 = 1 << (_Csizeof_digit * 8)
_digitmax = _digit2p2 - 1  # == (2 * PyLong_MASK + 1)
_digitlog = 1.0 / log(_digit2p2)

def _ilen(obj):
    '''Length of multi-precision int (aka long) in digits.
    '''
    n, i = 1, abs(obj)
    if i > _digitmax:
         # no log(x[, base]) in Python 2.2
        n += int(log(i) * _digitlog)
    return n

# [type] = (_ref(), _len(), basicsize, itemsize)
_types_data = {}

def _types(types, t, t4):
    '''Add type to types.
    '''
    if t in types:  # and types[t] != t4
        raise KeyError('asizeof type conflict: %r %r' % (t, t4))
    types[t] = t4

def _basic(t, r, n, i):
    '''Return basicsize of type and add type.
    '''
    s = getattr(t, '__basicsize__', 0)
    _types(_types_data, t, (r, n, s, getattr(t, '__itemsize__', 0) or i))
    return s

 # basicsize of data types
_basicsize_dict     = _basic(dict,          _dict, _len8, _Csizeof_ssize + (_Csizeof_voidp * 2))  # sizeof(PyDictEntry)
_basicsize_ellipsis = _basic(type(Ellipsis), None,  None,  0)
_basicsize_float    = _basic(float,          None,  None,  0)
_basicsize_int      = _basic(int,            None, _ilen,  int.__itemsize__)  # multi-precision in Python 3.0
_basicsize_list     = _basic(list,          _seq,  _len8, _Csizeof_voidp)  # sizeof(PyObject*)
_basicsize_none     = _basic(type(None),     None,  None,  0)
_basicsize_tuple    = _basic(tuple,         _seq,  _len,  _Csizeof_voidp)  # sizeof(PyObject*)

 # newer or obsolete data types
try:
    if isbuiltin(bool):  # Python 2.2
        _basicsize_bool = int.__basicsize__  # like int
    else:  # bool is a type
        _basicsize_bool = _basic(bool, None, None, 0)
except NameError:  # missing
    _basicsize_bool = 0

try:
    _basicsize_bytes = _basic(bytes, None, _len, 1)
except NameError:  # missing
    _basicsize_bytes = 0
try:  # XXX str8 == bytes + sentinel?
    _basic(str8, None, _len1, 1)
except NameError:  # missing
    pass

try:
    _basicsize_frozenset = _basic(frozenset, _seq, _len8, (_Csizeof_voidp + _Csizeof_long))  # sizeof(setentry)
except NameError:  # missing
    _basicsize_frozenset = 0
try:
    _basicsize_set = _basic(set, _seq, _len8, (_Csizeof_voidp + _Csizeof_long))  # sizeof(setentry)
except NameError:  # missing
    _basicsize_set = 0

try:
    _basicsize_long = _basic(long, None, _ilen, 0)  # multi-precision
except NameError:  # missing
    _basicsize_long = 0

try:
    _basic(type(NotImplemented), None, None, 0)
except NameError:  # missing
    pass

try:
    _basicsize_unicode = _basic(unicode, None, _len1, 2)  # sizeof(PY_UNICODE_TYPE) 2-byte short
    _basicsize_str     = _basic(str,     None, _len1, 1)  # 1-byte char
except NameError:  # missing
    _basicsize_unicode = 0  # str is unicode
    _basicsize_str     = _basic(str, None, _len1, 2)  # XXX 2-byte char?

try:
    if isbuiltin(xrange):  # Python 2.2
        _basicsize_xrange = 0
    else:  # xrange is a type
        _basicsize_xrange = _basic(xrange, _seq, _len, _basicsize_int)
except NameError:  # missing
    _basicsize_xrange = 0

 # basicsize of code and some special types
class _C:
    pass

class _M:
    def m(self):
        pass

def _F():
    pass

_basicsize_class    =   type(_C).__basicsize__
_basicsize_function =   type(_F).__basicsize__
_basicsize_instance = type(_C()).__basicsize__
_basicsize_method   = type(_M.m).__basicsize__

_basicsize_type     = _basic(type, _type, None, 0)

_basicsize_code     = _Csizeof_head + (10 * _Csizeof_voidp) + (5 * _Csizeof_int)  # sizeof(PyCodeObject)
_basicsize_frame    = _Csizeof_var_head + (13 * _Csizeof_voidp) + (21 * 3 * _Csizeof_int)  # sizeof(PyFrameObject)
_basicsize_module   = _Csizeof_head + _Csizeof_voidp  # sizeof(PyModuleObject)

del _C, _M, _F

 # keep code and data types separate
_types_code = _types_data.copy()

try:  # sum() builtin
    _sum = sum
except NameError:  # no sum() in Python 2.2
    def _sum(vals):
        t = 0
        for v in vals:
            t += v
        return t

def _sizer(obj, deep, code, mask, types, seen):
    '''Size an object, recursively.
    '''
    s, k = 0, id(obj)
    if k in seen:  # obj seen before
        seen[k] += 1
    else:
        seen[k]  = 1
        t = type(obj)
        try:  # get _ref(), _len(), basic- and itemsize
            r, n, s, i = types[t]
        except KeyError:  # new type
            r, n, i = None, _len, 0
            if isbuiltin(obj) or ismodule(obj):
                pass  # ignore
            elif isframe(obj):
                r, s = _frame, _basicsize_frame
            elif iscode(obj):
                if code:
                    r, s = _code, _basicsize_code
            elif hasattr(obj, '__call__'):  # no callable() in Python 3.0
                if code:
                    if isclass(obj):
                        r, s = _class, _basicsize_class
                    elif isfunction(obj):
                        r, s = _function, _basicsize_function
                    elif ismethod(obj):
                        r, s = _method, _basicsize_method
                    elif isinstance(obj, type):
                        r, s = _type, _basicsize_type
            else:  # assume some class inst
                r, s = _instance, _basicsize_instance
            if s:  # adjust size
                s = max(s, getattr(t, '__basicsize__', 0))
             # add new type
            _types(types, t, (r, n, s, getattr(t, '__itemsize__', i)))
        if n and i > 0:  # items size
            s += i * n(obj)
        if mask:  # align
            s = (s + mask) & ~mask
        if r and deep > 0:  # add sizes of ref'd objs
            try:
                deep -= 1
                s += _sum([_sizer(o, deep, code, mask, types, seen) for o in r(obj)])
                seen[0] = min(deep, seen[0])  # recursion depth
            except RuntimeError:  # XXX RecursionLimitExceeded:
                pass
    return s

def _obj(obj):
    '''Handle special cases.
    '''
    if ismodule(obj):
        return _module(obj)
    return obj

def _print(fmt, *args):
     if fmt and args:
         print(fmt % args)
     else:
         print(fmt)

def _repr(obj, clip=80):
    '''Clip long repr() string.
    '''
    try:  # safe repr()
        r = repr(obj)
    except TypeError:
        r = 'N/A'
    if 0 < clip < len(r):
        h = (clip // 2) - 2
        if h > 0:
            r = r[:h] + '....' + r[-h:]
    return r

def _SI(size, K=1024):
    '''Return size as SI string.
    '''
    if 1 < K < size:
       f = float(size)
       for si in iter('KMGPTE'):
           f /= K
           if f < K:
               return ' or %.1f %sB' % (f, si)
    return ''

def _asizeof(objs, deep=100, code=False, align=8, verbose=0):
    '''Combine size one or more objects.
    '''
    if align > 1:
        m = align - 1
        if (align & m) != 0:
            raise ValueError('asizeof invalid alignment: %r' % align)
    else:
        m = 0
    if code:
        t = _types_code
    else:
        t = _types_data
    v = {0: deep}  # recursion depth
     # positional arguments
    s = _sum([_sizer(_obj(o), deep, code, m, t, v) for o in objs])

     # print some details
    if verbose > 0:
        d = deep - v[0];  del v[0]
        w = 1 + len(str(s))
         # print size and stats
        _print('asizeof(%s, verbose=%d): ...', _repr(objs), verbose)
        if code:
           _print('%*d bytes%s (incl. code)', w, s, _SI(s))
        else:
           _print('%*d bytes%s', w, s, _SI(s))
        if m:
           _print('%*d byte aligned', w, m + 1)
        _print('%*d byte sizeof(void*)', w, _Csizeof_voidp)
        _print('%*d objects sized', w, len(v))
        _print('%*d objects seen', w, _sum(v.values()))
        if d > 0:
           _print('%*d deepest recursion', w, d)
        if verbose > 1:
             # print types
            def _sorted(args):
                try:  # XXX Python 3.0
                    args.sort()
                except AttributeError:
                    pass
                return args
            _print('%*d types:  (_ref(), _len(), basicsize, itemsize)', w, len(t))
            for t in _sorted(t.items()):
               _print('%*s %r:  %r', w, '', *t)
    return s

def asizeof(*objs, **kwds):
    '''Return the total size in bytes of all objects passed as
       positional argments.

       asizeof(obj, ..., deep=100,    # recursion limit
                         code=False,  # incl. size of callables
                         align=8,     # size alignment
                         verbose=0)   # verbosity level

       Set  deep to a positive value to accumulate the sizes of
       all objects referenced by obj, recursively.  Using  deep
       zero returns the basic size of the object, incl. the size
       for the dict, frozen/set, str, list or tuple items space.

       The code size of callable objects like classes, functions,
       methods, etc. is included only if  code is True.

       Set  align to a power of 2 to align sizes.  Any value less
       than 2 avoids size alignment.

       A positive value for  verbose prints some stats like the
       number and types of objects used.
    '''
    return _asizeof(objs, **kwds)  # (deep=100, code=False, align=8, verbose=0)


if __name__ == '__main__':

    MAX = getrecursionlimit() + 4

    def _aprint(obj, deep=MAX, **kwds):
        _print(" asizeof(%s) is %d, %d, %d", _repr(obj),
                 asizeof(obj, deep=0,    code=False),
                 asizeof(obj, deep=deep, code=False, **kwds),
                 asizeof(obj, deep=deep, code=True,  **kwds))

    _print('some C sizes:')
    s = [t for t in locals().items() if t[0].startswith('_Csizeof_')]
    s.sort()
    for k, v in s:
        _print(" sizeof(%s) is %d", k[9:], v)

    _print('')
    _print('type.__basicsize__ (0 means missing type):')
    s = [t for t in locals().items() if t[0].startswith('_basicsize_')]
    s.sort()
    for k, v in s:
        _print(" %s is %d", k[11:], v)

    class C: pass

    class D:
        _attr1 = None
        _attr2 = None

    class E(D):
        def __init__(self, a1=1, a2=2):
            _attr1 = a1
            _attr2 = a2

    class S:
        __slots__ = ('a', 'b')

    class T:
        __slots__ = ('a', 'b')
        def __init__(self):
            self.a = self.b = 0

    _print('')
    _print('asizeof(%s, deep=%s, code=%s)', '<non-callable>', '0/MAX', 'False/True')
    for o in (None,
              1.0, 1.0e100, 1024, 1000000000,
              MAX, 1<<32, 1<<64, 1<<256, -(1 << 256),
              '', 'a', 'abcdefg',
              type, {}, (), [], s,
              C(), C.__dict__,
              D(), D.__dict__,
              E(), E.__dict__,
              S(), S.__dict__,
              T(), T.__dict__,
             _types_data):
        _aprint(o)

    _print('')
    _print('asizeof(%s, deep=%s, code=%s)', '<callable>', '0/MAX', 'False/True')
    for o in (C, D, E, S, T,  # classes are callable
             _code, _dict, _instance, _ilen, _seq, lambda x: x,
            (_code, _dict, _instance, _ilen, _seq)):
        _aprint(o)

    _print('')
    _print('asizeof(%s, deep=%s, code=%s)', 'locals()', 'MAX', False)
    asizeof(locals(), deep=MAX, code=False, verbose=1)

    _print('')
    _print('asizeof(%s, deep=%s, code=%s)', 'globals()', 'MAX', False)
    asizeof(globals(), deep=MAX, code=False, verbose=2)

    _print('')
    _print('asizeof(deep=%s, code=%s, *%s)', 'MAX', True, 'sys.modules.values()')
    from sys import modules
    asizeof(deep=MAX, code=True, verbose=2, *modules.values())


# License file from an earlier version of asizeof() follows:

#---------------------------------------------------------------------
#       Copyright (c) 2002-2008 -- ProphICy Semiconductor, Inc.
#                        All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# - Redistributions of source code must retain the above copyright
#   notice, this list of conditions and the following disclaimer.
#
# - Redistributions in binary form must reproduce the above copyright
#   notice, this list of conditions and the following disclaimer in
#   the documentation and/or other materials provided with the
#   distribution.
# 
# - Neither the name of ProphICy Semiconductor, Inc. nor the names
#   of its contributors may be used to endorse or promote products
#   derived from this software without specific prior written
#   permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
# COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
# OF THE POSSIBILITY OF SUCH DAMAGE.
#---------------------------------------------------------------------