Welcome, guest | Sign In | My Account | Store | Cart

This recipe provides a quick way to search and retrieve information about installed Python packages and modules in your Python installation. It classifies the retrieved module information into types (built-in, source files, c-extensions etc) and prints module information under each type as a sorted list.

Python, 271 lines
  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
#!/usr/bin/env python

# pkgsearch.py - Print information about installed Python packages
# and modules.

import sys, os.path
import compiler
import site

from imp import *

__usage__="""Usage: %s [optional paths] % sys.argv[0])"""

class PkgUtil(object):
    """ Utility class for querying information about
    installed packages and modules """
    
    def __init__(self, paths=None):
        self.paths = sys.path
        if paths:
            self.paths = paths + self.paths

    def find_standard_package(self, pkgname):
        """Search in standard paths for a package/module """

	try:
	    result = find_module(pkgname)
	    return result
        except ImportError, e:
            return ()

    def get_package_init_path(self, pkgname, pkgdir):
        """ Return the init file path for the package.
        This has to be called only for directory packages """

        pkgdir = os.path.abspath(pkgdir)
        
        # Try __init__.py
        pkginitfile = os.path.join(pkgdir, '__init__.py')
        # If it does not exist, try <pkgname>.py
        if not os.path.isfile(pkginitfile):
            pkginitfile = os.path.join(pkgdir,pkgname + '.py')

        if os.path.isfile(pkginitfile):
            return pkginitfile
        else:
            # Everything failed, return pkgdir itself!
            return pkgdir
        
    def load_comments(self, pkgfile):
	""" Open the package and load comments if any. 
	Return the loaded comments """

        # Note: This has to be called with a Python
        # source file (.py) only!
        
	if not os.path.exists(pkgfile):
	    return ""
	
	comment = ""
	
	try:
	    of = open(pkgfile,'rb')
	    data = of.read()
	    if data:
		# Create code object
                try:
                    c = compiler.compile(data,pkgfile,'exec')
                    # Get the position of first line of code
                    if c:
                        lno = c.co_firstlineno
                        lnum = 0
                        # Read file till this line number
                        of.seek(0)
                        for line in of:
                            comment = "".join((comment, line))
                            lnum += 1
                            if lnum==lno or line=="\n": break
                except SyntaxError, e:
                    pass
                except Exception, e:
                    pass
	    of.close()
	except (OSError, IOError, TypeError), e:
            pass
	    
	return comment
	
    def find_package(self, pkgname):
        # Query for package/module and return a dictionary
        # with the following fields
        # 'name': Package/module name,
        # 'path' : Full path of the package/module,
        # 'type' : What kind of a package/module is it
        #          This has the following values
	# 'doc'  : Package documentation
        #
        #          PY_SOURCE: The module was found as a source file. 
        #          PY_COMPILED: The module was found as a compiled code
        #                       object file. 
        #          C_EXTENSION: The module was found as dynamically
        #                       loadable shared library. 
        #          PY_RESOURCE: The module was found as a Macintosh resource.
        #                       This value can only be returned on a Macintosh. 
        #          PKG_DIRECTORY: The module was found as a package directory. 
        #          C_BUILTIN:     The module was found as a built-in module. 
        #          PY_FROZEN:  The module was found as a frozen module.
        #
        # If no module/package is found, returns a null dictionary.
        d = {}
        packages = pkgname.split('.')
        top_level = packages[0]

        try:
            # First look for built-in modules
            result = self.find_standard_package(pkgname)
            if not result and self.paths:
                result = find_module(pkgname, self.paths)
            if result:
                of, pathname, desc = result
                # Last or only component of package
                if len(packages)==1:
		    # Load module
                    try:
                        M = load_module(pkgname, of, pathname, desc)
                    except Exception, e:
                        return d
                    
                    d['name'] = pkgname
                    d['type'] = desc[2]
		    d['doc']=''

                    if os.path.dirname(pathname):
                        d['path'] = self.get_package_init_path(pkgname, pathname)
                    else:
                        # For built-in modules
                        d['path']=pathname
		    if M:
			if M.__doc__:
			    # Set doc string
			    d['doc'] = M.__doc__
			else:
                            pkgfile = ''
			    # Load comments from the package file
			    # if any.
			    if d['type'] == PY_SOURCE:
				pkgfile = d['path']
			    elif d['type'] == PKG_DIRECTORY:
                                if os.path.isfile(d['path']):
                                    pkgfile = d['path']
				
			    if pkgfile:
				d['doc'] = self.load_comments(pkgfile)
			    
                    return d
                
        except ImportError, e:
            if len(packages)>1:
                try:
                    result = find_module(top_level, self.paths)
                    if result:
                        of, pathname, desc = result
                        try:
                            M = load_module(top_level, of, pathname, desc)
                            # Remove the top_level package from the name
                            pkgname = reduce(lambda x,y: x+'.'+y, packages[1:])
                            # Call this recursively
                            if hasattr(M, '__path__'):
                                return self.find_package(pkgname, M.__path__)
                        except ImportError, e:
                            pass
                        except Exception, e:
                            pass
                except ImportError, e:
                    pass

            else:
                pass


        return d

    def pkgTypeInfo(self, pkg_typ):
        """ Return information on the package - Version 2"""

        if pkg_typ is PY_SOURCE:
            return "PYTHON SOURCE FILE MODULES"
        elif pkg_typ is PY_COMPILED:
            return "PYTHON COMPILED CODE OBJECT MODULES "
        elif pkg_typ is C_EXTENSION:
            return "DYNAMICALLY LOADABLE SHARED LIBRARY (C-EXTENSION) MODULES"
        elif pkg_typ is PY_RESOURCE:
            return "MACINTOSH RESOURCE MODULES"
        elif pkg_typ is PKG_DIRECTORY:
            return "PYTHON PACKAGE DIRECTORY MODULES"
        elif pkg_typ is C_BUILTIN:
            return "BUILT-IN MODULES"
        elif pkg_typ is PY_FROZEN:
            return "FROZEN PYTHON MODULES"
        else:
            return "UNKNOWN MODULES"        
    
    def list_packages(self):
        """ An ambitious function which attempts to list all Python packages
        in your system, according to the configuration """

        # First extract loaded module names from sys.modules
        sys_modules = sys.modules.keys()

        packages = {}

        # First add moduels in sys.modules (built-ins,
        # preloads and already loaded ones)
        for name in sys_modules:
            d = self.find_package(name)
            if not d: continue
            try:
                pkginfo = packages[d['type']]
                pkginfo[d['name']] = d['path']
            except Exception, e:
                packages[d['type']] = { d['name'] : d['path'] }

        import site
        # Loop through all directories in sys.path and check for modules
        # Dont iterate through <prefix>/lib directory
        libdir = os.path.join(sys.prefix, 'lib')
        
        walked = []
        for top_level in self.paths:
            if not os.path.isdir(top_level):
                continue

            # Dont iterate through libdir
            if os.path.abspath(top_level) == os.path.abspath(libdir):
                continue
            
            walked.append(top_level)
            for item in os.listdir(top_level):

                fullpath = os.path.join(top_level, item)
                if fullpath in walked: continue

                walked.append(fullpath)
                # Remove the extension
                idx = item.find('.')
                if idx != -1: item = item[:idx]
                d = self.find_package(item)
                if not d: continue
                try:
                    pkginfo = packages[d['type']]
                    pkginfo[d['name']] = d['path']
                except Exception, e:
                    packages[d['type']] = { d['name'] : d['path'] }                
                    
        for key,item in packages.items():
            print
            print self.pkgTypeInfo(key)
            print
            
            # Print sorted
            listofitems = item.keys()
            listofitems.sort()

            for key2 in listofitems:
                print key2,':',item[key2]
        
if __name__=="__main__":
    u = PkgUtil(sys.argv)

    # List information about standard packages
    u.list_packages()

        

If invoked without any arguments it prints information on all modules inside your Python installation directory.

Example: % python pkgsearch.py

If you want to search in other directories, provide them as arguments to the program. Such as,

Example:

Search for packages in current directory and in /usr/local/python

and ~/programs/python, apart from the Python installation folder.

% python pkgsearch.py . /usr/local/python /home/user/programs/python

The search algorithm only performs a first level search (immediate children) for directories so this recipe cannot ferret out packages deeper inside directories. However most of the time this is not required so it is not a big drawback. It also loads comments for Python source files if it cannot find any documentation in the module's __doc__ attribute. However it is not printed.

3 comments

Scott David Daniels 18 years, 7 months ago  # | flag

Import of non-existant module. Why "import config"?

Anand Pillai (author) 18 years, 7 months ago  # | flag

Corrected it. This code was derived from a much larger codebase which allows you to query Python packages on a command line. It had a module named config which I forgot to remove when I rewrote it in one single module.

Gerry King 13 years, 11 months ago  # | flag

I am trying to get a listing of installed python packages using pkgsearch. Unfortunately it falls over with an import error. I added

print pathname, pkgname

just before line 125

M = load_module(pkgname, of, pathname, desc)

so I could see where it was happening. It looks as though it is having problems importing webkit(/usr/lib/pymodules/python2.6/gtk-2.0/webkit.so webkit). I am a Python newbie and have no idea how to resolve this. I guess I could just try and swallow the error but I would like to know why it occurs.

Gerry

Stacktrace:

/usr/lib/pymodules/python2.6/gtk-2.0/gtk gtk

/usr/lib/pymodules/python2.6/gtk-2.0/atk.so atk

/usr/lib/pymodules/python2.6/gtk-2.0/wnck.so wnck

/usr/lib/pymodules/python2.6/gtk-2.0/bugbuddy.py bugbuddy

/usr/lib/pymodules/python2.6/gtk-2.0/gnomedesktop gnomedesktop

/usr/lib/pymodules/python2.6/gtk-2.0/gnomeprint gnomeprint

/usr/lib/pymodules/python2.6/gtk-2.0/webkit.so webkit

ImportError: could not import gobject (error was: "name '_glib' is not defined")

Error in sys.excepthook:

Traceback (most recent call last):

File "/usr/lib/python2.6/dist-packages/apport_python_hook.py", line 39, in apport_excepthook from apport.packaging_impl import impl as packaging

File "/usr/lib/python2.6/dist-packages/apport/__init__.py", line 1, in <module> from apport.report import Report

File "/usr/lib/python2.6/dist-packages/apport/report.py", line 21, in <module> import fileutils

File "/usr/lib/python2.6/dist-packages/apport/fileutils.py", line 16, in <module> from packaging_impl import impl as packaging File "/usr/lib/python2.6/dist-packages/apport/packaging_impl.py", line 18, in <module> import apt

File "/usr/lib/python2.6/dist-packages/apt/__init__.py", line 25, in <module> from apt.package import Package

File "/usr/lib/python2.6/dist-packages/apt/package.py", line 518, in <module> class VersionList(Sequence):

File "/usr/lib/python2.6/abc.py", line 79, in __new__ cls = super(ABCMeta, mcls).__new__(mcls, name, bases, namespace)

TypeError: Error when calling the metaclass bases super(type, obj): obj must be an instance or subtype of type

Original exception was: ImportError: could not import gobject (error was: "name '_glib' is not defined")

Fatal Python error: can't initialise module gobject Aborted