ActiveState Code

#!/usr/bin/env python

"""Web Crawler/Spider

This module implements a web crawler. This is very _basic_ only
and needs to be extended to do anything usefull with the
traversed pages.
"""

import re
import sys
import time
import math
import urllib2
import urlparse
import optparse
from BeautifulSoup import BeautifulSoup

__version__ = "0.1"
__copyright__ = "CopyRight (C) 2008 by James Mills"
__license__ = "GPL"
__author__ = "James Mills"
__author_email__ = "James Mills, James dot Mills st dotred dot com dot au"

USAGE = "%prog [options] <url>"
VERSION = "%prog v" + __version__

AGENT = "%s/%s" % (__name__, __version__)

def encodeHTML(s=""):
    """encodeHTML(s) -> str

    Encode HTML special characters from their ASCII form to
    HTML entities.
    """

    return s.replace("&", "&amp;") \
            .replace("<", "&lt;") \
            .replace(">", "&gt;") \
            .replace("\"", "&quot;") \
            .replace("'", "&#039;") \
            .replace("--", "&mdash")

class Queue(object):

    def __init__(self, size=None):
        super(Queue, self).__init__()

        self._queue = []
        self._size = size

    def __len__(self):
        return len(self._queue)

    def __getitem__(self, n):
        if (not self.empty()) and (0 <= (n + 1) <= len(self._queue)):
            return self._queue[(len(self._queue) - (n + 1))]
        else:
            raise StopIteration

    def push(self, item):
        self._queue.insert(0, item)
        if self._size is not None:
            self._queue = self._queue[:self._size]

    def get(self, n=0, remove=False):
        if (not self.empty()) and (0 <= (n + 1) <= len(self._queue)):
            r = self._queue[(len(self._queue) - (n + 1))]
            if remove:
                del self._queue[(len(self._queue) - (n + 1))]
            return r
        else:
            return None

    def pop(self, n=0):
        return self.get(n, True)

    def peek(self, n=0):
        return self.get(n)

    def top(self):
        return self.peek()

    def bottom(self):
        return self.peek(len(self) - 1)

    def empty(self):
        return self._queue == []

    def size(self):
        return self._size

    def full(self):
        return len(self) == self.size()

class Crawler(object):

    def __init__(self, root, depth, locked=True):
        self.root = root
        self.depth = depth
        self.locked = locked
        self.host = urlparse.urlparse(root)[1]
        self.links = 0
        self.followed = 0

    def crawl(self):
        page = Fetcher(self.root)
        page.fetch()
        urls = Queue()
        for url in page.urls:
            urls.push(url)
        followed = [self.root]

        n = 0

        while not urls.empty():
            n += 1
            url = urls.pop()
            if url not in followed:
                try:
                    host = urlparse.urlparse(url)[1]
                    if self.locked and re.match(".*%s" % self.host, host):
                        followed.append(url)
                        self.followed += 1
                        page = Fetcher(url)
                        page.fetch()
                        for i, url in enumerate(page):
                            if url not in urls:
                                self.links += 1
                                urls.push(url)
                        if n > self.depth and self.depth > 0:
                            break
                except Exception, error:
                    print "Warning: Can't process url '%s'" % url

class Fetcher(object):

    def __init__(self, url):
        self.url = url
        self.urls = []

    def __contains__(self, x):
        return x in self.urls

    def __getitem__(self, x):
        return self.urls[x]

    def _addHeaders(self, request):
        request.add_header("User-Agent", AGENT)

    def open(self):
        url = self.url
        print "Following %s" % url
        try:
            request = urllib2.Request(url)
            handle = urllib2.build_opener()
        except IOError:
            return None
        return (request, handle)

    def fetch(self):
        request, handle = self.open()
        self._addHeaders(request)
        if handle:
            soup = BeautifulSoup()
            try:
                content = unicode(handle.open(request).read(), errors="ignore")
                soup.feed(content)
                tags = soup('a')
            except urllib2.HTTPError, error:
                if error.code == 404:
                    print >> sys.stderr, "ERROR: %s -> %s" % (error, error.url)
                else:
                    print >> sys.stderr, "ERROR: %s" % error
                tags = []
            except urllib2.URLError, error:
                print >> sys.stderr, "ERROR: %s" % error
                tags = []
            for tag in tags:
                try:
                    href = tag["href"]
                    if href is not None:
                        url = urlparse.urljoin(self.url, encodeHTML(href))
                        if url not in self:
                            print " Found: %s" % url
                            self.urls.append(url)
                except KeyError:
                    pass

def getLinks(url):
    page = Fetcher(url)
    page.fetch()
    for i, url in enumerate(page):
        print "%d. %s" % (i, url)

def parse_options():
    """parse_options() -> opts, args

    Parse any command-line options given returning both
    the parsed options and arguments.
    """

    parser = optparse.OptionParser(usage=USAGE, version=VERSION)

    parser.add_option("-q", "--quiet",
            action="store_true", default=False, dest="quiet",
            help="Enable quiet mode")

    parser.add_option("-l", "--links",
            action="store_true", default=False, dest="links",
            help="Get links for specified url only")

    parser.add_option("-d", "--depth",
            action="store", type="int", default=30, dest="depth",
            help="Maximum depth to traverse")

    opts, args = parser.parse_args()

    if len(args) < 1:
        parser.print_help()
        raise SystemExit, 1

    return opts, args

def main():
    opts, args = parse_options()

    url = args[0]

    if opts.links:
        getLinks(url)
        raise SystemExit, 0

    depth = opts.depth

    sTime = time.time()

    print "Crawling %s (Max Depth: %d)" % (url, depth)
    crawler = Crawler(url, depth)
    crawler.crawl()

    eTime = time.time()
    tTime = eTime - sTime

    print "Found:    %d" % crawler.links
    print "Followed: %d" % crawler.followed
    print "Stats:    (%d/s after %0.2fs)" % (
            int(math.ceil(float(crawler.links) / tTime)), tTime)

if __name__ == "__main__":
    main()