programming language popularity from sourceforge.net and freshmeat.net (Python recipe) by Scott Tsai

ActiveState Code (http://code.activestate.com/recipes/425529/)

import sys
import re
import mechanize
import pullparser

SOURCEFORGE_URL = 'http://sourceforge.net/softwaremap/trove_list.php?form_cat=160'
# sourceforge.net, projects per programming language, parser tokens:
"""
Token('starttag', 'a', [('href', 'trove_list.php?form_cat=163')]),
Token('starttag', 'img', [('src', 'http://images.sourceforge.net..'), ...)
Token('entityref', 'nbsp', None),
Token('data', ' Ada', None),
Token('endtag', 'a', None),
Token('data', ' ', None)]
Token('starttag', 'i', []),
Token('data', '(100 projects)', None),
"""
FRESHMEAT_URL = 'http://freshmeat.net/browse/160/?topic_id=160'
# projects per programming language, parser tokens:
"""
Token('data', 'Programming Language', None),
Token('endtag', 'b', None),
Token('data', '\r\n              ', None),
Token('starttag', 'br', []),
Token('data', '\r\n      \n\r\n      ', None),
Token('starttag', 'li', []),
Token('starttag', 'a', [('href', '/browse/163/')]),
Token('starttag', 'b', []),
Token('data', 'Ada', None),
Token('endtag', 'b', None),
Token('endtag', 'a', None),
Token('data', '\r\n          ', None),
Token('starttag', 'small', []),
Token('data', '(57 projects)', None),
Token('endtag', 'small', None),
Token('data', '\r\n              ', None),
Token('starttag', 'li', []),
Token('starttag', 'a', [('href', '/browse/161/')]),
"""

def get_n_project(s):
	return int(re.compile(r'([\d]*) projects').findall(s)[0])
def sourceforge_get_language_statistics(parser):
	'sourceforge_get_language_statistics(file_obj) -> [ (n_project, language), ... ]'
	def is_language_link(token):
		try:
			if token.type == 'starttag' and token.data == 'a' \
				and 'trove_list.php?form_cat=' in token.attrs[0][1]:
					return True
		except IndexError:
			return False
		return False
	for t in parser.tags():
		if is_language_link(t):
			l = [
			(parser.next(), 'img'),
			(parser.next(), 'nbsp'),
			(parser.next(), 'data(language)'),
			(parser.next(), '/a'),
			(parser.next(), 'data'),
			(parser.next(), 'i'),
			(parser.next(), 'data(n_project)'), ]

			language = l[2][0].data.strip()
			n = l[6][0].data.strip()
			try:
				n_project = get_n_project(n)
			except IndexError:
				continue
			else:
				yield (n_project, language)

def freshmeat_get_language_statistics(parser):
	def is_language_link(token):
		try:
			if token.type == 'starttag' and token.data == 'a' \
				and '/browse/' in token.attrs[0][1]:
					return True
		except IndexError:
			return False
		return False
	for t in parser.tags():
		if is_language_link(t):
			l = [
			(parser.next(), 'b'),
			(parser.next(), 'data(language)'),
			(parser.next(), '/b'),
			(parser.next(), 'a'),
			(parser.next(), 'data'),
			(parser.next(), 'small'),
			(parser.next(), 'data(n_project)'), ]
			language = l[1][0].data.strip()
			n = l[6][0].data
			try:
				n_project = get_n_project(n)
			except IndexError:
				continue
			else:
				yield (n_project, language)

for (name, url, get_statistics) in (
		('sourceforge.net', SOURCEFORGE_URL,
			sourceforge_get_language_statistics),
		('freshmeat.net', FRESHMEAT_URL,
			freshmeat_get_language_statistics),):
	b = mechanize.Browser()
	b.set_handle_robots(False)
	b.open(url)
	r = b.response()
	r.seek(0)
	p = pullparser.PullParser(r)
	l = list(get_statistics(p))
	l.sort()
	l.reverse()
	print name
	for (n_project, language) in l:
		print '%6d %s' % (n_project, language)
	print