stock prices historical data bulk download from internet (Python recipe) by gian paolo ciceri

ActiveState Code (http://code.activestate.com/recipes/511444/)

#! /usr/bin/env python
# -*- coding: utf-8 -*-


__author__ = 'gian paolo ciceri <gp.ciceri@gmail.com>'
__version__ = '0.1'
__date__ = '20070401'
__credits__ = "queue and MT code was shamelessly stolen from pycurl example retriever-multi.py"

#
# Usage: python grabYahooDataMt.py -h 
#
#
# for selecting tickers and starting date it uses an input file of this format 
# <ticker> <fromdate as YYYYMMDD>
# like
# ^GSPC 19500103 # S&P 500
# ^N225 19840104 # Nikkei 225

import sys, threading, Queue, datetime
import urllib
from optparse import OptionParser


# this thread ask the queue for job and does it!
class WorkerThread(threading.Thread):
    def __init__(self, queue):
        threading.Thread.__init__(self)
        self.queue = queue

    def run(self):
        while 1:
            try:
                # fetch a job from the queue
                ticker, fromdate, todate = self.queue.get_nowait()
            except Queue.Empty:
                raise SystemExit
            if ticker[0] == "^": 
                tick = ticker[1:]
            else:
                tick = ticker
            filename = downloadTo + "%s_%s.csv" % (tick, todate)
            fp = open(filename, "wb")
            if options.verbose:
                print "last date asked:", todate, todate[0:4], todate[4:6], todate[6:8] 
                print "first date asked:", fromdate, fromdate[0:4], fromdate[4:6], fromdate[6:8]
            quote = dict()
            quote['s'] = ticker
            quote['d'] = str(int(todate[4:6]) - 1)
            quote['e'] = str(int(todate[6:8]))
            quote['f'] = str(int(todate[0:4]))
            quote['g'] = "d" 
            quote['a'] = str(int(fromdate[4:6]) - 1)
            quote['b'] = str(int(fromdate[6:8]))
            quote['c'] = str(int(fromdate[0:4]))
            #print quote
            params = urllib.urlencode(quote)
            params += "&ignore=.csv"

            url = "http://ichart.yahoo.com/table.csv?%s" % params
            if options.verbose:
                print "fetching:", url           
            try:
                f = urllib.urlopen(url)
                fp.write(f.read())
            except:
                import traceback
                traceback.print_exc(file=sys.stderr)
                sys.stderr.flush()
            fp.close()
            if options.verbose:
                print url, "...fetched"
            else:
                sys.stdout.write(".")
                sys.stdout.flush()



if __name__ == '__main__':

    # today is
    today = datetime.datetime.now().strftime("%Y%m%d")
    
    # parse arguments
    parser = OptionParser()
    parser.add_option("-f", "--file", dest="tickerfile", action="store", default = "./tickers.txt",
    				  help="read ticker list from file, it uses ./tickers.txt as default")
    parser.add_option("-c", "--concurrent", type="int", dest="connections", default = 10, action="store",
    				  help="# of concurrent connections")
    parser.add_option("-d", "--dir", dest="downloadTo", action="store", default = "./rawdata/",
    				  help="save date to this directory, it uses ./rawdata/ as default")
    				  
    parser.add_option("-t", "--todate", dest="todate", default = today, action="store",
    				  help="most recent date needed")
    parser.add_option("-v", "--verbose",
    					  action="store_true", dest="verbose")
    parser.add_option("-q", "--quiet",
    					  action="store_false", dest="verbose")					 
    (options, args) = parser.parse_args()

    
    tickerfile = options.tickerfile
    downloadTo = options.downloadTo
    connections =  options.connections
    today = options.todate
    

    # get input list
    try:
    	tickers = open(tickerfile).readlines()
    except:
    	parser.error("ticker file %s not found" % (tickerfile,))
    	raise SystemExit
    
    
    # build a queue with (ticker, fromdate, todate) tuples
    queue = Queue.Queue()
    for tickerRow in tickers:
    	#print tickerRow
    	tickerRow = tickerRow.strip()
    	if not tickerRow or tickerRow[0] == "#":
    		continue
    	tickerSplit = tickerRow.split()
    	# ticker, fromdate, todate
    	queue.put((tickerSplit[0], tickerSplit[1], today))

    

    
    # Check args
    assert queue.queue, "no Tickers given"
    numTickers = len(queue.queue)
    connections = min(connections, numTickers)
    assert 1 <= connections <= 255, "too much concurrent connections asked"

    
    if options.verbose:
    	print "----- Getting", numTickers, "Tickers using", connections, "simultaneous connections -----"
    
    
    # start a bunch of threads, passing them the queue of jobs to do
    threads = []
    for dummy in range(connections):
    	t = WorkerThread(queue)
    	t.start()
    	threads.append(t)
    
    
    # wait for all threads to finish
    for thread in threads:
    	thread.join()
    sys.stdout.write("\n")
    sys.stdout.flush()
       	
    # tell something to the user before exiting
    if options.verbose:	   
    	print "all threads are finished - goodbye."