Welcome, guest | Sign In | My Account | Store | Cart
#!/usr/bin/env python
access_key = ''
secret_key = ''
import boto
import MySQLdb
from MySQLdb import cursors
import threading

db_user = ''
db_name = ''

db = MySQLdb.connect(user=db_user,db=db_name,cursorclass=cursors.DictCursor)
c = db.cursor()
sdb = boto.connect_sdb(access_key, secret_key)

def get_or_create_domain(domain):
        d = sdb.get_domain(domain)
    except boto.exception.SDBResponseError:
        d = sdb.create_domain(domain)
    return d

def get_primary_key(table_name, cursor):
    Returns a dictionary of fieldname -> infodict for the given table,
    where each infodict is in the format:
        {'primary_key': boolean representing whether it's the primary key,
         'unique': boolean representing whether it's a unique index}
    cursor.execute("SHOW INDEX FROM %s" % table_name)
    indexes = {}
    for row in cursor.fetchall():
        if row['Key_name'] == 'PRIMARY':
            return row['Column_name']
    raise("Table %s does not have a primary key" % table_name)

class BotoWorker(threading.Thread):
    def __init__(self, name, record, domain):
        self.domain = domain
        self.name = name
        self.record = record

    def run(self):
            print "inserting %s" % self.name
            item = self.domain.new_item(self.name)
            for key, value in self.record.items():
                    item[key] = value
                except UnicodeDecodeError:
                    item[key] = 'unicode error'

def main():
    c.execute("show tables;")
    for table in c.fetchall():
        table = table["Tables_in_%s" % db]
        print "loading data from %s" % table
        total = c.execute("select * from %s" % table)
        print "fetched %s items from mysql" % total

        for record in c.fetchall():
            name = record.pop(get_primary_key(table, c))
            thread_started = False
            while not thread_started:
                if threading.activeCount() < 10:
                    print "got a thread %s" % threading.activeCount()
                    BotoWorker(name=name, record=record, domain=get_or_create_domain(table)).start()
                    thread_started = True

if __name__ == '__main__':