import sys, os, time
from subprocess import Popen, list2cmdline
def cpu_count():
''' Returns the number of CPUs in the system
'''
num = 1
if sys.platform == 'win32':
try:
num = int(os.environ['NUMBER_OF_PROCESSORS'])
except (ValueError, KeyError):
pass
elif sys.platform == 'darwin':
try:
num = int(os.popen('sysctl -n hw.ncpu').read())
except ValueError:
pass
else:
try:
num = os.sysconf('SC_NPROCESSORS_ONLN')
except (ValueError, OSError, AttributeError):
pass
return num
def exec_commands(cmds):
''' Exec commands in parallel in multiple process
(as much as we have CPU)
'''
if not cmds: return # empty list
def done(p):
return p.poll() is not None
def success(p):
return p.returncode == 0
def fail():
sys.exit(1)
max_task = cpu_count()
processes = []
while True:
while cmds and len(processes) < max_task:
task = cmds.pop()
print list2cmdline(task)
processes.append(Popen(task))
for p in processes:
if done(p):
if success(p):
processes.remove(p)
else:
fail()
if not processes and not cmds:
break
else:
time.sleep(0.05)
commands = [
['curl', 'http://www.reddit.com/'],
['curl', 'http://en.wikipedia.org/'],
['curl', 'http://www.google.com/'],
['curl', 'http://www.yahoo.com/'],
['curl', 'http://news.ycombinator.com/']
]
exec_commands(commands)
Diff to Previous Revision
--- revision 3 2010-08-27 19:17:02
+++ revision 4 2010-08-31 00:11:00
@@ -57,10 +57,10 @@
time.sleep(0.05)
commands = [
- ['ls', '/bin'],
- ['ls', '/usr'],
- ['ls', '/etc'],
- ['ls', '/var'],
- ['ls', '/tmp']
+ ['curl', 'http://www.reddit.com/'],
+ ['curl', 'http://en.wikipedia.org/'],
+ ['curl', 'http://www.google.com/'],
+ ['curl', 'http://www.yahoo.com/'],
+ ['curl', 'http://news.ycombinator.com/']
]
exec_commands(commands)
