Quick http calls using Python without twisted
#Python, Performance, HttpRequest
from urlparse import urlparse
from threading import Thread
import httplib
import sys
from Queue import Queue
url_path = r'TheUrlFilePath'
concurrent = 200
def do_work():
while True:
url = q.get()
status, url = get_status(url)
do_something_with_result(status, url)
q.task_done()
def get_status(ourl):
try:
url = urlparse(ourl)
conn = httplib.HTTPConnection(url.netloc)
conn.request("HEAD", url.path)
res = conn.getresponse()
return res.status, ourl
except:
return "error", ourl
def do_something_with_result(status, url):
print status, url
q = Queue(concurrent * 2)
for i in range(concurrent):
t = Thread(target=do_work)
t.daemon = True
t.start()
try:
for url in open(url_path):
q.put(url.strip())
q.join()
except KeyboardInterrupt:
sys.exit(1)