skynyrd
10/5/2016 - 9:28 AM

Quick http calls using Python without twisted

Quick http calls using Python without twisted

#Python, Performance, HttpRequest

from urlparse import urlparse
from threading import Thread
import httplib
import sys
from Queue import Queue

url_path = r'TheUrlFilePath'
concurrent = 200


def do_work():
    while True:
        url = q.get()
        status, url = get_status(url)
        do_something_with_result(status, url)
        q.task_done()


def get_status(ourl):
    try:
        url = urlparse(ourl)
        conn = httplib.HTTPConnection(url.netloc)
        conn.request("HEAD", url.path)
        res = conn.getresponse()
        return res.status, ourl
    except:
        return "error", ourl


def do_something_with_result(status, url):
    print status, url


q = Queue(concurrent * 2)
for i in range(concurrent):
    t = Thread(target=do_work)
    t.daemon = True
    t.start()
try:
    for url in open(url_path):
        q.put(url.strip())
    q.join()
except KeyboardInterrupt:
    sys.exit(1)