пытаюсь создать клиент который получает одновременно содержимое большого количества урлов, в данном случае 1000 параллельных конектов, вот код: SimpleAsyncHTTP.py | Код |
import asyncore
import string, socket
import StringIO
import mimetools, urlparse
class AsyncHTTP(asyncore.dispatcher_with_send):
# HTTP requestor
def __init__(self, uri, consumer):
asyncore.dispatcher_with_send.__init__(self)
self.uri = uri
self.consumer = consumer
# turn the uri into a valid request
scheme, host, path, params, query, fragment = urlparse.urlparse(uri)
assert scheme == "http", "only supports HTTP requests"
try:
host, port = string.split(host, ":", 1)
port = int(port)
except (TypeError, ValueError):
port = 80 # default port
if not path:
path = "/"
if params:
path = path + ";" + params
if query:
path = path + "?" + query
self.request = "GET %s HTTP/1.0\r\nHost: %s\r\n\r\n" % (path, host)
self.host = host
self.port = port
self.status = None
self.header = None
self.data = ""
self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
try:
self.connect((host, port))
except:
print "can't connect"
self.handle_close()
def handle_connect(self):
# connection succeeded
self.send(self.request)
def handle_expt(self):
# connection failed; notify consumer (status is None)
self.close()
try:
http_header = self.consumer.http_header
except AttributeError:
pass
else:
http_header(self)
def handle_read(self):
data = self.recv(2048)
if not self.header:
self.data = self.data + data
try:
i = string.index(self.data, "\r\n\r\n")
except ValueError:
return # continue
else:
# parse header
fp = StringIO.StringIO(self.data[:i+4])
# status line is "HTTP/version status message"
status = fp.readline()
self.status = string.split(status, " ", 2)
# followed by a rfc822-style message header
self.header = mimetools.Message(fp)
# followed by a newline, and the payload (if any)
data = self.data[i+4:]
self.data = ""
# notify consumer (status is non-zero)
try:
http_header = self.consumer.http_header
except AttributeError:
pass
else:
http_header(self)
if not self.connected:
return # channel was closed by consumer
self.consumer.feed(data)
def handle_close(self):
self.consumer.close()
self.close()
|
| Код |
#!/usr/local/bin/python2.5
import SimpleAsyncHTTP
import asyncore
import sys, urlparse, socket
import time
# Get args
num_conn = 10
try:
if sys.argv[1] == "-":
urls = sys.stdin.readlines()
else:
urls = open(sys.argv[1]).readlines()
if len(sys.argv) >= 3:
num_conn = int(sys.argv[2])
except:
print "Usage: %s <file with URLs to fetch> [<# of concurrent connections>]" % sys.argv[0]
raise SystemExit
class DummyConsumer:
size = 0
def http_header(self, request):
# handle header
if request.status is None:
print "connection failed"
else:
print "status", "=>", request.status
for key, value in request.header.items():
print key, "=", value
def feed(self, data):
# handle incoming data
self.size = self.size + len(data)
def close(self):
# end of data
print self.size, "bytes in body"
#
# try it out
consumer = DummyConsumer()
for url in urls:
url = url.strip()
parts = urlparse.urlparse (url)
if parts[0] != 'http':
raise ValueError, "HTTP URL's only, please"
else:
start = time.time()
SimpleAsyncHTTP.AsyncHTTP(url, consumer)
end = time.time()
print "end [%s]" % (end-start)
asyncore.loop()
|
но до 1000 ему дойти никак не получается, ибо происходит задержка при инициализации AsyncHTTP, а точнее при коннекте сокета self.connect((host, port) отчего такое безобразие, что не так и кто виноват ? ) как запустить нужное мне количество потоков ? так чтобы не ждать конекта |
неблокирующие сокеты, asyncore 
Загрузка ...
