Request should be in the lock

This commit is contained in:
Bas Stottelaar
2014-09-21 18:44:02 +02:00
parent efdd51c361
commit c09c290ac8

View File

@@ -53,31 +53,33 @@ def request_response(url, method="get", auto_raise=True,
# requests to apply more magic per method. See lib/requests/api.py.
request_method = getattr(requests, method.lower())
# Enfore request rate limit if applicable. This uses the lock so there is
# synchronized access to the API.
if rate_limit:
lock, request_limit = rate_limit
with lock:
delta = time.time() - last_requests[lock]
if delta < request_limit:
logger.debug("Sleeping %.2f seconds for request, limit is %d " \
"req/sec.", request_limit - delta,
int(1.0 / request_limit))
# Sleep the remaining time
time.sleep(request_limit - delta)
try:
# Request the URL
# Enfore request rate limit if applicable. This uses the lock so there
# is synchronized access to the API. If no limit is enforced, just do
# it as usual.
logger.debug("Requesting URL via %s method: %s", method.upper(), url)
response = request_method(url, **kwargs)
# Update rate limit last access time here, because a request will also
# take time.
if rate_limit:
last_requests[lock] = time.time()
lock, request_limit = rate_limit
with lock:
delta = time.time() - last_requests[lock]
limit = int(1.0 / request_limit)
if delta < request_limit:
logger.debug("Sleeping %.2f seconds for request, limit " \
"is %d req/sec.", request_limit - delta, limit)
# Sleep the remaining time
time.sleep(request_limit - delta)
# Update last request time and start with request. Basically, if
# the request takes N seconds, next request will sleep N seconds
# less.
last_requests[lock] = time.time()
response = request_method(url, **kwargs)
else:
response = request_method(url, **kwargs)
# If status code != OK, then raise exception, except if the status code
# is white listed.