Merge pull request #1907 from basilfx/ratelimit2

Increase Last.FM rate limit to 5 req/sec. Improved rate limiting

headphones/lastfm.py

@@ -23,7 +23,7 @@ from headphones import db, logger, request
 from collections import defaultdict
 
 TIMEOUT = 60.0 # seconds
-REQUEST_LIMIT = 1.0 # seconds
+REQUEST_LIMIT = 1.0 / 5 # seconds
 ENTRY_POINT = "http://ws.audioscrobbler.com/2.0/"
 API_KEY = "395e6ec6bb557382fc41fde867bce66f"
 

headphones/request.py

@@ -53,33 +53,36 @@ def request_response(url, method="get", auto_raise=True,
     # requests to apply more magic per method. See lib/requests/api.py.
     request_method = getattr(requests, method.lower())
 
+    # Enfore request rate limit if applicable. This uses the lock so there
+    # is synchronized access to the API. When N threads enter this method, the
+    # first will pass trough, since there there was no last request recorded.
+    # The last request time will be set. Then, the second thread will unlock,
+    # and see that the last request was X seconds ago. It will sleep
+    # (request_limit - X) seconds, and then continue. Then the third one will
+    # unblock, and so on. After all threads finished, the total time will at
+    # least be (N * request_limit) seconds. If some request takes longer than
+    # request_limit seconds, the next unblocked thread will wait less.
+    if rate_limit:
+        lock, request_limit = rate_limit
+
+        with lock:
+            delta = time.time() - last_requests[lock]
+            limit = int(1.0 / request_limit)
+
+            if delta < request_limit:
+                logger.debug("Sleeping %.2f seconds for request, limit " \
+                    "is %d req/sec.", request_limit - delta, limit)
+
+                # Sleep the remaining time
+                time.sleep(request_limit - delta)
+
+            # Update last request time.
+            last_requests[lock] = time.time()
+
     try:
-        # Enfore request rate limit if applicable. This uses the lock so there
+        # Request URL and wait for response
-        # is synchronized access to the API. If no limit is enforced, just do
-        # it as usual.
         logger.debug("Requesting URL via %s method: %s", method.upper(), url)
-
+        response = request_method(url, **kwargs)
-        if rate_limit:
-            lock, request_limit = rate_limit
-
-            with lock:
-                delta = time.time() - last_requests[lock]
-                limit = int(1.0 / request_limit)
-
-                if delta < request_limit:
-                    logger.debug("Sleeping %.2f seconds for request, limit " \
-                        "is %d req/sec.", request_limit - delta, limit)
-
-                    # Sleep the remaining time
-                    time.sleep(request_limit - delta)
-
-                # Update last request time and start with request. Basically, if
-                # the request takes N seconds, next request will sleep N seconds
-                # less.
-                last_requests[lock] = time.time()
-                response = request_method(url, **kwargs)
-        else:
-            response = request_method(url, **kwargs)
 
         # If status code != OK, then raise exception, except if the status code
         # is white listed.