Initial version of rate limiting on requests. Should fix issue #1877

headphones/lastfm.py

@@ -13,18 +13,23 @@
 #  You should have received a copy of the GNU General Public License
 #  along with Headphones.  If not, see <http://www.gnu.org/licenses/>.
 
-import random
 import time
+import random
+import threading
 import headphones
 
 from headphones import db, logger, request
 
 from collections import defaultdict
 
-TIMEOUT = 60 # seconds
+TIMEOUT = 60.0 # seconds
+REQUEST_LIMIT = 1.0 # seconds
 ENTRY_POINT = "http://ws.audioscrobbler.com/2.0/"
 API_KEY = "395e6ec6bb557382fc41fde867bce66f"
 
+# Required for API request limit
+lock = threading.Lock()
+
 def request_lastfm(method, **kwargs):
     """
     Call a Last.FM API method. Automatically sets the method and API key. Method
@@ -43,7 +48,8 @@ def request_lastfm(method, **kwargs):
     logger.debug("Calling Last.FM method: %s", method)
     logger.debug("Last.FM call parameters: %s", kwargs)
 
-    data = request.request_json(ENTRY_POINT, timeout=TIMEOUT, params=kwargs)
+    data = request.request_json(ENTRY_POINT, timeout=TIMEOUT, params=kwargs,
+        rate_limit=(lock, REQUEST_LIMIT))
 
     # Parse response and check for errors.
     if not data:

headphones/request.py

@@ -18,16 +18,27 @@ from headphones import logger
 from xml.dom import minidom
 from bs4 import BeautifulSoup
 
+import time
 import requests
 import feedparser
 import headphones
+import collections
+
+# Dictionary with last request times, for rate limiting.
+last_requests = collections.defaultdict(int)
 
 def request_response(url, method="get", auto_raise=True,
-    whitelist_status_code=None, **kwargs):
+    whitelist_status_code=None, rate_limit=None, **kwargs):
     """
     Convenient wrapper for `requests.get', which will capture the exceptions and
     log them. On success, the Response object is returned. In case of a
     exception, None is returned.
+
+    Additionally, there is support for rate limiting. To use this feature,
+    supply a tuple of (lock, request_limit). The lock is used to make sure no
+    other request with the same lock is executed. The request limit is the
+    minimal time between two requests (and so 1/request_limit is the number of
+    requests per seconds).
     """
 
     # Convert whitelist_status_code to a list if needed
@@ -42,6 +53,25 @@ def request_response(url, method="get", auto_raise=True,
     # requests to apply more magic per method. See lib/requests/api.py.
     request_method = getattr(requests, method.lower())
 
+    # Enfore request rate limit if applicable. This uses the lock so there is
+    # synchronized access to the API.
+    if rate_limit:
+        lock, request_limit = rate_limit
+
+        with lock:
+            delta = time.time() - last_requests[lock]
+
+            if delta < request_limit:
+                logger.debug("Sleeping %.2f seconds for request, limit is %d " \
+                    "req/sec.", request_limit - delta,
+                    int(1.0 / request_limit))
+
+                # Sleep the remaining time and update time
+                time.sleep(request_limit - delta)
+
+            # Set last access time
+            last_requests[lock] = time.time()
+
     try:
         # Request the URL
         logger.debug("Requesting URL via %s method: %s", method.upper(), url)