Moved all request methods to seperate file, since it is getting a core function

headphones/albumart.py

@@ -13,7 +13,7 @@
 #  You should have received a copy of the GNU General Public License
 #  along with Headphones.  If not, see <http://www.gnu.org/licenses/>.
 
-from headphones import helpers, db
+from headphones import request, db
 
 def getAlbumArt(albumid):
     myDB = db.DBConnection()
@@ -32,7 +32,7 @@ def getCachedArt(albumid):
         return
 
     if artwork_path.startswith('http://'):
-        artwork = helpers.request_content(artwork_path, timeout=20)
+        artwork = request.request_content(artwork_path, timeout=20)
 
         if not artwork:
             logger.warn("Unable to open url: %s", artwork_path)

headphones/cache.py

@@ -14,12 +14,11 @@
 #  along with Headphones.  If not, see <http://www.gnu.org/licenses/>.
 
 import os
-import glob, urllib
+import glob
-
+import urllib
-import lib.simplejson as simplejson
-
 import headphones
-from headphones import db, helpers, logger, lastfm
+
+from headphones import db, helpers, logger, lastfm, request
 
 lastfm_apikey = "690e1ed3bc00bc91804cd8f7fe5ed6d4"
 
@@ -337,7 +336,7 @@ class Cache(object):
 
         # Should we grab the artwork here if we're just grabbing thumbs or info?? Probably not since the files can be quite big
         if image_url and self.query_type == 'artwork':
-            artwork = helpers.request_content(image_url, timeout=20)
+            artwork = request.request_content(image_url, timeout=20)
 
             if artwork:
                 # Make sure the artwork dir exists:
@@ -370,7 +369,7 @@ class Cache(object):
 
         # Grab the thumbnail as well if we're getting the full artwork (as long as it's missing/outdated
         if thumb_url and self.query_type in ['thumb','artwork'] and not (self.thumb_files and self._is_current(self.thumb_files[0])):
-            artwork = helpers.request_content(thumb_url, timeout=20)
+            artwork = request.request_content(thumb_url, timeout=20)
 
             if artwork:
                 # Make sure the artwork dir exists:

headphones/helpers.py

@@ -18,16 +18,9 @@ import re
 import time
 import shutil
 import datetime
-import requests
-import feedparser
 import headphones
 
-from headphones import logger
-
-from xml.dom import minidom
 from operator import itemgetter
-from bs4 import BeautifulSoup
-
 from beets.mediafile import MediaFile, FileTypeError, UnreadableFileError
 
 # Modified from https://github.com/Verrus/beets-plugin-featInTitle
@@ -243,6 +236,8 @@ def expand_subfolders(f):
     case, normal post processing will be better.
     """
 
+    from headphones import logger
+
     # Find all folders with media files in them
     media_folders = []
 
@@ -338,6 +333,8 @@ def extract_metadata(f):
     artists, albums and years found in the media files.
     """
 
+    from headphones import logger
+
     # Walk directory and scan all media files
     results = []
     count = 0
@@ -595,113 +592,4 @@ def create_https_certificates(ssl_cert, ssl_key):
         logger.error("Error creating SSL key and certificate: %s", e)
         return False
 
-    return True
+    return True
-
-def request_response(url, method="get", auto_raise=True, whitelist_status_code=None, **kwargs):
-    """
-    Convenient wrapper for `requests.get', which will capture the exceptions and
-    log them. On success, the Response object is returned. In case of a
-    exception, None is returned.
-    """
-
-    # Convert whitelist_status_code to a list if needed
-    if whitelist_status_code and type(whitelist_status_code) != list:
-        whitelist_status_code = [whitelist_status_code]
-
-    # Map method to the request.XXX method. This is a simple hack, but it allows
-    # requests to apply more magic per method. See lib/requests/api.py.
-    request_method = getattr(requests, method)
-
-    try:
-        # Request the URL
-        logger.debug("Requesting URL via %s method: %s", method.upper(), url)
-        response = request_method(url, **kwargs)
-
-        # If status code != OK, then raise exception, except if the status code
-        # is white listed.
-        if whitelist_status_code and auto_raise:
-            if response.status_code not in whitelist_status_code:
-                response.raise_for_status()
-            else:
-                logger.debug("Response Status code %d is white listed, not raising exception", response.status_code)
-        elif auto_raise:
-            response.raise_for_status()
-
-        return response
-    except requests.ConnectionError:
-        logger.error("Unable to connect to remote host.")
-    except requests.Timeout:
-        logger.error("Request timed out.")
-    except requests.HTTPError, e:
-        if e.response is not None:
-            logger.error("Request raise HTTP error with status code: %d", e.response.status_code)
-        else:
-            logger.error("Request raised HTTP error.")
-    except requests.RequestException, e:
-        logger.error("Request raised exception: %s", e)
-
-def request_soup(url, **kwargs):
-    """
-    Wrapper for `request_response', which will return a BeatifulSoup object if
-    no exceptions are raised.
-    """
-
-    parser = kwargs.pop("parser", "html5lib")
-    response = request_response(url, **kwargs)
-
-    if response is not None:
-        return BeautifulSoup(response.content, parser)
-
-def request_minidom(url, **kwargs):
-    """
-    Wrapper for `request_response', which will return a Minidom object if no
-    exceptions are raised.
-    """
-
-    response = request_response(url, **kwargs)
-
-    if response is not None:
-        return minidom.parseString(response.content)
-
-def request_json(url, **kwargs):
-    """
-    Wrapper for `request_response', which will decode the response as JSON
-    object and return the result, if no exceptions are raised.
-
-    As an option, a validator callback can be given, which should return True if
-    the result is valid.
-    """
-
-    validator = kwargs.pop("validator", None)
-    response = request_response(url, **kwargs)
-
-    if response is not None:
-        try:
-            result = response.json()
-
-            if validator and not validator(result):
-                logger.error("JSON validation result vailed")
-            else:
-                return result
-        except ValueError:
-            logger.error("Response returned invalid JSON data")
-
-def request_content(url, **kwargs):
-    """
-    Wrapper for `request_response', which will return the raw content.
-    """
-
-    response = request_response(url, **kwargs)
-
-    if response is not None:
-        return response.content
-
-def request_feed(url, **kwargs):
-    """
-    Wrapper for `request_response', which will return a feed object.
-    """
-
-    response = request_response(url, **kwargs)
-
-    if response is not None:
-        return feedparser.parse(response.content)

headphones/lastfm.py

@@ -17,7 +17,7 @@ import random
 import time
 import headphones
 
-from headphones import db, logger, helpers
+from headphones import db, logger, request
 
 from collections import defaultdict
 
@@ -40,7 +40,7 @@ def request_lastfm(method, **kwargs):
 
     # Send request
     logger.debug("Calling Last.FM method: %s", method)
-    data = helpers.request_json(ENTRY_POINT, timeout=20, params=kwargs)
+    data = request.request_json(ENTRY_POINT, timeout=20, params=kwargs)
 
     # Parse response and check for errors.
     if not data:

headphones/lyrics.py

@@ -16,7 +16,7 @@
 import re
 import htmlentitydefs
 
-from headphones import logger, helpers
+from headphones import logger, request
 
 def getLyrics(artist, song):
 
@@ -26,7 +26,7 @@ def getLyrics(artist, song):
                 }
 
     url = 'http://lyrics.wikia.com/api.php'
-    data = helpers.request_minidom(url, params)
+    data = request.request_minidom(url, params)
     
     if not data:
         return
@@ -39,7 +39,7 @@ def getLyrics(artist, song):
         logger.info('No lyrics found for %s - %s' % (artist, song))
         return
     
-    lyricspage = helpers.request_content(lyricsurl)
+    lyricspage = request.request_content(lyricsurl)
     
     if not lyricspage:
         logger.warn('Error fetching lyrics from: %s. Error: %s' % (lyricsurl, e))

headphones/notifiers.py

@@ -13,29 +13,31 @@
 #  You should have received a copy of the GNU General Public License
 #  along with Headphones.  If not, see <http://www.gnu.org/licenses/>.
 
-from headphones import logger, helpers, common
-from headphones.exceptions import ex
 import base64
 import cherrypy
 import urllib
 import urllib2
 import headphones
-from httplib import HTTPSConnection
+import simplejson
-from urllib import urlencode
 import os.path
 import subprocess
 import gntp.notifier
-import lib.simplejson as simplejson
-from xml.dom import minidom
 
-try:
+from xml.dom import minidom
-    from urlparse import parse_qsl #@UnusedImport
+from httplib import HTTPSConnection
-except:
+from urllib import urlencode
-    from cgi import parse_qsl #@Reimport
 
 import lib.oauth2 as oauth
 import lib.pythontwitter as twitter
 
+from headphones import logger, helpers, common, request
+from headphones.exceptions import ex
+
+try:
+    from urlparse import parse_qsl
+except:
+    from cgi import parse_qsl
+
 class GROWL:
 
     def __init__(self):
@@ -179,9 +181,9 @@ class XBMC:
         url = host + '/xbmcCmds/xbmcHttp/?' + url_command
 
         if self.password:
-            return helpers.request_content(url, auth=(self.username, self.password))
+            return request.request_content(url, auth=(self.username, self.password))
         else:
-            return helpers.request_content(url)
+            return request.request_content(url)
 
     def _sendjson(self, host, method, params={}):
         data = [{'id': 0, 'jsonrpc': '2.0', 'method': method, 'params': params}]
@@ -189,9 +191,9 @@ class XBMC:
         url = host + '/jsonrpc'
 
         if self.password:
-            response = helpers.request_json(req, method="POST", data=simplejson.dumps(data), headers=headers, auth=(self.username, self.password))
+            response = request.request_json(req, method="POST", data=simplejson.dumps(data), headers=headers, auth=(self.username, self.password))
         else:
-            response = helpers.request_json(req, method="POST", data=simplejson.dumps(data), headers=headers)
+            response = request.request_json(req, method="POST", data=simplejson.dumps(data), headers=headers)
 
         if response:
             return response[0]['result']
@@ -332,7 +334,7 @@ class NMA:
         self.priority = headphones.NMA_PRIORITY
         
     def _send(self, data):
-        return helpers.request_content('https://www.notifymyandroid.com/publicapi/notify', data=data)
+        return request.request_content('https://www.notifymyandroid.com/publicapi/notify', data=data)
         
     def notify(self, artist=None, album=None, snatched_nzb=None):
     

headphones/postprocessor.py

@@ -27,7 +27,7 @@ from beets import autotag
 from beets.mediafile import MediaFile
 
 import headphones
-from headphones import db, albumart, librarysync, lyrics, logger, helpers
+from headphones import db, albumart, librarysync, lyrics, logger, helpers, request
 from headphones.helpers import sab_replace_dots, sab_replace_spaces
 
 postprocessor_lock = threading.Lock()
@@ -360,7 +360,7 @@ def doPostProcessing(albumid, albumpath, release, tracks, downloaded_track_list,
     artwork = None
     album_art_path = albumart.getAlbumArt(albumid)
     if headphones.EMBED_ALBUM_ART or headphones.ADD_ALBUM_ART:
-        artwork = helpers.request_content(album_art_path)
+        artwork = request.request_content(album_art_path)
 
         if not album_art_path or not artwor or len(artwork) < 100:
             logger.info("No suitable album art found from Amazon. Checking Last.FM....")

headphones/request.py

@@ -0,0 +1,116 @@
+from headphones import logger
+
+from xml.dom import minidom
+from bs4 import BeautifulSoup
+
+import requests
+import feedparser
+
+def request_response(url, method="get", auto_raise=True, whitelist_status_code=None, **kwargs):
+    """
+    Convenient wrapper for `requests.get', which will capture the exceptions and
+    log them. On success, the Response object is returned. In case of a
+    exception, None is returned.
+    """
+
+    # Convert whitelist_status_code to a list if needed
+    if whitelist_status_code and type(whitelist_status_code) != list:
+        whitelist_status_code = [whitelist_status_code]
+
+    # Map method to the request.XXX method. This is a simple hack, but it allows
+    # requests to apply more magic per method. See lib/requests/api.py.
+    request_method = getattr(requests, method)
+
+    try:
+        # Request the URL
+        logger.debug("Requesting URL via %s method: %s", method.upper(), url)
+        response = request_method(url, **kwargs)
+
+        # If status code != OK, then raise exception, except if the status code
+        # is white listed.
+        if whitelist_status_code and auto_raise:
+            if response.status_code not in whitelist_status_code:
+                response.raise_for_status()
+            else:
+                logger.debug("Response Status code %d is white listed, not raising exception", response.status_code)
+        elif auto_raise:
+            response.raise_for_status()
+
+        return response
+    except requests.ConnectionError:
+        logger.error("Unable to connect to remote host.")
+    except requests.Timeout:
+        logger.error("Request timed out.")
+    except requests.HTTPError, e:
+        if e.response is not None:
+            logger.error("Request raise HTTP error with status code: %d", e.response.status_code)
+        else:
+            logger.error("Request raised HTTP error.")
+    except requests.RequestException, e:
+        logger.error("Request raised exception: %s", e)
+
+def request_soup(url, **kwargs):
+    """
+    Wrapper for `request_response', which will return a BeatifulSoup object if
+    no exceptions are raised.
+    """
+
+    parser = kwargs.pop("parser", "html5lib")
+    response = request_response(url, **kwargs)
+
+    if response is not None:
+        return BeautifulSoup(response.content, parser)
+
+def request_minidom(url, **kwargs):
+    """
+    Wrapper for `request_response', which will return a Minidom object if no
+    exceptions are raised.
+    """
+
+    response = request_response(url, **kwargs)
+
+    if response is not None:
+        return minidom.parseString(response.content)
+
+def request_json(url, **kwargs):
+    """
+    Wrapper for `request_response', which will decode the response as JSON
+    object and return the result, if no exceptions are raised.
+
+    As an option, a validator callback can be given, which should return True if
+    the result is valid.
+    """
+
+    validator = kwargs.pop("validator", None)
+    response = request_response(url, **kwargs)
+
+    if response is not None:
+        try:
+            result = response.json()
+
+            if validator and not validator(result):
+                logger.error("JSON validation result vailed")
+            else:
+                return result
+        except ValueError:
+            logger.error("Response returned invalid JSON data")
+
+def request_content(url, **kwargs):
+    """
+    Wrapper for `request_response', which will return the raw content.
+    """
+
+    response = request_response(url, **kwargs)
+
+    if response is not None:
+        return response.content
+
+def request_feed(url, **kwargs):
+    """
+    Wrapper for `request_response', which will return a feed object.
+    """
+
+    response = request_response(url, **kwargs)
+
+    if response is not None:
+        return feedparser.parse(response.content)

headphones/searcher.py

@@ -29,7 +29,7 @@ import requests
 
 import headphones
 from headphones.common import USER_AGENT
-from headphones import logger, db, helpers, classes, sab, nzbget
+from headphones import logger, db, helpers, classes, sab, nzbget, request
 from headphones import transmission
 
 import lib.bencode as bencode
@@ -311,7 +311,7 @@ def searchNZB(album, new=False, losslessOnly=False):
             "q": term
         }
 
-        data = helpers.request_feed(
+        data = request.request_feed(
             url="http://headphones.codeshy.com/newznab/api",
             params=params, headers=headers,
             auth=(headphones.HPUSER, headphones.HPPASS)
@@ -377,7 +377,7 @@ def searchNZB(album, new=False, losslessOnly=False):
                 "q": term
             }
 
-            data = helpers.request_feed(
+            data = request.request_feed(
                 url=newznab_host[0] + '/api?',
                 params=params, headers=headers
             )
@@ -424,7 +424,7 @@ def searchNZB(album, new=False, losslessOnly=False):
             "q": term
         }
 
-        data = helpers.request_feed(
+        data = request.request_feed(
             url='http://beta.nzbs.org/api',
             params=params, headers=headers,
             timeout=20
@@ -474,7 +474,7 @@ def searchNZB(album, new=False, losslessOnly=False):
             "searchtext": term
         }
 
-        data = helpers.request_json(
+        data = request.request_json(
             url='https://www.nzbsrus.com/api.php',
             params=params, headers=headers,
             validator=lambda x: type(x) == dict
@@ -524,7 +524,7 @@ def searchNZB(album, new=False, losslessOnly=False):
             "search": term
         }
 
-        data = helpers.request_json(
+        data = request.request_json(
             url='http://api.omgwtfnzbs.org/json/',
             params=params, headers=headers,
             validator=lambda x: type(x) == dict
@@ -739,7 +739,7 @@ def getresultNZB(result):
     nzb = None
 
     if result[3] == 'newzbin':
-        response = helpers.request_response(
+        response = request.request_response(
             url='https://www.newzbin2.es/api/dnzb/',
             auth=(headphones.HPUSER, headphones.HPPASS),
             params={"username": headphones.NEWZBIN_UID, "password": headphones.NEWZBIN_PASSWORD, "reportid": result[2]},
@@ -747,30 +747,31 @@ def getresultNZB(result):
             whitelist_status_code=400
         )
 
-        if response.status_code == 400:
+        if response is not None:
-            error_code = int(response.headers.header['X-DNZB-RCode'])
+            if response.status_code == 400:
+                error_code = int(response.headers.header['X-DNZB-RCode'])
 
-            if error_code == 450:
+                if error_code == 450:
-                result = re.search("wait (\d+) seconds", response.headers['X-DNZB-RText'])
+                    result = re.search("wait (\d+) seconds", response.headers['X-DNZB-RText'])
-                seconds = int(result.group(1))
+                    seconds = int(result.group(1))
 
-                logger.info("Newzbin throttled our NZB downloading, pausing for %d seconds", seconds)
+                    logger.info("Newzbin throttled our NZB downloading, pausing for %d seconds", seconds)
-                time.sleep(seconds)
+                    time.sleep(seconds)
 
-                # Try again -- possibly forever :(
+                    # Try again -- possibly forever :(
-                getresultNZB(result)
+                    getresultNZB(result)
+                else:
+                    logger.info("Newzbin error code %d", error_code)
             else:
-                logger.info("Newzbin error code %d", error_code)
+                nzb = response.content
-        else:
-            nzb = response.content
     elif result[3] == 'headphones':
-        nzb = helpers.request_content(
+        nzb = request.request_content(
             url=result[2],
             auth=(headphones.HPUSER, headphones.HPPASS),
             headers={'User-Agent': USER_AGENT}
         )
     else:
-        nzb = helpers.request_content(
+        nzb = request.request_content(
             url=result[2],
             headers={'User-Agent': USER_AGENT}
         )
@@ -861,7 +862,7 @@ def searchTorrent(album, new=False, losslessOnly=False):
             "rss": "1"
         }
 
-        data = helpers.request_feed(
+        data = request.request_feed(
             url=providerurl,
             params=params,
             timeout=20
@@ -880,7 +881,7 @@ def searchTorrent(album, new=False, losslessOnly=False):
                         url = item['links'][1]['href']
                         size = int(item['links'][1]['length'])
                         if format == "2":
-                            torrent = helpers.request_content(url)
+                            torrent = request.request_content(url)
                             if not torrent or (int(torrent.find(".mp3")) > 0 and int(torrent.find(".flac")) < 1):
                                 rightformat = False
                         if rightformat == True and size < maxsize and minimumseeders < int(seeders):
@@ -934,7 +935,7 @@ def searchTorrent(album, new=False, losslessOnly=False):
             "q": " ".join(query_items)
         }
 
-        data = helpers.request_feed(
+        data = request.request_feed(
             url=providerurl,
             params=params, headers=headers,
             timeout=20
@@ -1121,7 +1122,7 @@ def searchTorrent(album, new=False, losslessOnly=False):
             "sort": "seeds"
         }
 
-        data = helpers.request_soup(
+        data = request.request_soup(
             url=providerurl + category,
             params=params,
             timeout=20
@@ -1182,7 +1183,7 @@ def searchTorrent(album, new=False, losslessOnly=False):
             "sort": "seeds"
         }
 
-        data = helpers.request_feed(
+        data = request.request_feed(
             url=providerurl,
             params=params, headers=headers,
             auth=(headphones.HPUSER, headphones.HPPASS),
@@ -1207,7 +1208,7 @@ def searchTorrent(album, new=False, losslessOnly=False):
                         url = item.links[1]['url']
                         size = int(item.links[1]['length'])
                         if format == "2":
-                            torrent = helpers.request_content(url)
+                            torrent = request.request_content(url)
 
                             if not torrent or (int(torrent.find(".mp3")) > 0 and int(torrent.find(".flac")) < 1):
                                 rightformat = False
@@ -1242,7 +1243,7 @@ def searchTorrent(album, new=False, losslessOnly=False):
         # Requesting content
         logger.info('Parsing results from Mininova')
 
-        data = helpers.request_feed(
+        data = request.request_feed(
             url=providerurl,
             timeout=20
         )
@@ -1264,7 +1265,7 @@ def searchTorrent(album, new=False, losslessOnly=False):
                         url = item.links[1]['url']
                         size = int(item.links[1]['length'])
                         if format == "2":
-                            torrent = helpers.request_content(url)
+                            torrent = request.request_content(url)
 
                             if not torrent or (int(torrent.find(".mp3")) > 0 and int(torrent.find(".flac")) < 1):
                                 rightformat = False
@@ -1303,7 +1304,7 @@ def preprocess(resultlist):
             elif result[3] == 'What.cd':
                 headers['User-Agent'] = 'Headphones'
 
-            return helpers.request_content(url=result[2], headers=headers), result
+            return request.request_content(url=result[2], headers=headers), result
 
         else:
             usenet_retention = headphones.USENET_RETENTION or 2000

headphones/transmission.py

@@ -20,7 +20,7 @@ import headphones
 
 import simplejson as json
 
-from headphones import logger, notifiers, helpers
+from headphones import logger, notifiers, request
 
 # This is just a simple script to send torrents to transmission. The
 # intention is to turn this into a class where we can check the state
@@ -133,11 +133,11 @@ def torrentAction(method, arguments):
 
     # Retrieve session id
     if username and password:
-        response = helpers.request_response(host, auth=(username, password), whitelist_status_code=409)
+        response = request.request_response(host, auth=(username, password), whitelist_status_code=409)
     else:
-        response = helpers.request_response(host, whitelist_status_code=409)
+        response = request.request_response(host, whitelist_status_code=409)
 
-    if not response:
+    if response is None:
         logger.error("Error gettings Transmission session ID")
         return
 
@@ -153,7 +153,7 @@ def torrentAction(method, arguments):
     headers = { 'x-transmission-session-id': sessionid }
     data = { 'method': method, 'arguments': arguments }
 
-    response = helpers.request_content(host, data=json.dumps(data), headers=headers)
+    response = request.request_json(host, method="post", data=json.dumps(data), headers=headers)
 
     if not response:
         logger.error("Error sending torrent to Transmission")

headphones/versioncheck.py

@@ -16,7 +16,7 @@
 import platform, subprocess, re, os, tarfile
 
 import headphones
-from headphones import logger, version, helpers
+from headphones import logger, version, request
 from headphones.exceptions import ex
 
 import lib.simplejson as simplejson
@@ -122,9 +122,9 @@ def checkGithub():
     # Get the latest version available from github
     logger.info('Retrieving latest version information from GitHub')
     url = 'https://api.github.com/repos/%s/headphones/commits/%s' % (headphones.GIT_USER, headphones.GIT_BRANCH)
-    version = helpers.request_json(url, timeout=20, validator=lambda x: type(x) == dict)
+    version = request.request_json(url, timeout=20, validator=lambda x: type(x) == dict)
 
-    if not version:
+    if version is None:
         logger.warn('Could not get the latest version from GitHub. Are you running a local development version?')
         return headphones.CURRENT_VERSION
 
@@ -138,9 +138,9 @@ def checkGithub():
 
     logger.info('Comparing currently installed version with latest GitHub version')
     url = 'https://api.github.com/repos/%s/headphones/compare/%s...%s' % (headphones.GIT_USER, headphones.CURRENT_VERSION, headphones.LATEST_VERSION)
-    commits = helpers.request_json(url, timeout=20, whitelist_status_code=404, validator=lambda x: type(x) == dict)
+    commits = request.request_json(url, timeout=20, whitelist_status_code=404, validator=lambda x: type(x) == dict)
 
-    if not commits:
+    if commits is None:
         logger.warn('Could not get commits behind from GitHub.')
         return headphones.CURRENT_VERSION
 
@@ -179,7 +179,7 @@ def update():
         version_path = os.path.join(headphones.PROG_DIR, 'version.txt')
 
         logger.info('Downloading update from: '+tar_download_url)
-        data = helpers.request_content(tar_download_url)
+        data = request.request_content(tar_download_url)
 
         if not data:
             logger.error("Unable to retrieve new version from '%s', can't update", tar_download_url)