From 22151ff4b034891102437d3dec79d83bda84889e Mon Sep 17 00:00:00 2001 From: Bas Stottelaar Date: Mon, 7 Apr 2014 00:32:43 +0200 Subject: [PATCH] Moved all request methods to seperate file, since it is getting a core function --- headphones/albumart.py | 4 +- headphones/cache.py | 13 ++-- headphones/helpers.py | 122 ++---------------------------------- headphones/lastfm.py | 4 +- headphones/lyrics.py | 6 +- headphones/notifiers.py | 32 +++++----- headphones/postprocessor.py | 4 +- headphones/request.py | 116 ++++++++++++++++++++++++++++++++++ headphones/searcher.py | 61 +++++++++--------- headphones/transmission.py | 10 +-- headphones/versioncheck.py | 12 ++-- 11 files changed, 195 insertions(+), 189 deletions(-) create mode 100644 headphones/request.py diff --git a/headphones/albumart.py b/headphones/albumart.py index 69150924..daf137b0 100644 --- a/headphones/albumart.py +++ b/headphones/albumart.py @@ -13,7 +13,7 @@ # You should have received a copy of the GNU General Public License # along with Headphones. If not, see . -from headphones import helpers, db +from headphones import request, db def getAlbumArt(albumid): myDB = db.DBConnection() @@ -32,7 +32,7 @@ def getCachedArt(albumid): return if artwork_path.startswith('http://'): - artwork = helpers.request_content(artwork_path, timeout=20) + artwork = request.request_content(artwork_path, timeout=20) if not artwork: logger.warn("Unable to open url: %s", artwork_path) diff --git a/headphones/cache.py b/headphones/cache.py index 89754455..a762fd20 100644 --- a/headphones/cache.py +++ b/headphones/cache.py @@ -14,12 +14,11 @@ # along with Headphones. If not, see . import os -import glob, urllib - -import lib.simplejson as simplejson - +import glob +import urllib import headphones -from headphones import db, helpers, logger, lastfm + +from headphones import db, helpers, logger, lastfm, request lastfm_apikey = "690e1ed3bc00bc91804cd8f7fe5ed6d4" @@ -337,7 +336,7 @@ class Cache(object): # Should we grab the artwork here if we're just grabbing thumbs or info?? Probably not since the files can be quite big if image_url and self.query_type == 'artwork': - artwork = helpers.request_content(image_url, timeout=20) + artwork = request.request_content(image_url, timeout=20) if artwork: # Make sure the artwork dir exists: @@ -370,7 +369,7 @@ class Cache(object): # Grab the thumbnail as well if we're getting the full artwork (as long as it's missing/outdated if thumb_url and self.query_type in ['thumb','artwork'] and not (self.thumb_files and self._is_current(self.thumb_files[0])): - artwork = helpers.request_content(thumb_url, timeout=20) + artwork = request.request_content(thumb_url, timeout=20) if artwork: # Make sure the artwork dir exists: diff --git a/headphones/helpers.py b/headphones/helpers.py index f0f6d363..f2ec9ee6 100644 --- a/headphones/helpers.py +++ b/headphones/helpers.py @@ -18,16 +18,9 @@ import re import time import shutil import datetime -import requests -import feedparser import headphones -from headphones import logger - -from xml.dom import minidom from operator import itemgetter -from bs4 import BeautifulSoup - from beets.mediafile import MediaFile, FileTypeError, UnreadableFileError # Modified from https://github.com/Verrus/beets-plugin-featInTitle @@ -243,6 +236,8 @@ def expand_subfolders(f): case, normal post processing will be better. """ + from headphones import logger + # Find all folders with media files in them media_folders = [] @@ -338,6 +333,8 @@ def extract_metadata(f): artists, albums and years found in the media files. """ + from headphones import logger + # Walk directory and scan all media files results = [] count = 0 @@ -595,113 +592,4 @@ def create_https_certificates(ssl_cert, ssl_key): logger.error("Error creating SSL key and certificate: %s", e) return False - return True - -def request_response(url, method="get", auto_raise=True, whitelist_status_code=None, **kwargs): - """ - Convenient wrapper for `requests.get', which will capture the exceptions and - log them. On success, the Response object is returned. In case of a - exception, None is returned. - """ - - # Convert whitelist_status_code to a list if needed - if whitelist_status_code and type(whitelist_status_code) != list: - whitelist_status_code = [whitelist_status_code] - - # Map method to the request.XXX method. This is a simple hack, but it allows - # requests to apply more magic per method. See lib/requests/api.py. - request_method = getattr(requests, method) - - try: - # Request the URL - logger.debug("Requesting URL via %s method: %s", method.upper(), url) - response = request_method(url, **kwargs) - - # If status code != OK, then raise exception, except if the status code - # is white listed. - if whitelist_status_code and auto_raise: - if response.status_code not in whitelist_status_code: - response.raise_for_status() - else: - logger.debug("Response Status code %d is white listed, not raising exception", response.status_code) - elif auto_raise: - response.raise_for_status() - - return response - except requests.ConnectionError: - logger.error("Unable to connect to remote host.") - except requests.Timeout: - logger.error("Request timed out.") - except requests.HTTPError, e: - if e.response is not None: - logger.error("Request raise HTTP error with status code: %d", e.response.status_code) - else: - logger.error("Request raised HTTP error.") - except requests.RequestException, e: - logger.error("Request raised exception: %s", e) - -def request_soup(url, **kwargs): - """ - Wrapper for `request_response', which will return a BeatifulSoup object if - no exceptions are raised. - """ - - parser = kwargs.pop("parser", "html5lib") - response = request_response(url, **kwargs) - - if response is not None: - return BeautifulSoup(response.content, parser) - -def request_minidom(url, **kwargs): - """ - Wrapper for `request_response', which will return a Minidom object if no - exceptions are raised. - """ - - response = request_response(url, **kwargs) - - if response is not None: - return minidom.parseString(response.content) - -def request_json(url, **kwargs): - """ - Wrapper for `request_response', which will decode the response as JSON - object and return the result, if no exceptions are raised. - - As an option, a validator callback can be given, which should return True if - the result is valid. - """ - - validator = kwargs.pop("validator", None) - response = request_response(url, **kwargs) - - if response is not None: - try: - result = response.json() - - if validator and not validator(result): - logger.error("JSON validation result vailed") - else: - return result - except ValueError: - logger.error("Response returned invalid JSON data") - -def request_content(url, **kwargs): - """ - Wrapper for `request_response', which will return the raw content. - """ - - response = request_response(url, **kwargs) - - if response is not None: - return response.content - -def request_feed(url, **kwargs): - """ - Wrapper for `request_response', which will return a feed object. - """ - - response = request_response(url, **kwargs) - - if response is not None: - return feedparser.parse(response.content) \ No newline at end of file + return True \ No newline at end of file diff --git a/headphones/lastfm.py b/headphones/lastfm.py index 65fec683..4ea83280 100644 --- a/headphones/lastfm.py +++ b/headphones/lastfm.py @@ -17,7 +17,7 @@ import random import time import headphones -from headphones import db, logger, helpers +from headphones import db, logger, request from collections import defaultdict @@ -40,7 +40,7 @@ def request_lastfm(method, **kwargs): # Send request logger.debug("Calling Last.FM method: %s", method) - data = helpers.request_json(ENTRY_POINT, timeout=20, params=kwargs) + data = request.request_json(ENTRY_POINT, timeout=20, params=kwargs) # Parse response and check for errors. if not data: diff --git a/headphones/lyrics.py b/headphones/lyrics.py index c51e224d..c3e35f57 100644 --- a/headphones/lyrics.py +++ b/headphones/lyrics.py @@ -16,7 +16,7 @@ import re import htmlentitydefs -from headphones import logger, helpers +from headphones import logger, request def getLyrics(artist, song): @@ -26,7 +26,7 @@ def getLyrics(artist, song): } url = 'http://lyrics.wikia.com/api.php' - data = helpers.request_minidom(url, params) + data = request.request_minidom(url, params) if not data: return @@ -39,7 +39,7 @@ def getLyrics(artist, song): logger.info('No lyrics found for %s - %s' % (artist, song)) return - lyricspage = helpers.request_content(lyricsurl) + lyricspage = request.request_content(lyricsurl) if not lyricspage: logger.warn('Error fetching lyrics from: %s. Error: %s' % (lyricsurl, e)) diff --git a/headphones/notifiers.py b/headphones/notifiers.py index 1ca53595..df6aba1c 100644 --- a/headphones/notifiers.py +++ b/headphones/notifiers.py @@ -13,29 +13,31 @@ # You should have received a copy of the GNU General Public License # along with Headphones. If not, see . -from headphones import logger, helpers, common -from headphones.exceptions import ex import base64 import cherrypy import urllib import urllib2 import headphones -from httplib import HTTPSConnection -from urllib import urlencode +import simplejson import os.path import subprocess import gntp.notifier -import lib.simplejson as simplejson -from xml.dom import minidom -try: - from urlparse import parse_qsl #@UnusedImport -except: - from cgi import parse_qsl #@Reimport +from xml.dom import minidom +from httplib import HTTPSConnection +from urllib import urlencode import lib.oauth2 as oauth import lib.pythontwitter as twitter +from headphones import logger, helpers, common, request +from headphones.exceptions import ex + +try: + from urlparse import parse_qsl +except: + from cgi import parse_qsl + class GROWL: def __init__(self): @@ -179,9 +181,9 @@ class XBMC: url = host + '/xbmcCmds/xbmcHttp/?' + url_command if self.password: - return helpers.request_content(url, auth=(self.username, self.password)) + return request.request_content(url, auth=(self.username, self.password)) else: - return helpers.request_content(url) + return request.request_content(url) def _sendjson(self, host, method, params={}): data = [{'id': 0, 'jsonrpc': '2.0', 'method': method, 'params': params}] @@ -189,9 +191,9 @@ class XBMC: url = host + '/jsonrpc' if self.password: - response = helpers.request_json(req, method="POST", data=simplejson.dumps(data), headers=headers, auth=(self.username, self.password)) + response = request.request_json(req, method="POST", data=simplejson.dumps(data), headers=headers, auth=(self.username, self.password)) else: - response = helpers.request_json(req, method="POST", data=simplejson.dumps(data), headers=headers) + response = request.request_json(req, method="POST", data=simplejson.dumps(data), headers=headers) if response: return response[0]['result'] @@ -332,7 +334,7 @@ class NMA: self.priority = headphones.NMA_PRIORITY def _send(self, data): - return helpers.request_content('https://www.notifymyandroid.com/publicapi/notify', data=data) + return request.request_content('https://www.notifymyandroid.com/publicapi/notify', data=data) def notify(self, artist=None, album=None, snatched_nzb=None): diff --git a/headphones/postprocessor.py b/headphones/postprocessor.py index 1e4ad752..3af26280 100644 --- a/headphones/postprocessor.py +++ b/headphones/postprocessor.py @@ -27,7 +27,7 @@ from beets import autotag from beets.mediafile import MediaFile import headphones -from headphones import db, albumart, librarysync, lyrics, logger, helpers +from headphones import db, albumart, librarysync, lyrics, logger, helpers, request from headphones.helpers import sab_replace_dots, sab_replace_spaces postprocessor_lock = threading.Lock() @@ -360,7 +360,7 @@ def doPostProcessing(albumid, albumpath, release, tracks, downloaded_track_list, artwork = None album_art_path = albumart.getAlbumArt(albumid) if headphones.EMBED_ALBUM_ART or headphones.ADD_ALBUM_ART: - artwork = helpers.request_content(album_art_path) + artwork = request.request_content(album_art_path) if not album_art_path or not artwor or len(artwork) < 100: logger.info("No suitable album art found from Amazon. Checking Last.FM....") diff --git a/headphones/request.py b/headphones/request.py new file mode 100644 index 00000000..ed18f119 --- /dev/null +++ b/headphones/request.py @@ -0,0 +1,116 @@ +from headphones import logger + +from xml.dom import minidom +from bs4 import BeautifulSoup + +import requests +import feedparser + +def request_response(url, method="get", auto_raise=True, whitelist_status_code=None, **kwargs): + """ + Convenient wrapper for `requests.get', which will capture the exceptions and + log them. On success, the Response object is returned. In case of a + exception, None is returned. + """ + + # Convert whitelist_status_code to a list if needed + if whitelist_status_code and type(whitelist_status_code) != list: + whitelist_status_code = [whitelist_status_code] + + # Map method to the request.XXX method. This is a simple hack, but it allows + # requests to apply more magic per method. See lib/requests/api.py. + request_method = getattr(requests, method) + + try: + # Request the URL + logger.debug("Requesting URL via %s method: %s", method.upper(), url) + response = request_method(url, **kwargs) + + # If status code != OK, then raise exception, except if the status code + # is white listed. + if whitelist_status_code and auto_raise: + if response.status_code not in whitelist_status_code: + response.raise_for_status() + else: + logger.debug("Response Status code %d is white listed, not raising exception", response.status_code) + elif auto_raise: + response.raise_for_status() + + return response + except requests.ConnectionError: + logger.error("Unable to connect to remote host.") + except requests.Timeout: + logger.error("Request timed out.") + except requests.HTTPError, e: + if e.response is not None: + logger.error("Request raise HTTP error with status code: %d", e.response.status_code) + else: + logger.error("Request raised HTTP error.") + except requests.RequestException, e: + logger.error("Request raised exception: %s", e) + +def request_soup(url, **kwargs): + """ + Wrapper for `request_response', which will return a BeatifulSoup object if + no exceptions are raised. + """ + + parser = kwargs.pop("parser", "html5lib") + response = request_response(url, **kwargs) + + if response is not None: + return BeautifulSoup(response.content, parser) + +def request_minidom(url, **kwargs): + """ + Wrapper for `request_response', which will return a Minidom object if no + exceptions are raised. + """ + + response = request_response(url, **kwargs) + + if response is not None: + return minidom.parseString(response.content) + +def request_json(url, **kwargs): + """ + Wrapper for `request_response', which will decode the response as JSON + object and return the result, if no exceptions are raised. + + As an option, a validator callback can be given, which should return True if + the result is valid. + """ + + validator = kwargs.pop("validator", None) + response = request_response(url, **kwargs) + + if response is not None: + try: + result = response.json() + + if validator and not validator(result): + logger.error("JSON validation result vailed") + else: + return result + except ValueError: + logger.error("Response returned invalid JSON data") + +def request_content(url, **kwargs): + """ + Wrapper for `request_response', which will return the raw content. + """ + + response = request_response(url, **kwargs) + + if response is not None: + return response.content + +def request_feed(url, **kwargs): + """ + Wrapper for `request_response', which will return a feed object. + """ + + response = request_response(url, **kwargs) + + if response is not None: + return feedparser.parse(response.content) \ No newline at end of file diff --git a/headphones/searcher.py b/headphones/searcher.py index 7c247fca..37a3314f 100644 --- a/headphones/searcher.py +++ b/headphones/searcher.py @@ -29,7 +29,7 @@ import requests import headphones from headphones.common import USER_AGENT -from headphones import logger, db, helpers, classes, sab, nzbget +from headphones import logger, db, helpers, classes, sab, nzbget, request from headphones import transmission import lib.bencode as bencode @@ -311,7 +311,7 @@ def searchNZB(album, new=False, losslessOnly=False): "q": term } - data = helpers.request_feed( + data = request.request_feed( url="http://headphones.codeshy.com/newznab/api", params=params, headers=headers, auth=(headphones.HPUSER, headphones.HPPASS) @@ -377,7 +377,7 @@ def searchNZB(album, new=False, losslessOnly=False): "q": term } - data = helpers.request_feed( + data = request.request_feed( url=newznab_host[0] + '/api?', params=params, headers=headers ) @@ -424,7 +424,7 @@ def searchNZB(album, new=False, losslessOnly=False): "q": term } - data = helpers.request_feed( + data = request.request_feed( url='http://beta.nzbs.org/api', params=params, headers=headers, timeout=20 @@ -474,7 +474,7 @@ def searchNZB(album, new=False, losslessOnly=False): "searchtext": term } - data = helpers.request_json( + data = request.request_json( url='https://www.nzbsrus.com/api.php', params=params, headers=headers, validator=lambda x: type(x) == dict @@ -524,7 +524,7 @@ def searchNZB(album, new=False, losslessOnly=False): "search": term } - data = helpers.request_json( + data = request.request_json( url='http://api.omgwtfnzbs.org/json/', params=params, headers=headers, validator=lambda x: type(x) == dict @@ -739,7 +739,7 @@ def getresultNZB(result): nzb = None if result[3] == 'newzbin': - response = helpers.request_response( + response = request.request_response( url='https://www.newzbin2.es/api/dnzb/', auth=(headphones.HPUSER, headphones.HPPASS), params={"username": headphones.NEWZBIN_UID, "password": headphones.NEWZBIN_PASSWORD, "reportid": result[2]}, @@ -747,30 +747,31 @@ def getresultNZB(result): whitelist_status_code=400 ) - if response.status_code == 400: - error_code = int(response.headers.header['X-DNZB-RCode']) + if response is not None: + if response.status_code == 400: + error_code = int(response.headers.header['X-DNZB-RCode']) - if error_code == 450: - result = re.search("wait (\d+) seconds", response.headers['X-DNZB-RText']) - seconds = int(result.group(1)) + if error_code == 450: + result = re.search("wait (\d+) seconds", response.headers['X-DNZB-RText']) + seconds = int(result.group(1)) - logger.info("Newzbin throttled our NZB downloading, pausing for %d seconds", seconds) - time.sleep(seconds) + logger.info("Newzbin throttled our NZB downloading, pausing for %d seconds", seconds) + time.sleep(seconds) - # Try again -- possibly forever :( - getresultNZB(result) + # Try again -- possibly forever :( + getresultNZB(result) + else: + logger.info("Newzbin error code %d", error_code) else: - logger.info("Newzbin error code %d", error_code) - else: - nzb = response.content + nzb = response.content elif result[3] == 'headphones': - nzb = helpers.request_content( + nzb = request.request_content( url=result[2], auth=(headphones.HPUSER, headphones.HPPASS), headers={'User-Agent': USER_AGENT} ) else: - nzb = helpers.request_content( + nzb = request.request_content( url=result[2], headers={'User-Agent': USER_AGENT} ) @@ -861,7 +862,7 @@ def searchTorrent(album, new=False, losslessOnly=False): "rss": "1" } - data = helpers.request_feed( + data = request.request_feed( url=providerurl, params=params, timeout=20 @@ -880,7 +881,7 @@ def searchTorrent(album, new=False, losslessOnly=False): url = item['links'][1]['href'] size = int(item['links'][1]['length']) if format == "2": - torrent = helpers.request_content(url) + torrent = request.request_content(url) if not torrent or (int(torrent.find(".mp3")) > 0 and int(torrent.find(".flac")) < 1): rightformat = False if rightformat == True and size < maxsize and minimumseeders < int(seeders): @@ -934,7 +935,7 @@ def searchTorrent(album, new=False, losslessOnly=False): "q": " ".join(query_items) } - data = helpers.request_feed( + data = request.request_feed( url=providerurl, params=params, headers=headers, timeout=20 @@ -1121,7 +1122,7 @@ def searchTorrent(album, new=False, losslessOnly=False): "sort": "seeds" } - data = helpers.request_soup( + data = request.request_soup( url=providerurl + category, params=params, timeout=20 @@ -1182,7 +1183,7 @@ def searchTorrent(album, new=False, losslessOnly=False): "sort": "seeds" } - data = helpers.request_feed( + data = request.request_feed( url=providerurl, params=params, headers=headers, auth=(headphones.HPUSER, headphones.HPPASS), @@ -1207,7 +1208,7 @@ def searchTorrent(album, new=False, losslessOnly=False): url = item.links[1]['url'] size = int(item.links[1]['length']) if format == "2": - torrent = helpers.request_content(url) + torrent = request.request_content(url) if not torrent or (int(torrent.find(".mp3")) > 0 and int(torrent.find(".flac")) < 1): rightformat = False @@ -1242,7 +1243,7 @@ def searchTorrent(album, new=False, losslessOnly=False): # Requesting content logger.info('Parsing results from Mininova') - data = helpers.request_feed( + data = request.request_feed( url=providerurl, timeout=20 ) @@ -1264,7 +1265,7 @@ def searchTorrent(album, new=False, losslessOnly=False): url = item.links[1]['url'] size = int(item.links[1]['length']) if format == "2": - torrent = helpers.request_content(url) + torrent = request.request_content(url) if not torrent or (int(torrent.find(".mp3")) > 0 and int(torrent.find(".flac")) < 1): rightformat = False @@ -1303,7 +1304,7 @@ def preprocess(resultlist): elif result[3] == 'What.cd': headers['User-Agent'] = 'Headphones' - return helpers.request_content(url=result[2], headers=headers), result + return request.request_content(url=result[2], headers=headers), result else: usenet_retention = headphones.USENET_RETENTION or 2000 diff --git a/headphones/transmission.py b/headphones/transmission.py index 9b8027ab..060052f3 100644 --- a/headphones/transmission.py +++ b/headphones/transmission.py @@ -20,7 +20,7 @@ import headphones import simplejson as json -from headphones import logger, notifiers, helpers +from headphones import logger, notifiers, request # This is just a simple script to send torrents to transmission. The # intention is to turn this into a class where we can check the state @@ -133,11 +133,11 @@ def torrentAction(method, arguments): # Retrieve session id if username and password: - response = helpers.request_response(host, auth=(username, password), whitelist_status_code=409) + response = request.request_response(host, auth=(username, password), whitelist_status_code=409) else: - response = helpers.request_response(host, whitelist_status_code=409) + response = request.request_response(host, whitelist_status_code=409) - if not response: + if response is None: logger.error("Error gettings Transmission session ID") return @@ -153,7 +153,7 @@ def torrentAction(method, arguments): headers = { 'x-transmission-session-id': sessionid } data = { 'method': method, 'arguments': arguments } - response = helpers.request_content(host, data=json.dumps(data), headers=headers) + response = request.request_json(host, method="post", data=json.dumps(data), headers=headers) if not response: logger.error("Error sending torrent to Transmission") diff --git a/headphones/versioncheck.py b/headphones/versioncheck.py index 7fb76112..ee8c40b6 100644 --- a/headphones/versioncheck.py +++ b/headphones/versioncheck.py @@ -16,7 +16,7 @@ import platform, subprocess, re, os, tarfile import headphones -from headphones import logger, version, helpers +from headphones import logger, version, request from headphones.exceptions import ex import lib.simplejson as simplejson @@ -122,9 +122,9 @@ def checkGithub(): # Get the latest version available from github logger.info('Retrieving latest version information from GitHub') url = 'https://api.github.com/repos/%s/headphones/commits/%s' % (headphones.GIT_USER, headphones.GIT_BRANCH) - version = helpers.request_json(url, timeout=20, validator=lambda x: type(x) == dict) + version = request.request_json(url, timeout=20, validator=lambda x: type(x) == dict) - if not version: + if version is None: logger.warn('Could not get the latest version from GitHub. Are you running a local development version?') return headphones.CURRENT_VERSION @@ -138,9 +138,9 @@ def checkGithub(): logger.info('Comparing currently installed version with latest GitHub version') url = 'https://api.github.com/repos/%s/headphones/compare/%s...%s' % (headphones.GIT_USER, headphones.CURRENT_VERSION, headphones.LATEST_VERSION) - commits = helpers.request_json(url, timeout=20, whitelist_status_code=404, validator=lambda x: type(x) == dict) + commits = request.request_json(url, timeout=20, whitelist_status_code=404, validator=lambda x: type(x) == dict) - if not commits: + if commits is None: logger.warn('Could not get commits behind from GitHub.') return headphones.CURRENT_VERSION @@ -179,7 +179,7 @@ def update(): version_path = os.path.join(headphones.PROG_DIR, 'version.txt') logger.info('Downloading update from: '+tar_download_url) - data = helpers.request_content(tar_download_url) + data = request.request_content(tar_download_url) if not data: logger.error("Unable to retrieve new version from '%s', can't update", tar_download_url)