gronod/headphones
1
0
Fork 1
mirror of https://github.com/rembo10/headphones.git synced 2026-03-22 12:49:26 +00:00
Code Issues Packages Projects Releases Wiki Activity

Moved all request methods to seperate file, since it is getting a core function

Browse Source
This commit is contained in:
Bas Stottelaar
2014-04-07 00:32:43 +02:00
parent 21e1c100e7
commit 22151ff4b0
11 changed files with 195 additions and 189 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
headphones/albumart.py
View File

@@ -13,7 +13,7 @@
# You should have received a copy of the GNU General Public License
# along with Headphones. If not, see <http://www.gnu.org/licenses/>.
from headphones import helpers, db
from headphones import request, db
def getAlbumArt(albumid):
myDB = db.DBConnection()
@@ -32,7 +32,7 @@ def getCachedArt(albumid):
return
if artwork_path.startswith('http://'):
artwork = helpers.request_content(artwork_path, timeout=20)
artwork = request.request_content(artwork_path, timeout=20)
if not artwork:
logger.warn("Unable to open url: %s", artwork_path)

13
headphones/cache.py
View File

@@ -14,12 +14,11 @@
# along with Headphones. If not, see <http://www.gnu.org/licenses/>.
import os
import glob, urllib
import lib.simplejson as simplejson
import glob
import urllib
import headphones
from headphones import db, helpers, logger, lastfm
from headphones import db, helpers, logger, lastfm, request
lastfm_apikey = "690e1ed3bc00bc91804cd8f7fe5ed6d4"
@@ -337,7 +336,7 @@ class Cache(object):
# Should we grab the artwork here if we're just grabbing thumbs or info?? Probably not since the files can be quite big
if image_url and self.query_type == 'artwork':
artwork = helpers.request_content(image_url, timeout=20)
artwork = request.request_content(image_url, timeout=20)
if artwork:
# Make sure the artwork dir exists:
@@ -370,7 +369,7 @@ class Cache(object):
# Grab the thumbnail as well if we're getting the full artwork (as long as it's missing/outdated
if thumb_url and self.query_type in ['thumb','artwork'] and not (self.thumb_files and self._is_current(self.thumb_files[0])):
artwork = helpers.request_content(thumb_url, timeout=20)
artwork = request.request_content(thumb_url, timeout=20)
if artwork:
# Make sure the artwork dir exists:

122
headphones/helpers.py
View File

@@ -18,16 +18,9 @@ import re
import time
import shutil
import datetime
import requests
import feedparser
import headphones
from headphones import logger
from xml.dom import minidom
from operator import itemgetter
from bs4 import BeautifulSoup
from beets.mediafile import MediaFile, FileTypeError, UnreadableFileError
# Modified from https://github.com/Verrus/beets-plugin-featInTitle
@@ -243,6 +236,8 @@ def expand_subfolders(f):
case, normal post processing will be better.
"""
from headphones import logger
# Find all folders with media files in them
media_folders = []
@@ -338,6 +333,8 @@ def extract_metadata(f):
artists, albums and years found in the media files.
"""
from headphones import logger
# Walk directory and scan all media files
results = []
count = 0
@@ -595,113 +592,4 @@ def create_https_certificates(ssl_cert, ssl_key):
logger.error("Error creating SSL key and certificate: %s", e)
return False
return True
def request_response(url, method="get", auto_raise=True, whitelist_status_code=None, **kwargs):
"""
Convenient wrapper for `requests.get', which will capture the exceptions and
log them. On success, the Response object is returned. In case of a
exception, None is returned.
"""
# Convert whitelist_status_code to a list if needed
if whitelist_status_code and type(whitelist_status_code) != list:
whitelist_status_code = [whitelist_status_code]
# Map method to the request.XXX method. This is a simple hack, but it allows
# requests to apply more magic per method. See lib/requests/api.py.
request_method = getattr(requests, method)
try:
# Request the URL
logger.debug("Requesting URL via %s method: %s", method.upper(), url)
response = request_method(url, **kwargs)
# If status code != OK, then raise exception, except if the status code
# is white listed.
if whitelist_status_code and auto_raise:
if response.status_code not in whitelist_status_code:
response.raise_for_status()
else:
logger.debug("Response Status code %d is white listed, not raising exception", response.status_code)
elif auto_raise:
response.raise_for_status()
return response
except requests.ConnectionError:
logger.error("Unable to connect to remote host.")
except requests.Timeout:
logger.error("Request timed out.")
except requests.HTTPError, e:
if e.response is not None:
logger.error("Request raise HTTP error with status code: %d", e.response.status_code)
else:
logger.error("Request raised HTTP error.")
except requests.RequestException, e:
logger.error("Request raised exception: %s", e)
def request_soup(url, **kwargs):
"""
Wrapper for `request_response', which will return a BeatifulSoup object if
no exceptions are raised.
"""
parser = kwargs.pop("parser", "html5lib")
response = request_response(url, **kwargs)
if response is not None:
return BeautifulSoup(response.content, parser)
def request_minidom(url, **kwargs):
"""
Wrapper for `request_response', which will return a Minidom object if no
exceptions are raised.
"""
response = request_response(url, **kwargs)
if response is not None:
return minidom.parseString(response.content)
def request_json(url, **kwargs):
"""
Wrapper for `request_response', which will decode the response as JSON
object and return the result, if no exceptions are raised.
As an option, a validator callback can be given, which should return True if
the result is valid.
"""
validator = kwargs.pop("validator", None)
response = request_response(url, **kwargs)
if response is not None:
try:
result = response.json()
if validator and not validator(result):
logger.error("JSON validation result vailed")
else:
return result
except ValueError:
logger.error("Response returned invalid JSON data")
def request_content(url, **kwargs):
"""
Wrapper for `request_response', which will return the raw content.
"""
response = request_response(url, **kwargs)
if response is not None:
return response.content
def request_feed(url, **kwargs):
"""
Wrapper for `request_response', which will return a feed object.
"""
response = request_response(url, **kwargs)
if response is not None:
return feedparser.parse(response.content)
return True

4
headphones/lastfm.py
View File

@@ -17,7 +17,7 @@ import random
import time
import headphones
from headphones import db, logger, helpers
from headphones import db, logger, request
from collections import defaultdict
@@ -40,7 +40,7 @@ def request_lastfm(method, **kwargs):
# Send request
logger.debug("Calling Last.FM method: %s", method)
data = helpers.request_json(ENTRY_POINT, timeout=20, params=kwargs)
data = request.request_json(ENTRY_POINT, timeout=20, params=kwargs)
# Parse response and check for errors.
if not data:

6
headphones/lyrics.py
View File

@@ -16,7 +16,7 @@
import re
import htmlentitydefs
from headphones import logger, helpers
from headphones import logger, request
def getLyrics(artist, song):
@@ -26,7 +26,7 @@ def getLyrics(artist, song):
}
url = 'http://lyrics.wikia.com/api.php'
data = helpers.request_minidom(url, params)
data = request.request_minidom(url, params)
if not data:
return
@@ -39,7 +39,7 @@ def getLyrics(artist, song):
logger.info('No lyrics found for %s - %s' % (artist, song))
return
lyricspage = helpers.request_content(lyricsurl)
lyricspage = request.request_content(lyricsurl)
if not lyricspage:
logger.warn('Error fetching lyrics from: %s. Error: %s' % (lyricsurl, e))

32
headphones/notifiers.py
View File

@@ -13,29 +13,31 @@
# You should have received a copy of the GNU General Public License
# along with Headphones. If not, see <http://www.gnu.org/licenses/>.
from headphones import logger, helpers, common
from headphones.exceptions import ex
import base64
import cherrypy
import urllib
import urllib2
import headphones
from httplib import HTTPSConnection
from urllib import urlencode
import simplejson
import os.path
import subprocess
import gntp.notifier
import lib.simplejson as simplejson
from xml.dom import minidom
try:
from urlparse import parse_qsl #@UnusedImport
except:
from cgi import parse_qsl #@Reimport
from xml.dom import minidom
from httplib import HTTPSConnection
from urllib import urlencode
import lib.oauth2 as oauth
import lib.pythontwitter as twitter
from headphones import logger, helpers, common, request
from headphones.exceptions import ex
try:
from urlparse import parse_qsl
except:
from cgi import parse_qsl
class GROWL:
def __init__(self):
@@ -179,9 +181,9 @@ class XBMC:
url = host + '/xbmcCmds/xbmcHttp/?' + url_command
if self.password:
return helpers.request_content(url, auth=(self.username, self.password))
return request.request_content(url, auth=(self.username, self.password))
else:
return helpers.request_content(url)
return request.request_content(url)
def _sendjson(self, host, method, params={}):
data = [{'id': 0, 'jsonrpc': '2.0', 'method': method, 'params': params}]
@@ -189,9 +191,9 @@ class XBMC:
url = host + '/jsonrpc'
if self.password:
response = helpers.request_json(req, method="POST", data=simplejson.dumps(data), headers=headers, auth=(self.username, self.password))
response = request.request_json(req, method="POST", data=simplejson.dumps(data), headers=headers, auth=(self.username, self.password))
else:
response = helpers.request_json(req, method="POST", data=simplejson.dumps(data), headers=headers)
response = request.request_json(req, method="POST", data=simplejson.dumps(data), headers=headers)
if response:
return response[0]['result']
@@ -332,7 +334,7 @@ class NMA:
self.priority = headphones.NMA_PRIORITY
def _send(self, data):
return helpers.request_content('https://www.notifymyandroid.com/publicapi/notify', data=data)
return request.request_content('https://www.notifymyandroid.com/publicapi/notify', data=data)
def notify(self, artist=None, album=None, snatched_nzb=None):

4
headphones/postprocessor.py
View File

@@ -27,7 +27,7 @@ from beets import autotag
from beets.mediafile import MediaFile
import headphones
from headphones import db, albumart, librarysync, lyrics, logger, helpers
from headphones import db, albumart, librarysync, lyrics, logger, helpers, request
from headphones.helpers import sab_replace_dots, sab_replace_spaces
postprocessor_lock = threading.Lock()
@@ -360,7 +360,7 @@ def doPostProcessing(albumid, albumpath, release, tracks, downloaded_track_list,
artwork = None
album_art_path = albumart.getAlbumArt(albumid)
if headphones.EMBED_ALBUM_ART or headphones.ADD_ALBUM_ART:
artwork = helpers.request_content(album_art_path)
artwork = request.request_content(album_art_path)
if not album_art_path or not artwor or len(artwork) < 100:
logger.info("No suitable album art found from Amazon. Checking Last.FM....")

116
headphones/request.py Normal file
View File

@@ -0,0 +1,116 @@
from headphones import logger
from xml.dom import minidom
from bs4 import BeautifulSoup
import requests
import feedparser
def request_response(url, method="get", auto_raise=True, whitelist_status_code=None, **kwargs):
"""
Convenient wrapper for `requests.get', which will capture the exceptions and
log them. On success, the Response object is returned. In case of a
exception, None is returned.
"""
# Convert whitelist_status_code to a list if needed
if whitelist_status_code and type(whitelist_status_code) != list:
whitelist_status_code = [whitelist_status_code]
# Map method to the request.XXX method. This is a simple hack, but it allows
# requests to apply more magic per method. See lib/requests/api.py.
request_method = getattr(requests, method)
try:
# Request the URL
logger.debug("Requesting URL via %s method: %s", method.upper(), url)
response = request_method(url, **kwargs)
# If status code != OK, then raise exception, except if the status code
# is white listed.
if whitelist_status_code and auto_raise:
if response.status_code not in whitelist_status_code:
response.raise_for_status()
else:
logger.debug("Response Status code %d is white listed, not raising exception", response.status_code)
elif auto_raise:
response.raise_for_status()
return response
except requests.ConnectionError:
logger.error("Unable to connect to remote host.")
except requests.Timeout:
logger.error("Request timed out.")
except requests.HTTPError, e:
if e.response is not None:
logger.error("Request raise HTTP error with status code: %d", e.response.status_code)
else:
logger.error("Request raised HTTP error.")
except requests.RequestException, e:
logger.error("Request raised exception: %s", e)
def request_soup(url, **kwargs):
"""
Wrapper for `request_response', which will return a BeatifulSoup object if
no exceptions are raised.
"""
parser = kwargs.pop("parser", "html5lib")
response = request_response(url, **kwargs)
if response is not None:
return BeautifulSoup(response.content, parser)
def request_minidom(url, **kwargs):
"""
Wrapper for `request_response', which will return a Minidom object if no
exceptions are raised.
"""
response = request_response(url, **kwargs)
if response is not None:
return minidom.parseString(response.content)
def request_json(url, **kwargs):
"""
Wrapper for `request_response', which will decode the response as JSON
object and return the result, if no exceptions are raised.
As an option, a validator callback can be given, which should return True if
the result is valid.
"""
validator = kwargs.pop("validator", None)
response = request_response(url, **kwargs)
if response is not None:
try:
result = response.json()
if validator and not validator(result):
logger.error("JSON validation result vailed")
else:
return result
except ValueError:
logger.error("Response returned invalid JSON data")
def request_content(url, **kwargs):
"""
Wrapper for `request_response', which will return the raw content.
"""
response = request_response(url, **kwargs)
if response is not None:
return response.content
def request_feed(url, **kwargs):
"""
Wrapper for `request_response', which will return a feed object.
"""
response = request_response(url, **kwargs)
if response is not None:
return feedparser.parse(response.content)

61
headphones/searcher.py
View File

@@ -29,7 +29,7 @@ import requests
import headphones
from headphones.common import USER_AGENT
from headphones import logger, db, helpers, classes, sab, nzbget
from headphones import logger, db, helpers, classes, sab, nzbget, request
from headphones import transmission
import lib.bencode as bencode
@@ -311,7 +311,7 @@ def searchNZB(album, new=False, losslessOnly=False):
"q": term
}
data = helpers.request_feed(
data = request.request_feed(
url="http://headphones.codeshy.com/newznab/api",
params=params, headers=headers,
auth=(headphones.HPUSER, headphones.HPPASS)
@@ -377,7 +377,7 @@ def searchNZB(album, new=False, losslessOnly=False):
"q": term
}
data = helpers.request_feed(
data = request.request_feed(
url=newznab_host[0] + '/api?',
params=params, headers=headers
)
@@ -424,7 +424,7 @@ def searchNZB(album, new=False, losslessOnly=False):
"q": term
}
data = helpers.request_feed(
data = request.request_feed(
url='http://beta.nzbs.org/api',
params=params, headers=headers,
timeout=20
@@ -474,7 +474,7 @@ def searchNZB(album, new=False, losslessOnly=False):
"searchtext": term
}
data = helpers.request_json(
data = request.request_json(
url='https://www.nzbsrus.com/api.php',
params=params, headers=headers,
validator=lambda x: type(x) == dict
@@ -524,7 +524,7 @@ def searchNZB(album, new=False, losslessOnly=False):
"search": term
}
data = helpers.request_json(
data = request.request_json(
url='http://api.omgwtfnzbs.org/json/',
params=params, headers=headers,
validator=lambda x: type(x) == dict
@@ -739,7 +739,7 @@ def getresultNZB(result):
nzb = None
if result[3] == 'newzbin':
response = helpers.request_response(
response = request.request_response(
url='https://www.newzbin2.es/api/dnzb/',
auth=(headphones.HPUSER, headphones.HPPASS),
params={"username": headphones.NEWZBIN_UID, "password": headphones.NEWZBIN_PASSWORD, "reportid": result[2]},
@@ -747,30 +747,31 @@ def getresultNZB(result):
whitelist_status_code=400
)
if response.status_code == 400:
error_code = int(response.headers.header['X-DNZB-RCode'])
if response is not None:
if response.status_code == 400:
error_code = int(response.headers.header['X-DNZB-RCode'])
if error_code == 450:
result = re.search("wait (\d+) seconds", response.headers['X-DNZB-RText'])
seconds = int(result.group(1))
if error_code == 450:
result = re.search("wait (\d+) seconds", response.headers['X-DNZB-RText'])
seconds = int(result.group(1))
logger.info("Newzbin throttled our NZB downloading, pausing for %d seconds", seconds)
time.sleep(seconds)
logger.info("Newzbin throttled our NZB downloading, pausing for %d seconds", seconds)
time.sleep(seconds)
# Try again -- possibly forever :(
getresultNZB(result)
# Try again -- possibly forever :(
getresultNZB(result)
else:
logger.info("Newzbin error code %d", error_code)
else:
logger.info("Newzbin error code %d", error_code)
else:
nzb = response.content
nzb = response.content
elif result[3] == 'headphones':
nzb = helpers.request_content(
nzb = request.request_content(
url=result[2],
auth=(headphones.HPUSER, headphones.HPPASS),
headers={'User-Agent': USER_AGENT}
)
else:
nzb = helpers.request_content(
nzb = request.request_content(
url=result[2],
headers={'User-Agent': USER_AGENT}
)
@@ -861,7 +862,7 @@ def searchTorrent(album, new=False, losslessOnly=False):
"rss": "1"
}
data = helpers.request_feed(
data = request.request_feed(
url=providerurl,
params=params,
timeout=20
@@ -880,7 +881,7 @@ def searchTorrent(album, new=False, losslessOnly=False):
url = item['links'][1]['href']
size = int(item['links'][1]['length'])
if format == "2":
torrent = helpers.request_content(url)
torrent = request.request_content(url)
if not torrent or (int(torrent.find(".mp3")) > 0 and int(torrent.find(".flac")) < 1):
rightformat = False
if rightformat == True and size < maxsize and minimumseeders < int(seeders):
@@ -934,7 +935,7 @@ def searchTorrent(album, new=False, losslessOnly=False):
"q": " ".join(query_items)
}
data = helpers.request_feed(
data = request.request_feed(
url=providerurl,
params=params, headers=headers,
timeout=20
@@ -1121,7 +1122,7 @@ def searchTorrent(album, new=False, losslessOnly=False):
"sort": "seeds"
}
data = helpers.request_soup(
data = request.request_soup(
url=providerurl + category,
params=params,
timeout=20
@@ -1182,7 +1183,7 @@ def searchTorrent(album, new=False, losslessOnly=False):
"sort": "seeds"
}
data = helpers.request_feed(
data = request.request_feed(
url=providerurl,
params=params, headers=headers,
auth=(headphones.HPUSER, headphones.HPPASS),
@@ -1207,7 +1208,7 @@ def searchTorrent(album, new=False, losslessOnly=False):
url = item.links[1]['url']
size = int(item.links[1]['length'])
if format == "2":
torrent = helpers.request_content(url)
torrent = request.request_content(url)
if not torrent or (int(torrent.find(".mp3")) > 0 and int(torrent.find(".flac")) < 1):
rightformat = False
@@ -1242,7 +1243,7 @@ def searchTorrent(album, new=False, losslessOnly=False):
# Requesting content
logger.info('Parsing results from Mininova')
data = helpers.request_feed(
data = request.request_feed(
url=providerurl,
timeout=20
)
@@ -1264,7 +1265,7 @@ def searchTorrent(album, new=False, losslessOnly=False):
url = item.links[1]['url']
size = int(item.links[1]['length'])
if format == "2":
torrent = helpers.request_content(url)
torrent = request.request_content(url)
if not torrent or (int(torrent.find(".mp3")) > 0 and int(torrent.find(".flac")) < 1):
rightformat = False
@@ -1303,7 +1304,7 @@ def preprocess(resultlist):
elif result[3] == 'What.cd':
headers['User-Agent'] = 'Headphones'
return helpers.request_content(url=result[2], headers=headers), result
return request.request_content(url=result[2], headers=headers), result
else:
usenet_retention = headphones.USENET_RETENTION or 2000

10
headphones/transmission.py
View File

@@ -20,7 +20,7 @@ import headphones
import simplejson as json
from headphones import logger, notifiers, helpers
from headphones import logger, notifiers, request
# This is just a simple script to send torrents to transmission. The
# intention is to turn this into a class where we can check the state
@@ -133,11 +133,11 @@ def torrentAction(method, arguments):
# Retrieve session id
if username and password:
response = helpers.request_response(host, auth=(username, password), whitelist_status_code=409)
response = request.request_response(host, auth=(username, password), whitelist_status_code=409)
else:
response = helpers.request_response(host, whitelist_status_code=409)
response = request.request_response(host, whitelist_status_code=409)
if not response:
if response is None:
logger.error("Error gettings Transmission session ID")
return
@@ -153,7 +153,7 @@ def torrentAction(method, arguments):
headers = { 'x-transmission-session-id': sessionid }
data = { 'method': method, 'arguments': arguments }
response = helpers.request_content(host, data=json.dumps(data), headers=headers)
response = request.request_json(host, method="post", data=json.dumps(data), headers=headers)
if not response:
logger.error("Error sending torrent to Transmission")

12
headphones/versioncheck.py
View File

@@ -16,7 +16,7 @@
import platform, subprocess, re, os, tarfile
import headphones
from headphones import logger, version, helpers
from headphones import logger, version, request
from headphones.exceptions import ex
import lib.simplejson as simplejson
@@ -122,9 +122,9 @@ def checkGithub():
# Get the latest version available from github
logger.info('Retrieving latest version information from GitHub')
url = 'https://api.github.com/repos/%s/headphones/commits/%s' % (headphones.GIT_USER, headphones.GIT_BRANCH)
version = helpers.request_json(url, timeout=20, validator=lambda x: type(x) == dict)
version = request.request_json(url, timeout=20, validator=lambda x: type(x) == dict)
if not version:
if version is None:
logger.warn('Could not get the latest version from GitHub. Are you running a local development version?')
return headphones.CURRENT_VERSION
@@ -138,9 +138,9 @@ def checkGithub():
logger.info('Comparing currently installed version with latest GitHub version')
url = 'https://api.github.com/repos/%s/headphones/compare/%s...%s' % (headphones.GIT_USER, headphones.CURRENT_VERSION, headphones.LATEST_VERSION)
commits = helpers.request_json(url, timeout=20, whitelist_status_code=404, validator=lambda x: type(x) == dict)
commits = request.request_json(url, timeout=20, whitelist_status_code=404, validator=lambda x: type(x) == dict)
if not commits:
if commits is None:
logger.warn('Could not get commits behind from GitHub.')
return headphones.CURRENT_VERSION
@@ -179,7 +179,7 @@ def update():
version_path = os.path.join(headphones.PROG_DIR, 'version.txt')
logger.info('Downloading update from: '+tar_download_url)
data = helpers.request_content(tar_download_url)
data = request.request_content(tar_download_url)
if not data:
logger.error("Unable to retrieve new version from '%s', can't update", tar_download_url)