gronod/headphones
1
0
Fork 0
mirror of https://github.com/rembo10/headphones.git synced 2026-07-21 08:24:00 +01:00
Code Issues Packages Projects Releases Wiki Activity

Replaced urllib with requests, part 1

Browse Source
This commit is contained in:
Bas Stottelaar
2014-04-05 21:01:18 +02:00
parent 667f768749
commit f757ce0bde
1 changed files with 336 additions and 389 deletions
Download Patch File Download Diff File Expand all files Collapse all files
headphones/searcher.py
+312 -365
View File
@@ -15,7 +15,7 @@
# NZBGet support added by CurlyMo <curlymoo1@gmail.com> as a part of XBian - XBMC on the Raspberry Pi # NZBGet support added by CurlyMo <curlymoo1@gmail.com> as a part of XBian - XBMC on the Raspberry Pi
import urllib, urllib2, urlparse, httplib import urllib, urlparse
import lib.feedparser as feedparser import lib.feedparser as feedparser
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
from lib.pygazelle import api as gazelleapi from lib.pygazelle import api as gazelleapi
@@ -31,6 +31,7 @@ import gzip, base64
import os, re, time import os, re, time
import string import string
import shutil import shutil
import requests
import headphones, exceptions import headphones, exceptions
from headphones.common import USER_AGENT from headphones.common import USER_AGENT
@@ -45,49 +46,93 @@ rutracker = rutrackersearch.Rutracker()
# Persistent What.cd API object # Persistent What.cd API object
gazelle = None gazelle = None
def request_response(url, method="GET", auto_raise=True, status_pass=None, *args, **kwargs):
"""
Convenient wrapper for `requests.get', which will capture the exceptions and
log them. On success, the Response object is returned. In case of a
exception, None is returned.
"""
class NewzbinDownloader(urllib.FancyURLopener):
def __init__(self):
urllib.FancyURLopener.__init__(self)
def http_error_default(self, url, fp, errcode, errmsg, headers):
# if newzbin is throttling us, wait seconds and try again
if errcode == 400:
newzbinErrCode = int(headers.getheader('X-DNZB-RCode'))
if newzbinErrCode == 450:
rtext = str(headers.getheader('X-DNZB-RText'))
result = re.search("wait (\d+) seconds", rtext)
logger.info("Newzbin throttled our NZB downloading, pausing for " + result.group(1) + " seconds")
time.sleep(int(result.group(1)))
raise exceptions.NewzbinAPIThrottled()
elif newzbinErrCode == 401:
logger.info("Newzbin error 401")
#raise exceptions.AuthException("Newzbin username or password incorrect")
elif newzbinErrCode == 402:
#raise exceptions.AuthException("Newzbin account not premium status, can't download NZBs")
logger.info("Newzbin error 402")
#this should be in a class somewhere
def getNewzbinURL(url):
myOpener = classes.AuthURLOpener(headphones.NEWZBIN_UID, headphones.NEWZBIN_PASSWORD)
try: try:
f = myOpener.openit(url) # Request the URL
except (urllib.ContentTooShortError, IOError), e: logger.debug("Requesting URL via %s method: %s", method, url)
logger.info("Error loading search results: ContentTooShortError ") response = requests.request(method, url, *args, **kwargs)
return None
data = f.read() # If status code != OK, then raise exception, except if the status code
f.close() # is white listed.
if status_pass and auto_raise:
if response.status_code not in status_pass:
response.raise_for_status()
else:
logger.debug("Response code %d is white listed, not raising exception", response.status_code)
elif auto_raise:
response.raise_for_status()
return data return response
except requests.ConnectionError:
logger.error("Unable to connect to remote host.")
except requests.Timeout:
logger.error("Request timed out.")
except requests.HTTPError, e:
if e.response:
logger.error("Request returned unexpected status code: %d", e.status_code)
else:
logger.error("Request raised an HTTP error.")
except requests.RequestException, e:
logger.error("Request raised exception: %s", e)
def request_soup(*args, **kwargs):
"""
Wrapper for `request_response', which will return a BeatifulSoup object if
no exceptions are raised.
"""
response = request_response(*args, **kwargs)
if response:
return BeautifulSoup(response.content)
def request_json(validator=None, *args, **kwargs):
"""
Wrapper for `request_response', which will decode the response as JSON
object and return the result, if no exceptions are raised.
As an option, a validator callback can be given, which should return True if
the result is valid.
"""
response = request_response(*args, **kwargs)
if response:
try:
result = response.json()
if validator and not validator(result):
logger.error("JSON validation result vailed")
else:
return result
except ValueError:
logger.error("Response returned invalid JSON data")
def request_content(*args, **kwargs):
"""
Wrapper for `request_response', which will return the raw content.
"""
response = request_response(*args, **kwargs)
if response:
return response.content
def request_feed(*args, **kwargs):
"""
Wrapper for `request_response', which will return a feed object.
"""
response = request_response(*args, **kwargs)
if response:
return feedparser.parse(response.content)
def url_fix(s, charset='utf-8'): def url_fix(s, charset='utf-8'):
if isinstance(s, unicode): if isinstance(s, unicode):
@@ -97,19 +142,7 @@ def url_fix(s, charset='utf-8'):
qs = urllib.quote_plus(qs, ':&=') qs = urllib.quote_plus(qs, ':&=')
return urlparse.urlunsplit((scheme, netloc, path, qs, anchor)) return urlparse.urlunsplit((scheme, netloc, path, qs, anchor))
def patch_http_response_read(func):
def inner(*args):
try:
return func(*args)
except httplib.IncompleteRead, e:
return e.partial
return inner
httplib.HTTPResponse.read = patch_http_response_read(httplib.HTTPResponse.read)
def searchforalbum(albumid=None, new=False, losslessOnly=False, choose_specific_download=False): def searchforalbum(albumid=None, new=False, losslessOnly=False, choose_specific_download=False):
myDB = db.DBConnection() myDB = db.DBConnection()
if not albumid: if not albumid:
@@ -348,8 +381,8 @@ def searchNZB(album, new=False, losslessOnly=False):
resultlist = [] resultlist = []
if headphones.HEADPHONES_INDEXER: if headphones.HEADPHONES_INDEXER:
provider = "headphones" provider = "headphones"
if headphones.PREFERRED_QUALITY == 3 or losslessOnly: if headphones.PREFERRED_QUALITY == 3 or losslessOnly:
categories = "3040" categories = "3040"
elif headphones.PREFERRED_QUALITY: elif headphones.PREFERRED_QUALITY:
@@ -361,41 +394,30 @@ def searchNZB(album, new=False, losslessOnly=False):
categories = "3030" categories = "3030"
logger.info("Album type is audiobook/spokenword. Using audiobook category") logger.info("Album type is audiobook/spokenword. Using audiobook category")
params = { "t": "search", # Request results
"cat": categories, logger.info('Parsing results from Headphones Indexer')
"apikey": '89edf227c1de9b3de50383fff11466c6',
"maxage": headphones.USENET_RETENTION,
"q": term
}
searchURL = 'http://headphones.codeshy.com/newznab/api?' + urllib.urlencode(params) headers = { 'User-Agent', USER_AGENT }
params = {
"t": "search",
"cat": categories,
"apikey": '89edf227c1de9b3de50383fff11466c6',
"maxage": headphones.USENET_RETENTION,
"q": term
}
# Add a user-agent data = request_feed(
request = urllib2.Request(searchURL) url="http://headphones.codeshy.com/newznab/api",
request.add_header('User-Agent', USER_AGENT) params=params, headers=headers,
base64string = base64.encodestring('%s:%s' % (headphones.HPUSER, headphones.HPPASS)).replace('\n', '') auth=(headphones.HPUSER, headphones.HPPASS)
request.add_header("Authorization", "Basic %s" % base64string) )
opener = urllib2.build_opener()
logger.info(u'Parsing results from <a href="%s">%s</a>' % (searchURL, 'Headphones Index'))
try:
data = opener.open(request).read()
except Exception, e:
logger.warn('Error fetching data from %s: %s' % ('Headphones Index', e))
data = False
# Process feed
if data: if data:
if not len(data.entries):
d = feedparser.parse(data)
if not len(d.entries):
logger.info(u"No results found from %s for %s" % ('Headphones Index', term)) logger.info(u"No results found from %s for %s" % ('Headphones Index', term))
pass
else: else:
for item in d.entries: for item in data.entries:
try: try:
url = item.link url = item.link
title = item.title title = item.title
@@ -408,14 +430,13 @@ def searchNZB(album, new=False, losslessOnly=False):
logger.error(u"An unknown error occurred trying to parse the feed: %s" % e) logger.error(u"An unknown error occurred trying to parse the feed: %s" % e)
if headphones.NEWZNAB: if headphones.NEWZNAB:
provider = "newznab"
newznab_hosts = [] newznab_hosts = []
for newznab_host in headphones.EXTRA_NEWZNABS: for newznab_host in headphones.EXTRA_NEWZNABS:
if newznab_host[2] == '1' or newznab_host[2] == 1: if newznab_host[2] == '1' or newznab_host[2] == 1:
newznab_hosts.append(newznab_host) newznab_hosts.append(newznab_host)
provider = "newznab"
if headphones.PREFERRED_QUALITY == 3 or losslessOnly: if headphones.PREFERRED_QUALITY == 3 or losslessOnly:
categories = "3040" categories = "3040"
elif headphones.PREFERRED_QUALITY: elif headphones.PREFERRED_QUALITY:
@@ -428,7 +449,6 @@ def searchNZB(album, new=False, losslessOnly=False):
logger.info("Album type is audiobook/spokenword. Using audiobook category") logger.info("Album type is audiobook/spokenword. Using audiobook category")
for newznab_host in newznab_hosts: for newznab_host in newznab_hosts:
# Add a little mod for kere.ws # Add a little mod for kere.ws
if newznab_host[0] == "http://kere.ws": if newznab_host[0] == "http://kere.ws":
if categories == "3040": if categories == "3040":
@@ -440,35 +460,29 @@ def searchNZB(album, new=False, losslessOnly=False):
else: else:
categories = categories + ",4050" categories = categories + ",4050"
params = { "t": "search", # Request results
"apikey": newznab_host[1], logger.info('Parsing results from %s', newznab_host[0])
"cat": categories,
"maxage": headphones.USENET_RETENTION,
"q": term
}
searchURL = newznab_host[0] + '/api?' + urllib.urlencode(params) headers = { 'User-Agent', USER_AGENT }
params = {
"t": "search",
"apikey": newznab_host[1],
"cat": categories,
"maxage": headphones.USENET_RETENTION,
"q": term
}
# Add a user-agent data = request_feed(
request = urllib2.Request(searchURL) url=newznab_host[0] + '/api?',
request.add_header('User-Agent', USER_AGENT) params=params, headers=headers
opener = urllib2.build_opener() )
logger.info(u'Parsing results from <a href="%s">%s</a>', searchURL, newznab_host[0])
try:
data = opener.open(request).read()
except Exception, e:
logger.warn('Error fetching data from %s: %s', newznab_host[0], e)
data = False
# Process feed
if data: if data:
d = feedparser.parse(data) if not len(data.entries):
if not len(d.entries):
logger.info(u"No results found from %s for %s", newznab_host[0], term) logger.info(u"No results found from %s for %s", newznab_host[0], term)
else: else:
for item in d.entries: for item in data.entries:
try: try:
url = item.link url = item.link
title = item.title title = item.title
@@ -478,7 +492,7 @@ def searchNZB(album, new=False, losslessOnly=False):
logger.info('Found %s. Size: %s' % (title, helpers.bytes_to_mb(size))) logger.info('Found %s. Size: %s' % (title, helpers.bytes_to_mb(size)))
except Exception, e: except Exception, e:
logger.error(u"An unknown error occurred trying to parse the feed: %s" % e) logger.exception("An unknown error occurred trying to parse the feed: %s" % e)
if headphones.NZBSORG: if headphones.NZBSORG:
provider = "nzbsorg" provider = "nzbsorg"
@@ -493,32 +507,30 @@ def searchNZB(album, new=False, losslessOnly=False):
categories = "3030" categories = "3030"
logger.info("Album type is audiobook/spokenword. Using audiobook category") logger.info("Album type is audiobook/spokenword. Using audiobook category")
params = { "t": "search", # Request results
"apikey": headphones.NZBSORG_HASH, logger.info('Parsing results from nzbs.org')
"cat": categories,
"maxage": headphones.USENET_RETENTION,
"q": term
}
searchURL = 'http://beta.nzbs.org/api?' + urllib.urlencode(params) headers = { 'User-Agent', USER_AGENT }
params = {
"t": "search",
"apikey": headphones.NZBSORG_HASH,
"cat": categories,
"maxage": headphones.USENET_RETENTION,
"q": term
}
logger.info(u'Parsing results from <a href="%s">nzbs.org</a>' % searchURL) data = request_feed(
url='http://beta.nzbs.org/api',
try: params=params, headers=headers,
data = urllib2.urlopen(searchURL, timeout=20).read() timeout=20
except urllib2.URLError, e: )
logger.warn('Error fetching data from nzbs.org: %s' % e)
data = False
# Process feed
if data: if data:
d = feedparser.parse(data) if not len(data.entries):
if not len(d.entries):
logger.info(u"No results found from nzbs.org for %s" % term) logger.info(u"No results found from nzbs.org for %s" % term)
pass
else: else:
for item in d.entries: for item in data.entries:
try: try:
url = item.link url = item.link
title = item.title title = item.title
@@ -544,37 +556,32 @@ def searchNZB(album, new=False, losslessOnly=False):
sub = "" sub = ""
logger.info("Album type is audiobook/spokenword. Searching all music categories") logger.info("Album type is audiobook/spokenword. Searching all music categories")
params = { "uid": headphones.NZBSRUS_UID, # Request results
"key": headphones.NZBSRUS_APIKEY, logger.info('Parsing results from NZBsRus')
"cat": categories,
"sub": sub,
"age": headphones.USENET_RETENTION,
"searchtext": term
}
searchURL = 'https://www.nzbsrus.com/api.php?' + urllib.urlencode(params) headers = { 'User-Agent', USER_AGENT }
params = {
"uid": headphones.NZBSRUS_UID,
"key": headphones.NZBSRUS_APIKEY,
"cat": categories,
"sub": sub,
"age": headphones.USENET_RETENTION,
"searchtext": term
}
# Add a user-agent data = request_json(
request = urllib2.Request(searchURL) url='https://www.nzbsrus.com/api.php',
request.add_header('User-Agent', USER_AGENT) params=params, headers=headers,
opener = urllib2.build_opener() validator=lambda x: type(x) == dict
)
logger.info(u'Parsing results from <a href="%s">NZBsRus</a>' % searchURL)
try:
data = opener.open(request).read()
except Exception, e:
logger.warn('Error fetching data from NZBsRus: %s', e)
data = False
# Parse response
if data: if data:
d = json.loads(data) if data.get('matches', 0) == 0:
logger.info(u"No results found from NZBsRus for %s", term)
if d['matches'] <= 0:
logger.info(u"No results found from NZBsRus for %s" % term)
pass pass
else: else:
for item in d['results']: for item in data['results']:
try: try:
url = "http://www.nzbsrus.com/nzbdownload_rss.php/" + item['id'] + "/" + headphones.NZBSRUS_UID + "/" + item['key'] url = "http://www.nzbsrus.com/nzbdownload_rss.php/" + item['id'] + "/" + headphones.NZBSRUS_UID + "/" + item['key']
title = item['name'] title = item['name']
@@ -600,36 +607,28 @@ def searchNZB(album, new=False, losslessOnly=False):
categories = "29" categories = "29"
logger.info("Album type is audiobook/spokenword. Searching all music categories") logger.info("Album type is audiobook/spokenword. Searching all music categories")
params = { "user": headphones.OMGWTFNZBS_UID, # Request results
"api": headphones.OMGWTFNZBS_APIKEY, logger.info('Parsing results from omgwtfnzbs')
"catid": categories,
"retention": headphones.USENET_RETENTION,
"search": term
}
searchURL = 'http://api.omgwtfnzbs.org/json/?' + urllib.urlencode(params) headers = { 'User-Agent', USER_AGENT }
params = {
"user": headphones.OMGWTFNZBS_UID,
"api": headphones.OMGWTFNZBS_APIKEY,
"catid": categories,
"retention": headphones.USENET_RETENTION,
"search": term
}
# Add a user-agent data = request_json(
request = urllib2.Request(searchURL) url='http://api.omgwtfnzbs.org/json/',
request.add_header('User-Agent', USER_AGENT) params=params, headers=headers,
opener = urllib2.build_opener() validator=lambda x: type(x) == dict
)
logger.info(u'Parsing results from <a href="%s">omgwtfnzbs</a>' % searchURL)
try:
data = opener.open(request).read()
except Exception, e:
logger.warn('Error fetching data from omgwtfnzbs: %s', e)
data = False
# Parse response
if data: if data:
d = json.loads(data)
if 'notice' in data: if 'notice' in data:
logger.info(u"No results returned from omgwtfnzbs: %s", d['notice']) logger.info(u"No results returned from omgwtfnzbs: %s", d['notice'])
pass
else: else:
for item in d: for item in d:
try: try:
@@ -639,7 +638,6 @@ def searchNZB(album, new=False, losslessOnly=False):
resultlist.append((title, size, url, provider, 'nzb')) resultlist.append((title, size, url, provider, 'nzb'))
logger.info('Found %s. Size: %s', title, helpers.bytes_to_mb(size)) logger.info('Found %s. Size: %s', title, helpers.bytes_to_mb(size))
except Exception, e: except Exception, e:
logger.exception("Unhandled exception") logger.exception("Unhandled exception")
@@ -689,16 +687,17 @@ def send_to_downloader(data, bestqual, album):
folder_name = helpers.sab_replace_spaces(folder_name) folder_name = helpers.sab_replace_spaces(folder_name)
else: else:
nzb_name = folder_name + '.nzb' nzb_name = folder_name + '.nzb'
download_path = os.path.join(headphones.BLACKHOLE_DIR, nzb_name) download_path = os.path.join(headphones.BLACKHOLE_DIR, nzb_name)
try: try:
prev = os.umask(headphones.UMASK) prev = os.umask(headphones.UMASK)
f = open(download_path, 'w')
f.write(data) with open(download_path, 'w') as fp:
f.close() fp.write(data)
os.umask(prev) os.umask(prev)
logger.info('File saved to: %s' % nzb_name) logger.info('File saved to: %s', nzb_name)
except Exception, e: except Exception, e:
logger.error('Couldn\'t write NZB file: %s', e) logger.error('Couldn\'t write NZB file: %s', e)
return return
@@ -713,9 +712,9 @@ def send_to_downloader(data, bestqual, album):
return return
# Get torrent name from .torrent, this is usually used by the torrent client as the folder name # Get torrent name from .torrent, this is usually used by the torrent client as the folder name
torrent_name = folder_name + '.torrent' torrent_name = folder_name + '.torrent'
download_path = os.path.join(headphones.TORRENTBLACKHOLE_DIR, torrent_name) download_path = os.path.join(headphones.TORRENTBLACKHOLE_DIR, torrent_name)
try: try:
if bestqual[3] == 'rutracker.org': if bestqual[3] == 'rutracker.org':
download_path = rutracker.get_torrent(bestqual[2], headphones.TORRENTBLACKHOLE_DIR) download_path = rutracker.get_torrent(bestqual[2], headphones.TORRENTBLACKHOLE_DIR)
@@ -724,17 +723,16 @@ def send_to_downloader(data, bestqual, album):
else: else:
#Write the torrent file to a path derived from the TORRENTBLACKHOLE_DIR and file name. #Write the torrent file to a path derived from the TORRENTBLACKHOLE_DIR and file name.
prev = os.umask(headphones.UMASK) prev = os.umask(headphones.UMASK)
torrent_file = open(download_path, 'wb') with open(download_path, 'wb') as fp:
torrent_file.write(data) fp.write(data)
torrent_file.close()
os.umask(prev) os.umask(prev)
#Open the fresh torrent file again so we can extract the proper torrent name #Open the fresh torrent file again so we can extract the proper torrent name
#Used later in post-processing. #Used later in post-processing.
torrent_file = open(download_path, 'rb') with open(download_path, 'rb') as fp:
torrent_info = bencode.bdecode(torrent_file.read()) torrent_info = bencode.bdecode(fp.read())
torrent_file.close()
torrent_folder_name = torrent_info['info'].get('name','').decode('utf-8') torrent_folder_name = torrent_info['info'].get('name', '')
logger.info('Torrent folder name: %s' % torrent_folder_name) logger.info('Torrent folder name: %s' % torrent_folder_name)
except Exception, e: except Exception, e:
logger.error('Couldn\'t get name from Torrent file: %s' % e) logger.error('Couldn\'t get name from Torrent file: %s' % e)
@@ -767,7 +765,7 @@ def send_to_downloader(data, bestqual, album):
try: try:
shutil.rmtree(os.path.split(file_or_url)[0]) shutil.rmtree(os.path.split(file_or_url)[0])
except Exception, e: except Exception, e:
logger.warning('Couldn\'t remove temp dir %s' % e) logger.exception("Unhandled exception")
myDB = db.DBConnection() myDB = db.DBConnection()
myDB.action('UPDATE albums SET status = "Snatched" WHERE AlbumID=?', [album['AlbumID']]) myDB.action('UPDATE albums SET status = "Snatched" WHERE AlbumID=?', [album['AlbumID']])
@@ -839,44 +837,45 @@ def verifyresult(title, artistterm, term, lossless):
return True return True
def getresultNZB(result): def getresultNZB(result):
nzb = None nzb = None
if result[3] == 'newzbin': if result[3] == 'newzbin':
params = urllib.urlencode({"username": headphones.NEWZBIN_UID, "password": headphones.NEWZBIN_PASSWORD, "reportid": result[2]}) response = request_response(
url = "https://www.newzbin2.es" + "/api/dnzb/" url='https://www.newzbin2.es/api/dnzb/',
urllib._urlopener = NewzbinDownloader() auth=(headphones.HPUSER, headphones.HPPASS),
try: params={"username": headphones.NEWZBIN_UID, "password": headphones.NEWZBIN_PASSWORD, "reportid": result[2]},
nzb = urllib.urlopen(url, data=params).read() headers={'User-Agent': USER_AGENT},
except urllib2.URLError, e: status_pass=[400]
logger.warn('Error fetching nzb from url: %s. Error: %s' % (url, e)) )
except exceptions.NewzbinAPIThrottled:
#TODO: This has created a potentially infinite loop? As long as they keep throttling we keep trying. if response.status_code == 400:
logger.info("Done waiting for Newzbin API throttle limit, starting downloads again") error_code = int(response.headers.header['X-DNZB-RCode'])
getresultNZB(result)
except AttributeError: if error_code == 450:
logger.warn("AttributeError in getresultNZB.") result = re.search("wait (\d+) seconds", response.headers['X-DNZB-RText'])
seconds = int(result.group(1))
logger.info("Newzbin throttled our NZB downloading, pausing for %d seconds", seconds)
time.sleep(seconds)
# Try again -- possibly forever :(
getresultNZB(result)
else:
logger.info("Newzbin error code %d", error_code)
else:
nzb = response.content
elif result[3] == 'headphones': elif result[3] == 'headphones':
request = urllib2.Request(result[2]) nzb = request_content(
request.add_header('User-Agent', USER_AGENT) url=result[2],
base64string = base64.encodestring('%s:%s' % (headphones.HPUSER, headphones.HPPASS)).replace('\n', '') auth=(headphones.HPUSER, headphones.HPPASS),
request.add_header("Authorization", "Basic %s" % base64string) headers={'User-Agent': USER_AGENT}
)
opener = urllib2.build_opener()
try:
nzb = opener.open(request).read()
except Exception, e:
logger.warn('Error fetching NZB from url: %s, %s', result[2], e)
else: else:
request = urllib2.Request(result[2]) nzb = request_content(
request.add_header('User-Agent', USER_AGENT) url=result[2],
opener = urllib2.build_opener() headers={'User-Agent': USER_AGENT}
)
try:
nzb = opener.open(request).read()
except Exception, e:
logger.warn('Error fetching NZB from url: %s, %s', result[2], e)
return nzb return nzb
def searchTorrent(album, new=False, losslessOnly=False): def searchTorrent(album, new=False, losslessOnly=False):
@@ -953,25 +952,24 @@ def searchTorrent(album, new=False, losslessOnly=False):
format = "8" #mp3 format = "8" #mp3
maxsize = 300000000 maxsize = 300000000
# Requesting content
logger.info('Parsing results from KAT')
params = { params = {
"categories[0]": "music", "categories[0]": "music",
"field": "seeders", "field": "seeders",
"sorder": "desc", "sorder": "desc",
"rss": "1" "rss": "1"
} }
searchURL = providerurl + "/?%s" % urllib.urlencode(params)
try: data = request_feed(
data = urllib2.urlopen(searchURL, timeout=20) url=providerurl,
except urllib2.URLError, e: params=params,
logger.warn('Error fetching data from %s: %s', provider, e) timeout=20
data = False )
# Process feed
if data: if data:
logger.info(u'Parsing results from <a href="%s">KAT</a>', searchURL)
d = feedparser.parse(data)
if not len(d.entries): if not len(d.entries):
logger.info(u"No results found from %s for %s" % provider, term) logger.info(u"No results found from %s for %s" % provider, term)
else: else:
@@ -982,22 +980,10 @@ def searchTorrent(album, new=False, losslessOnly=False):
seeders = item['torrent_seeds'] seeders = item['torrent_seeds']
url = item['links'][1]['href'] url = item['links'][1]['href']
size = int(item['links'][1]['length']) size = int(item['links'][1]['length'])
try: if format == "2":
if format == "2": torrent = request_content(url)
request = urllib2.Request(url) if not torrent or (int(torrent.find(".mp3")) > 0 and int(torrent.find(".flac")) < 1):
request.add_header('Accept-encoding', 'gzip') rightformat = False
request.add_header('Referer', 'http://kat.ph/')
response = urllib2.urlopen(request)
if response.info().get('Content-Encoding') == 'gzip':
buf = StringIO( response.read())
f = gzip.GzipFile(fileobj=buf)
torrent = f.read()
else:
torrent = response.read()
if int(torrent.find(".mp3")) > 0 and int(torrent.find(".flac")) < 1:
rightformat = False
except Exception, e:
rightformat = False
if rightformat == True and size < maxsize and minimumseeders < int(seeders): if rightformat == True and size < maxsize and minimumseeders < int(seeders):
resultlist.append((title, size, url, provider, 'torrent')) resultlist.append((title, size, url, provider, 'torrent'))
logger.info('Found %s. Size: %s' % (title, helpers.bytes_to_mb(size))) logger.info('Found %s. Size: %s' % (title, helpers.bytes_to_mb(size)))
@@ -1036,33 +1022,29 @@ def searchTorrent(album, new=False, losslessOnly=False):
if bitrate: if bitrate:
query_items.append('bitrate:"%s"' % bitrate) query_items.append('bitrate:"%s"' % bitrate)
# Requesting content
logger.info('Parsing results from Waffles')
params = { params = {
"uid": headphones.WAFFLES_UID, "uid": headphones.WAFFLES_UID,
"passkey": headphones.WAFFLES_PASSKEY, "passkey": headphones.WAFFLES_PASSKEY,
"rss": "1", "rss": "1",
"c0": "1", "c0": "1",
"s": "seeders", # sort by "s": "seeders", # sort by
"d": "desc" # direction "d": "desc", # direction
"q": " ".join(query_items)
} }
searchURL = "%s?%s&q=%s" % (providerurl, urllib.urlencode(params), urllib.quote(" ".join(query_items))) data = request_feed(
url=providerurl,
try: params=params, headers=headers,
data = urllib2.urlopen(searchURL, timeout=20).read() timeout=20
except urllib2.URLError, e: )
logger.warn('Error fetching data from %s: %s' % (provider, e))
data = False
# Process feed
if data: if data:
logger.info(u'Parsing results from <a href="%s">Waffles.fm</a>' % searchURL)
d = feedparser.parse(data)
if not len(d.entries): if not len(d.entries):
logger.info(u"No results found from %s for %s" % (provider, term)) logger.info(u"No results found from %s for %s" % (provider, term))
pass
else: else:
for item in d.entries: for item in d.entries:
@@ -1233,23 +1215,24 @@ def searchTorrent(album, new=False, losslessOnly=False):
category = '101' #mp3 category = '101' #mp3
maxsize = 300000000 maxsize = 300000000
searchURL = providerurl + category # Requesting content
logger.info('Parsing results from The Pirate Bay')
try: params = {
data = urllib2.urlopen(searchURL, timeout=20).read() "iht": "2",
except urllib2.URLError, e: "sort": "seeds"
logger.warn('Error fetching data from The Pirate Bay: %s' % e) }
data = False
data = request_soup(
url=providerurl + category,
params=params,
timeout=20
)
# Process feed
if data: if data:
table = data.find('table')
logger.info(u'Parsing results from <a href="%s">The Pirate Bay</a>' % searchURL) rows = table.findAll('tr')
soup = BeautifulSoup(data)
table = soup.find('table')
rows = None
if table:
rows = table.findAll('tr')
if not rows or len(rows) == '1': if not rows or len(rows) == '1':
logger.info(u"No results found from %s for %s" % (provider, term)) logger.info(u"No results found from %s for %s" % (provider, term))
@@ -1293,26 +1276,28 @@ def searchTorrent(album, new=False, losslessOnly=False):
format = "8" #mp3 format = "8" #mp3
maxsize = 300000000 maxsize = 300000000
# Requesting content
logger.info('Parsing results from ISOHunt')
headers = { 'User-Agent', USER_AGENT }
params = { params = {
"iht": "2", "iht": "2",
"sort": "seeds" "sort": "seeds"
} }
searchURL = providerurl + "?%s" % urllib.urlencode(params)
try: data = request_feed(
data = urllib2.urlopen(searchURL, timeout=20).read() url=providerurl,
except urllib2.URLError, e: params=params, headers=headers,
logger.warn('Error fetching data from %s: %s', provider, e) auth=(headphones.HPUSER, headphones.HPPASS),
data = False timeout=20
)
# Process feed
if data: if data:
logger.info(u'Parsing results from <a href="%s">isoHunt</a>' % searchURL) if not len(data.entries):
d = feedparser.parse(data)
if not len(d.entries):
logger.info(u"No results found from %s for %s", provider, term) logger.info(u"No results found from %s for %s", provider, term)
else: else:
for item in d.entries: for item in data.entries:
try: try:
rightformat = True rightformat = True
title = re.sub(r"(?<= \[)(.+)(?=\])","",item.title) title = re.sub(r"(?<= \[)(.+)(?=\])","",item.title)
@@ -1324,21 +1309,11 @@ def searchTorrent(album, new=False, losslessOnly=False):
sxstart = sxstart + 1 sxstart = sxstart + 1
url = item.links[1]['url'] url = item.links[1]['url']
size = int(item.links[1]['length']) size = int(item.links[1]['length'])
try: if format == "2":
if format == "2": torrent = request_content(url)
request = urllib2.Request(url)
request.add_header('Accept-encoding', 'gzip') if not torrent or (int(torrent.find(".mp3")) > 0 and int(torrent.find(".flac")) < 1):
response = urllib2.urlopen(request) rightformat = False
if response.info().get('Content-Encoding') == 'gzip':
buf = StringIO( response.read())
f = gzip.GzipFile(fileobj=buf)
torrent = f.read()
else:
torrent = response.read()
if int(torrent.find(".mp3")) > 0 and int(torrent.find(".flac")) < 1:
rightformat = False
except Exception, e:
rightformat = False
for findterm in term.split(" "): for findterm in term.split(" "):
if not findterm in title: if not findterm in title:
rightformat = False rightformat = False
@@ -1347,12 +1322,13 @@ def searchTorrent(album, new=False, losslessOnly=False):
logger.info('Found %s. Size: %s' % (title, helpers.bytes_to_mb(size))) logger.info('Found %s. Size: %s' % (title, helpers.bytes_to_mb(size)))
else: else:
logger.info('%s is larger than the maxsize, the wrong format or has too little seeders for this category, skipping. (Size: %i bytes, Seeders: %i, Format: %s)' % (title, size, int(seeds), rightformat)) logger.info('%s is larger than the maxsize, the wrong format or has too little seeders for this category, skipping. (Size: %i bytes, Seeders: %i, Format: %s)' % (title, size, int(seeds), rightformat))
except Exception, e: except Exception:
logger.exception(u"Unhandled exception in isoHunt parser: %s") logger.exception("Unhandled exception in isoHunt parser")
if headphones.MININOVA: if headphones.MININOVA:
provider = "Mininova" provider = "Mininova"
providerurl = url_fix("http://www.mininova.org/rss/" + term + "/5") providerurl = url_fix("http://www.mininova.org/rss/" + term + "/5")
if headphones.PREFERRED_QUALITY == 3 or losslessOnly: if headphones.PREFERRED_QUALITY == 3 or losslessOnly:
categories = "7" #music categories = "7" #music
format = "2" #flac format = "2" #flac
@@ -1366,18 +1342,16 @@ def searchTorrent(album, new=False, losslessOnly=False):
format = "8" #mp3 format = "8" #mp3
maxsize = 300000000 maxsize = 300000000
searchURL = providerurl # Requesting content
logger.info('Parsing results from Mininova')
try: data = request_feed(
data = urllib2.urlopen(searchURL, timeout=20).read() url=providerurl,
except urllib2.URLError, e: timeout=20
logger.warn('Error fetching data from %s: %s', provider, e) )
data = False
# Process feed
if data: if data:
logger.info(u'Parsing results from <a href="%s">Mininova</a>' % searchURL)
d = feedparser.parse(data)
if not len(d.entries): if not len(d.entries):
logger.info(u"No results found from %s for %s" % (provider, term)) logger.info(u"No results found from %s for %s" % (provider, term))
else: else:
@@ -1392,21 +1366,11 @@ def searchTorrent(album, new=False, losslessOnly=False):
sxstart = sxstart + 1 sxstart = sxstart + 1
url = item.links[1]['url'] url = item.links[1]['url']
size = int(item.links[1]['length']) size = int(item.links[1]['length'])
try: if format == "2":
if format == "2": torrent = request_content(url)
request = urllib2.Request(url)
request.add_header('Accept-encoding', 'gzip') if not torrent or (int(torrent.find(".mp3")) > 0 and int(torrent.find(".flac")) < 1):
response = urllib2.urlopen(request) rightformat = False
if response.info().get('Content-Encoding') == 'gzip':
buf = StringIO( response.read())
f = gzip.GzipFile(fileobj=buf)
torrent = f.read()
else:
torrent = response.read()
if int(torrent.find(".mp3")) > 0 and int(torrent.find(".flac")) < 1:
rightformat = False
except Exception, e:
rightformat = False
if rightformat == True and size < maxsize and minimumseeders < seeds: if rightformat == True and size < maxsize and minimumseeders < seeds:
resultlist.append((title, size, url, provider, 'torrent')) resultlist.append((title, size, url, provider, 'torrent'))
logger.info('Found %s. Size: %s' % (title, helpers.bytes_to_mb(size))) logger.info('Found %s. Size: %s' % (title, helpers.bytes_to_mb(size)))
@@ -1414,7 +1378,7 @@ def searchTorrent(album, new=False, losslessOnly=False):
logger.info('%s is larger than the maxsize, the wrong format or has too little seeders for this category, skipping. (Size: %i bytes, Seeders: %i, Format: %s)' % (title, size, int(seeds), rightformat)) logger.info('%s is larger than the maxsize, the wrong format or has too little seeders for this category, skipping. (Size: %i bytes, Seeders: %i, Format: %s)' % (title, size, int(seeds), rightformat))
except Exception, e: except Exception, e:
logger.error(u"An unknown error occurred in the Mininova Parser: %s" % e) logger.exception("Unhandled exception in Mininova Parser")
#attempt to verify that this isn't a substring result #attempt to verify that this isn't a substring result
#when looking for "Foo - Foo" we don't want "Foobar" #when looking for "Foo - Foo" we don't want "Foobar"
@@ -1438,33 +1402,16 @@ def preprocess(resultlist):
if result[3] == 'rutracker.org': if result[3] == 'rutracker.org':
return True, result return True, result
try: # Download the torrent file
request = urllib2.Request(result[2]) if result[3] == 'Kick Ass Torrent':
request.add_header('Accept-encoding', 'gzip') headers = { 'Referer': 'http://kat.ph/' }
if result[3] == 'Kick Ass Torrent': elif result[3] == 'What.cd':
request.add_header('Referer', 'http://kat.ph/') headers = { 'User-Agent': 'Headphones' }
if result[3] == 'What.cd': return request_content(url=result[2], headers=headers), result
request.add_header('User-Agent', 'Headphones')
response = urllib2.urlopen(request)
if response.info().get('Content-Encoding') == 'gzip':
buf = StringIO(response.read())
f = gzip.GzipFile(fileobj=buf)
torrent = f.read()
else:
torrent = response.read()
except ExpatError:
logger.error('Unable to torrent %s' % result[2])
continue
return torrent, result
else: else:
if not headphones.USENET_RETENTION: usenet_retention = headphones.USENET_RETENTION or 2000
usenet_retention = 2000
else:
usenet_retention = int(headphones.USENET_RETENTION)
nzb = getresultNZB(result) nzb = getresultNZB(result)
if nzb: if nzb: