gronod/headphones
1
0
Fork 1
mirror of https://github.com/rembo10/headphones.git synced 2026-05-21 19:07:44 +01:00
Code Issues Packages Projects Releases Wiki Activity

A little cleanup in searcher.py

Browse Source
This commit is contained in:
rembo10
2022-02-15 14:14:06 +05:30
parent b91206c64a
commit 654f923a8d
1 changed files with 266 additions and 194 deletions
Download Patch File Download Diff File Expand all files Collapse all files
headphones/searcher.py
+266 -194
View File
@@ -15,8 +15,8 @@
# NZBGet support added by CurlyMo <curlymoo1@gmail.com> as a part of XBian - XBMC on the Raspberry Pi
from base64 import b16encode, b32decode
from hashlib import sha1
import os
import re
import string
import random
import urllib.request, urllib.parse, urllib.error
@@ -24,19 +24,23 @@ import datetime
import subprocess
import unicodedata
import urllib.parse
from base64 import b16encode, b32decode
from dataclasses import dataclass
from hashlib import sha1
import os
import re
from bencode import encode as bencode
from bencode import decode as bdecode
from pygazelle import api as gazelleapi
from pygazelle import encoding as gazelleencoding
from pygazelle import format as gazelleformat
from pygazelle import release_type as gazellerelease_type
from unidecode import unidecode
import headphones
from headphones.common import USER_AGENT
from headphones import logger, db, helpers, classes, sab, nzbget, request
from headphones import utorrent, transmission, notifiers, rutracker, deluge, qbittorrent
from bencode import encode as bencode
from bencode import decode as bdecode
# Magnet to torrent services, for Black hole. Stolen from CouchPotato.
TORRENT_TO_MAGNET_SERVICES = [
@@ -52,6 +56,16 @@ ruobj = None
redobj = None
@dataclass(frozen=True)
class Result:
title: str
size: int
url: str
provider: str
url_type: str
matches: bool
def fix_url(s, charset="utf-8"):
"""
Fix the URL so it is proper formatted and encoded.
@@ -318,19 +332,25 @@ def do_sorted_search(album, new, losslessOnly, choose_specific_download=False):
return results
# Filter all results that do not comply
results = [result for result in results if result[5]]
results = [result for result in results if result.matches]
# Sort the remaining results
print(f"Sorting {len(results)} search results")
sorted_search_results = sort_search_results(results, album, new, albumlength)
print(f"Sorted")
if not sorted_search_results:
print(len(sorted_search_results))
return
logger.info("Making sure we can download the best result")
(data, bestqual) = preprocess(sorted_search_results)
logger.info(
"Making sure we can download the best result: "
f"{sorted_search_results[0].title} from {sorted_search_results[0].provider}"
)
(data, result) = preprocess(sorted_search_results)
if data and bestqual:
send_to_downloader(data, bestqual, album)
if data and result:
send_to_downloader(data, result, album)
def more_filtering(results, album, albumlength, new):
@@ -366,36 +386,46 @@ def more_filtering(results, album, albumlength, new):
for result in results:
if low_size_limit and (int(result[1]) < low_size_limit):
if low_size_limit and result.size < low_size_limit:
logger.info(
"%s from %s is too small for this album - not considering it. (Size: %s, Minsize: %s)",
result[0], result[3], helpers.bytes_to_mb(result[1]),
helpers.bytes_to_mb(low_size_limit))
f"{result.title} from {result.provider} is too small for this album. "
f"(Size: {result.size}, MinSize: {helpers.bytes_to_mb(low_size_limit)})"
)
continue
if high_size_limit and (int(result[1]) > high_size_limit):
if high_size_limit and result.size > high_size_limit:
logger.info(
"%s from %s is too large for this album - not considering it. (Size: %s, Maxsize: %s)",
result[0], result[3], helpers.bytes_to_mb(result[1]),
helpers.bytes_to_mb(high_size_limit))
f"{result.title} from {result.provider} is too large for this album. "
f"(Size: {result.size}, MaxSize: {helpers.bytes_to_mb(high_size_limit)})"
)
# Keep lossless results if there are no good lossy matches
if not (allow_lossless and 'flac' in result[0].lower()):
if not (allow_lossless and 'flac' in result.title.lower()):
continue
if new:
alreadydownloaded = myDB.select('SELECT * from snatched WHERE URL=?', [result[2]])
alreadydownloaded = myDB.select(
"SELECT * from snatched WHERE URL=?", [result.url]
)
if len(alreadydownloaded):
logger.info(
'%s has already been downloaded from %s. Skipping.' % (result[0], result[3]))
f"{result.title} has already been downloaded from "
f"{result.provider}. Skipping."
)
continue
newlist.append(result)
results = newlist
return newlist
return results
def sort_by_priority_then_size(rs):
return list(map(lambda x: x[0],
sorted(
rs,
key=lambda x: (x[0].matches, x[1], x[0].size),
reverse=True
)
))
def sort_search_results(resultlist, album, new, albumlength):
@@ -405,83 +435,74 @@ def sort_search_results(resultlist, album, new, albumlength):
return None
# Add a priority if it has any of the preferred words
temp_list = []
preferred_words = None
if headphones.CONFIG.PREFERRED_WORDS:
preferred_words = helpers.split_string(headphones.CONFIG.PREFERRED_WORDS)
results_with_priority = []
preferred_words = helpers.split_string(headphones.CONFIG.PREFERRED_WORDS)
for result in resultlist:
priority = 0
if preferred_words:
if any(word.lower() in result[0].lower() for word in preferred_words):
priority = 1
# add a search provider priority (weighted based on position)
i = next((i for i, word in enumerate(preferred_words) if word in result[3].lower()),
None)
if i is not None:
priority += round((len(preferred_words) - i) / float(len(preferred_words)), 2)
for word in preferred_words:
if word.lower() in [result.title.lower(), result.provider.lower()]:
priority += len(preferred_words) - preferred_words.index(word)
results_with_priority.append((result, priority))
temp_list.append((result[0], result[1], result[2], result[3], result[4], priority))
resultlist = temp_list
# if headphones.CONFIG.PREFERRED_QUALITY == 2 and headphones.CONFIG.PREFERRED_BITRATE and result[3] != 'Orpheus.network':
if headphones.CONFIG.PREFERRED_QUALITY == 2 and headphones.CONFIG.PREFERRED_BITRATE:
try:
targetsize = albumlength / 1000 * int(headphones.CONFIG.PREFERRED_BITRATE) * 128
if not targetsize:
logger.info('No track information for %s - %s. Defaulting to highest quality' % (
album['ArtistName'], album['AlbumTitle']))
finallist = sorted(resultlist, key=lambda title: (title[5], int(title[1])),
reverse=True)
logger.info(
f"No track information for {album['ArtistName']} - "
f"{album['AlbumTitle']}. Defaulting to highest quality"
)
return sort_by_priority_then_size(results_with_priority)
else:
newlist = []
flac_list = []
lossy_results_with_delta = []
lossless_results = []
for result in resultlist:
for result, priority in results_with_priority:
# Add lossless results to the "flac list" which we can use if there are no good lossy matches
if 'flac' in result[0].lower():
flac_list.append(
(result[0], result[1], result[2], result[3], result[4], result[5]))
continue
if 'flac' in result.title.lower():
lossless_results.append((result, priority))
else:
delta = abs(targetsize - result.size)
lossy_results_with_delta.append((result, priority, delta))
delta = abs(targetsize - int(result[1]))
newlist.append(
(result[0], result[1], result[2], result[3], result[4], result[5], delta))
return list(map(lambda x: x[0],
sorted(
lossy_results_with_delta,
key=lambda x: (-x[0].matches, -x[1], x[2])
)
))
finallist = sorted(newlist, key=lambda title: (-title[5], title[6]))
if not len(finallist) and len(
flac_list) and headphones.CONFIG.PREFERRED_BITRATE_ALLOW_LOSSLESS:
if (
not len(lossy_results_with_delta)
and len(lossless_results)
and headphones.CONFIG.PREFERRED_BITRATE_ALLOW_LOSSLESS
):
logger.info(
"Since there were no appropriate lossy matches (and at least one lossless match), going to use lossless instead")
finallist = sorted(flac_list, key=lambda title: (title[5], int(title[1])),
reverse=True)
"Since there were no appropriate lossy matches "
"(and at least one lossless match), going to use "
"lossless instead"
)
return sort_by_priority_then_size(results_with_priority)
except Exception:
logger.exception('Unhandled exception')
logger.info('No track information for %s - %s. Defaulting to highest quality',
album['ArtistName'], album['AlbumTitle'])
finallist = sorted(resultlist, key=lambda title: (title[5], int(title[1])),
reverse=True)
logger.info(
f"No track information for {album['ArtistName']} - "
f"{album['AlbumTitle']}. Defaulting to highest quality"
)
return sort_by_priority_then_size(results_with_priority)
else:
return sort_by_priority_then_size(results_with_priority)
finallist = sorted(resultlist, key=lambda title: (title[5], int(title[1])), reverse=True)
# keep number of seeders order for Orpheus.network
# if result[3] == 'Orpheus.network':
# finallist = resultlist
if not len(finallist):
logger.info('No appropriate matches found for %s - %s', album['ArtistName'],
album['AlbumTitle'])
return None
return finallist
logger.info(
f"No appropriate matches found for {album['ArtistName']} - "
f"{album['AlbumTitle']}"
)
return None
def get_year_from_release_date(release_date):
@@ -498,11 +519,22 @@ def searchNZB(album, new=False, losslessOnly=False, albumlength=None,
reldate = album['ReleaseDate']
year = get_year_from_release_date(reldate)
dic = {'...': '', ' & ': ' ', ' = ': ' ', '?': '', '$': 's', ' + ': ' ', '"': '', ',': '',
'*': '', '.': '', ':': ''}
replacements = {
'...': '',
' & ': ' ',
' = ': ' ',
'?': '',
'$': 's',
' + ': ' ',
'"': '',
',': '',
'*': '',
'.': '',
':': ''
}
cleanalbum = helpers.latinToAscii(helpers.replace_all(album['AlbumTitle'], dic)).strip()
cleanartist = helpers.latinToAscii(helpers.replace_all(album['ArtistName'], dic)).strip()
cleanalbum = unidecode(helpers.replace_all(album['AlbumTitle'], replacements)).strip()
cleanartist = unidecode(helpers.replace_all(album['ArtistName'], replacements)).strip()
# Use the provided search term if available, otherwise build a search term
if album['SearchTerm']:
@@ -578,7 +610,7 @@ def searchNZB(album, new=False, losslessOnly=False, albumlength=None,
title = item.title
size = int(item.links[1]['length'])
resultlist.append((title, size, url, provider, 'nzb', True))
resultlist.append(Result(title, size, url, provider, 'nzb', True))
logger.info('Found %s. Size: %s' % (title, helpers.bytes_to_mb(size)))
except Exception as e:
logger.error("An unknown error occurred trying to parse the feed: %s" % e)
@@ -651,7 +683,7 @@ def searchNZB(album, new=False, losslessOnly=False, albumlength=None,
if all(word.lower() in title.lower() for word in term.split()):
logger.info(
'Found %s. Size: %s' % (title, helpers.bytes_to_mb(size)))
resultlist.append((title, size, url, provider, 'nzb', True))
resultlist.append(Result(title, size, url, provider, 'nzb', True))
else:
logger.info('Skipping %s, not all search term words found' % title)
@@ -699,7 +731,7 @@ def searchNZB(album, new=False, losslessOnly=False, albumlength=None,
title = item.title
size = int(item.links[1]['length'])
resultlist.append((title, size, url, provider, 'nzb', True))
resultlist.append(Result(title, size, url, provider, 'nzb', True))
logger.info('Found %s. Size: %s' % (title, helpers.bytes_to_mb(size)))
except Exception as e:
logger.exception("Unhandled exception while parsing feed")
@@ -746,7 +778,7 @@ def searchNZB(album, new=False, losslessOnly=False, albumlength=None,
title = item['release']
size = int(item['sizebytes'])
resultlist.append((title, size, url, provider, 'nzb', True))
resultlist.append(Result(title, size, url, provider, 'nzb', True))
logger.info('Found %s. Size: %s', title, helpers.bytes_to_mb(size))
except Exception as e:
logger.exception("Unhandled exception")
@@ -758,7 +790,7 @@ def searchNZB(album, new=False, losslessOnly=False, albumlength=None,
# Also will filter flac & remix albums if not specifically looking for it
# This code also checks the ignored words and required words
results = [result for result in resultlist if
verifyresult(result[0], artistterm, term, losslessOnly)]
verifyresult(result.title, artistterm, term, losslessOnly)]
# Additional filtering for size etc
if results and not choose_specific_download:
@@ -767,16 +799,18 @@ def searchNZB(album, new=False, losslessOnly=False, albumlength=None,
return results
def send_to_downloader(data, bestqual, album):
logger.info('Found best result from %s: <a href="%s">%s</a> - %s', bestqual[3], bestqual[2],
bestqual[0], helpers.bytes_to_mb(bestqual[1]))
def send_to_downloader(data, result, album):
logger.info(
f"Found best result from {result.provider}: <a href=\"{result.url}\">"
f"{result.title}</a> - {helpers.bytes_to_mb(result.size)}"
)
# Get rid of any dodgy chars here so we can prevent sab from renaming our downloads
kind = bestqual[4]
kind = result.url_type
seed_ratio = None
torrentid = None
if kind == 'nzb':
folder_name = helpers.sab_sanitize_foldername(bestqual[0])
folder_name = helpers.sab_sanitize_foldername(result.title)
if headphones.CONFIG.NZB_DOWNLOADER == 1:
@@ -819,8 +853,8 @@ def send_to_downloader(data, bestqual, album):
return
else:
folder_name = '%s - %s [%s]' % (
helpers.latinToAscii(album['ArtistName']).replace('/', '_'),
helpers.latinToAscii(album['AlbumTitle']).replace('/', '_'),
unidecode(album['ArtistName']).replace('/', '_'),
unidecode(album['AlbumTitle']).replace('/', '_'),
get_year_from_release_date(album['ReleaseDate']))
# Blackhole
@@ -830,26 +864,26 @@ def send_to_downloader(data, bestqual, album):
torrent_name = helpers.replace_illegal_chars(folder_name) + '.torrent'
download_path = os.path.join(headphones.CONFIG.TORRENTBLACKHOLE_DIR, torrent_name)
if bestqual[2].lower().startswith("magnet:"):
if result.url.lower().startswith("magnet:"):
if headphones.CONFIG.MAGNET_LINKS == 1:
try:
if headphones.SYS_PLATFORM == 'win32':
os.startfile(bestqual[2])
os.startfile(result.url)
elif headphones.SYS_PLATFORM == 'darwin':
subprocess.Popen(["open", bestqual[2]], stdout=subprocess.PIPE,
subprocess.Popen(["open", result.url], stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
else:
subprocess.Popen(["xdg-open", bestqual[2]], stdout=subprocess.PIPE,
subprocess.Popen(["xdg-open", result.url], stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
# Gonna just take a guess at this..... Is there a better way to find this out?
folder_name = bestqual[0]
folder_name = result.title
except Exception as e:
logger.error("Error opening magnet link: %s" % str(e))
return
elif headphones.CONFIG.MAGNET_LINKS == 2:
# Procedure adapted from CouchPotato
torrent_hash = calculate_torrent_hash(bestqual[2])
torrent_hash = calculate_torrent_hash(result.url)
# Randomize list of services
services = TORRENT_TO_MAGNET_SERVICES[:]
@@ -863,8 +897,9 @@ def send_to_downloader(data, bestqual, album):
if not torrent_to_file(download_path, data):
return
# Extract folder name from torrent
folder_name = read_torrent_name(download_path,
bestqual[0])
folder_name = read_torrent_name(
download_path,
result.title)
# Break for loop
break
@@ -888,7 +923,7 @@ def send_to_downloader(data, bestqual, album):
return
# Extract folder name from torrent
folder_name = read_torrent_name(download_path, bestqual[0])
folder_name = read_torrent_name(download_path, result.title)
if folder_name:
logger.info('Torrent folder name: %s' % folder_name)
@@ -896,10 +931,10 @@ def send_to_downloader(data, bestqual, album):
logger.info("Sending torrent to Transmission")
# Add torrent
if bestqual[3] == 'rutracker.org':
if result.provider == 'rutracker.org':
torrentid = transmission.addTorrent('', data)
else:
torrentid = transmission.addTorrent(bestqual[2])
torrentid = transmission.addTorrent(result.url)
if not torrentid:
logger.error("Error sending torrent to Transmission. Are you sure it's running?")
@@ -913,7 +948,7 @@ def send_to_downloader(data, bestqual, album):
return
# Set Seed Ratio
seed_ratio = get_seed_ratio(bestqual[3])
seed_ratio = get_seed_ratio(result.provider)
if seed_ratio is not None:
transmission.setSeedRatio(torrentid, seed_ratio)
@@ -922,10 +957,10 @@ def send_to_downloader(data, bestqual, album):
try:
# Add torrent
if bestqual[3] == 'rutracker.org':
if result.provider == 'rutracker.org':
torrentid = deluge.addTorrent('', data)
else:
torrentid = deluge.addTorrent(bestqual[2])
torrentid = deluge.addTorrent(result.url)
if not torrentid:
logger.error("Error sending torrent to Deluge. Are you sure it's running? Maybe the torrent already exists?")
@@ -940,7 +975,7 @@ def send_to_downloader(data, bestqual, album):
deluge.setTorrentLabel({'hash': torrentid})
# Set Seed Ratio
seed_ratio = get_seed_ratio(bestqual[3])
seed_ratio = get_seed_ratio(result.provider)
if seed_ratio is not None:
deluge.setSeedRatio({'hash': torrentid, 'ratio': seed_ratio})
@@ -963,13 +998,13 @@ def send_to_downloader(data, bestqual, album):
logger.info("Sending torrent to uTorrent")
# Add torrent
if bestqual[3] == 'rutracker.org':
if result.provider == 'rutracker.org':
ruobj.utorrent_add_file(data)
else:
utorrent.addTorrent(bestqual[2])
utorrent.addTorrent(result.url)
# Get hash
torrentid = calculate_torrent_hash(bestqual[2], data)
torrentid = calculate_torrent_hash(result.url, data)
if not torrentid:
logger.error('Torrent id could not be determined')
return
@@ -987,23 +1022,23 @@ def send_to_downloader(data, bestqual, album):
utorrent.labelTorrent(torrentid)
# Set Seed Ratio
seed_ratio = get_seed_ratio(bestqual[3])
seed_ratio = get_seed_ratio(result.provider)
if seed_ratio is not None:
utorrent.setSeedRatio(torrentid, seed_ratio)
else: # if headphones.CONFIG.TORRENT_DOWNLOADER == 4:
logger.info("Sending torrent to QBiTorrent")
# Add torrent
if bestqual[3] == 'rutracker.org':
if result.provider == 'rutracker.org':
if qbittorrent.apiVersion2:
qbittorrent.addFile(data)
else:
ruobj.qbittorrent_add_file(data)
else:
qbittorrent.addTorrent(bestqual[2])
qbittorrent.addTorrent(result.url)
# Get hash
torrentid = calculate_torrent_hash(bestqual[2], data)
torrentid = calculate_torrent_hash(result.url, data)
torrentid = torrentid.lower()
if not torrentid:
logger.error('Torrent id could not be determined')
@@ -1018,29 +1053,33 @@ def send_to_downloader(data, bestqual, album):
return
# Set Seed Ratio
seed_ratio = get_seed_ratio(bestqual[3])
# Oh my god why is this repeated again for the 100th time
seed_ratio = get_seed_ratio(result.provider)
if seed_ratio is not None:
qbittorrent.setSeedRatio(torrentid, seed_ratio)
myDB = db.DBConnection()
myDB.action('UPDATE albums SET status = "Snatched" WHERE AlbumID=?', [album['AlbumID']])
myDB.action('INSERT INTO snatched VALUES( ?, ?, ?, ?, DATETIME("NOW", "localtime"), ?, ?, ?, ?)',
[album['AlbumID'], bestqual[0], bestqual[1], bestqual[2], "Snatched", folder_name,
kind, torrentid])
# Store the torrent id so we can check later if it's finished seeding and can be removed
if seed_ratio is not None and seed_ratio != 0 and torrentid:
myDB.action(
'INSERT INTO snatched VALUES( ?, ?, ?, ?, DATETIME("NOW", "localtime"), ?, ?, ?, ?)',
[album['AlbumID'], bestqual[0], bestqual[1], bestqual[2], "Seed_Snatched", folder_name,
kind, torrentid])
myDB.action(
"INSERT INTO snatched VALUES (?, ?, ?, ?, DATETIME('NOW', 'localtime'), "
"?, ?, ?, ?)", [
album['AlbumID'],
result.title,
result.size,
result.url,
"Seed_Snatched" if seed_ratio and torrentid else "Snatched",
folder_name,
kind,
torrentid
]
)
# notify
artist = album[1]
albumname = album[2]
rgid = album[6]
title = artist + ' - ' + albumname
provider = bestqual[3]
provider = result.provider
if provider.startswith(("http://", "https://")):
provider = provider.split("//")[1]
name = folder_name if folder_name else None
@@ -1209,13 +1248,22 @@ def searchTorrent(album, new=False, losslessOnly=False, albumlength=None,
year = get_year_from_release_date(reldate)
# MERGE THIS WITH THE TERM CLEANUP FROM searchNZB
dic = {'...': '', ' & ': ' ', ' = ': ' ', '?': '', '$': 's', ' + ': ' ', '"': '', ',': ' ',
'*': ''}
replacements = {
'...': '',
' & ': ' ',
' = ': ' ',
'?': '',
'$': 's',
' + ': ' ',
'"': '',
',': ' ',
'*': ''
}
semi_cleanalbum = helpers.replace_all(album['AlbumTitle'], dic)
cleanalbum = helpers.latinToAscii(semi_cleanalbum)
semi_cleanartist = helpers.replace_all(album['ArtistName'], dic)
cleanartist = helpers.latinToAscii(semi_cleanartist)
semi_cleanalbum = helpers.replace_all(album['AlbumTitle'], replacements)
cleanalbum = unidecode(semi_cleanalbum)
semi_cleanartist = helpers.replace_all(album['ArtistName'], replacements)
cleanartist = unidecode(semi_cleanartist)
# Use provided term if available, otherwise build our own (this code needs to be cleaned up since a lot
# of these torrent providers are just using cleanartist/cleanalbum terms
@@ -1350,7 +1398,7 @@ def searchTorrent(album, new=False, losslessOnly=False, albumlength=None,
if all(word.lower() in title.lower() for word in term.split()):
if size < maxsize and minimumseeders < seeders:
logger.info('Found %s. Size: %s' % (title, helpers.bytes_to_mb(size)))
resultlist.append((title, size, url, provider, 'torrent', True))
resultlist.append(Result(title, size, url, provider, 'torrent', True))
else:
logger.info(
'%s is larger than the maxsize or has too little seeders for this category, '
@@ -1424,7 +1472,7 @@ def searchTorrent(album, new=False, losslessOnly=False, albumlength=None,
desc_match = re.search(r"Size: (\d+)<", item.description)
size = int(desc_match.group(1))
url = item.link
resultlist.append((title, size, url, provider, 'torrent', True))
resultlist.append(Result(title, size, url, provider, 'torrent', True))
logger.info('Found %s. Size: %s', title, helpers.bytes_to_mb(size))
except Exception as e:
logger.error(
@@ -1589,11 +1637,16 @@ def searchTorrent(album, new=False, losslessOnly=False, albumlength=None,
for torrent in match_torrents:
if not torrent.file_path:
torrent.group.update_group_data() # will load the file_path for the individual torrents
resultlist.append((torrent.file_path,
torrent.size,
orpheusobj.generate_torrent_link(torrent.id),
provider,
'torrent', True))
resultlist.append(
Result(
torrent.file_path,
torrent.size,
orpheusobj.generate_torrent_link(torrent.id),
provider,
'torrent',
True
)
)
# Redacted - Using same logic as What.CD as it's also Gazelle, so should really make this into something reusable
if headphones.CONFIG.REDACTED:
@@ -1690,11 +1743,16 @@ def searchTorrent(album, new=False, losslessOnly=False, albumlength=None,
if not torrent.file_path:
torrent.group.update_group_data() # will load the file_path for the individual torrents
use_token = headphones.CONFIG.REDACTED_USE_FLTOKEN and torrent.can_use_token
resultlist.append((torrent.file_path,
torrent.size,
redobj.generate_torrent_link(torrent.id, use_token),
provider,
'torrent', True))
resultlist.append(
Result(
torrent.file_path,
torrent.size,
redobj.generate_torrent_link(torrent.id, use_token),
provider,
'torrent',
True
)
)
# Pirate Bay
if headphones.CONFIG.PIRATEBAY:
@@ -1768,7 +1826,7 @@ def searchTorrent(album, new=False, losslessOnly=False, albumlength=None,
logger.info('%s is larger than the maxsize or has too little seeders for this category, '
'skipping. (Size: %i bytes, Seeders: %i)' % (title, size, int(seeds)))
resultlist.append((title, size, url, provider, "torrent", match))
resultlist.append(Result(title, size, url, provider, "torrent", match))
except Exception as e:
logger.error("An unknown error occurred in the Pirate Bay parser: %s" % e)
@@ -1822,7 +1880,7 @@ def searchTorrent(album, new=False, losslessOnly=False, albumlength=None,
logger.info('%s is larger than the maxsize or has too little seeders for this category, '
'skipping. (Size: %i bytes, Seeders: %i)' % (title, size, int(seeds)))
resultlist.append((title, size, url, provider, "torrent", match))
resultlist.append(Result(title, size, url, provider, "torrent", match))
except Exception as e:
logger.error(
"An unknown error occurred in the Old Pirate Bay parser: %s" % e)
@@ -1830,10 +1888,9 @@ def searchTorrent(album, new=False, losslessOnly=False, albumlength=None,
# attempt to verify that this isn't a substring result
# when looking for "Foo - Foo" we don't want "Foobar"
# this should be less of an issue when it isn't a self-titled album so we'll only check vs artist
results = [result for result in resultlist if verifyresult(result[0], artistterm, term, losslessOnly)]
results = [result for result in resultlist if verifyresult(result.title, artistterm, term, losslessOnly)]
# Additional filtering for size etc
# if results and not choose_specific_download and result[3] != 'Orpheus.network':
if results and not choose_specific_download:
results = more_filtering(results, album, albumlength, new)
@@ -1845,60 +1902,75 @@ def searchTorrent(album, new=False, losslessOnly=False, albumlength=None,
def preprocess(resultlist):
for result in resultlist:
if result[4] == 'torrent':
headers = {}
# rutracker always needs the torrent data
if result[3] == 'rutracker.org':
return ruobj.get_torrent_data(result[2]), result
# Jackett sometimes redirects
jackett_content = None
if result[3].startswith('Jackett_') or 'torznab' in result[3].lower():
r = request.request_response(url=result[2], headers=headers, allow_redirects=False)
if r:
jackett_content = r.content
link = r.headers.get('Location')
if link and link != result[2]:
if link.startswith('magnet:'):
result = (result[0], result[1], link, result[3], "magnet", result[5])
return "d10:magnet-uri%d:%se" % (len(link), link), result
else:
result = (result[0], result[1], link, result[3], result[4], result[5])
return True, result
if result.url_type == 'torrent':
# Get out of here if we're using Transmission or Deluge
# if not a magnet link still need the .torrent to generate hash... uTorrent support labeling
if headphones.CONFIG.TORRENT_DOWNLOADER in [1, 3]:
return True, result
# Get out of here if it's a magnet link
if result[2].lower().startswith("magnet:"):
if result.url.lower().startswith("magnet:"):
return True, result
# rutracker always needs the torrent data
if result.provider == 'rutracker.org':
return ruobj.get_torrent_data(result.url), result
# Jackett sometimes redirects
if result.provider.startswith('Jackett_') or 'torznab' in result.provider.lower():
r = request.request_response(url=result.url, headers=headers, allow_redirects=False)
if r:
link = r.headers.get('Location')
if link and link != result.url:
if link.startswith('magnet:'):
result = Result(
result.url,
result.size,
link,
result.provider,
"magnet",
result.matches
)
return "d10:magnet-uri%d:%se" % (len(link), link), result
else:
result = Result(
result.url,
result.size,
link,
result.provider,
result.url_type,
result.matches
)
return True, result
else:
return True, result
# Download the torrent file
if result.provider in ["The Pirate Bay", "Old Pirate Bay"]:
headers = {
'User-Agent':
'Mozilla/5.0 (Windows NT 6.3; Win64; x64) \
AppleWebKit/537.36 (KHTML, like Gecko) \
Chrome/41.0.2243.2 Safari/537.36'
}
else:
headers = {'User-Agent': USER_AGENT}
if result[3] == 'Orpheus.network':
headers['User-Agent'] = 'Headphones'
elif result[3] == 'Redacted':
headers['User-Agent'] = 'Headphones'
elif result[3] == "The Pirate Bay" or result[3] == "Old Pirate Bay":
headers['User-Agent'] = 'Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2243.2 Safari/537.36'
elif jackett_content:
return jackett_content, result
return request.request_content(url=result.url, headers=headers), result
return request.request_content(url=result[2], headers=headers), result
if result[4] == 'magnet':
magnet_link = result[2]
if result.url_type == 'magnet':
magnet_link = result.url
return "d10:magnet-uri%d:%se" % (len(magnet_link), magnet_link), result
else:
headers = {'User-Agent': USER_AGENT}
if result[3] == 'headphones':
return request.request_content(url=result[2], headers=headers,
auth=(headphones.CONFIG.HPUSER, headphones.CONFIG.HPPASS)), result
if result.provider == 'headphones':
return request.request_content(
url=result.url,
headers=headers,
auth=(headphones.CONFIG.HPUSER, headphones.CONFIG.HPPASS)
), result
else:
return request.request_content(url=result[2], headers=headers), result
return request.request_content(url=result.url, headers=headers), result