From 654f923a8dcc811757070421016f33edf5a1488d Mon Sep 17 00:00:00 2001 From: rembo10 Date: Tue, 15 Feb 2022 14:14:06 +0530 Subject: [PATCH] A little cleanup in searcher.py --- headphones/searcher.py | 460 ++++++++++++++++++++++++----------------- 1 file changed, 266 insertions(+), 194 deletions(-) diff --git a/headphones/searcher.py b/headphones/searcher.py index 1753944f..b9a16a47 100644 --- a/headphones/searcher.py +++ b/headphones/searcher.py @@ -15,8 +15,8 @@ # NZBGet support added by CurlyMo as a part of XBian - XBMC on the Raspberry Pi -from base64 import b16encode, b32decode -from hashlib import sha1 +import os +import re import string import random import urllib.request, urllib.parse, urllib.error @@ -24,19 +24,23 @@ import datetime import subprocess import unicodedata import urllib.parse +from base64 import b16encode, b32decode +from dataclasses import dataclass +from hashlib import sha1 -import os -import re +from bencode import encode as bencode +from bencode import decode as bdecode from pygazelle import api as gazelleapi from pygazelle import encoding as gazelleencoding from pygazelle import format as gazelleformat from pygazelle import release_type as gazellerelease_type +from unidecode import unidecode + import headphones from headphones.common import USER_AGENT from headphones import logger, db, helpers, classes, sab, nzbget, request from headphones import utorrent, transmission, notifiers, rutracker, deluge, qbittorrent -from bencode import encode as bencode -from bencode import decode as bdecode + # Magnet to torrent services, for Black hole. Stolen from CouchPotato. TORRENT_TO_MAGNET_SERVICES = [ @@ -52,6 +56,16 @@ ruobj = None redobj = None +@dataclass(frozen=True) +class Result: + title: str + size: int + url: str + provider: str + url_type: str + matches: bool + + def fix_url(s, charset="utf-8"): """ Fix the URL so it is proper formatted and encoded. @@ -318,19 +332,25 @@ def do_sorted_search(album, new, losslessOnly, choose_specific_download=False): return results # Filter all results that do not comply - results = [result for result in results if result[5]] + results = [result for result in results if result.matches] # Sort the remaining results + print(f"Sorting {len(results)} search results") sorted_search_results = sort_search_results(results, album, new, albumlength) + print(f"Sorted") if not sorted_search_results: + print(len(sorted_search_results)) return - logger.info("Making sure we can download the best result") - (data, bestqual) = preprocess(sorted_search_results) + logger.info( + "Making sure we can download the best result: " + f"{sorted_search_results[0].title} from {sorted_search_results[0].provider}" + ) + (data, result) = preprocess(sorted_search_results) - if data and bestqual: - send_to_downloader(data, bestqual, album) + if data and result: + send_to_downloader(data, result, album) def more_filtering(results, album, albumlength, new): @@ -366,36 +386,46 @@ def more_filtering(results, album, albumlength, new): for result in results: - if low_size_limit and (int(result[1]) < low_size_limit): + if low_size_limit and result.size < low_size_limit: logger.info( - "%s from %s is too small for this album - not considering it. (Size: %s, Minsize: %s)", - result[0], result[3], helpers.bytes_to_mb(result[1]), - helpers.bytes_to_mb(low_size_limit)) + f"{result.title} from {result.provider} is too small for this album. " + f"(Size: {result.size}, MinSize: {helpers.bytes_to_mb(low_size_limit)})" + ) continue - if high_size_limit and (int(result[1]) > high_size_limit): + if high_size_limit and result.size > high_size_limit: logger.info( - "%s from %s is too large for this album - not considering it. (Size: %s, Maxsize: %s)", - result[0], result[3], helpers.bytes_to_mb(result[1]), - helpers.bytes_to_mb(high_size_limit)) - + f"{result.title} from {result.provider} is too large for this album. " + f"(Size: {result.size}, MaxSize: {helpers.bytes_to_mb(high_size_limit)})" + ) # Keep lossless results if there are no good lossy matches - if not (allow_lossless and 'flac' in result[0].lower()): + if not (allow_lossless and 'flac' in result.title.lower()): continue if new: - alreadydownloaded = myDB.select('SELECT * from snatched WHERE URL=?', [result[2]]) - + alreadydownloaded = myDB.select( + "SELECT * from snatched WHERE URL=?", [result.url] + ) if len(alreadydownloaded): logger.info( - '%s has already been downloaded from %s. Skipping.' % (result[0], result[3])) + f"{result.title} has already been downloaded from " + f"{result.provider}. Skipping." + ) continue newlist.append(result) - results = newlist + return newlist - return results + +def sort_by_priority_then_size(rs): + return list(map(lambda x: x[0], + sorted( + rs, + key=lambda x: (x[0].matches, x[1], x[0].size), + reverse=True + ) + )) def sort_search_results(resultlist, album, new, albumlength): @@ -405,83 +435,74 @@ def sort_search_results(resultlist, album, new, albumlength): return None # Add a priority if it has any of the preferred words - temp_list = [] - preferred_words = None - if headphones.CONFIG.PREFERRED_WORDS: - preferred_words = helpers.split_string(headphones.CONFIG.PREFERRED_WORDS) + results_with_priority = [] + preferred_words = helpers.split_string(headphones.CONFIG.PREFERRED_WORDS) for result in resultlist: priority = 0 - if preferred_words: - if any(word.lower() in result[0].lower() for word in preferred_words): - priority = 1 - # add a search provider priority (weighted based on position) - i = next((i for i, word in enumerate(preferred_words) if word in result[3].lower()), - None) - if i is not None: - priority += round((len(preferred_words) - i) / float(len(preferred_words)), 2) + for word in preferred_words: + if word.lower() in [result.title.lower(), result.provider.lower()]: + priority += len(preferred_words) - preferred_words.index(word) + results_with_priority.append((result, priority)) - temp_list.append((result[0], result[1], result[2], result[3], result[4], priority)) - - resultlist = temp_list - - # if headphones.CONFIG.PREFERRED_QUALITY == 2 and headphones.CONFIG.PREFERRED_BITRATE and result[3] != 'Orpheus.network': if headphones.CONFIG.PREFERRED_QUALITY == 2 and headphones.CONFIG.PREFERRED_BITRATE: try: targetsize = albumlength / 1000 * int(headphones.CONFIG.PREFERRED_BITRATE) * 128 - if not targetsize: - logger.info('No track information for %s - %s. Defaulting to highest quality' % ( - album['ArtistName'], album['AlbumTitle'])) - finallist = sorted(resultlist, key=lambda title: (title[5], int(title[1])), - reverse=True) + logger.info( + f"No track information for {album['ArtistName']} - " + f"{album['AlbumTitle']}. Defaulting to highest quality" + ) + return sort_by_priority_then_size(results_with_priority) else: - newlist = [] - flac_list = [] + lossy_results_with_delta = [] + lossless_results = [] - for result in resultlist: + for result, priority in results_with_priority: # Add lossless results to the "flac list" which we can use if there are no good lossy matches - if 'flac' in result[0].lower(): - flac_list.append( - (result[0], result[1], result[2], result[3], result[4], result[5])) - continue + if 'flac' in result.title.lower(): + lossless_results.append((result, priority)) + else: + delta = abs(targetsize - result.size) + lossy_results_with_delta.append((result, priority, delta)) - delta = abs(targetsize - int(result[1])) - newlist.append( - (result[0], result[1], result[2], result[3], result[4], result[5], delta)) + return list(map(lambda x: x[0], + sorted( + lossy_results_with_delta, + key=lambda x: (-x[0].matches, -x[1], x[2]) + ) + )) - finallist = sorted(newlist, key=lambda title: (-title[5], title[6])) - - if not len(finallist) and len( - flac_list) and headphones.CONFIG.PREFERRED_BITRATE_ALLOW_LOSSLESS: + if ( + not len(lossy_results_with_delta) + and len(lossless_results) + and headphones.CONFIG.PREFERRED_BITRATE_ALLOW_LOSSLESS + ): logger.info( - "Since there were no appropriate lossy matches (and at least one lossless match), going to use lossless instead") - finallist = sorted(flac_list, key=lambda title: (title[5], int(title[1])), - reverse=True) + "Since there were no appropriate lossy matches " + "(and at least one lossless match), going to use " + "lossless instead" + ) + return sort_by_priority_then_size(results_with_priority) + except Exception: logger.exception('Unhandled exception') - logger.info('No track information for %s - %s. Defaulting to highest quality', - album['ArtistName'], album['AlbumTitle']) - - finallist = sorted(resultlist, key=lambda title: (title[5], int(title[1])), - reverse=True) + logger.info( + f"No track information for {album['ArtistName']} - " + f"{album['AlbumTitle']}. Defaulting to highest quality" + ) + return sort_by_priority_then_size(results_with_priority) else: + return sort_by_priority_then_size(results_with_priority) - finallist = sorted(resultlist, key=lambda title: (title[5], int(title[1])), reverse=True) - - # keep number of seeders order for Orpheus.network - # if result[3] == 'Orpheus.network': - # finallist = resultlist - - if not len(finallist): - logger.info('No appropriate matches found for %s - %s', album['ArtistName'], - album['AlbumTitle']) - return None - - return finallist + logger.info( + f"No appropriate matches found for {album['ArtistName']} - " + f"{album['AlbumTitle']}" + ) + return None def get_year_from_release_date(release_date): @@ -498,11 +519,22 @@ def searchNZB(album, new=False, losslessOnly=False, albumlength=None, reldate = album['ReleaseDate'] year = get_year_from_release_date(reldate) - dic = {'...': '', ' & ': ' ', ' = ': ' ', '?': '', '$': 's', ' + ': ' ', '"': '', ',': '', - '*': '', '.': '', ':': ''} + replacements = { + '...': '', + ' & ': ' ', + ' = ': ' ', + '?': '', + '$': 's', + ' + ': ' ', + '"': '', + ',': '', + '*': '', + '.': '', + ':': '' + } - cleanalbum = helpers.latinToAscii(helpers.replace_all(album['AlbumTitle'], dic)).strip() - cleanartist = helpers.latinToAscii(helpers.replace_all(album['ArtistName'], dic)).strip() + cleanalbum = unidecode(helpers.replace_all(album['AlbumTitle'], replacements)).strip() + cleanartist = unidecode(helpers.replace_all(album['ArtistName'], replacements)).strip() # Use the provided search term if available, otherwise build a search term if album['SearchTerm']: @@ -578,7 +610,7 @@ def searchNZB(album, new=False, losslessOnly=False, albumlength=None, title = item.title size = int(item.links[1]['length']) - resultlist.append((title, size, url, provider, 'nzb', True)) + resultlist.append(Result(title, size, url, provider, 'nzb', True)) logger.info('Found %s. Size: %s' % (title, helpers.bytes_to_mb(size))) except Exception as e: logger.error("An unknown error occurred trying to parse the feed: %s" % e) @@ -651,7 +683,7 @@ def searchNZB(album, new=False, losslessOnly=False, albumlength=None, if all(word.lower() in title.lower() for word in term.split()): logger.info( 'Found %s. Size: %s' % (title, helpers.bytes_to_mb(size))) - resultlist.append((title, size, url, provider, 'nzb', True)) + resultlist.append(Result(title, size, url, provider, 'nzb', True)) else: logger.info('Skipping %s, not all search term words found' % title) @@ -699,7 +731,7 @@ def searchNZB(album, new=False, losslessOnly=False, albumlength=None, title = item.title size = int(item.links[1]['length']) - resultlist.append((title, size, url, provider, 'nzb', True)) + resultlist.append(Result(title, size, url, provider, 'nzb', True)) logger.info('Found %s. Size: %s' % (title, helpers.bytes_to_mb(size))) except Exception as e: logger.exception("Unhandled exception while parsing feed") @@ -746,7 +778,7 @@ def searchNZB(album, new=False, losslessOnly=False, albumlength=None, title = item['release'] size = int(item['sizebytes']) - resultlist.append((title, size, url, provider, 'nzb', True)) + resultlist.append(Result(title, size, url, provider, 'nzb', True)) logger.info('Found %s. Size: %s', title, helpers.bytes_to_mb(size)) except Exception as e: logger.exception("Unhandled exception") @@ -758,7 +790,7 @@ def searchNZB(album, new=False, losslessOnly=False, albumlength=None, # Also will filter flac & remix albums if not specifically looking for it # This code also checks the ignored words and required words results = [result for result in resultlist if - verifyresult(result[0], artistterm, term, losslessOnly)] + verifyresult(result.title, artistterm, term, losslessOnly)] # Additional filtering for size etc if results and not choose_specific_download: @@ -767,16 +799,18 @@ def searchNZB(album, new=False, losslessOnly=False, albumlength=None, return results -def send_to_downloader(data, bestqual, album): - logger.info('Found best result from %s: %s - %s', bestqual[3], bestqual[2], - bestqual[0], helpers.bytes_to_mb(bestqual[1])) +def send_to_downloader(data, result, album): + logger.info( + f"Found best result from {result.provider}: " + f"{result.title} - {helpers.bytes_to_mb(result.size)}" + ) # Get rid of any dodgy chars here so we can prevent sab from renaming our downloads - kind = bestqual[4] + kind = result.url_type seed_ratio = None torrentid = None if kind == 'nzb': - folder_name = helpers.sab_sanitize_foldername(bestqual[0]) + folder_name = helpers.sab_sanitize_foldername(result.title) if headphones.CONFIG.NZB_DOWNLOADER == 1: @@ -819,8 +853,8 @@ def send_to_downloader(data, bestqual, album): return else: folder_name = '%s - %s [%s]' % ( - helpers.latinToAscii(album['ArtistName']).replace('/', '_'), - helpers.latinToAscii(album['AlbumTitle']).replace('/', '_'), + unidecode(album['ArtistName']).replace('/', '_'), + unidecode(album['AlbumTitle']).replace('/', '_'), get_year_from_release_date(album['ReleaseDate'])) # Blackhole @@ -830,26 +864,26 @@ def send_to_downloader(data, bestqual, album): torrent_name = helpers.replace_illegal_chars(folder_name) + '.torrent' download_path = os.path.join(headphones.CONFIG.TORRENTBLACKHOLE_DIR, torrent_name) - if bestqual[2].lower().startswith("magnet:"): + if result.url.lower().startswith("magnet:"): if headphones.CONFIG.MAGNET_LINKS == 1: try: if headphones.SYS_PLATFORM == 'win32': - os.startfile(bestqual[2]) + os.startfile(result.url) elif headphones.SYS_PLATFORM == 'darwin': - subprocess.Popen(["open", bestqual[2]], stdout=subprocess.PIPE, + subprocess.Popen(["open", result.url], stdout=subprocess.PIPE, stderr=subprocess.PIPE) else: - subprocess.Popen(["xdg-open", bestqual[2]], stdout=subprocess.PIPE, + subprocess.Popen(["xdg-open", result.url], stdout=subprocess.PIPE, stderr=subprocess.PIPE) # Gonna just take a guess at this..... Is there a better way to find this out? - folder_name = bestqual[0] + folder_name = result.title except Exception as e: logger.error("Error opening magnet link: %s" % str(e)) return elif headphones.CONFIG.MAGNET_LINKS == 2: # Procedure adapted from CouchPotato - torrent_hash = calculate_torrent_hash(bestqual[2]) + torrent_hash = calculate_torrent_hash(result.url) # Randomize list of services services = TORRENT_TO_MAGNET_SERVICES[:] @@ -863,8 +897,9 @@ def send_to_downloader(data, bestqual, album): if not torrent_to_file(download_path, data): return # Extract folder name from torrent - folder_name = read_torrent_name(download_path, - bestqual[0]) + folder_name = read_torrent_name( + download_path, + result.title) # Break for loop break @@ -888,7 +923,7 @@ def send_to_downloader(data, bestqual, album): return # Extract folder name from torrent - folder_name = read_torrent_name(download_path, bestqual[0]) + folder_name = read_torrent_name(download_path, result.title) if folder_name: logger.info('Torrent folder name: %s' % folder_name) @@ -896,10 +931,10 @@ def send_to_downloader(data, bestqual, album): logger.info("Sending torrent to Transmission") # Add torrent - if bestqual[3] == 'rutracker.org': + if result.provider == 'rutracker.org': torrentid = transmission.addTorrent('', data) else: - torrentid = transmission.addTorrent(bestqual[2]) + torrentid = transmission.addTorrent(result.url) if not torrentid: logger.error("Error sending torrent to Transmission. Are you sure it's running?") @@ -913,7 +948,7 @@ def send_to_downloader(data, bestqual, album): return # Set Seed Ratio - seed_ratio = get_seed_ratio(bestqual[3]) + seed_ratio = get_seed_ratio(result.provider) if seed_ratio is not None: transmission.setSeedRatio(torrentid, seed_ratio) @@ -922,10 +957,10 @@ def send_to_downloader(data, bestqual, album): try: # Add torrent - if bestqual[3] == 'rutracker.org': + if result.provider == 'rutracker.org': torrentid = deluge.addTorrent('', data) else: - torrentid = deluge.addTorrent(bestqual[2]) + torrentid = deluge.addTorrent(result.url) if not torrentid: logger.error("Error sending torrent to Deluge. Are you sure it's running? Maybe the torrent already exists?") @@ -940,7 +975,7 @@ def send_to_downloader(data, bestqual, album): deluge.setTorrentLabel({'hash': torrentid}) # Set Seed Ratio - seed_ratio = get_seed_ratio(bestqual[3]) + seed_ratio = get_seed_ratio(result.provider) if seed_ratio is not None: deluge.setSeedRatio({'hash': torrentid, 'ratio': seed_ratio}) @@ -963,13 +998,13 @@ def send_to_downloader(data, bestqual, album): logger.info("Sending torrent to uTorrent") # Add torrent - if bestqual[3] == 'rutracker.org': + if result.provider == 'rutracker.org': ruobj.utorrent_add_file(data) else: - utorrent.addTorrent(bestqual[2]) + utorrent.addTorrent(result.url) # Get hash - torrentid = calculate_torrent_hash(bestqual[2], data) + torrentid = calculate_torrent_hash(result.url, data) if not torrentid: logger.error('Torrent id could not be determined') return @@ -987,23 +1022,23 @@ def send_to_downloader(data, bestqual, album): utorrent.labelTorrent(torrentid) # Set Seed Ratio - seed_ratio = get_seed_ratio(bestqual[3]) + seed_ratio = get_seed_ratio(result.provider) if seed_ratio is not None: utorrent.setSeedRatio(torrentid, seed_ratio) else: # if headphones.CONFIG.TORRENT_DOWNLOADER == 4: logger.info("Sending torrent to QBiTorrent") # Add torrent - if bestqual[3] == 'rutracker.org': + if result.provider == 'rutracker.org': if qbittorrent.apiVersion2: qbittorrent.addFile(data) else: ruobj.qbittorrent_add_file(data) else: - qbittorrent.addTorrent(bestqual[2]) + qbittorrent.addTorrent(result.url) # Get hash - torrentid = calculate_torrent_hash(bestqual[2], data) + torrentid = calculate_torrent_hash(result.url, data) torrentid = torrentid.lower() if not torrentid: logger.error('Torrent id could not be determined') @@ -1018,29 +1053,33 @@ def send_to_downloader(data, bestqual, album): return # Set Seed Ratio - seed_ratio = get_seed_ratio(bestqual[3]) + # Oh my god why is this repeated again for the 100th time + seed_ratio = get_seed_ratio(result.provider) if seed_ratio is not None: qbittorrent.setSeedRatio(torrentid, seed_ratio) myDB = db.DBConnection() myDB.action('UPDATE albums SET status = "Snatched" WHERE AlbumID=?', [album['AlbumID']]) - myDB.action('INSERT INTO snatched VALUES( ?, ?, ?, ?, DATETIME("NOW", "localtime"), ?, ?, ?, ?)', - [album['AlbumID'], bestqual[0], bestqual[1], bestqual[2], "Snatched", folder_name, - kind, torrentid]) - - # Store the torrent id so we can check later if it's finished seeding and can be removed - if seed_ratio is not None and seed_ratio != 0 and torrentid: - myDB.action( - 'INSERT INTO snatched VALUES( ?, ?, ?, ?, DATETIME("NOW", "localtime"), ?, ?, ?, ?)', - [album['AlbumID'], bestqual[0], bestqual[1], bestqual[2], "Seed_Snatched", folder_name, - kind, torrentid]) + myDB.action( + "INSERT INTO snatched VALUES (?, ?, ?, ?, DATETIME('NOW', 'localtime'), " + "?, ?, ?, ?)", [ + album['AlbumID'], + result.title, + result.size, + result.url, + "Seed_Snatched" if seed_ratio and torrentid else "Snatched", + folder_name, + kind, + torrentid + ] + ) # notify artist = album[1] albumname = album[2] rgid = album[6] title = artist + ' - ' + albumname - provider = bestqual[3] + provider = result.provider if provider.startswith(("http://", "https://")): provider = provider.split("//")[1] name = folder_name if folder_name else None @@ -1209,13 +1248,22 @@ def searchTorrent(album, new=False, losslessOnly=False, albumlength=None, year = get_year_from_release_date(reldate) # MERGE THIS WITH THE TERM CLEANUP FROM searchNZB - dic = {'...': '', ' & ': ' ', ' = ': ' ', '?': '', '$': 's', ' + ': ' ', '"': '', ',': ' ', - '*': ''} + replacements = { + '...': '', + ' & ': ' ', + ' = ': ' ', + '?': '', + '$': 's', + ' + ': ' ', + '"': '', + ',': ' ', + '*': '' + } - semi_cleanalbum = helpers.replace_all(album['AlbumTitle'], dic) - cleanalbum = helpers.latinToAscii(semi_cleanalbum) - semi_cleanartist = helpers.replace_all(album['ArtistName'], dic) - cleanartist = helpers.latinToAscii(semi_cleanartist) + semi_cleanalbum = helpers.replace_all(album['AlbumTitle'], replacements) + cleanalbum = unidecode(semi_cleanalbum) + semi_cleanartist = helpers.replace_all(album['ArtistName'], replacements) + cleanartist = unidecode(semi_cleanartist) # Use provided term if available, otherwise build our own (this code needs to be cleaned up since a lot # of these torrent providers are just using cleanartist/cleanalbum terms @@ -1350,7 +1398,7 @@ def searchTorrent(album, new=False, losslessOnly=False, albumlength=None, if all(word.lower() in title.lower() for word in term.split()): if size < maxsize and minimumseeders < seeders: logger.info('Found %s. Size: %s' % (title, helpers.bytes_to_mb(size))) - resultlist.append((title, size, url, provider, 'torrent', True)) + resultlist.append(Result(title, size, url, provider, 'torrent', True)) else: logger.info( '%s is larger than the maxsize or has too little seeders for this category, ' @@ -1424,7 +1472,7 @@ def searchTorrent(album, new=False, losslessOnly=False, albumlength=None, desc_match = re.search(r"Size: (\d+)<", item.description) size = int(desc_match.group(1)) url = item.link - resultlist.append((title, size, url, provider, 'torrent', True)) + resultlist.append(Result(title, size, url, provider, 'torrent', True)) logger.info('Found %s. Size: %s', title, helpers.bytes_to_mb(size)) except Exception as e: logger.error( @@ -1589,11 +1637,16 @@ def searchTorrent(album, new=False, losslessOnly=False, albumlength=None, for torrent in match_torrents: if not torrent.file_path: torrent.group.update_group_data() # will load the file_path for the individual torrents - resultlist.append((torrent.file_path, - torrent.size, - orpheusobj.generate_torrent_link(torrent.id), - provider, - 'torrent', True)) + resultlist.append( + Result( + torrent.file_path, + torrent.size, + orpheusobj.generate_torrent_link(torrent.id), + provider, + 'torrent', + True + ) + ) # Redacted - Using same logic as What.CD as it's also Gazelle, so should really make this into something reusable if headphones.CONFIG.REDACTED: @@ -1690,11 +1743,16 @@ def searchTorrent(album, new=False, losslessOnly=False, albumlength=None, if not torrent.file_path: torrent.group.update_group_data() # will load the file_path for the individual torrents use_token = headphones.CONFIG.REDACTED_USE_FLTOKEN and torrent.can_use_token - resultlist.append((torrent.file_path, - torrent.size, - redobj.generate_torrent_link(torrent.id, use_token), - provider, - 'torrent', True)) + resultlist.append( + Result( + torrent.file_path, + torrent.size, + redobj.generate_torrent_link(torrent.id, use_token), + provider, + 'torrent', + True + ) + ) # Pirate Bay if headphones.CONFIG.PIRATEBAY: @@ -1768,7 +1826,7 @@ def searchTorrent(album, new=False, losslessOnly=False, albumlength=None, logger.info('%s is larger than the maxsize or has too little seeders for this category, ' 'skipping. (Size: %i bytes, Seeders: %i)' % (title, size, int(seeds))) - resultlist.append((title, size, url, provider, "torrent", match)) + resultlist.append(Result(title, size, url, provider, "torrent", match)) except Exception as e: logger.error("An unknown error occurred in the Pirate Bay parser: %s" % e) @@ -1822,7 +1880,7 @@ def searchTorrent(album, new=False, losslessOnly=False, albumlength=None, logger.info('%s is larger than the maxsize or has too little seeders for this category, ' 'skipping. (Size: %i bytes, Seeders: %i)' % (title, size, int(seeds))) - resultlist.append((title, size, url, provider, "torrent", match)) + resultlist.append(Result(title, size, url, provider, "torrent", match)) except Exception as e: logger.error( "An unknown error occurred in the Old Pirate Bay parser: %s" % e) @@ -1830,10 +1888,9 @@ def searchTorrent(album, new=False, losslessOnly=False, albumlength=None, # attempt to verify that this isn't a substring result # when looking for "Foo - Foo" we don't want "Foobar" # this should be less of an issue when it isn't a self-titled album so we'll only check vs artist - results = [result for result in resultlist if verifyresult(result[0], artistterm, term, losslessOnly)] + results = [result for result in resultlist if verifyresult(result.title, artistterm, term, losslessOnly)] # Additional filtering for size etc - # if results and not choose_specific_download and result[3] != 'Orpheus.network': if results and not choose_specific_download: results = more_filtering(results, album, albumlength, new) @@ -1845,60 +1902,75 @@ def searchTorrent(album, new=False, losslessOnly=False, albumlength=None, def preprocess(resultlist): for result in resultlist: - if result[4] == 'torrent': - - headers = {} - - # rutracker always needs the torrent data - if result[3] == 'rutracker.org': - return ruobj.get_torrent_data(result[2]), result - - # Jackett sometimes redirects - jackett_content = None - if result[3].startswith('Jackett_') or 'torznab' in result[3].lower(): - r = request.request_response(url=result[2], headers=headers, allow_redirects=False) - if r: - jackett_content = r.content - link = r.headers.get('Location') - if link and link != result[2]: - if link.startswith('magnet:'): - result = (result[0], result[1], link, result[3], "magnet", result[5]) - return "d10:magnet-uri%d:%se" % (len(link), link), result - else: - result = (result[0], result[1], link, result[3], result[4], result[5]) - return True, result - + if result.url_type == 'torrent': # Get out of here if we're using Transmission or Deluge # if not a magnet link still need the .torrent to generate hash... uTorrent support labeling if headphones.CONFIG.TORRENT_DOWNLOADER in [1, 3]: return True, result # Get out of here if it's a magnet link - if result[2].lower().startswith("magnet:"): + if result.url.lower().startswith("magnet:"): return True, result + # rutracker always needs the torrent data + if result.provider == 'rutracker.org': + return ruobj.get_torrent_data(result.url), result + + # Jackett sometimes redirects + if result.provider.startswith('Jackett_') or 'torznab' in result.provider.lower(): + r = request.request_response(url=result.url, headers=headers, allow_redirects=False) + if r: + link = r.headers.get('Location') + if link and link != result.url: + if link.startswith('magnet:'): + result = Result( + result.url, + result.size, + link, + result.provider, + "magnet", + result.matches + ) + return "d10:magnet-uri%d:%se" % (len(link), link), result + else: + result = Result( + result.url, + result.size, + link, + result.provider, + result.url_type, + result.matches + ) + return True, result + else: + return True, result + + # Download the torrent file + if result.provider in ["The Pirate Bay", "Old Pirate Bay"]: + headers = { + 'User-Agent': + 'Mozilla/5.0 (Windows NT 6.3; Win64; x64) \ + AppleWebKit/537.36 (KHTML, like Gecko) \ + Chrome/41.0.2243.2 Safari/537.36' + } + else: + headers = {'User-Agent': USER_AGENT} - if result[3] == 'Orpheus.network': - headers['User-Agent'] = 'Headphones' - elif result[3] == 'Redacted': - headers['User-Agent'] = 'Headphones' - elif result[3] == "The Pirate Bay" or result[3] == "Old Pirate Bay": - headers['User-Agent'] = 'Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2243.2 Safari/537.36' - elif jackett_content: - return jackett_content, result + return request.request_content(url=result.url, headers=headers), result - return request.request_content(url=result[2], headers=headers), result - - if result[4] == 'magnet': - magnet_link = result[2] + if result.url_type == 'magnet': + magnet_link = result.url return "d10:magnet-uri%d:%se" % (len(magnet_link), magnet_link), result else: headers = {'User-Agent': USER_AGENT} - if result[3] == 'headphones': - return request.request_content(url=result[2], headers=headers, - auth=(headphones.CONFIG.HPUSER, headphones.CONFIG.HPPASS)), result + if result.provider == 'headphones': + return request.request_content( + url=result.url, + headers=headers, + auth=(headphones.CONFIG.HPUSER, headphones.CONFIG.HPPASS) + ), result else: - return request.request_content(url=result[2], headers=headers), result + return request.request_content(url=result.url, headers=headers), result