From 39c5242f30b367a67dd8d05c3fbd77e51f7ed57a Mon Sep 17 00:00:00 2001 From: Bas Stottelaar Date: Fri, 19 Dec 2014 23:02:13 +0100 Subject: [PATCH] Add support for Old Pirate Bay. Note that this only downloads magnet links and you have to provide the URL yourself, because OPB is meant to be 'open'. --- data/interfaces/default/config.html | 19 +++++++- headphones/config.py | 3 ++ headphones/helpers.py | 9 +++- headphones/searcher.py | 70 ++++++++++++++++++++++++++--- headphones/webserve.py | 11 +++-- 5 files changed, 100 insertions(+), 12 deletions(-) diff --git a/data/interfaces/default/config.html b/data/interfaces/default/config.html index c3f58e0a..2cd51488 100644 --- a/data/interfaces/default/config.html +++ b/data/interfaces/default/config.html @@ -533,7 +533,24 @@
- + +
+ + + +
+ Old Pirate Bay +
+ +
+
+
+ + +
+
+ +
diff --git a/headphones/config.py b/headphones/config.py index ebae5b62..4071edd7 100644 --- a/headphones/config.py +++ b/headphones/config.py @@ -150,6 +150,9 @@ _CONFIG_DEFINITIONS = { 'PIRATEBAY': (int, 'Piratebay', 0), 'PIRATEBAY_PROXY_URL': (str, 'Piratebay', ''), 'PIRATEBAY_RATIO': (str, 'Piratebay', ''), + 'OLDPIRATEBAY': (int, 'Old Piratebay', 0), + 'OLDPIRATEBAY_URL': (str, 'Old Piratebay', ''), + 'OLDPIRATEBAY_RATIO': (str, 'Old Piratebay', ''), 'PLEX_CLIENT_HOST': (str, 'Plex', ''), 'PLEX_ENABLED': (int, 'Plex', 0), 'PLEX_NOTIFY': (int, 'Plex', 0), diff --git a/headphones/helpers.py b/headphones/helpers.py index f94d015f..27b9b2e4 100644 --- a/headphones/helpers.py +++ b/headphones/helpers.py @@ -171,13 +171,20 @@ def mb_to_bytes(mb_str): def piratesize(size): split = size.split(" ") factor = float(split[0]) - unit = split[1] + unit = split[1].upper() + if unit == 'MiB': size = factor * 1048576 + elif unit == 'MB': + size = factor * 1000000 elif unit == 'GiB': size = factor * 1073741824 + elif unit == 'GB': + size = factor * 1000000000 elif unit == 'KiB': size = factor * 1024 + elif unit == 'KB': + size = factor * 1000 elif unit == "B": size = factor else: diff --git a/headphones/searcher.py b/headphones/searcher.py index 1cafccd6..531ca87d 100644 --- a/headphones/searcher.py +++ b/headphones/searcher.py @@ -28,6 +28,7 @@ import re import string import shutil import random +import urllib import headphones import subprocess import unicodedata @@ -160,6 +161,8 @@ def get_seed_ratio(provider): seed_ratio = headphones.CONFIG.WHATCD_RATIO elif provider == 'The Pirate Bay': seed_ratio = headphones.CONFIG.PIRATEBAY_RATIO + elif provider == 'Old Pirate Bay': + seed_ratio = headphones.CONFIG.OLDPIRATEBAY_RATIO elif provider == 'Waffles.fm': seed_ratio = headphones.CONFIG.WAFFLES_RATIO elif provider == 'Mininova': @@ -186,7 +189,6 @@ def searchforalbum(albumid=None, new=False, losslessOnly=False, results = myDB.select('SELECT * from albums WHERE Status="Wanted" OR Status="Wanted Lossless"') for album in results: - if not album['AlbumTitle'] or not album['ArtistName']: logger.warn('Skipping release %s. No title available', album['AlbumID']) continue @@ -217,7 +219,7 @@ def do_sorted_search(album, new, losslessOnly, choose_specific_download=False): NZB_PROVIDERS = (headphones.CONFIG.HEADPHONES_INDEXER or headphones.CONFIG.NEWZNAB or headphones.CONFIG.NZBSORG or headphones.CONFIG.OMGWTFNZBS) NZB_DOWNLOADERS = (headphones.CONFIG.SAB_HOST or headphones.CONFIG.BLACKHOLE_DIR or headphones.CONFIG.NZBGET_HOST) - TORRENT_PROVIDERS = (headphones.CONFIG.KAT or headphones.CONFIG.PIRATEBAY or headphones.CONFIG.MININOVA or headphones.CONFIG.WAFFLES or headphones.CONFIG.RUTRACKER or headphones.CONFIG.WHATCD) + TORRENT_PROVIDERS = (headphones.CONFIG.KAT or headphones.CONFIG.PIRATEBAY or headphones.CONFIG.OLDPIRATEBAY or headphones.CONFIG.MININOVA or headphones.CONFIG.WAFFLES or headphones.CONFIG.RUTRACKER or headphones.CONFIG.WHATCD) results = [] myDB = db.DBConnection() @@ -1347,7 +1349,7 @@ def searchTorrent(album, new=False, losslessOnly=False, albumlength=None): # Request content logger.info("Searching The Pirate Bay using term: %s", tpb_term) - headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2049.0 Safari/537.36'} + headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2243.2 Safari/537.36'} data = request.request_soup(url=providerurl + category, headers=headers) # Process content @@ -1383,13 +1385,69 @@ def searchTorrent(album, new=False, losslessOnly=False, albumlength=None): size = helpers.piratesize(formatted_size) if size < maxsize and minimumseeders < seeds and url is not None: - resultlist.append((title, size, url, provider, "torrent")) + match = True logger.info('Found %s. Size: %s' % (title, formatted_size)) else: + match = False logger.info('%s is larger than the maxsize or has too little seeders for this category, skipping. (Size: %i bytes, Seeders: %i)' % (title, size, int(seeds))) + + resultlist.append((title, size, url, provider, "torrent", match)) except Exception as e: logger.error(u"An unknown error occurred in the Pirate Bay parser: %s" % e) + # Old Pirate Bay Compatible + if headphones.CONFIG.OLDPIRATEBAY: + provider = "Old Pirate Bay" + tpb_term = term.replace("!", "") + + # Pick category for torrents + if headphones.CONFIG.PREFERRED_QUALITY == 3 or losslessOnly: + maxsize = 10000000000 + elif headphones.CONFIG.PREFERRED_QUALITY == 1 or allow_lossless: + maxsize = 10000000000 + else: + maxsize = 300000000 + + # Requesting content + logger.info("Parsing results from Old Pirate Bay") + + headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2243.2 Safari/537.36'} + provider_url = fix_url(headphones.CONFIG.OLDPIRATEBAY_URL) + \ + "/search.php?" + urllib.urlencode({"q": tpb_term, "iht": 6}) + + data = request.request_soup(url=provider_url, headers=headers) + + # Process content + if data: + rows = data.select('table tbody tr') + + if not rows: + logger.info("No results found") + else: + for item in rows: + try: + links = item.select("td.title-row a") + + rightformat = True + title = links[1].text + seeds = int(item.select("td.seeders-row")[0].text) + url = links[0]["href"] # Magnet link. The actual download link is not based on the URL + + formatted_size = item.select("td.size-row")[0].text + size = helpers.piratesize(formatted_size) + + if size < maxsize and minimumseeders < seeds and url is not None: + match = True + logger.info('Found %s. Size: %s' % (title, formatted_size)) + else: + match = False + logger.info('%s is larger than the maxsize or has too little seeders for this category, skipping. (Size: %i bytes, Seeders: %i)' % (title, size, int(seeds))) + + resultlist.append((title, size, url, provider, "torrent", match)) + except Exception as e: + logger.error(u"An unknown error occurred in the Old Pirate Bay parser: %s" % e) + + # Mininova if headphones.CONFIG.MININOVA: provider = "Mininova" providerurl = fix_url("http://www.mininova.org/rss/" + term + "/5") @@ -1479,8 +1537,8 @@ def preprocess(resultlist): headers['Referer'] = 'http://kat.ph/' elif result[3] == 'What.cd': headers['User-Agent'] = 'Headphones' - elif result[3] == "The Pirate Bay": - headers['User-Agent'] = 'Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2049.0 Safari/537.36' + elif result[3] == "The Pirate Bay" or result[3] == "Old Pirate Bay": + headers['User-Agent'] = 'Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2243.2 Safari/537.36' return request.request_content(url=result[2], headers=headers), result else: diff --git a/headphones/webserve.py b/headphones/webserve.py index a6def563..50cfcff3 100644 --- a/headphones/webserve.py +++ b/headphones/webserve.py @@ -998,6 +998,9 @@ class WebInterface(object): "use_piratebay": checked(headphones.CONFIG.PIRATEBAY), "piratebay_proxy_url": headphones.CONFIG.PIRATEBAY_PROXY_URL, "piratebay_ratio": headphones.CONFIG.PIRATEBAY_RATIO, + "use_oldpiratebay": checked(headphones.CONFIG.OLDPIRATEBAY), + "oldpiratebay_url": headphones.CONFIG.OLDPIRATEBAY_URL, + "oldpiratebay_ratio": headphones.CONFIG.OLDPIRATEBAY_RATIO, "use_mininova": checked(headphones.CONFIG.MININOVA), "mininova_ratio": headphones.CONFIG.MININOVA_RATIO, "use_waffles": checked(headphones.CONFIG.WAFFLES), @@ -1172,10 +1175,10 @@ class WebInterface(object): checked_configs = [ "launch_browser", "enable_https", "api_enabled", "use_blackhole", "headphones_indexer", "use_newznab", "newznab_enabled", - "use_nzbsorg", "use_omgwtfnzbs", "use_kat", "use_piratebay", "use_mininova", "use_waffles", "use_rutracker", "use_whatcd", - "preferred_bitrate_allow_lossless", "detect_bitrate", "freeze_db", "cue_split", "move_files", "rename_files", "correct_metadata", - "cleanup_files", "keep_nfo", "add_album_art", "embed_album_art", "embed_lyrics", "replace_existing_folders", "file_underscores", - "include_extras", "autowant_upcoming", "autowant_all", "autowant_manually_added", "keep_torrent_files", "music_encoder", + "use_nzbsorg", "use_omgwtfnzbs", "use_kat", "use_piratebay", "use_oldpiratebay", "use_mininova", "use_waffles", "use_rutracker", + "use_whatcd", "preferred_bitrate_allow_lossless", "detect_bitrate", "freeze_db", "cue_split", "move_files", "rename_files", + "correct_metadata", "cleanup_files", "keep_nfo", "add_album_art", "embed_album_art", "embed_lyrics", "replace_existing_folders", + "file_underscores", "include_extras", "autowant_upcoming", "autowant_all", "autowant_manually_added", "keep_torrent_files", "music_encoder", "encoderlossless", "encoder_multicore", "delete_lossless_files", "growl_enabled", "growl_onsnatch", "prowl_enabled", "prowl_onsnatch", "xbmc_enabled", "xbmc_update", "xbmc_notify", "lms_enabled", "plex_enabled", "plex_update", "plex_notify", "nma_enabled", "nma_onsnatch", "pushalot_enabled", "pushalot_onsnatch", "synoindex_enabled", "pushover_enabled",