From 7fec54289b03d88bb9d994e2b84ca7ed44398bdf Mon Sep 17 00:00:00 2001 From: AdeHub Date: Fri, 21 Apr 2017 19:45:37 +1200 Subject: [PATCH] Revert "Add deezloader provider and aria2 downloader" --- data/interfaces/default/config.html | 61 +- headphones/aria2.py | 979 ---------------------------- headphones/config.py | 7 - headphones/deezloader.py | 428 ------------ headphones/postprocessor.py | 40 +- headphones/searcher.py | 167 +---- headphones/webserve.py | 13 +- 7 files changed, 21 insertions(+), 1674 deletions(-) delete mode 100644 headphones/aria2.py delete mode 100644 headphones/deezloader.py diff --git a/data/interfaces/default/config.html b/data/interfaces/default/config.html index 43aead6a..841df0d1 100644 --- a/data/interfaces/default/config.html +++ b/data/interfaces/default/config.html @@ -304,45 +304,6 @@ - -
- Direct Download - Aria2 -
-
-
- - - usually http://localhost:6800/jsonrpc -
-
- - -
-
- - -
-
- - -
-
- - - Full path where ddl client downloads your music, e.g. /Users/name/Downloads/music -
-
@@ -500,7 +461,6 @@ NZBs Torrents - Direct Download No Preference
@@ -613,17 +573,6 @@ - -
- Direct Download - -
-
- -
-
- -
@@ -855,6 +804,7 @@
+ @@ -2455,11 +2405,6 @@ $("#deluge_options").show(); } - if ($("#ddl_downloader_aria").is(":checked")) - { - $("#ddl_aria_options").show(); - } - $('input[type=radio]').change(function(){ if ($("#preferred_bitrate").is(":checked")) { @@ -2513,9 +2458,6 @@ { $("#torrent_blackhole_options,#utorrent_options,#transmission_options,#qbittorrent_options").fadeOut("fast", function() { $("#deluge_options").fadeIn() }); } - if ($("#ddl_downloader_aria").is(":checked")) - { - } }); $("#mirror").change(handleNewServerSelection); @@ -2618,7 +2560,6 @@ initConfigCheckbox("#enable_https"); initConfigCheckbox("#customauth"); initConfigCheckbox("#use_tquattrecentonze"); - initConfigCheckbox("#use_deezloader"); $('#twitterStep1').click(function () { diff --git a/headphones/aria2.py b/headphones/aria2.py deleted file mode 100644 index 06d8e4e0..00000000 --- a/headphones/aria2.py +++ /dev/null @@ -1,979 +0,0 @@ -# -*- coding: utf8 -*- -# Copyright (C) 2012-2016 Xyne -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# (version 2) as published by the Free Software Foundation. -# -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -from __future__ import with_statement -import base64 -import json -import math -import os -import ssl -import string -import time -import httplib -import urllib2 - -from headphones import logger - -# ################################ Constants ################################### - -DEFAULT_PORT = 6800 -SERVER_URI_FORMAT = '{}://{}:{:d}/jsonrpc' - -# Status values for unfinished downloads. -TEMPORARY_STATUS = ('active', 'waiting', 'paused') -# Status values for finished downloads. -FINAL_STATUS = ('complete', 'error') - -ARIA2_CONTROL_FILE_EXT = '.aria2' - -# ########################## Convenience Functions ############################# - - -def to_json_list(objs): - ''' - Wrap strings in lists. Other iterables are converted to lists directly. - ''' - if isinstance(objs, str): - return [objs] - elif not isinstance(objs, list): - return list(objs) - else: - return objs - - -def add_options_and_position(params, options=None, position=None): - ''' - Convenience method for adding options and position to parameters. - ''' - if options: - params.append(options) - if position: - if not isinstance(position, int): - try: - position = int(position) - except ValueError: - position = -1 - if position >= 0: - params.append(position) - return params - - -def get_status(response): - ''' - Process a status response. - ''' - if response: - try: - return response['status'] - except KeyError: - logger.error('no status returned from Aria2 RPC server') - return 'error' - else: - logger.error('no response from server') - return 'error' - - -def random_token(length, valid_chars=None): - ''' - Get a random secret token for the Aria2 RPC server. - - length: - The length of the token - - valid_chars: - A list or other ordered and indexable iterable of valid characters. If not - given of None, asciinumberic characters with some punctuation characters - will be used. - ''' - if not valid_chars: - valid_chars = string.ascii_letters + string.digits + '!@#$%^&*()-_=+' - number_of_chars = len(valid_chars) - bytes_to_read = math.ceil(math.log(number_of_chars) / math.log(0x100)) - max_value = 0x100**bytes_to_read - max_index = number_of_chars - 1 - token = '' - for _ in range(length): - value = int.from_bytes(os.urandom(bytes_to_read), byteorder='little') - index = round((value * max_index) / max_value) - token += valid_chars[index] - return token - -# ################ From python3-aur's ThreadedServers.common ################### - - -def format_bytes(size): - '''Format bytes for inferior humans.''' - if size < 0x400: - return '{:d} B'.format(size) - else: - size = float(size) / 0x400 - for prefix in ('KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB'): - if size < 0x400: - return '{:0.02f} {}'.format(size, prefix) - else: - size /= 0x400 - return '{:0.02f} YiB'.format(size) - - -def format_seconds(s): - '''Format seconds for inferior humans.''' - string = '' - for base, char in ( - (60, 's'), - (60, 'm'), - (24, 'h') - ): - s, r = divmod(s, base) - if s == 0: - return '{:d}{}{}'.format(r, char, string) - elif r != 0: - string = '{:02d}{}{}'.format(r, char, string) - else: - return '{:d}d{}'.format(s, string) - -# ############################ Aria2JsonRpcError ############################### - - -class Aria2JsonRpcError(Exception): - def __init__(self, msg, connection_error=False): - super(self.__class__, self).__init__(self, msg) - self.connection_error = connection_error - -# ############################ Aria2JsonRpc Class ############################## - - -class Aria2JsonRpc(object): - ''' - Interface class for interacting with an Aria2 RPC server. - ''' - # TODO: certificate options, etc. - def __init__( - self, ID, uri, - mode='normal', - token=None, - http_user=None, http_passwd=None, - server_cert=None, client_cert=None, client_cert_password=None, - ssl_protocol=None, - setup_function=None - ): - ''' - ID: the ID to send to the RPC interface - - uri: the URI of the RPC interface - - mode: - normal - process requests immediately - batch - queue requests (run with "process_queue") - format - return RPC request objects - - token: - RPC method-level authorization token (set using `--rpc-secret`) - - http_user, http_password: - HTTP Basic authentication credentials (deprecated) - - server_cert: - server certificate for HTTPS connections - - client_cert: - client certificate for HTTPS connections - - client_cert_password: - prompt for client certificate password - - ssl_protocol: - SSL protocol from the ssl module - - setup_function: - A function to invoke prior to the first server call. This could be the - launch() method of an Aria2RpcServer instance, for example. This attribute - is set automatically in instances returned from Aria2RpcServer.get_a2jr() - ''' - self.id = ID - self.uri = uri - self.mode = mode - self.queue = [] - self.handlers = dict() - self.token = token - self.setup_function = setup_function - - if None not in (http_user, http_passwd): - self.add_HTTPBasicAuthHandler(http_user, http_passwd) - - if server_cert or client_cert: - self.add_HTTPSHandler( - server_cert=server_cert, - client_cert=client_cert, - client_cert_password=client_cert_password, - protocol=ssl_protocol - ) - - self.update_opener() - - def iter_handlers(self): - ''' - Iterate over handlers. - ''' - for name in ('HTTPS', 'HTTPBasicAuth'): - try: - yield self.handlers[name] - except KeyError: - pass - - def update_opener(self): - ''' - Build an opener from the current handlers. - ''' - self.opener = urllib2.build_opener(*self.iter_handlers()) - - def remove_handler(self, name): - ''' - Remove a handler. - ''' - try: - del self.handlers[name] - except KeyError: - pass - - def add_HTTPBasicAuthHandler(self, user, passwd): - ''' - Add a handler for HTTP Basic authentication. - - If either user or passwd are None, the handler is removed. - ''' - handler = urllib2.HTTPBasicAuthHandler() - handler.add_password( - realm='aria2', - uri=self.uri, - user=user, - passwd=passwd, - ) - self.handlers['HTTPBasicAuth'] = handler - - def remove_HTTPBasicAuthHandler(self): - self.remove_handler('HTTPBasicAuth') - - def add_HTTPSHandler( - self, - server_cert=None, - client_cert=None, - client_cert_password=None, - protocol=None, - ): - ''' - Add a handler for HTTPS connections with optional server and client - certificates. - ''' - if not protocol: - protocol = ssl.PROTOCOL_TLSv1 -# protocol = ssl.PROTOCOL_TLSv1_1 # openssl 1.0.1+ -# protocol = ssl.PROTOCOL_TLSv1_2 # Python 3.4+ - context = ssl.SSLContext(protocol) - - if server_cert: - context.verify_mode = ssl.CERT_REQUIRED - context.load_verify_locations(cafile=server_cert) - else: - context.verify_mode = ssl.CERT_OPTIONAL - - if client_cert: - context.load_cert_chain(client_cert, password=client_cert_password) - - self.handlers['HTTPS'] = urllib2.HTTPSHandler( - context=context, - check_hostname=False - ) - - def remove_HTTPSHandler(self): - self.remove_handler('HTTPS') - - def send_request(self, req_obj): - ''' - Send the request and return the response. - ''' - if self.setup_function: - self.setup_function() - self.setup_function = None - logger.debug("Aria req_obj: %s" % json.dumps(req_obj, indent=2, sort_keys=True)) - req = json.dumps(req_obj).encode('UTF-8') - try: - f = self.opener.open(self.uri, req) - obj = json.loads(f.read()) - try: - return obj['result'] - except KeyError: - raise Aria2JsonRpcError('unexpected result: {}'.format(obj)) - except (urllib2.URLError) as e: - # This should work but URLError does not set the errno attribute: - # e.errno == errno.ECONNREFUSED - raise Aria2JsonRpcError( - str(e), - connection_error=( - '111' in str(e) - ) - ) - except httplib.BadStatusLine as e: - raise Aria2JsonRpcError('{}: BadStatusLine: {} (HTTPS error?)'.format( - self.__class__.__name__, e - )) - - def jsonrpc(self, method, params=None, prefix='aria2.'): - ''' - POST a request to the RPC interface. - ''' - if not params: - params = [] - - if self.token is not None: - token_str = 'token:{}'.format(self.token) - if method == 'multicall': - for p in params[0]: - try: - p['params'].insert(0, token_str) - except KeyError: - p['params'] = [token_str] - else: - params.insert(0, token_str) - - req_obj = { - 'jsonrpc': '2.0', - 'id': self.id, - 'method': prefix + method, - 'params': params, - } - if self.mode == 'batch': - self.queue.append(req_obj) - return None - elif self.mode == 'format': - return req_obj - else: - return self.send_request(req_obj) - - def process_queue(self): - ''' - Processed queued requests. - ''' - req_obj = self.queue - self.queue = [] - return self.send_request(req_obj) - - -# ############################# Standard Methods ############################### - def addUri(self, uris, options=None, position=None): - ''' - aria2.addUri method - - uris: list of URIs - - options: dictionary of additional options - - position: position in queue - - Returns a GID - ''' - params = [uris] - params = add_options_and_position(params, options, position) - return self.jsonrpc('addUri', params) - - def addTorrent(self, torrent, uris=None, options=None, position=None): - ''' - aria2.addTorrent method - - torrent: base64-encoded torrent file - - uris: list of webseed URIs - - options: dictionary of additional options - - position: position in queue - - Returns a GID. - ''' - params = [torrent] - if uris: - params.append(uris) - params = add_options_and_position(params, options, position) - return self.jsonrpc('addTorrent', params) - - def addMetalink(self, metalink, options=None, position=None): - ''' - aria2.addMetalink method - - metalink: base64-encoded metalink file - - options: dictionary of additional options - - position: position in queue - - Returns an array of GIDs. - ''' - params = [metalink] - params = add_options_and_position(params, options, position) - return self.jsonrpc('addTorrent', params) - - def remove(self, gid): - ''' - aria2.remove method - - gid: GID to remove - ''' - params = [gid] - return self.jsonrpc('remove', params) - - def forceRemove(self, gid): - ''' - aria2.forceRemove method - - gid: GID to remove - ''' - params = [gid] - return self.jsonrpc('forceRemove', params) - - def pause(self, gid): - ''' - aria2.pause method - - gid: GID to pause - ''' - params = [gid] - return self.jsonrpc('pause', params) - - def pauseAll(self): - ''' - aria2.pauseAll method - ''' - return self.jsonrpc('pauseAll') - - def forcePause(self, gid): - ''' - aria2.forcePause method - - gid: GID to pause - ''' - params = [gid] - return self.jsonrpc('forcePause', params) - - def forcePauseAll(self): - ''' - aria2.forcePauseAll method - ''' - return self.jsonrpc('forcePauseAll') - - def unpause(self, gid): - ''' - aria2.unpause method - - gid: GID to unpause - ''' - params = [gid] - return self.jsonrpc('unpause', params) - - def unpauseAll(self): - ''' - aria2.unpauseAll method - ''' - return self.jsonrpc('unpauseAll') - - def tellStatus(self, gid, keys=None): - ''' - aria2.tellStatus method - - gid: GID to query - - keys: subset of status keys to return (all keys are returned otherwise) - - Returns a dictionary. - ''' - params = [gid] - if keys: - params.append(keys) - return self.jsonrpc('tellStatus', params) - - def getUris(self, gid): - ''' - aria2.getUris method - - gid: GID to query - - Returns a list of dictionaries. - ''' - params = [gid] - return self.jsonrpc('getUris', params) - - def getFiles(self, gid): - ''' - aria2.getFiles method - - gid: GID to query - - Returns a list of dictionaries. - ''' - params = [gid] - return self.jsonrpc('getFiles', params) - - def getPeers(self, gid): - ''' - aria2.getPeers method - - gid: GID to query - - Returns a list of dictionaries. - ''' - params = [gid] - return self.jsonrpc('getPeers', params) - - def getServers(self, gid): - ''' - aria2.getServers method - - gid: GID to query - - Returns a list of dictionaries. - ''' - params = [gid] - return self.jsonrpc('getServers', params) - - def tellActive(self, keys=None): - ''' - aria2.tellActive method - - keys: same as tellStatus - - Returns a list of dictionaries. The dictionaries are the same as those - returned by tellStatus. - ''' - if keys: - params = [keys] - else: - params = None - return self.jsonrpc('tellActive', params) - - def tellWaiting(self, offset, num, keys=None): - ''' - aria2.tellWaiting method - - offset: offset from start of waiting download queue - (negative values are counted from the end of the queue) - - num: number of downloads to return - - keys: same as tellStatus - - Returns a list of dictionaries. The dictionaries are the same as those - returned by tellStatus. - ''' - params = [offset, num] - if keys: - params.append(keys) - return self.jsonrpc('tellWaiting', params) - - def tellStopped(self, offset, num, keys=None): - ''' - aria2.tellStopped method - - offset: offset from oldest download (same semantics as tellWaiting) - - num: same as tellWaiting - - keys: same as tellStatus - - Returns a list of dictionaries. The dictionaries are the same as those - returned by tellStatus. - ''' - params = [offset, num] - if keys: - params.append(keys) - return self.jsonrpc('tellStopped', params) - - def changePosition(self, gid, pos, how): - ''' - aria2.changePosition method - - gid: GID to change - - pos: the position - - how: "POS_SET", "POS_CUR" or "POS_END" - ''' - params = [gid, pos, how] - return self.jsonrpc('changePosition', params) - - def changeUri(self, gid, fileIndex, delUris, addUris, position=None): - ''' - aria2.changePosition method - - gid: GID to change - - fileIndex: file to affect (1-based) - - delUris: URIs to remove - - addUris: URIs to add - - position: where URIs are inserted, after URIs have been removed - ''' - params = [gid, fileIndex, delUris, addUris] - if position: - params.append(position) - return self.jsonrpc('changePosition', params) - - def getOption(self, gid): - ''' - aria2.getOption method - - gid: GID to query - - Returns a dictionary of options. - ''' - params = [gid] - return self.jsonrpc('getOption', params) - - def changeOption(self, gid, options): - ''' - aria2.changeOption method - - gid: GID to change - - options: dictionary of new options - (not all options can be changed for active downloads) - ''' - params = [gid, options] - return self.jsonrpc('changeOption', params) - - def getGlobalOption(self): - ''' - aria2.getGlobalOption method - - Returns a dictionary. - ''' - return self.jsonrpc('getGlobalOption') - - def changeGlobalOption(self, options): - ''' - aria2.changeGlobalOption method - - options: dictionary of new options - ''' - params = [options] - return self.jsonrpc('changeGlobalOption', params) - - def getGlobalStat(self): - ''' - aria2.getGlobalStat method - - Returns a dictionary. - ''' - return self.jsonrpc('getGlobalStat') - - def purgeDownloadResult(self): - ''' - aria2.purgeDownloadResult method - ''' - self.jsonrpc('purgeDownloadResult') - - def removeDownloadResult(self, gid): - ''' - aria2.removeDownloadResult method - - gid: GID to remove - ''' - params = [gid] - return self.jsonrpc('removeDownloadResult', params) - - def getVersion(self): - ''' - aria2.getVersion method - - Returns a dictionary. - ''' - return self.jsonrpc('getVersion') - - def getSessionInfo(self): - ''' - aria2.getSessionInfo method - - Returns a dictionary. - ''' - return self.jsonrpc('getSessionInfo') - - def shutdown(self): - ''' - aria2.shutdown method - ''' - return self.jsonrpc('shutdown') - - def forceShutdown(self): - ''' - aria2.forceShutdown method - ''' - return self.jsonrpc('forceShutdown') - - def multicall(self, methods): - ''' - aria2.multicall method - - methods: list of dictionaries (keys: methodName, params) - - The method names must be those used by Aria2c, e.g. "aria2.tellStatus". - ''' - return self.jsonrpc('multicall', [methods], prefix='system.') - - -# ########################### Convenience Methods ############################## - def add_torrent(self, path, uris=None, options=None, position=None): - ''' - A wrapper around addTorrent for loading files. - ''' - with open(path, 'r') as f: - torrent = base64.encode(f.read()) - return self.addTorrent(torrent, uris, options, position) - - def add_metalink(self, path, uris=None, options=None, position=None): - ''' - A wrapper around addMetalink for loading files. - ''' - with open(path, 'r') as f: - metalink = base64.encode(f.read()) - return self.addMetalink(metalink, uris, options, position) - - def get_status(self, gid): - ''' - Get the status of a single GID. - ''' - response = self.tellStatus(gid, ['status']) - return get_status(response) - - def wait_for_final_status(self, gid, interval=1): - ''' - Wait for a GID to complete or fail and return its status. - ''' - if not interval or interval < 0: - interval = 1 - while True: - status = self.get_status(gid) - if status in TEMPORARY_STATUS: - time.sleep(interval) - else: - return status - - def get_statuses(self, gids): - ''' - Get the status of multiple GIDs. The status of each is yielded in order. - ''' - methods = [ - { - 'methodName': 'aria2.tellStatus', - 'params': [gid, ['gid', 'status']] - } - for gid in gids - ] - results = self.multicall(methods) - if results: - status = dict((r[0]['gid'], r[0]['status']) for r in results) - for gid in gids: - try: - yield status[gid] - except KeyError: - logger.error('Aria2 RPC server returned no status for GID {}'.format(gid)) - yield 'error' - else: - logger.error('no response from Aria2 RPC server') - for gid in gids: - yield 'error' - - def wait_for_final_statuses(self, gids, interval=1): - ''' - Wait for multiple GIDs to complete or fail and return their statuses in - order. - - gids: - A flat list of GIDs. - ''' - if not interval or interval < 0: - interval = 1 - statusmap = dict((g, None) for g in gids) - remaining = list( - g for g, s in statusmap.items() if s is None - ) - while remaining: - for g, s in zip(remaining, self.get_statuses(remaining)): - if s in TEMPORARY_STATUS: - continue - else: - statusmap[g] = s - remaining = list( - g for g, s in statusmap.items() if s is None - ) - if remaining: - time.sleep(interval) - for g in gids: - yield statusmap[g] - - def print_global_status(self): - ''' - Print global status of the RPC server. - ''' - status = self.getGlobalStat() - if status: - numWaiting = int(status['numWaiting']) - numStopped = int(status['numStopped']) - keys = ['totalLength', 'completedLength'] - total = self.tellActive(keys) - waiting = self.tellWaiting(0, numWaiting, keys) - if waiting: - total += waiting - stopped = self.tellStopped(0, numStopped, keys) - if stopped: - total += stopped - - downloadSpeed = int(status['downloadSpeed']) - uploadSpeed = int(status['uploadSpeed']) - totalLength = sum(int(x['totalLength']) for x in total) - completedLength = sum(int(x['completedLength']) for x in total) - remaining = totalLength - completedLength - - status['downloadSpeed'] = format_bytes(downloadSpeed) + '/s' - status['uploadSpeed'] = format_bytes(uploadSpeed) + '/s' - - preordered = ('downloadSpeed', 'uploadSpeed') - - rows = list() - for k in sorted(status): - if k in preordered: - continue - rows.append((k, status[k])) - - rows.extend((x, status[x]) for x in preordered) - - if totalLength > 0: - rows.append(('total', format(format_bytes(totalLength)))) - rows.append(('completed', format(format_bytes(completedLength)))) - rows.append(('remaining', format(format_bytes(remaining)))) - if completedLength == totalLength: - eta = 'finished' - else: - try: - eta = format_seconds(remaining // downloadSpeed) - except ZeroDivisionError: - eta = 'never' - rows.append(('ETA', eta)) - - l = max(len(r[0]) for r in rows) - r = max(len(r[1]) for r in rows) - r = max(r, len(self.uri) - (l + 2)) - fmt = '{:<' + str(l) + 's} {:>' + str(r) + 's}' - - print(self.uri) - for k, v in rows: - print(fmt.format(k, v)) - - def queue_uris(self, uris, options, interval=None): - ''' - Enqueue URIs and wait for download to finish while printing status at - regular intervals. - ''' - gid = self.addUri(uris, options) - print('GID: {}'.format(gid)) - - if gid and interval is not None: - blanker = '' - while True: - response = self.tellStatus(gid, ['status']) - if response: - try: - status = response['status'] - except KeyError: - print('error: no status returned from Aria2 RPC server') - break - print('{}\rstatus: {}'.format(blanker, status)), - blanker = ' ' * len(status) - if status in TEMPORARY_STATUS: - time.sleep(interval) - else: - break - else: - print('error: no response from server') - break - - -# ####################### Polymethod download handlers ######################### - def polymethod_enqueue_many(self, downloads): - ''' - Enqueue downloads. - - downloads: Same as polymethod_download(). - ''' - methods = list( - { - 'methodName': 'aria2.{}'.format(d[0]), - 'params': list(d[1:]) - } for d in downloads - ) - return self.multicall(methods) - - def polymethod_wait_many(self, gids, interval=1): - ''' - Wait for the GIDs to complete or fail and return their statuses. - - gids: - A list of lists of GIDs. - ''' - # The flattened list of GIDs - gs = list(g for gs in gids for g in gs) - statusmap = dict(tuple(zip( - gs, - self.wait_for_final_statuses(gs, interval=interval) - ))) - for gs in gids: - yield list(statusmap.get(g, 'error') for g in gs) - - def polymethod_enqueue_one(self, download): - ''' - Same as polymethod_enqueue_many but for one element. - ''' - return getattr(self, download[0])(*download[1:]) - - def polymethod_download(self, downloads, interval=1): - ''' - Enqueue a series of downloads and wait for them to finish. Iterate over the - status of each, in order. - - downloads: - An iterable over (, , ...) where indicates the "add" - method to use ('addUri', 'addTorrent', 'addMetalink') and everything that - follows are arguments to pass to that method. - - interval: - The status check interval while waiting. - - Iterates over the download status of finished downloads. "complete" - indicates success. Lists of statuses will be returned for downloads that - create multiple GIDs (e.g. metalinks). - ''' - gids = self.polymethod_enqueue_many(downloads) - return self.polymethod_wait_many(gids, interval=interval) - - def polymethod_download_bool(self, *args, **kwargs): - ''' - A wrapper around polymethod_download() which returns a boolean for each - download to indicate success (True) or failure (False). - ''' -# for status in self.polymethod_download(*args, **kwargs): -# yield all(s == 'complete' for s in status) - return list( - all(s == 'complete' for s in status) - for status in self.polymethod_download(*args, **kwargs) - ) diff --git a/headphones/config.py b/headphones/config.py index 86aa9216..57af4c16 100644 --- a/headphones/config.py +++ b/headphones/config.py @@ -46,10 +46,6 @@ _CONFIG_DEFINITIONS = { 'APOLLO_RATIO': (str, 'Apollo.rip', ''), 'APOLLO_USERNAME': (str, 'Apollo.rip', ''), 'APOLLO_URL': (str, 'Apollo.rip', 'https://apollo.rip'), - 'ARIA_HOST': (str, 'Aria2', ''), - 'ARIA_PASSWORD': (str, 'Aria2', ''), - 'ARIA_TOKEN': (str, 'Aria2', ''), - 'ARIA_USERNAME': (str, 'Aria2', ''), 'AUTOWANT_ALL': (int, 'General', 0), 'AUTOWANT_MANUALLY_ADDED': (int, 'General', 1), 'AUTOWANT_UPCOMING': (int, 'General', 1), @@ -77,8 +73,6 @@ _CONFIG_DEFINITIONS = { 'CUSTOMPORT': (int, 'General', 5000), 'CUSTOMSLEEP': (int, 'General', 1), 'CUSTOMUSER': (str, 'General', ''), - 'DDL_DOWNLOADER': (int, 'General', 0), - 'DEEZLOADER': (int, 'DeezLoader', 0), 'DELETE_LOSSLESS_FILES': (int, 'General', 1), 'DELUGE_HOST': (str, 'Deluge', ''), 'DELUGE_CERT': (str, 'Deluge', ''), @@ -92,7 +86,6 @@ _CONFIG_DEFINITIONS = { 'DOWNLOAD_DIR': (path, 'General', ''), 'DOWNLOAD_SCAN_INTERVAL': (int, 'General', 5), 'DOWNLOAD_TORRENT_DIR': (path, 'General', ''), - 'DOWNLOAD_DDL_DIR': (path, 'General', ''), 'DO_NOT_OVERRIDE_GIT_BRANCH': (int, 'General', 0), 'EMAIL_ENABLED': (int, 'Email', 0), 'EMAIL_FROM': (str, 'Email', ''), diff --git a/headphones/deezloader.py b/headphones/deezloader.py deleted file mode 100644 index 528cded6..00000000 --- a/headphones/deezloader.py +++ /dev/null @@ -1,428 +0,0 @@ -# -*- coding: utf-8 -*- -# Deezloader (c) 2016 by ParadoxalManiak -# -# Deezloader is licensed under a -# Creative Commons Attribution-NonCommercial-ShareAlike 3.0 Unported License. -# -# You should have received a copy of the license along with this -# work. If not, see . -# -# Version 2.1.0 -# Maintained by ParadoxalManiak -# Original work by ZzMTV -# -# Author's disclaimer: -# I am not responsible for the usage of this program by other people. -# I do not recommend you doing this illegally or against Deezer's terms of service. -# This project is licensed under CC BY-NC-SA 4.0 - -import re -import os -from datetime import datetime -from Crypto.Cipher import AES, Blowfish -from hashlib import md5 -import binascii - -from beets.mediafile import MediaFile -from headphones import logger, request, helpers -import headphones - -# Public constants -PROVIDER_NAME = 'Deezer' - -# Internal constants -__API_URL = "http://www.deezer.com/ajax/gw-light.php" -__API_INFO_URL = "http://api.deezer.com/" -__HTTP_HEADERS = { - "User-Agent": "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36", - "Content-Language": "en-US", - "Cache-Control": "max-age=0", - "Accept": "*/*", - "Accept-Charset": "utf-8,ISO-8859-1;q=0.7,*;q=0.3", - "Accept-Language": "de-DE,de;q=0.8,en-US;q=0.6,en;q=0.4" -} - -# Internal variables -__api_queries = { - 'api_version': "1.0", - 'api_token': "None", - 'input': "3" -} -__cookies = None -__tracks_cache = {} -__albums_cache = {} - - -def __getApiToken(): - global __cookies - response = request.request_response(url="http://www.deezer.com/", headers=__HTTP_HEADERS) - __cookies = response.cookies - data = response.content - if data: - matches = re.search(r"checkForm\s*=\s*['|\"](.*)[\"|'];", data) - if matches: - token = matches.group(1) - __api_queries['api_token'] = token - logger.debug(u"Deezloader : api token loeaded ('%s')" % token) - - if not token: - logger.error(u"Deezloader: Unable to get api token") - - -def getAlbumByLink(album_link): - """Returns deezer album infos using album link url - - :param str album_link: deezer album link url (eg: 'http://www.deezer.com/album/1234567/') - """ - matches = re.search(r"album\/([0-9]+)\/?$", album_link) - if matches: - return getAlbum(matches.group(1)) - - -def getAlbum(album_id): - """Returns deezer album infos - - :param int album_id: deezer album id - """ - global __albums_cache - - if str(album_id) in __albums_cache: - return __albums_cache[str(album_id)] - - url = __API_INFO_URL + "album/" + str(album_id) - data = request.request_json(url=url, headers=__HTTP_HEADERS, cookies=__cookies) - - if data and 'error' not in data: - __albums_cache[str(album_id)] = data - return data - else: - logger.debug("Deezloader: Can't load album infos") - return None - - -def searchAlbums(search_term): - """Search for deezer albums using search term - - :param str search_term: search term to search album for - """ - logger.info(u'Searching Deezer using term: "%s"' % search_term) - - url = __API_INFO_URL + "search/album?q=" + search_term - data = request.request_json(url=url, headers=__HTTP_HEADERS, cookies=__cookies) - - albums = [] - - # Process content - if data and 'total' in data and data['total'] > 0 and 'data' in data: - for item in data['data']: - try: - albums.append(getAlbum(item['id'])) - except Exception as e: - logger.error(u"An unknown error occurred in the Deezer parser: %s" % e) - else: - logger.info(u'No results found from Deezer using term: "%s"' % search_term) - - return albums - - -def __matchAlbums(albums, artist_name, album_title, album_length): - resultlist = [] - - for album in albums: - total_size = 0 - tracks_found = 0 - - for track in album['tracks']['data']: - t = getTrack(track['id']) - if t: - if t["FILESIZE_MP3_320"] > 0: - size = t["FILESIZE_MP3_320"] - elif t["FILESIZE_MP3_256"] > 0: - size = t["FILESIZE_MP3_256"] - elif t["FILESIZE_MP3_128"] > 0: - size = t["FILESIZE_MP3_128"] - else: - size = t["FILESIZE_MP3_64"] - - size = int(size) - total_size += size - tracks_found += 1 - logger.debug(u'Found song "%s". Size: %s' % (t['SNG_TITLE'], helpers.bytes_to_mb(size))) - - matched = True - mismatch_reason = 'matched!' - - if album_length > 0 and abs(int(album['duration']) - album_length) > 240: - matched = False - mismatch_reason = (u'duration mismatch: %i not in [%i, %i]' % (int(album['duration']), (album_length - 240), (album_length + 240))) - - elif (helpers.latinToAscii(re.sub(r"\W", "", album_title, 0, re.UNICODE)).lower() != - helpers.latinToAscii(re.sub(r"\W", "", album['title'], 0, re.UNICODE)).lower()): - matched = False - mismatch_reason = (u'album name mismatch: %s != %s' % (album['title'], album_title)) - - elif (helpers.latinToAscii(re.sub(r"\W", "", artist_name, 0, re.UNICODE)).lower() != - helpers.latinToAscii(re.sub(r"\W", "", album['artist']['name'], 0, re.UNICODE)).lower()): - matched = False - mismatch_reason = (u'artist name mismatch: %s != %s' % (album['artist']['name'], artist_name)) - - resultlist.append( - (album['artist']['name'] + ' - ' + album['title'] + ' [' + album['release_date'][:4] + '] (' + str(tracks_found) + '/' + str(album['nb_tracks']) + ')', - total_size, album['link'], PROVIDER_NAME, "ddl", matched) - ) - logger.info(u'Found "%s". Tracks %i/%i. Size: %s (%s)' % (album['title'], tracks_found, album['nb_tracks'], helpers.bytes_to_mb(total_size), mismatch_reason)) - - return resultlist - - -def searchAlbum(artist_name, album_title, user_search_term=None, album_length=None): - """Search for deezer specific album. - This will iterate over deezer albums and try to find best matches - - :param str artist_name: album artist name - :param str album_title: album title - :param str user_search_term: search terms provided by user - :param int album_length: targeted album duration in seconds - """ - # User search term by-pass normal search - if user_search_term: - return __matchAlbums(searchAlbums(user_search_term), artist_name, album_title, album_length) - - resultlist = __matchAlbums(searchAlbums((artist_name + ' ' + album_title).strip()), artist_name, album_title, album_length) - if resultlist: - return resultlist - - # Deezer API supports unicode, so just remove non alphanumeric characters - clean_artist_name = re.sub(r"[^\w\s]", " ", artist_name, 0, re.UNICODE).strip() - clean_album_name = re.sub(r"[^\w\s]", " ", album_title, 0, re.UNICODE).strip() - - resultlist = __matchAlbums(searchAlbums((clean_artist_name + ' ' + clean_album_name).strip()), artist_name, album_title, album_length) - if resultlist: - return resultlist - - resultlist = __matchAlbums(searchAlbums(clean_artist_name), artist_name, album_title, album_length) - if resultlist: - return resultlist - - return resultlist - - -def getTrack(sng_id, try_reload_api=True): - """Returns deezer track infos - - :param int sng_id: deezer song id - :param bool try_reload_api: whether or not try reloading API if session expired - """ - global __tracks_cache - - if str(sng_id) in __tracks_cache: - return __tracks_cache[str(sng_id)] - - data = "[{\"method\":\"song.getListData\",\"params\":{\"sng_ids\":[" + str(sng_id) + "]}}]" - json = request.request_json(url=__API_URL, headers=__HTTP_HEADERS, method='post', params=__api_queries, data=data, cookies=__cookies) - - results = [] - error = None - invalid_token = False - - if json: - # Check for errors - if 'error' in json: - error = json['error'] - if 'GATEWAY_ERROR' in json['error'] and json['error']['GATEWAY_ERROR'] == u"invalid api token": - invalid_token = True - - elif 'error' in json[0] and json[0]['error']: - error = json[0]['error'] - if 'VALID_TOKEN_REQUIRED' in json[0]['error'] and json[0]['error']['VALID_TOKEN_REQUIRED'] == u"Invalid CSRF token": - invalid_token = True - - # Got invalid token error - if error: - if invalid_token and try_reload_api: - __getApiToken() - return getTrack(sng_id, False) - else: - logger.error(u"An unknown error occurred in the Deezer parser: %s" % error) - else: - try: - results = json[0]['results'] - item = results['data'][0] - if 'token' in item: - logger.error(u"An unknown error occurred in the Deezer parser: Uploaded Files are currently not supported") - return - - sng_id = item["SNG_ID"] - md5Origin = item["MD5_ORIGIN"] - sng_format = 3 - - if item["FILESIZE_MP3_320"] <= 0: - if item["FILESIZE_MP3_256"] > 0: - sng_format = 5 - else: - sng_format = 1 - - mediaVersion = int(item["MEDIA_VERSION"]) - item['downloadUrl'] = __getDownloadUrl(md5Origin, sng_id, sng_format, mediaVersion) - - __tracks_cache[sng_id] = item - return item - - except Exception as e: - logger.error(u"An unknown error occurred in the Deezer parser: %s" % e) - - -def __getDownloadUrl(md5Origin, sng_id, sng_format, mediaVersion): - urlPart = md5Origin.encode('utf-8') + b'\xa4' + str(sng_format) + b'\xa4' + str(sng_id) + b'\xa4' + str(mediaVersion) - md5val = md5(urlPart).hexdigest() - urlPart = md5val + b'\xa4' + urlPart + b'\xa4' - cipher = AES.new('jo6aey6haid2Teih', AES.MODE_ECB) - ciphertext = cipher.encrypt(__pad(urlPart, AES.block_size)) - return "http://e-cdn-proxy-" + md5Origin[:1] + ".deezer.com/mobile/1/" + binascii.hexlify(ciphertext).lower() - - -def __pad(raw, block_size): - if (len(raw) % block_size == 0): - return raw - padding_required = block_size - (len(raw) % block_size) - padChar = b'\x00' - data = raw + padding_required * padChar - return data - - -def __tagTrack(path, track): - try: - album = getAlbum(track['ALB_ID']) - - f = MediaFile(path) - f.artist = track['ART_NAME'] - f.album = track['ALB_TITLE'] - f.title = track['SNG_TITLE'] - f.track = track['TRACK_NUMBER'] - f.tracktotal = album['nb_tracks'] - f.disc = track['DISK_NUMBER'] - f.bpm = track['BPM'] - f.date = datetime.strptime(album['release_date'], '%Y-%m-%d').date() - f.albumartist = album['artist']['name'] - if u'genres' in album and u'data' in album['genres']: - f.genres = [genre['name'] for genre in album['genres']['data']] - - f.save() - - except Exception as e: - logger.error(u'Unable to tag deezer track "%s": %s' % (path, e)) - - -def decryptTracks(paths): - """Decrypt downloaded deezer tracks. - - :param paths: list of path to deezer tracks (*.dzr files). - """ - # Note: tracks can be from different albums - decrypted_tracks = {} - - # First pass: load tracks data - for path in paths: - try: - album_folder = os.path.dirname(path) - sng_id = os.path.splitext(os.path.basename(path))[0] - track = getTrack(sng_id) - track_number = int(track['TRACK_NUMBER']) - disk_number = int(track['DISK_NUMBER']) - - if album_folder not in decrypted_tracks: - decrypted_tracks[album_folder] = {} - - if disk_number not in decrypted_tracks[album_folder]: - decrypted_tracks[album_folder][disk_number] = {} - - decrypted_tracks[album_folder][disk_number][track_number] = track - - except Exception as e: - logger.error(u'Unable to load deezer track infos "%s": %s' % (path, e)) - - # Second pass: decrypt tracks - for album_folder in decrypted_tracks: - multi_disks = len(decrypted_tracks[album_folder]) > 1 - for disk_number in decrypted_tracks[album_folder]: - for track_number, track in decrypted_tracks[album_folder][disk_number].items(): - try: - filename = helpers.replace_illegal_chars(track['SNG_TITLE']).strip() - filename = ('{:02d}'.format(track_number) + '_' + filename + '.mp3') - - # Add a 'cd x' sub-folder if album has more than one disk - disk_folder = os.path.join(album_folder, 'cd ' + str(disk_number)) if multi_disks else album_folder - - dest = os.path.join(disk_folder, filename).encode(headphones.SYS_ENCODING, 'replace') - - # Decrypt track if not already done - if not os.path.exists(dest): - try: - __decryptDownload(path, sng_id, dest) - __tagTrack(dest, track) - except Exception as e: - logger.error(u'Unable to decrypt deezer track "%s": %s' % (path, e)) - if os.path.exists(dest): - os.remove(dest) - decrypted_tracks[album_folder][disk_number].pop(track_number) - continue - - decrypted_tracks[album_folder][disk_number][track_number]['path'] = dest - - except Exception as e: - logger.error(u'Unable to decrypt deezer track "%s": %s' % (path, e)) - - return decrypted_tracks - - -def __decryptDownload(source, sng_id, dest): - interval_chunk = 3 - chunk_size = 2048 - blowFishKey = __getBlowFishKey(sng_id) - i = 0 - iv = "\x00\x01\x02\x03\x04\x05\x06\x07" - - dest_folder = os.path.dirname(dest) - if not os.path.exists(dest_folder): - os.makedirs(dest_folder) - - f = open(source, "rb") - fout = open(dest, "wb") - try: - chunk = f.read(chunk_size) - while chunk: - if(i % interval_chunk == 0): - cipher = Blowfish.new(blowFishKey, Blowfish.MODE_CBC, iv) - chunk = cipher.decrypt(__pad(chunk, Blowfish.block_size)) - - fout.write(chunk) - i += 1 - chunk = f.read(chunk_size) - finally: - f.close() - fout.close() - - -def __getBlowFishKey(encryptionKey): - if encryptionKey < 1: - encryptionKey *= -1 - - hashcode = md5(str(encryptionKey)).hexdigest() - hPart = hashcode[0:16] - lPart = hashcode[16:32] - parts = ['g4el58wc0zvf9na1', hPart, lPart] - - return __xorHex(parts) - - -def __xorHex(parts): - data = "" - for i in range(0, 16): - character = ord(parts[0][i]) - - for j in range(1, len(parts)): - character ^= ord(parts[j][i]) - - data += chr(character) - - return data diff --git a/headphones/postprocessor.py b/headphones/postprocessor.py index c9621506..e3deee23 100755 --- a/headphones/postprocessor.py +++ b/headphones/postprocessor.py @@ -29,7 +29,7 @@ from beetsplug import lyrics as beetslyrics from headphones import notifiers, utorrent, transmission, deluge, qbittorrent from headphones import db, albumart, librarysync from headphones import logger, helpers, mb, music_encoder -from headphones import metadata, deezloader +from headphones import metadata postprocessor_lock = threading.Lock() @@ -47,8 +47,6 @@ def checkFolder(): single = False if album['Kind'] == 'nzb': download_dir = headphones.CONFIG.DOWNLOAD_DIR - elif album['Kind'] == 'ddl': - download_dir = headphones.CONFIG.DOWNLOAD_DDL_DIR else: if headphones.CONFIG.DELUGE_DONE_DIRECTORY and headphones.CONFIG.TORRENT_DOWNLOADER == 3: download_dir = headphones.CONFIG.DELUGE_DONE_DIRECTORY @@ -206,7 +204,6 @@ def verify(albumid, albumpath, Kind=None, forced=False, keep_original_folder=Fal tracks = myDB.select('SELECT * from tracks WHERE AlbumID=?', [albumid]) downloaded_track_list = [] - downloaded_deezer_list = [] downloaded_cuecount = 0 for r, d, f in os.walk(albumpath): @@ -215,10 +212,8 @@ def verify(albumid, albumpath, Kind=None, forced=False, keep_original_folder=Fal downloaded_track_list.append(os.path.join(r, files)) elif files.lower().endswith('.cue'): downloaded_cuecount += 1 - elif files.lower().endswith('.dzr'): - downloaded_deezer_list.append(os.path.join(r, files)) # if any of the files end in *.part, we know the torrent isn't done yet. Process if forced, though - elif files.lower().endswith(('.part', '.utpart', '.aria2')) and not forced: + elif files.lower().endswith(('.part', '.utpart')) and not forced: logger.info( "Looks like " + os.path.basename(albumpath).decode(headphones.SYS_ENCODING, 'replace') + " isn't complete yet. Will try again on the next run") @@ -257,37 +252,6 @@ def verify(albumid, albumpath, Kind=None, forced=False, keep_original_folder=Fal downloaded_track_list = helpers.get_downloaded_track_list(albumpath) keep_original_folder = False - # Decrypt deezer tracks - if downloaded_deezer_list: - logger.info('Decrypting deezer tracks') - decrypted_deezer_list = deezloader.decryptTracks(downloaded_deezer_list) - - # Check if album is complete based on album duration only - # (total track numbers is not determinant enough due to hidden tracks for eg) - db_track_duration = 0 - downloaded_track_duration = 0 - try: - for track in tracks: - db_track_duration += track['TrackDuration'] / 1000 - except: - downloaded_track_duration = False - - try: - for disk_number in decrypted_deezer_list[albumpath]: - for track in decrypted_deezer_list[albumpath][disk_number].values(): - downloaded_track_list.append(track['path']) - downloaded_track_duration += int(track['DURATION']) - except: - downloaded_track_duration = False - - if not downloaded_track_duration or not db_track_duration or abs(downloaded_track_duration - db_track_duration) > 240: - logger.info("Looks like " + - os.path.basename(albumpath).decode(headphones.SYS_ENCODING, 'replace') + - " isn't complete yet (duration mismatch). Will try again on the next run") - return - - downloaded_track_list = list(set(downloaded_track_list)) # Remove duplicates - # test #1: metadata - usually works logger.debug('Verifying metadata...') diff --git a/headphones/searcher.py b/headphones/searcher.py index f6554cb1..b0652f7f 100644 --- a/headphones/searcher.py +++ b/headphones/searcher.py @@ -36,7 +36,6 @@ import headphones from headphones.common import USER_AGENT from headphones import logger, db, helpers, classes, sab, nzbget, request from headphones import utorrent, transmission, notifiers, rutracker, deluge, qbittorrent -from headphones import deezloader, aria2 from bencode import bencode, bdecode # Magnet to torrent services, for Black hole. Stolen from CouchPotato. @@ -52,27 +51,6 @@ ruobj = None # Persistent RED API object redobj = None -# Persistent Aria2 RPC object -__aria2rpc_obj = None - - -def getAria2RPC(): - global __aria2rpc_obj - if not __aria2rpc_obj: - __aria2rpc_obj = aria2.Aria2JsonRpc( - ID='headphones', - uri=headphones.CONFIG.ARIA_HOST, - token=headphones.CONFIG.ARIA_TOKEN if headphones.CONFIG.ARIA_TOKEN else None, - http_user=headphones.CONFIG.ARIA_USERNAME if headphones.CONFIG.ARIA_USERNAME else None, - http_passwd=headphones.CONFIG.ARIA_PASSWORD if headphones.CONFIG.ARIA_PASSWORD else None - ) - return __aria2rpc_obj - - -def reconfigure(): - global __aria2rpc_obj - __aria2rpc_obj = None - def fix_url(s, charset="utf-8"): """ @@ -303,53 +281,32 @@ def do_sorted_search(album, new, losslessOnly, choose_specific_download=False): headphones.CONFIG.STRIKE or headphones.CONFIG.TQUATTRECENTONZE) - DDL_PROVIDERS = (headphones.CONFIG.DEEZLOADER) - + results = [] myDB = db.DBConnection() albumlength = myDB.select('SELECT sum(TrackDuration) from tracks WHERE AlbumID=?', [album['AlbumID']])[0][0] - nzb_results = None - torrent_results = None - ddl_results = None - if headphones.CONFIG.PREFER_TORRENTS == 0 and not choose_specific_download: if NZB_PROVIDERS and NZB_DOWNLOADERS: - nzb_results = searchNZB(album, new, losslessOnly, albumlength) + results = searchNZB(album, new, losslessOnly, albumlength) - if not nzb_results: - if DDL_PROVIDERS: - ddl_results = searchDdl(album, new, losslessOnly, albumlength) - - if TORRENT_PROVIDERS: - torrent_results = searchTorrent(album, new, losslessOnly, albumlength) + if not results and TORRENT_PROVIDERS: + results = searchTorrent(album, new, losslessOnly, albumlength) elif headphones.CONFIG.PREFER_TORRENTS == 1 and not choose_specific_download: if TORRENT_PROVIDERS: - torrent_results = searchTorrent(album, new, losslessOnly, albumlength) + results = searchTorrent(album, new, losslessOnly, albumlength) - if not torrent_results: - if DDL_PROVIDERS: - ddl_results = searchDdl(album, new, losslessOnly, albumlength) - - if NZB_PROVIDERS and NZB_DOWNLOADERS: - nzb_results = searchNZB(album, new, losslessOnly, albumlength) - - elif headphones.CONFIG.PREFER_TORRENTS == 3 and not choose_specific_download: - - if DDL_PROVIDERS: - ddl_results = searchDdl(album, new, losslessOnly, albumlength) - - if not ddl_results: - if TORRENT_PROVIDERS: - torrent_results = searchTorrent(album, new, losslessOnly, albumlength) - - if NZB_PROVIDERS and NZB_DOWNLOADERS: - nzb_results = searchNZB(album, new, losslessOnly, albumlength) + if not results and NZB_PROVIDERS and NZB_DOWNLOADERS: + results = searchNZB(album, new, losslessOnly, albumlength) else: + + nzb_results = None + torrent_results = None + if NZB_PROVIDERS and NZB_DOWNLOADERS: nzb_results = searchNZB(album, new, losslessOnly, albumlength, choose_specific_download) @@ -357,19 +314,13 @@ def do_sorted_search(album, new, losslessOnly, choose_specific_download=False): torrent_results = searchTorrent(album, new, losslessOnly, albumlength, choose_specific_download) - if DDL_PROVIDERS: - ddl_results = searchDdl(album, new, losslessOnly, albumlength, choose_specific_download) + if not nzb_results: + nzb_results = [] - if not nzb_results: - nzb_results = [] + if not torrent_results: + torrent_results = [] - if not torrent_results: - torrent_results = [] - - if not ddl_results: - ddl_results = [] - - results = nzb_results + torrent_results + ddl_results + results = nzb_results + torrent_results if choose_specific_download: return results @@ -875,31 +826,6 @@ def send_to_downloader(data, bestqual, album): except Exception as e: logger.error('Couldn\'t write NZB file: %s', e) return - - elif kind == 'ddl': - folder_name = '%s - %s [%s]' % ( - helpers.latinToAscii(album['ArtistName']).encode('UTF-8').replace('/', '_'), - helpers.latinToAscii(album['AlbumTitle']).encode('UTF-8').replace('/', '_'), - get_year_from_release_date(album['ReleaseDate'])) - - # Aria2 downloader - if headphones.CONFIG.DDL_DOWNLOADER == 0: - logger.info("Sending download to Aria2") - - try: - deezer_album = deezloader.getAlbumByLink(bestqual[2]) - - for album_track in deezer_album['tracks']['data']: - track = deezloader.getTrack(album_track['id']) - if track: - filename = track['SNG_ID'] + '.dzr' - logger.debug(u'Sending song "%s" to Aria' % track['SNG_TITLE']) - getAria2RPC().addUri([track['downloadUrl']], {'out': filename, 'auto-file-renaming': 'false', 'continue': 'true', 'dir': folder_name}) - - except Exception as e: - logger.error(u'Error sending torrent to Aria2. Are you sure it\'s running? (%s)' % e) - return - else: folder_name = '%s - %s [%s]' % ( helpers.latinToAscii(album['ArtistName']).encode('UTF-8').replace('/', '_'), @@ -1278,62 +1204,6 @@ def verifyresult(title, artistterm, term, lossless): return True -def searchDdl(album, new=False, losslessOnly=False, albumlength=None, - choose_specific_download=False): - reldate = album['ReleaseDate'] - year = get_year_from_release_date(reldate) - - # MERGE THIS WITH THE TERM CLEANUP FROM searchNZB - dic = {'...': '', ' & ': ' ', ' = ': ' ', '?': '', '$': 's', ' + ': ' ', '"': '', ',': ' ', - '*': ''} - - semi_cleanalbum = helpers.replace_all(album['AlbumTitle'], dic) - cleanalbum = helpers.latinToAscii(semi_cleanalbum) - semi_cleanartist = helpers.replace_all(album['ArtistName'], dic) - cleanartist = helpers.latinToAscii(semi_cleanartist) - - # Use provided term if available, otherwise build our own (this code needs to be cleaned up since a lot - # of these torrent providers are just using cleanartist/cleanalbum terms - if album['SearchTerm']: - term = album['SearchTerm'] - elif album['Type'] == 'part of': - term = cleanalbum + " " + year - else: - # FLAC usually doesn't have a year for some reason so I'll leave it out - # Various Artist albums might be listed as VA, so I'll leave that out too - # Only use the year if the term could return a bunch of different albums, i.e. self-titled albums - if album['ArtistName'] in album['AlbumTitle'] or len(album['ArtistName']) < 4 or len( - album['AlbumTitle']) < 4: - term = cleanartist + ' ' + cleanalbum + ' ' + year - elif album['ArtistName'] == 'Various Artists': - term = cleanalbum + ' ' + year - else: - term = cleanartist + ' ' + cleanalbum - - # Replace bad characters in the term and unicode it - term = re.sub('[\.\-\/]', ' ', term).encode('utf-8') - artistterm = re.sub('[\.\-\/]', ' ', cleanartist).encode('utf-8', 'replace') - - logger.debug(u'Using search term: "%s"' % helpers.latinToAscii(term)) - - resultlist = [] - - # Deezer only provides lossy - if headphones.CONFIG.DEEZLOADER and not losslessOnly: - resultlist += deezloader.searchAlbum(album['ArtistName'], album['AlbumTitle'], album['SearchTerm'], int(albumlength / 1000)) - - # attempt to verify that this isn't a substring result - # when looking for "Foo - Foo" we don't want "Foobar" - # this should be less of an issue when it isn't a self-titled album so we'll only check vs artist - results = [result for result in resultlist if verifyresult(result[0], artistterm, term, losslessOnly)] - - # Additional filtering for size etc - if results and not choose_specific_download: - results = more_filtering(results, album, albumlength, new) - - return results - - def searchTorrent(album, new=False, losslessOnly=False, albumlength=None, choose_specific_download=False): global apolloobj # persistent apollo.rip api object to reduce number of login attempts @@ -2166,10 +2036,7 @@ def searchTorrent(album, new=False, losslessOnly=False, albumlength=None, def preprocess(resultlist): for result in resultlist: - if result[3] == deezloader.PROVIDER_NAME: - return True, result - - elif result[4] == 'torrent': + if result[4] == 'torrent': # rutracker always needs the torrent data if result[3] == 'rutracker.org': diff --git a/headphones/webserve.py b/headphones/webserve.py index 7e97cefb..855fed24 100644 --- a/headphones/webserve.py +++ b/headphones/webserve.py @@ -1174,10 +1174,6 @@ class WebInterface(object): "utorrent_username": headphones.CONFIG.UTORRENT_USERNAME, "utorrent_password": headphones.CONFIG.UTORRENT_PASSWORD, "utorrent_label": headphones.CONFIG.UTORRENT_LABEL, - "aria_host": headphones.CONFIG.ARIA_HOST, - "aria_password": headphones.CONFIG.ARIA_PASSWORD, - "aria_token": headphones.CONFIG.ARIA_TOKEN, - "aria_username": headphones.CONFIG.ARIA_USERNAME, "nzb_downloader_sabnzbd": radio(headphones.CONFIG.NZB_DOWNLOADER, 0), "nzb_downloader_nzbget": radio(headphones.CONFIG.NZB_DOWNLOADER, 1), "nzb_downloader_blackhole": radio(headphones.CONFIG.NZB_DOWNLOADER, 2), @@ -1186,7 +1182,6 @@ class WebInterface(object): "torrent_downloader_utorrent": radio(headphones.CONFIG.TORRENT_DOWNLOADER, 2), "torrent_downloader_deluge": radio(headphones.CONFIG.TORRENT_DOWNLOADER, 3), "torrent_downloader_qbittorrent": radio(headphones.CONFIG.TORRENT_DOWNLOADER, 4), - "ddl_downloader_aria": radio(headphones.CONFIG.DDL_DOWNLOADER, 0), "download_dir": headphones.CONFIG.DOWNLOAD_DIR, "use_blackhole": checked(headphones.CONFIG.BLACKHOLE), "blackhole_dir": headphones.CONFIG.BLACKHOLE_DIR, @@ -1248,8 +1243,6 @@ class WebInterface(object): "use_tquattrecentonze": checked(headphones.CONFIG.TQUATTRECENTONZE), "tquattrecentonze_user": headphones.CONFIG.TQUATTRECENTONZE_USER, "tquattrecentonze_password": headphones.CONFIG.TQUATTRECENTONZE_PASSWORD, - "download_ddl_dir": headphones.CONFIG.DOWNLOAD_DDL_DIR, - "use_deezloader": checked(headphones.CONFIG.DEEZLOADER), "pref_qual_0": radio(headphones.CONFIG.PREFERRED_QUALITY, 0), "pref_qual_1": radio(headphones.CONFIG.PREFERRED_QUALITY, 1), "pref_qual_2": radio(headphones.CONFIG.PREFERRED_QUALITY, 2), @@ -1295,7 +1288,6 @@ class WebInterface(object): "prefer_torrents_0": radio(headphones.CONFIG.PREFER_TORRENTS, 0), "prefer_torrents_1": radio(headphones.CONFIG.PREFER_TORRENTS, 1), "prefer_torrents_2": radio(headphones.CONFIG.PREFER_TORRENTS, 2), - "prefer_torrents_3": radio(headphones.CONFIG.PREFER_TORRENTS, 3), "magnet_links_0": radio(headphones.CONFIG.MAGNET_LINKS, 0), "magnet_links_1": radio(headphones.CONFIG.MAGNET_LINKS, 1), "magnet_links_2": radio(headphones.CONFIG.MAGNET_LINKS, 2), @@ -1454,7 +1446,7 @@ class WebInterface(object): "launch_browser", "enable_https", "api_enabled", "use_blackhole", "headphones_indexer", "use_newznab", "newznab_enabled", "use_torznab", "torznab_enabled", "use_nzbsorg", "use_omgwtfnzbs", "use_kat", "use_piratebay", "use_oldpiratebay", - "use_mininova", "use_waffles", "use_rutracker", "use_deezloader", + "use_mininova", "use_waffles", "use_rutracker", "use_apollo", "use_redacted", "use_strike", "use_tquattrecentonze", "preferred_bitrate_allow_lossless", "detect_bitrate", "ignore_clean_releases", "freeze_db", "cue_split", "move_files", "rename_files", "correct_metadata", "cleanup_files", "keep_nfo", "add_album_art", @@ -1588,9 +1580,6 @@ class WebInterface(object): # Reconfigure musicbrainz database connection with the new values mb.startmb() - # Reconfigure Aria2 - searcher.reconfigure() - raise cherrypy.HTTPRedirect("config") @cherrypy.expose