From 4eabc1c614afd4df9ad09e3218668b1d3545e018 Mon Sep 17 00:00:00 2001 From: Ade Date: Fri, 14 Apr 2017 08:47:09 +1200 Subject: [PATCH 01/27] opus format fixes https://github.com/rembo10/headphones/issues/2909 --- headphones/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/headphones/__init__.py b/headphones/__init__.py index 700caf47..20476169 100644 --- a/headphones/__init__.py +++ b/headphones/__init__.py @@ -86,7 +86,7 @@ CURRENT_VERSION = None LATEST_VERSION = None COMMITS_BEHIND = None -LOSSY_MEDIA_FORMATS = ["mp3", "aac", "ogg", "ape", "m4a", "asf", "wma"] +LOSSY_MEDIA_FORMATS = ["mp3", "aac", "ogg", "ape", "m4a", "asf", "wma", "opus"] LOSSLESS_MEDIA_FORMATS = ["flac", "aiff"] MEDIA_FORMATS = LOSSY_MEDIA_FORMATS + LOSSLESS_MEDIA_FORMATS From f0fe23899ad8bd43ce22bf3e11832e7c73072564 Mon Sep 17 00:00:00 2001 From: Kallys Date: Wed, 19 Apr 2017 18:59:27 +0200 Subject: [PATCH 02/27] Add deezloader provider Add aria2 downloader --- data/interfaces/default/config.html | 61 +- headphones/aria2.py | 1095 +++++++++++++++++++++++++++ headphones/config.py | 7 + headphones/deezloader.py | 415 ++++++++++ headphones/postprocessor.py | 40 +- headphones/searcher.py | 163 +++- headphones/webserve.py | 13 +- 7 files changed, 1773 insertions(+), 21 deletions(-) create mode 100644 headphones/aria2.py create mode 100644 headphones/deezloader.py diff --git a/data/interfaces/default/config.html b/data/interfaces/default/config.html index 841df0d1..43aead6a 100644 --- a/data/interfaces/default/config.html +++ b/data/interfaces/default/config.html @@ -304,6 +304,45 @@ + +
+ Direct Download + Aria2 +
+
+
+ + + usually http://localhost:6800/jsonrpc +
+
+ + +
+
+ + +
+
+ + +
+
+ + + Full path where ddl client downloads your music, e.g. /Users/name/Downloads/music +
+
@@ -461,6 +500,7 @@ NZBs Torrents + Direct Download No Preference
@@ -573,6 +613,17 @@ + +
+ Direct Download + +
+
+ +
+
+ +
@@ -804,7 +855,6 @@
- @@ -2405,6 +2455,11 @@ $("#deluge_options").show(); } + if ($("#ddl_downloader_aria").is(":checked")) + { + $("#ddl_aria_options").show(); + } + $('input[type=radio]').change(function(){ if ($("#preferred_bitrate").is(":checked")) { @@ -2458,6 +2513,9 @@ { $("#torrent_blackhole_options,#utorrent_options,#transmission_options,#qbittorrent_options").fadeOut("fast", function() { $("#deluge_options").fadeIn() }); } + if ($("#ddl_downloader_aria").is(":checked")) + { + } }); $("#mirror").change(handleNewServerSelection); @@ -2560,6 +2618,7 @@ initConfigCheckbox("#enable_https"); initConfigCheckbox("#customauth"); initConfigCheckbox("#use_tquattrecentonze"); + initConfigCheckbox("#use_deezloader"); $('#twitterStep1').click(function () { diff --git a/headphones/aria2.py b/headphones/aria2.py new file mode 100644 index 00000000..68c4cb0a --- /dev/null +++ b/headphones/aria2.py @@ -0,0 +1,1095 @@ +# -*- coding: utf8 -*- +# Copyright (C) 2012-2016 Xyne +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# (version 2) as published by the Free Software Foundation. +# +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +from __future__ import with_statement +import base64 +import json +import math +import os +import ssl +import string +import time +import httplib +import urllib2 + +from headphones import logger + +################################## Constants ################################### + +DEFAULT_PORT = 6800 +SERVER_URI_FORMAT = '{}://{}:{:d}/jsonrpc' + +# Status values for unfinished downloads. +TEMPORARY_STATUS = ('active', 'waiting', 'paused') +# Status values for finished downloads. +FINAL_STATUS = ('complete', 'error') + +ARIA2_CONTROL_FILE_EXT = '.aria2' + +############################ Convenience Functions ############################# + +def to_json_list(objs): + ''' + Wrap strings in lists. Other iterables are converted to lists directly. + ''' + if isinstance(objs, str): + return [objs] + elif not isinstance(objs, list): + return list(objs) + else: + return objs + + + +def add_options_and_position(params, options=None, position=None): + ''' + Convenience method for adding options and position to parameters. + ''' + if options: + params.append(options) + if position: + if not isinstance(position, int): + try: + position = int(position) + except ValueError: + position = -1 + if position >= 0: + params.append(position) + return params + + + +def get_status(response): + ''' + Process a status response. + ''' + if response: + try: + return response['status'] + except KeyError: + logger.error('no status returned from Aria2 RPC server') + return 'error' + else: + logger.error('no response from server') + return 'error' + + + +def random_token(length, valid_chars=None): + ''' + Get a random secret token for the Aria2 RPC server. + + length: + The length of the token + + valid_chars: + A list or other ordered and indexable iterable of valid characters. If not + given of None, asciinumberic characters with some punctuation characters + will be used. + ''' + if not valid_chars: + valid_chars = string.ascii_letters + string.digits + '!@#$%^&*()-_=+' + number_of_chars = len(valid_chars) + bytes_to_read = math.ceil(math.log(number_of_chars) / math.log(0x100)) + max_value = 0x100**bytes_to_read + max_index = number_of_chars - 1 + token = '' + for _ in range(length): + value = int.from_bytes(os.urandom(bytes_to_read), byteorder='little') + index = round((value * max_index)/max_value) + token += valid_chars[index] + return token + + +################## From python3-aur's ThreadedServers.common ################### + +def format_bytes(size): + '''Format bytes for inferior humans.''' + if size < 0x400: + return '{:d} B'.format(size) + else: + size = float(size) / 0x400 + for prefix in ('KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB'): + if size < 0x400: + return '{:0.02f} {}'.format(size, prefix) + else: + size /= 0x400 + return '{:0.02f} YiB'.format(size) + + + +def format_seconds(s): + '''Format seconds for inferior humans.''' + string = '' + for base, char in ( + (60, 's'), + (60, 'm'), + (24, 'h') + ): + s, r = divmod(s, base) + if s == 0: + return '{:d}{}{}'.format(r, char, string) + elif r != 0: + string = '{:02d}{}{}'.format(r, char, string) + else: + return '{:d}d{}'.format(s, string) + +############################## Aria2JsonRpcError ############################### + +class Aria2JsonRpcError(Exception): + def __init__(self, msg, connection_error=False): + super(self.__class__, self).__init__(self, msg) + self.connection_error = connection_error + +############################## Aria2JsonRpc Class ############################## + +class Aria2JsonRpc(object): + ''' + Interface class for interacting with an Aria2 RPC server. + ''' + # TODO: certificate options, etc. + def __init__( + self, ID, uri, + mode='normal', + token=None, + http_user=None, http_passwd=None, + server_cert=None, client_cert=None, client_cert_password=None, + ssl_protocol=None, + setup_function=None + ): + ''' + ID: the ID to send to the RPC interface + + uri: the URI of the RPC interface + + mode: + normal - process requests immediately + batch - queue requests (run with "process_queue") + format - return RPC request objects + + token: + RPC method-level authorization token (set using `--rpc-secret`) + + http_user, http_password: + HTTP Basic authentication credentials (deprecated) + + server_cert: + server certificate for HTTPS connections + + client_cert: + client certificate for HTTPS connections + + client_cert_password: + prompt for client certificate password + + ssl_protocol: + SSL protocol from the ssl module + + setup_function: + A function to invoke prior to the first server call. This could be the + launch() method of an Aria2RpcServer instance, for example. This attribute + is set automatically in instances returned from Aria2RpcServer.get_a2jr() + ''' + self.id = ID + self.uri = uri + self.mode = mode + self.queue = [] + self.handlers = dict() + self.token = token + self.setup_function = setup_function + + if None not in (http_user, http_passwd): + self.add_HTTPBasicAuthHandler(http_user, http_passwd) + + if server_cert or client_cert: + self.add_HTTPSHandler( + server_cert=server_cert, + client_cert=client_cert, + client_cert_password=client_cert_password, + protocol=ssl_protocol + ) + + self.update_opener() + + + + def iter_handlers(self): + ''' + Iterate over handlers. + ''' + for name in ('HTTPS', 'HTTPBasicAuth'): + try: + yield self.handlers[name] + except KeyError: + pass + + + + def update_opener(self): + ''' + Build an opener from the current handlers. + ''' + self.opener = urllib2.build_opener(*self.iter_handlers()) + + + + def remove_handler(self, name): + ''' + Remove a handler. + ''' + try: + del self.handlers[name] + except KeyError: + pass + + + + def add_HTTPBasicAuthHandler(self, user, passwd): + ''' + Add a handler for HTTP Basic authentication. + + If either user or passwd are None, the handler is removed. + ''' + handler = urllib2.HTTPBasicAuthHandler() + handler.add_password( + realm='aria2', + uri=self.uri, + user=user, + passwd=passwd, + ) + self.handlers['HTTPBasicAuth'] = handler + + + + def remove_HTTPBasicAuthHandler(self): + self.remove_handler('HTTPBasicAuth') + + + + def add_HTTPSHandler( + self, + server_cert=None, + client_cert=None, + client_cert_password=None, + protocol=None, + ): + ''' + Add a handler for HTTPS connections with optional server and client + certificates. + ''' + if not protocol: + protocol = ssl.PROTOCOL_TLSv1 +# protocol = ssl.PROTOCOL_TLSv1_1 # openssl 1.0.1+ +# protocol = ssl.PROTOCOL_TLSv1_2 # Python 3.4+ + context = ssl.SSLContext(protocol) + + if server_cert: + context.verify_mode = ssl.CERT_REQUIRED + context.load_verify_locations(cafile=server_cert) + else: + context.verify_mode = ssl.CERT_OPTIONAL + + if client_cert: + context.load_cert_chain(client_cert, password=client_cert_password) + + self.handlers['HTTPS'] = urllib2.HTTPSHandler( + context=context, + check_hostname=False + ) + + + + def remove_HTTPSHandler(self): + self.remove_handler('HTTPS') + + + + def send_request(self, req_obj): + ''' + Send the request and return the response. + ''' + if self.setup_function: + self.setup_function() + self.setup_function = None + logger.debug("Aria req_obj: %s" % json.dumps(req_obj, indent=2, sort_keys=True)) + req = json.dumps(req_obj).encode('UTF-8') + try: + f = self.opener.open(self.uri, req) + obj = json.loads(f.read()) + try: + return obj['result'] + except KeyError: + raise Aria2JsonRpcError('unexpected result: {}'.format(obj)) + except (urllib2.URLError) as e: + # This should work but URLError does not set the errno attribute: + # e.errno == errno.ECONNREFUSED + raise Aria2JsonRpcError( + str(e), + connection_error=( + '111' in str(e) + ) + ) + except httplib.BadStatusLine as e: + raise Aria2JsonRpcError('{}: BadStatusLine: {} (HTTPS error?)'.format( + self.__class__.__name__, e + )) + + + + def jsonrpc(self, method, params=None, prefix='aria2.'): + ''' + POST a request to the RPC interface. + ''' + if not params: + params = [] + + if self.token is not None: + token_str = 'token:{}'.format(self.token) + if method == 'multicall': + for p in params[0]: + try: + p['params'].insert(0, token_str) + except KeyError: + p['params'] = [token_str] + else: + params.insert(0, token_str) + + req_obj = { + 'jsonrpc' : '2.0', + 'id' : self.id, + 'method' : prefix + method, + 'params' : params, + } + if self.mode == 'batch': + self.queue.append(req_obj) + return None + elif self.mode == 'format': + return req_obj + else: + return self.send_request(req_obj) + + + + def process_queue(self): + ''' + Processed queued requests. + ''' + req_obj = self.queue + self.queue = [] + return self.send_request(req_obj) + + + +############################### Standard Methods ############################### + + def addUri(self, uris, options=None, position=None): + ''' + aria2.addUri method + + uris: list of URIs + + options: dictionary of additional options + + position: position in queue + + Returns a GID + ''' + params = [uris] + params = add_options_and_position(params, options, position) + return self.jsonrpc('addUri', params) + + + + def addTorrent(self, torrent, uris=None, options=None, position=None): + ''' + aria2.addTorrent method + + torrent: base64-encoded torrent file + + uris: list of webseed URIs + + options: dictionary of additional options + + position: position in queue + + Returns a GID. + ''' + params = [torrent] + if uris: + params.append(uris) + params = add_options_and_position(params, options, position) + return self.jsonrpc('addTorrent', params) + + + + def addMetalink(self, metalink, options=None, position=None): + ''' + aria2.addMetalink method + + metalink: base64-encoded metalink file + + options: dictionary of additional options + + position: position in queue + + Returns an array of GIDs. + ''' + params = [metalink] + params = add_options_and_position(params, options, position) + return self.jsonrpc('addTorrent', params) + + + + def remove(self, gid): + ''' + aria2.remove method + + gid: GID to remove + ''' + params = [gid] + return self.jsonrpc('remove', params) + + + + def forceRemove(self, gid): + ''' + aria2.forceRemove method + + gid: GID to remove + ''' + params = [gid] + return self.jsonrpc('forceRemove', params) + + + + def pause(self, gid): + ''' + aria2.pause method + + gid: GID to pause + ''' + params = [gid] + return self.jsonrpc('pause', params) + + + + def pauseAll(self): + ''' + aria2.pauseAll method + ''' + return self.jsonrpc('pauseAll') + + + + def forcePause(self, gid): + ''' + aria2.forcePause method + + gid: GID to pause + ''' + params = [gid] + return self.jsonrpc('forcePause', params) + + + + def forcePauseAll(self): + ''' + aria2.forcePauseAll method + ''' + return self.jsonrpc('forcePauseAll') + + + + def unpause(self, gid): + ''' + aria2.unpause method + + gid: GID to unpause + ''' + params = [gid] + return self.jsonrpc('unpause', params) + + + + def unpauseAll(self): + ''' + aria2.unpauseAll method + ''' + return self.jsonrpc('unpauseAll') + + + + def tellStatus(self, gid, keys=None): + ''' + aria2.tellStatus method + + gid: GID to query + + keys: subset of status keys to return (all keys are returned otherwise) + + Returns a dictionary. + ''' + params = [gid] + if keys: + params.append(keys) + return self.jsonrpc('tellStatus', params) + + + + def getUris(self, gid): + ''' + aria2.getUris method + + gid: GID to query + + Returns a list of dictionaries. + ''' + params = [gid] + return self.jsonrpc('getUris', params) + + + + def getFiles(self, gid): + ''' + aria2.getFiles method + + gid: GID to query + + Returns a list of dictionaries. + ''' + params = [gid] + return self.jsonrpc('getFiles', params) + + + + def getPeers(self, gid): + ''' + aria2.getPeers method + + gid: GID to query + + Returns a list of dictionaries. + ''' + params = [gid] + return self.jsonrpc('getPeers', params) + + + + def getServers(self, gid): + ''' + aria2.getServers method + + gid: GID to query + + Returns a list of dictionaries. + ''' + params = [gid] + return self.jsonrpc('getServers', params) + + + + def tellActive(self, keys=None): + ''' + aria2.tellActive method + + keys: same as tellStatus + + Returns a list of dictionaries. The dictionaries are the same as those + returned by tellStatus. + ''' + if keys: + params = [keys] + else: + params = None + return self.jsonrpc('tellActive', params) + + + + def tellWaiting(self, offset, num, keys=None): + ''' + aria2.tellWaiting method + + offset: offset from start of waiting download queue + (negative values are counted from the end of the queue) + + num: number of downloads to return + + keys: same as tellStatus + + Returns a list of dictionaries. The dictionaries are the same as those + returned by tellStatus. + ''' + params = [offset, num] + if keys: + params.append(keys) + return self.jsonrpc('tellWaiting', params) + + + + def tellStopped(self, offset, num, keys=None): + ''' + aria2.tellStopped method + + offset: offset from oldest download (same semantics as tellWaiting) + + num: same as tellWaiting + + keys: same as tellStatus + + Returns a list of dictionaries. The dictionaries are the same as those + returned by tellStatus. + ''' + params = [offset, num] + if keys: + params.append(keys) + return self.jsonrpc('tellStopped', params) + + + + def changePosition(self, gid, pos, how): + ''' + aria2.changePosition method + + gid: GID to change + + pos: the position + + how: "POS_SET", "POS_CUR" or "POS_END" + ''' + params = [gid, pos, how] + return self.jsonrpc('changePosition', params) + + + + def changeUri(self, gid, fileIndex, delUris, addUris, position=None): + ''' + aria2.changePosition method + + gid: GID to change + + fileIndex: file to affect (1-based) + + delUris: URIs to remove + + addUris: URIs to add + + position: where URIs are inserted, after URIs have been removed + ''' + params = [gid, fileIndex, delUris, addUris] + if position: + params.append(position) + return self.jsonrpc('changePosition', params) + + + + def getOption(self, gid): + ''' + aria2.getOption method + + gid: GID to query + + Returns a dictionary of options. + ''' + params = [gid] + return self.jsonrpc('getOption', params) + + + + def changeOption(self, gid, options): + ''' + aria2.changeOption method + + gid: GID to change + + options: dictionary of new options + (not all options can be changed for active downloads) + ''' + params = [gid, options] + return self.jsonrpc('changeOption', params) + + + + def getGlobalOption(self): + ''' + aria2.getGlobalOption method + + Returns a dictionary. + ''' + return self.jsonrpc('getGlobalOption') + + + + def changeGlobalOption(self, options): + ''' + aria2.changeGlobalOption method + + options: dictionary of new options + ''' + params = [options] + return self.jsonrpc('changeGlobalOption', params) + + + + def getGlobalStat(self): + ''' + aria2.getGlobalStat method + + Returns a dictionary. + ''' + return self.jsonrpc('getGlobalStat') + + + + def purgeDownloadResult(self): + ''' + aria2.purgeDownloadResult method + ''' + self.jsonrpc('purgeDownloadResult') + + + + def removeDownloadResult(self, gid): + ''' + aria2.removeDownloadResult method + + gid: GID to remove + ''' + params = [gid] + return self.jsonrpc('removeDownloadResult', params) + + + + def getVersion(self): + ''' + aria2.getVersion method + + Returns a dictionary. + ''' + return self.jsonrpc('getVersion') + + + + def getSessionInfo(self): + ''' + aria2.getSessionInfo method + + Returns a dictionary. + ''' + return self.jsonrpc('getSessionInfo') + + + + def shutdown(self): + ''' + aria2.shutdown method + ''' + return self.jsonrpc('shutdown') + + + + def forceShutdown(self): + ''' + aria2.forceShutdown method + ''' + return self.jsonrpc('forceShutdown') + + + + def multicall(self, methods): + ''' + aria2.multicall method + + methods: list of dictionaries (keys: methodName, params) + + The method names must be those used by Aria2c, e.g. "aria2.tellStatus". + ''' + return self.jsonrpc('multicall', [methods], prefix='system.') + + + + +############################# Convenience Methods ############################## + + def add_torrent(self, path, uris=None, options=None, position=None): + ''' + A wrapper around addTorrent for loading files. + ''' + with open(path, 'r') as f: + torrent = base64.encode(f.read()) + return self.addTorrent(torrent, uris, options, position) + + + + def add_metalink(self, path, uris=None, options=None, position=None): + ''' + A wrapper around addMetalink for loading files. + ''' + with open(path, 'r') as f: + metalink = base64.encode(f.read()) + return self.addMetalink(metalink, uris, options, position) + + + + def get_status(self, gid): + ''' + Get the status of a single GID. + ''' + response = self.tellStatus(gid, ['status']) + return get_status(response) + + + + def wait_for_final_status(self, gid, interval=1): + ''' + Wait for a GID to complete or fail and return its status. + ''' + if not interval or interval < 0: + interval = 1 + while True: + status = self.get_status(gid) + if status in TEMPORARY_STATUS: + time.sleep(interval) + else: + return status + + + + def get_statuses(self, gids): + ''' + Get the status of multiple GIDs. The status of each is yielded in order. + ''' + methods = [ + { + 'methodName' : 'aria2.tellStatus', + 'params' : [gid, ['gid', 'status']] + } + for gid in gids + ] + results = self.multicall(methods) + if results: + status = dict((r[0]['gid'], r[0]['status']) for r in results) + for gid in gids: + try: + yield status[gid] + except KeyError: + logger.error('Aria2 RPC server returned no status for GID {}'.format(gid)) + yield 'error' + else: + logger.error('no response from Aria2 RPC server') + for gid in gids: + yield 'error' + + + + def wait_for_final_statuses(self, gids, interval=1): + ''' + Wait for multiple GIDs to complete or fail and return their statuses in + order. + + gids: + A flat list of GIDs. + ''' + if not interval or interval < 0: + interval = 1 + statusmap = dict((g, None) for g in gids) + remaining = list( + g for g,s in statusmap.items() if s is None + ) + while remaining: + for g, s in zip(remaining, self.get_statuses(remaining)): + if s in TEMPORARY_STATUS: + continue + else: + statusmap[g] = s + remaining = list( + g for g,s in statusmap.items() if s is None + ) + if remaining: + time.sleep(interval) + for g in gids: + yield statusmap[g] + + + + def print_global_status(self): + ''' + Print global status of the RPC server. + ''' + status = self.getGlobalStat() + if status: + numWaiting = int(status['numWaiting']) + numStopped = int(status['numStopped']) + keys = ['totalLength', 'completedLength'] + total = self.tellActive(keys) + waiting = self.tellWaiting(0, numWaiting, keys) + if waiting: + total += waiting + stopped = self.tellStopped(0, numStopped, keys) + if stopped: + total += stopped + + downloadSpeed = int(status['downloadSpeed']) + uploadSpeed = int(status['uploadSpeed']) + totalLength = sum(int(x['totalLength']) for x in total) + completedLength = sum(int(x['completedLength']) for x in total) + remaining = totalLength - completedLength + + status['downloadSpeed'] = format_bytes(downloadSpeed) + '/s' + status['uploadSpeed'] = format_bytes(uploadSpeed) + '/s' + + preordered = ('downloadSpeed', 'uploadSpeed') + + rows = list() + for k in sorted(status): + if k in preordered: + continue + rows.append((k, status[k])) + + rows.extend((x, status[x]) for x in preordered) + + if totalLength > 0: + rows.append(('total', format(format_bytes(totalLength)))) + rows.append(('completed', format(format_bytes(completedLength)))) + rows.append(('remaining', format(format_bytes(remaining)))) + if completedLength == totalLength: + eta = 'finished' + else: + try: + eta = format_seconds(remaining // downloadSpeed) + except ZeroDivisionError: + eta = 'never' + rows.append(('ETA', eta)) + + l = max(len(r[0]) for r in rows) + r = max(len(r[1]) for r in rows) + r = max(r, len(self.uri) - (l + 2)) + fmt = '{:<' + str(l) + 's} {:>' + str(r) + 's}' + + print(self.uri) + for k, v in rows: + print(fmt.format(k, v)) + + + + def queue_uris(self, uris, options, interval=None): + ''' + Enqueue URIs and wait for download to finish while printing status at + regular intervals. + ''' + gid = self.addUri(uris, options) + print('GID: {}'.format(gid)) + + if gid and interval is not None: + blanker = '' + while True: + response = self.tellStatus(gid, ['status']) + if response: + try: + status = response['status'] + except KeyError: + print('error: no status returned from Aria2 RPC server') + break + print('{}\rstatus: {}'.format(blanker, status)), + blanker = ' ' * len(status) + if status in TEMPORARY_STATUS: + time.sleep(interval) + else: + break + else: + print('error: no response from server') + break + + + +######################### Polymethod download handlers ######################### + + def polymethod_enqueue_many(self, downloads): + ''' + Enqueue downloads. + + downloads: Same as polymethod_download(). + ''' + methods = list( + { + 'methodName': 'aria2.{}'.format(d[0]), + 'params': list(d[1:]) + } for d in downloads + ) + return self.multicall(methods) + + + + def polymethod_wait_many(self, gids, interval=1): + ''' + Wait for the GIDs to complete or fail and return their statuses. + + gids: + A list of lists of GIDs. + ''' + # The flattened list of GIDs + gs = list(g for gs in gids for g in gs) + statusmap = dict(tuple(zip( + gs, + self.wait_for_final_statuses(gs, interval=interval) + ))) + for gs in gids: + yield list(statusmap.get(g, 'error') for g in gs) + + + + + def polymethod_enqueue_one(self, download): + ''' + Same as polymethod_enqueue_many but for one element. + ''' + return getattr(self, download[0])(*download[1:]) + + + + def polymethod_download(self, downloads, interval=1): + ''' + Enqueue a series of downloads and wait for them to finish. Iterate over the + status of each, in order. + + downloads: + An iterable over (, , ...) where indicates the "add" + method to use ('addUri', 'addTorrent', 'addMetalink') and everything that + follows are arguments to pass to that method. + + interval: + The status check interval while waiting. + + Iterates over the download status of finished downloads. "complete" + indicates success. Lists of statuses will be returned for downloads that + create multiple GIDs (e.g. metalinks). + ''' + gids = self.polymethod_enqueue_many(downloads) + return self.polymethod_wait_many(gids, interval=interval) + + + + def polymethod_download_bool(self, *args, **kwargs): + ''' + A wrapper around polymethod_download() which returns a boolean for each + download to indicate success (True) or failure (False). + ''' +# for status in self.polymethod_download(*args, **kwargs): +# yield all(s == 'complete' for s in status) + return list( + all(s == 'complete' for s in status) + for status in self.polymethod_download(*args, **kwargs) + ) + \ No newline at end of file diff --git a/headphones/config.py b/headphones/config.py index 57af4c16..86aa9216 100644 --- a/headphones/config.py +++ b/headphones/config.py @@ -46,6 +46,10 @@ _CONFIG_DEFINITIONS = { 'APOLLO_RATIO': (str, 'Apollo.rip', ''), 'APOLLO_USERNAME': (str, 'Apollo.rip', ''), 'APOLLO_URL': (str, 'Apollo.rip', 'https://apollo.rip'), + 'ARIA_HOST': (str, 'Aria2', ''), + 'ARIA_PASSWORD': (str, 'Aria2', ''), + 'ARIA_TOKEN': (str, 'Aria2', ''), + 'ARIA_USERNAME': (str, 'Aria2', ''), 'AUTOWANT_ALL': (int, 'General', 0), 'AUTOWANT_MANUALLY_ADDED': (int, 'General', 1), 'AUTOWANT_UPCOMING': (int, 'General', 1), @@ -73,6 +77,8 @@ _CONFIG_DEFINITIONS = { 'CUSTOMPORT': (int, 'General', 5000), 'CUSTOMSLEEP': (int, 'General', 1), 'CUSTOMUSER': (str, 'General', ''), + 'DDL_DOWNLOADER': (int, 'General', 0), + 'DEEZLOADER': (int, 'DeezLoader', 0), 'DELETE_LOSSLESS_FILES': (int, 'General', 1), 'DELUGE_HOST': (str, 'Deluge', ''), 'DELUGE_CERT': (str, 'Deluge', ''), @@ -86,6 +92,7 @@ _CONFIG_DEFINITIONS = { 'DOWNLOAD_DIR': (path, 'General', ''), 'DOWNLOAD_SCAN_INTERVAL': (int, 'General', 5), 'DOWNLOAD_TORRENT_DIR': (path, 'General', ''), + 'DOWNLOAD_DDL_DIR': (path, 'General', ''), 'DO_NOT_OVERRIDE_GIT_BRANCH': (int, 'General', 0), 'EMAIL_ENABLED': (int, 'Email', 0), 'EMAIL_FROM': (str, 'Email', ''), diff --git a/headphones/deezloader.py b/headphones/deezloader.py new file mode 100644 index 00000000..db14ae5d --- /dev/null +++ b/headphones/deezloader.py @@ -0,0 +1,415 @@ +# -*- coding: utf-8 -*- +# Deezloader (c) 2016 by ParadoxalManiak +# +# Deezloader is licensed under a +# Creative Commons Attribution-NonCommercial-ShareAlike 3.0 Unported License. +# +# You should have received a copy of the license along with this +# work. If not, see . +# +# Version 2.1.0 +# Maintained by ParadoxalManiak +# Original work by ZzMTV +# +# Author's disclaimer: +# I am not responsible for the usage of this program by other people. +# I do not recommend you doing this illegally or against Deezer's terms of service. +# This project is licensed under CC BY-NC-SA 4.0 + +import re +import os +from datetime import datetime +from Crypto.Cipher import AES, Blowfish +from hashlib import md5 +import binascii + +from beets.mediafile import MediaFile +from headphones import logger, request, helpers +import headphones +from twisted.conch.insults import helper + +# Public constants +PROVIDER_NAME = 'Deezer' + +# Internal constants +__API_URL = "http://www.deezer.com/ajax/gw-light.php" +__API_INFO_URL = "http://api.deezer.com/" +__HTTP_HEADERS = { + "User-Agent": "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36", + "Content-Language": "en-US", + "Cache-Control": "max-age=0", + "Accept": "*/*", + "Accept-Charset": "utf-8,ISO-8859-1;q=0.7,*;q=0.3", + "Accept-Language": "de-DE,de;q=0.8,en-US;q=0.6,en;q=0.4" +} + +# Internal variables +__api_queries = { + 'api_version': "1.0", + 'api_token': "None", + 'input': "3" +} +__cookies = None +__tracks_cache = {} +__albums_cache = {} + +def __getApiToken(): + global __cookies + response = request.request_response(url="http://www.deezer.com/", headers=__HTTP_HEADERS) + __cookies = response.cookies + data = response.content + if data: + matches = re.search(r"checkForm\s*=\s*['|\"](.*)[\"|'];", data) + if matches: + token = matches.group(1) + __api_queries['api_token'] = token + logger.debug(u"Deezloader : api token loeaded ('%s')" % token) + + if not token: + logger.error(u"Deezloader: Unable to get api token") + +def getAlbumByLink(album_link): + """Returns deezer album infos using album link url + + :param str album_link: deezer album link url (eg: 'http://www.deezer.com/album/1234567/') + """ + matches = re.search(r"album\/([0-9]+)\/?$", album_link) + if matches: + return getAlbum(matches.group(1)) + +def getAlbum(album_id): + """Returns deezer album infos + + :param int album_id: deezer album id + """ + global __albums_cache + + if str(album_id) in __albums_cache: + return __albums_cache[str(album_id)] + + url = __API_INFO_URL + "album/" + str(album_id) + data = request.request_json(url=url, headers=__HTTP_HEADERS, cookies=__cookies) + + if data and 'error' not in data: + __albums_cache[str(album_id)] = data + return data + else: + logger.debug("Deezloader: Can't load album infos") + return None + +def searchAlbums(search_term): + """Search for deezer albums using search term + + :param str search_term: search term to search album for + """ + logger.info(u'Searching Deezer using term: "%s"' % search_term) + + url = __API_INFO_URL + "search/album?q=" + search_term + data = request.request_json(url=url, headers=__HTTP_HEADERS, cookies=__cookies) + + albums = [] + + # Process content + if data and 'total' in data and data['total'] > 0 and 'data' in data: + for item in data['data']: + try: + albums.append(getAlbum(item['id'])) + except Exception as e: + logger.error(u"An unknown error occurred in the Deezer parser: %s" % e) + else: + logger.info(u'No results found from Deezer using term: "%s"' % search_term) + + return albums + +def __matchAlbums(albums, artist_name, album_title, album_length): + resultlist = [] + + for album in albums: + total_size = 0 + tracks_found = 0 + + for track in album['tracks']['data']: + t = getTrack(track['id']) + if t: + if t["FILESIZE_MP3_320"] > 0: + size = t["FILESIZE_MP3_320"] + elif t["FILESIZE_MP3_256"] > 0: + size = t["FILESIZE_MP3_256"] + elif t["FILESIZE_MP3_128"] > 0: + size = t["FILESIZE_MP3_128"] + else: + size = t["FILESIZE_MP3_64"] + + size = int(size) + total_size += size + tracks_found += 1 + logger.debug(u'Found song "%s". Size: %s' % (t['SNG_TITLE'], helpers.bytes_to_mb(size))) + + matched = True + mismatch_reason = 'matched!' + + if album_length > 0 and abs(int(album['duration']) - album_length) > 240: + matched = False + mismatch_reason = (u'duration mismatch: %i not in [%i, %i]' % (int(album['duration']), (album_length - 240), (album_length + 240))) + + elif (helpers.latinToAscii(re.sub(r"\W", "", album_title, 0, re.UNICODE)).lower() != + helpers.latinToAscii(re.sub(r"\W", "", album['title'], 0, re.UNICODE)).lower()): + matched = False + mismatch_reason = (u'album name mismatch: %s != %s' % (album['title'], album_title)) + + elif (helpers.latinToAscii(re.sub(r"\W", "", artist_name, 0, re.UNICODE)).lower() != + helpers.latinToAscii(re.sub(r"\W", "", album['artist']['name'], 0, re.UNICODE)).lower()): + matched = False + mismatch_reason = (u'artist name mismatch: %s != %s' % (album['artist']['name'], artist_name)) + + resultlist.append( + (album['artist']['name'] + ' - ' + album['title'] + ' [' + album['release_date'][:4] + '] (' + str(tracks_found) + '/' + str(album['nb_tracks']) + ')', + total_size, album['link'], PROVIDER_NAME, "ddl", matched) + ) + logger.info(u'Found "%s". Tracks %i/%i. Size: %s (%s)' % (album['title'], tracks_found, album['nb_tracks'], helpers.bytes_to_mb(total_size), mismatch_reason)) + + return resultlist + +def searchAlbum(artist_name, album_title, user_search_term=None, album_length=None): + """Search for deezer specific album. + This will iterate over deezer albums and try to find best matches + + :param str artist_name: album artist name + :param str album_title: album title + :param str user_search_term: search terms provided by user + :param int album_length: targeted album duration in seconds + """ + # User search term by-pass normal search + if user_search_term: + return __matchAlbums(searchAlbums(user_search_term), artist_name, album_title, album_length) + + resultlist = __matchAlbums(searchAlbums((artist_name + ' ' + album_title).strip()), artist_name, album_title, album_length) + if resultlist: + return resultlist + + # Deezer API supports unicode, so just remove non alphanumeric characters + clean_artist_name = re.sub(r"[^\w\s]", " ", artist_name, 0, re.UNICODE).strip() + clean_album_name = re.sub(r"[^\w\s]", " ", album_title, 0, re.UNICODE).strip() + + resultlist = __matchAlbums(searchAlbums((clean_artist_name + ' ' + clean_album_name).strip()), artist_name, album_title, album_length) + if resultlist: + return resultlist + + resultlist = __matchAlbums(searchAlbums(clean_artist_name), artist_name, album_title, album_length) + if resultlist: + return resultlist + + return resultlist + +def getTrack(sng_id, try_reload_api=True): + """Returns deezer track infos + + :param int sng_id: deezer song id + :param bool try_reload_api: whether or not try reloading API if session expired + """ + global __tracks_cache + + if str(sng_id) in __tracks_cache: + return __tracks_cache[str(sng_id)] + + data = "[{\"method\":\"song.getListData\",\"params\":{\"sng_ids\":[" + str(sng_id) + "]}}]" + json = request.request_json(url=__API_URL, headers=__HTTP_HEADERS, method='post', params=__api_queries, data=data, cookies=__cookies) + + results = [] + error = None + invalid_token = False + + if json: + # Check for errors + if 'error' in json: + error = json['error'] + if 'GATEWAY_ERROR' in json['error'] and json['error']['GATEWAY_ERROR'] == u"invalid api token": + invalid_token = True + + elif 'error' in json[0] and json[0]['error']: + error = json[0]['error'] + if 'VALID_TOKEN_REQUIRED' in json[0]['error'] and json[0]['error']['VALID_TOKEN_REQUIRED'] == u"Invalid CSRF token": + invalid_token = True + + # Got invalid token error + if error: + if invalid_token and try_reload_api: + __getApiToken() + return getTrack(sng_id, False) + else: + logger.error(u"An unknown error occurred in the Deezer parser: %s" % error) + else: + try: + results = json[0]['results'] + item = results['data'][0] + if 'token' in item: + logger.error(u"An unknown error occurred in the Deezer parser: Uploaded Files are currently not supported") + return + + sng_id = item["SNG_ID"] + md5Origin = item["MD5_ORIGIN"] + sng_format = 3 + + if item["FILESIZE_MP3_320"] <= 0: + if item["FILESIZE_MP3_256"] > 0: + sng_format = 5 + else: + sng_format = 1 + + mediaVersion = int(item["MEDIA_VERSION"]) + item['downloadUrl'] = __getDownloadUrl(md5Origin, sng_id, sng_format, mediaVersion) + + __tracks_cache[sng_id] = item + return item + + except Exception as e: + logger.error(u"An unknown error occurred in the Deezer parser: %s" % e) + +def __getDownloadUrl(md5Origin, sng_id, sng_format, mediaVersion): + urlPart = md5Origin.encode('utf-8') + b'\xa4' + str(sng_format) + b'\xa4' + str(sng_id) + b'\xa4' + str(mediaVersion) + md5val = md5(urlPart).hexdigest() + urlPart = md5val + b'\xa4' + urlPart + b'\xa4' + cipher = AES.new('jo6aey6haid2Teih', AES.MODE_ECB) + ciphertext = cipher.encrypt(__pad(urlPart, AES.block_size)) + return "http://e-cdn-proxy-" + md5Origin[:1] + ".deezer.com/mobile/1/" + binascii.hexlify(ciphertext).lower() + +def __pad(raw, block_size): + if (len(raw) % block_size == 0): + return raw + padding_required = block_size - (len(raw) % block_size) + padChar = b'\x00' + data = raw + padding_required * padChar + return data + +def __tagTrack(path, track): + try: + album = getAlbum(track['ALB_ID']) + + f = MediaFile(path) + f.artist = track['ART_NAME'] + f.album = track['ALB_TITLE'] + f.title = track['SNG_TITLE'] + f.track = track['TRACK_NUMBER'] + f.tracktotal = album['nb_tracks'] + f.disc = track['DISK_NUMBER'] + f.bpm = track['BPM'] + f.date = datetime.strptime(album['release_date'], '%Y-%m-%d').date() + f.albumartist = album['artist']['name'] + if u'genres' in album and u'data' in album['genres']: + f.genres = [genre['name'] for genre in album['genres']['data']] + + f.save() + + except Exception as e: + logger.error(u'Unable to tag deezer track "%s": %s' % (path, e)) + +def decryptTracks(paths): + """Decrypt downloaded deezer tracks. + + :param paths: list of path to deezer tracks (*.dzr files). + """ + # Note: tracks can be from different albums + decrypted_tracks = {} + + # First pass: load tracks data + for path in paths: + try: + album_folder = os.path.dirname(path) + sng_id = os.path.splitext(os.path.basename(path))[0] + track = getTrack(sng_id) + track_number = int(track['TRACK_NUMBER']) + disk_number = int(track['DISK_NUMBER']) + + if album_folder not in decrypted_tracks: + decrypted_tracks[album_folder] = {} + + if disk_number not in decrypted_tracks[album_folder]: + decrypted_tracks[album_folder][disk_number] = {} + + decrypted_tracks[album_folder][disk_number][track_number] = track + + except Exception as e: + logger.error(u'Unable to load deezer track infos "%s": %s' % (path, e)) + + # Second pass: decrypt tracks + for album_folder in decrypted_tracks: + multi_disks = len(decrypted_tracks[album_folder]) > 1 + for disk_number in decrypted_tracks[album_folder]: + for track_number, track in decrypted_tracks[album_folder][disk_number].items(): + try: + filename = helpers.replace_illegal_chars(track['SNG_TITLE']).strip() + filename = ('{:02d}'.format(track_number) + '_' + filename + '.mp3') + + # Add a 'cd x' sub-folder if album has more than one disk + disk_folder = os.path.join(album_folder, 'cd ' + str(disk_number)) if multi_disks else album_folder + + dest = os.path.join(disk_folder, filename).encode(headphones.SYS_ENCODING, 'replace') + + # Decrypt track if not already done + if not os.path.exists(dest): + try: + __decryptDownload(path, sng_id, dest) + __tagTrack(dest, track) + except Exception as e: + logger.error(u'Unable to decrypt deezer track "%s": %s' % (path, e)) + if os.path.exists(dest): + os.remove(dest) + decrypted_tracks[album_folder][disk_number].pop(track_number) + continue + + decrypted_tracks[album_folder][disk_number][track_number]['path'] = dest + + except Exception as e: + logger.error(u'Unable to decrypt deezer track "%s": %s' % (path, e)) + + return decrypted_tracks + +def __decryptDownload(source, sng_id, dest): + interval_chunk = 3 + chunk_size = 2048 + blowFishKey = __getBlowFishKey(sng_id) + i = 0 + iv = "\x00\x01\x02\x03\x04\x05\x06\x07" + + dest_folder = os.path.dirname(dest) + if not os.path.exists(dest_folder): + os.makedirs(dest_folder) + + f = open(source, "rb") + fout = open(dest, "wb") + try: + chunk = f.read(chunk_size) + while chunk: + if(i % interval_chunk == 0): + cipher = Blowfish.new(blowFishKey, Blowfish.MODE_CBC, iv) + chunk = cipher.decrypt(__pad(chunk, Blowfish.block_size)) + + fout.write(chunk) + i += 1 + chunk = f.read(chunk_size) + finally: + f.close() + fout.close() + +def __getBlowFishKey(encryptionKey): + if encryptionKey < 1: + encryptionKey *= -1 + + hashcode = md5(str(encryptionKey)).hexdigest() + hPart = hashcode[0:16] + lPart = hashcode[16:32] + parts = ['g4el58wc0zvf9na1', hPart, lPart] + + return __xorHex(parts) + +def __xorHex(parts): + data = "" + for i in range(0, 16): + character = ord(parts[0][i]) + + for j in range(1, len(parts)): + character ^= ord(parts[j][i]) + + data += chr(character) + + return data diff --git a/headphones/postprocessor.py b/headphones/postprocessor.py index e3deee23..336030af 100755 --- a/headphones/postprocessor.py +++ b/headphones/postprocessor.py @@ -29,7 +29,7 @@ from beetsplug import lyrics as beetslyrics from headphones import notifiers, utorrent, transmission, deluge, qbittorrent from headphones import db, albumart, librarysync from headphones import logger, helpers, mb, music_encoder -from headphones import metadata +from headphones import metadata, deezloader postprocessor_lock = threading.Lock() @@ -47,6 +47,8 @@ def checkFolder(): single = False if album['Kind'] == 'nzb': download_dir = headphones.CONFIG.DOWNLOAD_DIR + elif album['Kind'] == 'ddl': + download_dir = headphones.CONFIG.DOWNLOAD_DDL_DIR else: if headphones.CONFIG.DELUGE_DONE_DIRECTORY and headphones.CONFIG.TORRENT_DOWNLOADER == 3: download_dir = headphones.CONFIG.DELUGE_DONE_DIRECTORY @@ -204,6 +206,7 @@ def verify(albumid, albumpath, Kind=None, forced=False, keep_original_folder=Fal tracks = myDB.select('SELECT * from tracks WHERE AlbumID=?', [albumid]) downloaded_track_list = [] + downloaded_deezer_list = [] downloaded_cuecount = 0 for r, d, f in os.walk(albumpath): @@ -212,8 +215,10 @@ def verify(albumid, albumpath, Kind=None, forced=False, keep_original_folder=Fal downloaded_track_list.append(os.path.join(r, files)) elif files.lower().endswith('.cue'): downloaded_cuecount += 1 + elif files.lower().endswith('.dzr'): + downloaded_deezer_list.append(os.path.join(r, files)) # if any of the files end in *.part, we know the torrent isn't done yet. Process if forced, though - elif files.lower().endswith(('.part', '.utpart')) and not forced: + elif files.lower().endswith(('.part', '.utpart', '.aria2')) and not forced: logger.info( "Looks like " + os.path.basename(albumpath).decode(headphones.SYS_ENCODING, 'replace') + " isn't complete yet. Will try again on the next run") @@ -252,6 +257,37 @@ def verify(albumid, albumpath, Kind=None, forced=False, keep_original_folder=Fal downloaded_track_list = helpers.get_downloaded_track_list(albumpath) keep_original_folder = False + # Decrypt deezer tracks + if downloaded_deezer_list: + logger.info('Decrypting deezer tracks') + decrypted_deezer_list = deezloader.decryptTracks(downloaded_deezer_list) + + # Check if album is complete based on album duration only + # (total track numbers is not determinant enough due to hidden tracks for eg) + db_track_duration = 0 + downloaded_track_duration = 0 + try: + for track in tracks: + db_track_duration += track['TrackDuration'] / 1000 + except: + downloaded_track_duration = False + + try: + for disk_number in decrypted_deezer_list[albumpath]: + for track in decrypted_deezer_list[albumpath][disk_number].values(): + downloaded_track_list.append(track['path']) + downloaded_track_duration += int(track['DURATION']) + except: + downloaded_track_duration = False + + if not downloaded_track_duration or not db_track_duration or abs(downloaded_track_duration - db_track_duration) > 240: + logger.info("Looks like " + + os.path.basename(albumpath).decode(headphones.SYS_ENCODING, 'replace') + + " isn't complete yet (duration mismatch). Will try again on the next run") + return + + downloaded_track_list = list(set(downloaded_track_list)) # Remove duplicates + # test #1: metadata - usually works logger.debug('Verifying metadata...') diff --git a/headphones/searcher.py b/headphones/searcher.py index b0652f7f..2516bc31 100644 --- a/headphones/searcher.py +++ b/headphones/searcher.py @@ -36,6 +36,7 @@ import headphones from headphones.common import USER_AGENT from headphones import logger, db, helpers, classes, sab, nzbget, request from headphones import utorrent, transmission, notifiers, rutracker, deluge, qbittorrent +from headphones import deezloader, aria2 from bencode import bencode, bdecode # Magnet to torrent services, for Black hole. Stolen from CouchPotato. @@ -51,6 +52,24 @@ ruobj = None # Persistent RED API object redobj = None +# Persistent Aria2 RPC object +__aria2rpc_obj = None + +def getAria2RPC(): + global __aria2rpc_obj + if not __aria2rpc_obj: + __aria2rpc_obj = aria2.Aria2JsonRpc( + ID='headphones', + uri=headphones.CONFIG.ARIA_HOST, + token=headphones.CONFIG.ARIA_TOKEN if headphones.CONFIG.ARIA_TOKEN else None, + http_user=headphones.CONFIG.ARIA_USERNAME if headphones.CONFIG.ARIA_USERNAME else None, + http_passwd=headphones.CONFIG.ARIA_PASSWORD if headphones.CONFIG.ARIA_PASSWORD else None + ) + return __aria2rpc_obj + +def reconfigure(): + global __aria2rpc_obj + __aria2rpc_obj = None def fix_url(s, charset="utf-8"): """ @@ -281,32 +300,53 @@ def do_sorted_search(album, new, losslessOnly, choose_specific_download=False): headphones.CONFIG.STRIKE or headphones.CONFIG.TQUATTRECENTONZE) - results = [] + DDL_PROVIDERS = (headphones.CONFIG.DEEZLOADER) + myDB = db.DBConnection() albumlength = myDB.select('SELECT sum(TrackDuration) from tracks WHERE AlbumID=?', [album['AlbumID']])[0][0] + nzb_results = None + torrent_results = None + ddl_results = None + if headphones.CONFIG.PREFER_TORRENTS == 0 and not choose_specific_download: if NZB_PROVIDERS and NZB_DOWNLOADERS: - results = searchNZB(album, new, losslessOnly, albumlength) + nzb_results = searchNZB(album, new, losslessOnly, albumlength) - if not results and TORRENT_PROVIDERS: - results = searchTorrent(album, new, losslessOnly, albumlength) + if not nzb_results: + if DDL_PROVIDERS: + ddl_results = searchDdl(album, new, losslessOnly, albumlength) + + if TORRENT_PROVIDERS: + torrent_results = searchTorrent(album, new, losslessOnly, albumlength) elif headphones.CONFIG.PREFER_TORRENTS == 1 and not choose_specific_download: if TORRENT_PROVIDERS: - results = searchTorrent(album, new, losslessOnly, albumlength) + torrent_results = searchTorrent(album, new, losslessOnly, albumlength) - if not results and NZB_PROVIDERS and NZB_DOWNLOADERS: - results = searchNZB(album, new, losslessOnly, albumlength) + if not torrent_results: + if DDL_PROVIDERS: + ddl_results = searchDdl(album, new, losslessOnly, albumlength) + + if NZB_PROVIDERS and NZB_DOWNLOADERS: + nzb_results = searchNZB(album, new, losslessOnly, albumlength) + + elif headphones.CONFIG.PREFER_TORRENTS == 3 and not choose_specific_download: + + if DDL_PROVIDERS: + ddl_results = searchDdl(album, new, losslessOnly, albumlength) + + if not ddl_results: + if TORRENT_PROVIDERS: + torrent_results = searchTorrent(album, new, losslessOnly, albumlength) + + if NZB_PROVIDERS and NZB_DOWNLOADERS: + nzb_results = searchNZB(album, new, losslessOnly, albumlength) else: - - nzb_results = None - torrent_results = None - if NZB_PROVIDERS and NZB_DOWNLOADERS: nzb_results = searchNZB(album, new, losslessOnly, albumlength, choose_specific_download) @@ -314,13 +354,19 @@ def do_sorted_search(album, new, losslessOnly, choose_specific_download=False): torrent_results = searchTorrent(album, new, losslessOnly, albumlength, choose_specific_download) - if not nzb_results: - nzb_results = [] + if DDL_PROVIDERS: + ddl_results = searchDdl(album, new, losslessOnly, albumlength, choose_specific_download) - if not torrent_results: - torrent_results = [] + if not nzb_results: + nzb_results = [] - results = nzb_results + torrent_results + if not torrent_results: + torrent_results = [] + + if not ddl_results: + ddl_results = [] + + results = nzb_results + torrent_results + ddl_results if choose_specific_download: return results @@ -826,6 +872,31 @@ def send_to_downloader(data, bestqual, album): except Exception as e: logger.error('Couldn\'t write NZB file: %s', e) return + + elif kind == 'ddl': + folder_name = '%s - %s [%s]' % ( + helpers.latinToAscii(album['ArtistName']).encode('UTF-8').replace('/', '_'), + helpers.latinToAscii(album['AlbumTitle']).encode('UTF-8').replace('/', '_'), + get_year_from_release_date(album['ReleaseDate'])) + + # Aria2 downloader + if headphones.CONFIG.DDL_DOWNLOADER == 0: + logger.info("Sending download to Aria2") + + try: + deezer_album = deezloader.getAlbumByLink(bestqual[2]) + + for album_track in deezer_album['tracks']['data']: + track = deezloader.getTrack(album_track['id']) + if track: + filename = track['SNG_ID'] + '.dzr' + logger.debug(u'Sending song "%s" to Aria' % track['SNG_TITLE']) + getAria2RPC().addUri([track['downloadUrl']], {'out': filename, 'auto-file-renaming': 'false', 'continue': 'true', 'dir': folder_name}) + + except Exception as e: + logger.error(u'Error sending torrent to Aria2. Are you sure it\'s running? (%s)' % e) + return + else: folder_name = '%s - %s [%s]' % ( helpers.latinToAscii(album['ArtistName']).encode('UTF-8').replace('/', '_'), @@ -1203,6 +1274,61 @@ def verifyresult(title, artistterm, term, lossless): return True +def searchDdl(album, new=False, losslessOnly=False, albumlength=None, + choose_specific_download=False): + reldate = album['ReleaseDate'] + year = get_year_from_release_date(reldate) + + # MERGE THIS WITH THE TERM CLEANUP FROM searchNZB + dic = {'...': '', ' & ': ' ', ' = ': ' ', '?': '', '$': 's', ' + ': ' ', '"': '', ',': ' ', + '*': ''} + + semi_cleanalbum = helpers.replace_all(album['AlbumTitle'], dic) + cleanalbum = helpers.latinToAscii(semi_cleanalbum) + semi_cleanartist = helpers.replace_all(album['ArtistName'], dic) + cleanartist = helpers.latinToAscii(semi_cleanartist) + + # Use provided term if available, otherwise build our own (this code needs to be cleaned up since a lot + # of these torrent providers are just using cleanartist/cleanalbum terms + if album['SearchTerm']: + term = album['SearchTerm'] + elif album['Type'] == 'part of': + term = cleanalbum + " " + year + else: + # FLAC usually doesn't have a year for some reason so I'll leave it out + # Various Artist albums might be listed as VA, so I'll leave that out too + # Only use the year if the term could return a bunch of different albums, i.e. self-titled albums + if album['ArtistName'] in album['AlbumTitle'] or len(album['ArtistName']) < 4 or len( + album['AlbumTitle']) < 4: + term = cleanartist + ' ' + cleanalbum + ' ' + year + elif album['ArtistName'] == 'Various Artists': + term = cleanalbum + ' ' + year + else: + term = cleanartist + ' ' + cleanalbum + + # Replace bad characters in the term and unicode it + term = re.sub('[\.\-\/]', ' ', term).encode('utf-8') + artistterm = re.sub('[\.\-\/]', ' ', cleanartist).encode('utf-8', 'replace') + + logger.debug(u'Using search term: "%s"' % helpers.latinToAscii(term)) + + resultlist = [] + + # Deezer only provides lossy + if headphones.CONFIG.DEEZLOADER and not losslessOnly: + resultlist += deezloader.searchAlbum(album['ArtistName'], album['AlbumTitle'], album['SearchTerm'], int(albumlength / 1000)) + + # attempt to verify that this isn't a substring result + # when looking for "Foo - Foo" we don't want "Foobar" + # this should be less of an issue when it isn't a self-titled album so we'll only check vs artist + results = [result for result in resultlist if verifyresult(result[0], artistterm, term, losslessOnly)] + + # Additional filtering for size etc + if results and not choose_specific_download: + results = more_filtering(results, album, albumlength, new) + + return results + def searchTorrent(album, new=False, losslessOnly=False, albumlength=None, choose_specific_download=False): @@ -2036,7 +2162,10 @@ def searchTorrent(album, new=False, losslessOnly=False, albumlength=None, def preprocess(resultlist): for result in resultlist: - if result[4] == 'torrent': + if result[3] == deezloader.PROVIDER_NAME: + return True, result + + elif result[4] == 'torrent': # rutracker always needs the torrent data if result[3] == 'rutracker.org': diff --git a/headphones/webserve.py b/headphones/webserve.py index 855fed24..19e2ffb3 100644 --- a/headphones/webserve.py +++ b/headphones/webserve.py @@ -1174,6 +1174,10 @@ class WebInterface(object): "utorrent_username": headphones.CONFIG.UTORRENT_USERNAME, "utorrent_password": headphones.CONFIG.UTORRENT_PASSWORD, "utorrent_label": headphones.CONFIG.UTORRENT_LABEL, + "aria_host": headphones.CONFIG.ARIA_HOST, + "aria_password": headphones.CONFIG.ARIA_PASSWORD, + "aria_token": headphones.CONFIG.ARIA_TOKEN, + "aria_username": headphones.CONFIG.ARIA_USERNAME, "nzb_downloader_sabnzbd": radio(headphones.CONFIG.NZB_DOWNLOADER, 0), "nzb_downloader_nzbget": radio(headphones.CONFIG.NZB_DOWNLOADER, 1), "nzb_downloader_blackhole": radio(headphones.CONFIG.NZB_DOWNLOADER, 2), @@ -1182,6 +1186,7 @@ class WebInterface(object): "torrent_downloader_utorrent": radio(headphones.CONFIG.TORRENT_DOWNLOADER, 2), "torrent_downloader_deluge": radio(headphones.CONFIG.TORRENT_DOWNLOADER, 3), "torrent_downloader_qbittorrent": radio(headphones.CONFIG.TORRENT_DOWNLOADER, 4), + "ddl_downloader_aria": radio(headphones.CONFIG.DDL_DOWNLOADER, 0), "download_dir": headphones.CONFIG.DOWNLOAD_DIR, "use_blackhole": checked(headphones.CONFIG.BLACKHOLE), "blackhole_dir": headphones.CONFIG.BLACKHOLE_DIR, @@ -1243,6 +1248,8 @@ class WebInterface(object): "use_tquattrecentonze": checked(headphones.CONFIG.TQUATTRECENTONZE), "tquattrecentonze_user": headphones.CONFIG.TQUATTRECENTONZE_USER, "tquattrecentonze_password": headphones.CONFIG.TQUATTRECENTONZE_PASSWORD, + "download_ddl_dir": headphones.CONFIG.DOWNLOAD_DDL_DIR, + "use_deezloader": checked(headphones.CONFIG.DEEZLOADER), "pref_qual_0": radio(headphones.CONFIG.PREFERRED_QUALITY, 0), "pref_qual_1": radio(headphones.CONFIG.PREFERRED_QUALITY, 1), "pref_qual_2": radio(headphones.CONFIG.PREFERRED_QUALITY, 2), @@ -1288,6 +1295,7 @@ class WebInterface(object): "prefer_torrents_0": radio(headphones.CONFIG.PREFER_TORRENTS, 0), "prefer_torrents_1": radio(headphones.CONFIG.PREFER_TORRENTS, 1), "prefer_torrents_2": radio(headphones.CONFIG.PREFER_TORRENTS, 2), + "prefer_torrents_3": radio(headphones.CONFIG.PREFER_TORRENTS, 3), "magnet_links_0": radio(headphones.CONFIG.MAGNET_LINKS, 0), "magnet_links_1": radio(headphones.CONFIG.MAGNET_LINKS, 1), "magnet_links_2": radio(headphones.CONFIG.MAGNET_LINKS, 2), @@ -1446,7 +1454,7 @@ class WebInterface(object): "launch_browser", "enable_https", "api_enabled", "use_blackhole", "headphones_indexer", "use_newznab", "newznab_enabled", "use_torznab", "torznab_enabled", "use_nzbsorg", "use_omgwtfnzbs", "use_kat", "use_piratebay", "use_oldpiratebay", - "use_mininova", "use_waffles", "use_rutracker", + "use_mininova", "use_waffles", "use_rutracker", "use_deezloader", "use_apollo", "use_redacted", "use_strike", "use_tquattrecentonze", "preferred_bitrate_allow_lossless", "detect_bitrate", "ignore_clean_releases", "freeze_db", "cue_split", "move_files", "rename_files", "correct_metadata", "cleanup_files", "keep_nfo", "add_album_art", @@ -1579,6 +1587,9 @@ class WebInterface(object): # Reconfigure musicbrainz database connection with the new values mb.startmb() + + # Reconfigure Aria2 + searcher.reconfigure() raise cherrypy.HTTPRedirect("config") From 34b8a11145e7d8150f0008da2cd79bae601b165d Mon Sep 17 00:00:00 2001 From: Kallys Date: Wed, 19 Apr 2017 19:55:41 +0200 Subject: [PATCH 03/27] Fix travis errors --- headphones/aria2.py | 1798 ++++++++++++++++------------------- headphones/deezloader.py | 117 ++- headphones/helpers_test.py | 4 +- headphones/postprocessor.py | 10 +- headphones/searcher.py | 8 +- headphones/webserve.py | 2 +- 6 files changed, 920 insertions(+), 1019 deletions(-) diff --git a/headphones/aria2.py b/headphones/aria2.py index 68c4cb0a..06d8e4e0 100644 --- a/headphones/aria2.py +++ b/headphones/aria2.py @@ -27,7 +27,7 @@ import urllib2 from headphones import logger -################################## Constants ################################### +# ################################ Constants ################################### DEFAULT_PORT = 6800 SERVER_URI_FORMAT = '{}://{}:{:d}/jsonrpc' @@ -39,1057 +39,941 @@ FINAL_STATUS = ('complete', 'error') ARIA2_CONTROL_FILE_EXT = '.aria2' -############################ Convenience Functions ############################# +# ########################## Convenience Functions ############################# + def to_json_list(objs): - ''' - Wrap strings in lists. Other iterables are converted to lists directly. - ''' - if isinstance(objs, str): - return [objs] - elif not isinstance(objs, list): - return list(objs) - else: - return objs - + ''' + Wrap strings in lists. Other iterables are converted to lists directly. + ''' + if isinstance(objs, str): + return [objs] + elif not isinstance(objs, list): + return list(objs) + else: + return objs def add_options_and_position(params, options=None, position=None): - ''' - Convenience method for adding options and position to parameters. - ''' - if options: - params.append(options) - if position: - if not isinstance(position, int): - try: - position = int(position) - except ValueError: - position = -1 - if position >= 0: - params.append(position) - return params - + ''' + Convenience method for adding options and position to parameters. + ''' + if options: + params.append(options) + if position: + if not isinstance(position, int): + try: + position = int(position) + except ValueError: + position = -1 + if position >= 0: + params.append(position) + return params def get_status(response): - ''' - Process a status response. - ''' - if response: - try: - return response['status'] - except KeyError: - logger.error('no status returned from Aria2 RPC server') - return 'error' - else: - logger.error('no response from server') - return 'error' - + ''' + Process a status response. + ''' + if response: + try: + return response['status'] + except KeyError: + logger.error('no status returned from Aria2 RPC server') + return 'error' + else: + logger.error('no response from server') + return 'error' def random_token(length, valid_chars=None): - ''' - Get a random secret token for the Aria2 RPC server. + ''' + Get a random secret token for the Aria2 RPC server. - length: - The length of the token + length: + The length of the token - valid_chars: - A list or other ordered and indexable iterable of valid characters. If not - given of None, asciinumberic characters with some punctuation characters - will be used. - ''' - if not valid_chars: - valid_chars = string.ascii_letters + string.digits + '!@#$%^&*()-_=+' - number_of_chars = len(valid_chars) - bytes_to_read = math.ceil(math.log(number_of_chars) / math.log(0x100)) - max_value = 0x100**bytes_to_read - max_index = number_of_chars - 1 - token = '' - for _ in range(length): - value = int.from_bytes(os.urandom(bytes_to_read), byteorder='little') - index = round((value * max_index)/max_value) - token += valid_chars[index] - return token + valid_chars: + A list or other ordered and indexable iterable of valid characters. If not + given of None, asciinumberic characters with some punctuation characters + will be used. + ''' + if not valid_chars: + valid_chars = string.ascii_letters + string.digits + '!@#$%^&*()-_=+' + number_of_chars = len(valid_chars) + bytes_to_read = math.ceil(math.log(number_of_chars) / math.log(0x100)) + max_value = 0x100**bytes_to_read + max_index = number_of_chars - 1 + token = '' + for _ in range(length): + value = int.from_bytes(os.urandom(bytes_to_read), byteorder='little') + index = round((value * max_index) / max_value) + token += valid_chars[index] + return token +# ################ From python3-aur's ThreadedServers.common ################### -################## From python3-aur's ThreadedServers.common ################### def format_bytes(size): - '''Format bytes for inferior humans.''' - if size < 0x400: - return '{:d} B'.format(size) - else: - size = float(size) / 0x400 - for prefix in ('KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB'): - if size < 0x400: - return '{:0.02f} {}'.format(size, prefix) - else: - size /= 0x400 - return '{:0.02f} YiB'.format(size) - + '''Format bytes for inferior humans.''' + if size < 0x400: + return '{:d} B'.format(size) + else: + size = float(size) / 0x400 + for prefix in ('KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB'): + if size < 0x400: + return '{:0.02f} {}'.format(size, prefix) + else: + size /= 0x400 + return '{:0.02f} YiB'.format(size) def format_seconds(s): - '''Format seconds for inferior humans.''' - string = '' - for base, char in ( - (60, 's'), - (60, 'm'), - (24, 'h') - ): - s, r = divmod(s, base) - if s == 0: - return '{:d}{}{}'.format(r, char, string) - elif r != 0: - string = '{:02d}{}{}'.format(r, char, string) - else: - return '{:d}d{}'.format(s, string) + '''Format seconds for inferior humans.''' + string = '' + for base, char in ( + (60, 's'), + (60, 'm'), + (24, 'h') + ): + s, r = divmod(s, base) + if s == 0: + return '{:d}{}{}'.format(r, char, string) + elif r != 0: + string = '{:02d}{}{}'.format(r, char, string) + else: + return '{:d}d{}'.format(s, string) + +# ############################ Aria2JsonRpcError ############################### -############################## Aria2JsonRpcError ############################### class Aria2JsonRpcError(Exception): - def __init__(self, msg, connection_error=False): - super(self.__class__, self).__init__(self, msg) - self.connection_error = connection_error + def __init__(self, msg, connection_error=False): + super(self.__class__, self).__init__(self, msg) + self.connection_error = connection_error + +# ############################ Aria2JsonRpc Class ############################## -############################## Aria2JsonRpc Class ############################## class Aria2JsonRpc(object): - ''' - Interface class for interacting with an Aria2 RPC server. - ''' - # TODO: certificate options, etc. - def __init__( - self, ID, uri, - mode='normal', - token=None, - http_user=None, http_passwd=None, - server_cert=None, client_cert=None, client_cert_password=None, - ssl_protocol=None, - setup_function=None - ): - ''' - ID: the ID to send to the RPC interface + ''' + Interface class for interacting with an Aria2 RPC server. + ''' + # TODO: certificate options, etc. + def __init__( + self, ID, uri, + mode='normal', + token=None, + http_user=None, http_passwd=None, + server_cert=None, client_cert=None, client_cert_password=None, + ssl_protocol=None, + setup_function=None + ): + ''' + ID: the ID to send to the RPC interface - uri: the URI of the RPC interface + uri: the URI of the RPC interface - mode: - normal - process requests immediately - batch - queue requests (run with "process_queue") - format - return RPC request objects + mode: + normal - process requests immediately + batch - queue requests (run with "process_queue") + format - return RPC request objects - token: - RPC method-level authorization token (set using `--rpc-secret`) + token: + RPC method-level authorization token (set using `--rpc-secret`) - http_user, http_password: - HTTP Basic authentication credentials (deprecated) + http_user, http_password: + HTTP Basic authentication credentials (deprecated) - server_cert: - server certificate for HTTPS connections + server_cert: + server certificate for HTTPS connections - client_cert: - client certificate for HTTPS connections + client_cert: + client certificate for HTTPS connections - client_cert_password: - prompt for client certificate password + client_cert_password: + prompt for client certificate password - ssl_protocol: - SSL protocol from the ssl module + ssl_protocol: + SSL protocol from the ssl module - setup_function: - A function to invoke prior to the first server call. This could be the - launch() method of an Aria2RpcServer instance, for example. This attribute - is set automatically in instances returned from Aria2RpcServer.get_a2jr() - ''' - self.id = ID - self.uri = uri - self.mode = mode - self.queue = [] - self.handlers = dict() - self.token = token - self.setup_function = setup_function + setup_function: + A function to invoke prior to the first server call. This could be the + launch() method of an Aria2RpcServer instance, for example. This attribute + is set automatically in instances returned from Aria2RpcServer.get_a2jr() + ''' + self.id = ID + self.uri = uri + self.mode = mode + self.queue = [] + self.handlers = dict() + self.token = token + self.setup_function = setup_function - if None not in (http_user, http_passwd): - self.add_HTTPBasicAuthHandler(http_user, http_passwd) + if None not in (http_user, http_passwd): + self.add_HTTPBasicAuthHandler(http_user, http_passwd) - if server_cert or client_cert: - self.add_HTTPSHandler( - server_cert=server_cert, - client_cert=client_cert, - client_cert_password=client_cert_password, - protocol=ssl_protocol - ) + if server_cert or client_cert: + self.add_HTTPSHandler( + server_cert=server_cert, + client_cert=client_cert, + client_cert_password=client_cert_password, + protocol=ssl_protocol + ) - self.update_opener() + self.update_opener() + def iter_handlers(self): + ''' + Iterate over handlers. + ''' + for name in ('HTTPS', 'HTTPBasicAuth'): + try: + yield self.handlers[name] + except KeyError: + pass + def update_opener(self): + ''' + Build an opener from the current handlers. + ''' + self.opener = urllib2.build_opener(*self.iter_handlers()) - def iter_handlers(self): - ''' - Iterate over handlers. - ''' - for name in ('HTTPS', 'HTTPBasicAuth'): - try: - yield self.handlers[name] - except KeyError: - pass + def remove_handler(self, name): + ''' + Remove a handler. + ''' + try: + del self.handlers[name] + except KeyError: + pass + def add_HTTPBasicAuthHandler(self, user, passwd): + ''' + Add a handler for HTTP Basic authentication. + If either user or passwd are None, the handler is removed. + ''' + handler = urllib2.HTTPBasicAuthHandler() + handler.add_password( + realm='aria2', + uri=self.uri, + user=user, + passwd=passwd, + ) + self.handlers['HTTPBasicAuth'] = handler - def update_opener(self): - ''' - Build an opener from the current handlers. - ''' - self.opener = urllib2.build_opener(*self.iter_handlers()) + def remove_HTTPBasicAuthHandler(self): + self.remove_handler('HTTPBasicAuth') - - - def remove_handler(self, name): - ''' - Remove a handler. - ''' - try: - del self.handlers[name] - except KeyError: - pass - - - - def add_HTTPBasicAuthHandler(self, user, passwd): - ''' - Add a handler for HTTP Basic authentication. - - If either user or passwd are None, the handler is removed. - ''' - handler = urllib2.HTTPBasicAuthHandler() - handler.add_password( - realm='aria2', - uri=self.uri, - user=user, - passwd=passwd, - ) - self.handlers['HTTPBasicAuth'] = handler - - - - def remove_HTTPBasicAuthHandler(self): - self.remove_handler('HTTPBasicAuth') - - - - def add_HTTPSHandler( - self, - server_cert=None, - client_cert=None, - client_cert_password=None, - protocol=None, - ): - ''' - Add a handler for HTTPS connections with optional server and client - certificates. - ''' - if not protocol: - protocol = ssl.PROTOCOL_TLSv1 + def add_HTTPSHandler( + self, + server_cert=None, + client_cert=None, + client_cert_password=None, + protocol=None, + ): + ''' + Add a handler for HTTPS connections with optional server and client + certificates. + ''' + if not protocol: + protocol = ssl.PROTOCOL_TLSv1 # protocol = ssl.PROTOCOL_TLSv1_1 # openssl 1.0.1+ # protocol = ssl.PROTOCOL_TLSv1_2 # Python 3.4+ - context = ssl.SSLContext(protocol) - - if server_cert: - context.verify_mode = ssl.CERT_REQUIRED - context.load_verify_locations(cafile=server_cert) - else: - context.verify_mode = ssl.CERT_OPTIONAL - - if client_cert: - context.load_cert_chain(client_cert, password=client_cert_password) - - self.handlers['HTTPS'] = urllib2.HTTPSHandler( - context=context, - check_hostname=False - ) - - - - def remove_HTTPSHandler(self): - self.remove_handler('HTTPS') - - - - def send_request(self, req_obj): - ''' - Send the request and return the response. - ''' - if self.setup_function: - self.setup_function() - self.setup_function = None - logger.debug("Aria req_obj: %s" % json.dumps(req_obj, indent=2, sort_keys=True)) - req = json.dumps(req_obj).encode('UTF-8') - try: - f = self.opener.open(self.uri, req) - obj = json.loads(f.read()) - try: - return obj['result'] - except KeyError: - raise Aria2JsonRpcError('unexpected result: {}'.format(obj)) - except (urllib2.URLError) as e: - # This should work but URLError does not set the errno attribute: - # e.errno == errno.ECONNREFUSED - raise Aria2JsonRpcError( - str(e), - connection_error=( - '111' in str(e) - ) - ) - except httplib.BadStatusLine as e: - raise Aria2JsonRpcError('{}: BadStatusLine: {} (HTTPS error?)'.format( - self.__class__.__name__, e - )) + context = ssl.SSLContext(protocol) + + if server_cert: + context.verify_mode = ssl.CERT_REQUIRED + context.load_verify_locations(cafile=server_cert) + else: + context.verify_mode = ssl.CERT_OPTIONAL + + if client_cert: + context.load_cert_chain(client_cert, password=client_cert_password) + + self.handlers['HTTPS'] = urllib2.HTTPSHandler( + context=context, + check_hostname=False + ) + + def remove_HTTPSHandler(self): + self.remove_handler('HTTPS') + + def send_request(self, req_obj): + ''' + Send the request and return the response. + ''' + if self.setup_function: + self.setup_function() + self.setup_function = None + logger.debug("Aria req_obj: %s" % json.dumps(req_obj, indent=2, sort_keys=True)) + req = json.dumps(req_obj).encode('UTF-8') + try: + f = self.opener.open(self.uri, req) + obj = json.loads(f.read()) + try: + return obj['result'] + except KeyError: + raise Aria2JsonRpcError('unexpected result: {}'.format(obj)) + except (urllib2.URLError) as e: + # This should work but URLError does not set the errno attribute: + # e.errno == errno.ECONNREFUSED + raise Aria2JsonRpcError( + str(e), + connection_error=( + '111' in str(e) + ) + ) + except httplib.BadStatusLine as e: + raise Aria2JsonRpcError('{}: BadStatusLine: {} (HTTPS error?)'.format( + self.__class__.__name__, e + )) + + def jsonrpc(self, method, params=None, prefix='aria2.'): + ''' + POST a request to the RPC interface. + ''' + if not params: + params = [] + + if self.token is not None: + token_str = 'token:{}'.format(self.token) + if method == 'multicall': + for p in params[0]: + try: + p['params'].insert(0, token_str) + except KeyError: + p['params'] = [token_str] + else: + params.insert(0, token_str) + + req_obj = { + 'jsonrpc': '2.0', + 'id': self.id, + 'method': prefix + method, + 'params': params, + } + if self.mode == 'batch': + self.queue.append(req_obj) + return None + elif self.mode == 'format': + return req_obj + else: + return self.send_request(req_obj) + + def process_queue(self): + ''' + Processed queued requests. + ''' + req_obj = self.queue + self.queue = [] + return self.send_request(req_obj) + + +# ############################# Standard Methods ############################### + def addUri(self, uris, options=None, position=None): + ''' + aria2.addUri method + + uris: list of URIs + + options: dictionary of additional options + + position: position in queue + + Returns a GID + ''' + params = [uris] + params = add_options_and_position(params, options, position) + return self.jsonrpc('addUri', params) + + def addTorrent(self, torrent, uris=None, options=None, position=None): + ''' + aria2.addTorrent method + + torrent: base64-encoded torrent file + + uris: list of webseed URIs + + options: dictionary of additional options + + position: position in queue + + Returns a GID. + ''' + params = [torrent] + if uris: + params.append(uris) + params = add_options_and_position(params, options, position) + return self.jsonrpc('addTorrent', params) + + def addMetalink(self, metalink, options=None, position=None): + ''' + aria2.addMetalink method + + metalink: base64-encoded metalink file + + options: dictionary of additional options + + position: position in queue + + Returns an array of GIDs. + ''' + params = [metalink] + params = add_options_and_position(params, options, position) + return self.jsonrpc('addTorrent', params) + + def remove(self, gid): + ''' + aria2.remove method + + gid: GID to remove + ''' + params = [gid] + return self.jsonrpc('remove', params) + + def forceRemove(self, gid): + ''' + aria2.forceRemove method + + gid: GID to remove + ''' + params = [gid] + return self.jsonrpc('forceRemove', params) + + def pause(self, gid): + ''' + aria2.pause method + + gid: GID to pause + ''' + params = [gid] + return self.jsonrpc('pause', params) + + def pauseAll(self): + ''' + aria2.pauseAll method + ''' + return self.jsonrpc('pauseAll') + def forcePause(self, gid): + ''' + aria2.forcePause method + gid: GID to pause + ''' + params = [gid] + return self.jsonrpc('forcePause', params) - def jsonrpc(self, method, params=None, prefix='aria2.'): - ''' - POST a request to the RPC interface. - ''' - if not params: - params = [] + def forcePauseAll(self): + ''' + aria2.forcePauseAll method + ''' + return self.jsonrpc('forcePauseAll') - if self.token is not None: - token_str = 'token:{}'.format(self.token) - if method == 'multicall': - for p in params[0]: - try: - p['params'].insert(0, token_str) - except KeyError: - p['params'] = [token_str] - else: - params.insert(0, token_str) + def unpause(self, gid): + ''' + aria2.unpause method - req_obj = { - 'jsonrpc' : '2.0', - 'id' : self.id, - 'method' : prefix + method, - 'params' : params, - } - if self.mode == 'batch': - self.queue.append(req_obj) - return None - elif self.mode == 'format': - return req_obj - else: - return self.send_request(req_obj) + gid: GID to unpause + ''' + params = [gid] + return self.jsonrpc('unpause', params) + def unpauseAll(self): + ''' + aria2.unpauseAll method + ''' + return self.jsonrpc('unpauseAll') + def tellStatus(self, gid, keys=None): + ''' + aria2.tellStatus method - def process_queue(self): - ''' - Processed queued requests. - ''' - req_obj = self.queue - self.queue = [] - return self.send_request(req_obj) + gid: GID to query + keys: subset of status keys to return (all keys are returned otherwise) + Returns a dictionary. + ''' + params = [gid] + if keys: + params.append(keys) + return self.jsonrpc('tellStatus', params) + + def getUris(self, gid): + ''' + aria2.getUris method -############################### Standard Methods ############################### + gid: GID to query - def addUri(self, uris, options=None, position=None): - ''' - aria2.addUri method + Returns a list of dictionaries. + ''' + params = [gid] + return self.jsonrpc('getUris', params) - uris: list of URIs + def getFiles(self, gid): + ''' + aria2.getFiles method - options: dictionary of additional options + gid: GID to query - position: position in queue + Returns a list of dictionaries. + ''' + params = [gid] + return self.jsonrpc('getFiles', params) - Returns a GID - ''' - params = [uris] - params = add_options_and_position(params, options, position) - return self.jsonrpc('addUri', params) + def getPeers(self, gid): + ''' + aria2.getPeers method + gid: GID to query + Returns a list of dictionaries. + ''' + params = [gid] + return self.jsonrpc('getPeers', params) - def addTorrent(self, torrent, uris=None, options=None, position=None): - ''' - aria2.addTorrent method + def getServers(self, gid): + ''' + aria2.getServers method - torrent: base64-encoded torrent file + gid: GID to query - uris: list of webseed URIs + Returns a list of dictionaries. + ''' + params = [gid] + return self.jsonrpc('getServers', params) - options: dictionary of additional options + def tellActive(self, keys=None): + ''' + aria2.tellActive method - position: position in queue + keys: same as tellStatus - Returns a GID. - ''' - params = [torrent] - if uris: - params.append(uris) - params = add_options_and_position(params, options, position) - return self.jsonrpc('addTorrent', params) + Returns a list of dictionaries. The dictionaries are the same as those + returned by tellStatus. + ''' + if keys: + params = [keys] + else: + params = None + return self.jsonrpc('tellActive', params) + def tellWaiting(self, offset, num, keys=None): + ''' + aria2.tellWaiting method + offset: offset from start of waiting download queue + (negative values are counted from the end of the queue) - def addMetalink(self, metalink, options=None, position=None): - ''' - aria2.addMetalink method + num: number of downloads to return - metalink: base64-encoded metalink file + keys: same as tellStatus - options: dictionary of additional options + Returns a list of dictionaries. The dictionaries are the same as those + returned by tellStatus. + ''' + params = [offset, num] + if keys: + params.append(keys) + return self.jsonrpc('tellWaiting', params) - position: position in queue + def tellStopped(self, offset, num, keys=None): + ''' + aria2.tellStopped method - Returns an array of GIDs. - ''' - params = [metalink] - params = add_options_and_position(params, options, position) - return self.jsonrpc('addTorrent', params) + offset: offset from oldest download (same semantics as tellWaiting) + num: same as tellWaiting + keys: same as tellStatus - def remove(self, gid): - ''' - aria2.remove method + Returns a list of dictionaries. The dictionaries are the same as those + returned by tellStatus. + ''' + params = [offset, num] + if keys: + params.append(keys) + return self.jsonrpc('tellStopped', params) - gid: GID to remove - ''' - params = [gid] - return self.jsonrpc('remove', params) + def changePosition(self, gid, pos, how): + ''' + aria2.changePosition method + gid: GID to change + pos: the position - def forceRemove(self, gid): - ''' - aria2.forceRemove method + how: "POS_SET", "POS_CUR" or "POS_END" + ''' + params = [gid, pos, how] + return self.jsonrpc('changePosition', params) - gid: GID to remove - ''' - params = [gid] - return self.jsonrpc('forceRemove', params) + def changeUri(self, gid, fileIndex, delUris, addUris, position=None): + ''' + aria2.changePosition method + gid: GID to change + fileIndex: file to affect (1-based) - def pause(self, gid): - ''' - aria2.pause method + delUris: URIs to remove - gid: GID to pause - ''' - params = [gid] - return self.jsonrpc('pause', params) + addUris: URIs to add + position: where URIs are inserted, after URIs have been removed + ''' + params = [gid, fileIndex, delUris, addUris] + if position: + params.append(position) + return self.jsonrpc('changePosition', params) + def getOption(self, gid): + ''' + aria2.getOption method - def pauseAll(self): - ''' - aria2.pauseAll method - ''' - return self.jsonrpc('pauseAll') + gid: GID to query + Returns a dictionary of options. + ''' + params = [gid] + return self.jsonrpc('getOption', params) + def changeOption(self, gid, options): + ''' + aria2.changeOption method - def forcePause(self, gid): - ''' - aria2.forcePause method - - gid: GID to pause - ''' - params = [gid] - return self.jsonrpc('forcePause', params) - - - - def forcePauseAll(self): - ''' - aria2.forcePauseAll method - ''' - return self.jsonrpc('forcePauseAll') - - - - def unpause(self, gid): - ''' - aria2.unpause method - - gid: GID to unpause - ''' - params = [gid] - return self.jsonrpc('unpause', params) - - - - def unpauseAll(self): - ''' - aria2.unpauseAll method - ''' - return self.jsonrpc('unpauseAll') - - - - def tellStatus(self, gid, keys=None): - ''' - aria2.tellStatus method - - gid: GID to query - - keys: subset of status keys to return (all keys are returned otherwise) - - Returns a dictionary. - ''' - params = [gid] - if keys: - params.append(keys) - return self.jsonrpc('tellStatus', params) - - - - def getUris(self, gid): - ''' - aria2.getUris method - - gid: GID to query - - Returns a list of dictionaries. - ''' - params = [gid] - return self.jsonrpc('getUris', params) - - - - def getFiles(self, gid): - ''' - aria2.getFiles method - - gid: GID to query - - Returns a list of dictionaries. - ''' - params = [gid] - return self.jsonrpc('getFiles', params) - - - - def getPeers(self, gid): - ''' - aria2.getPeers method - - gid: GID to query - - Returns a list of dictionaries. - ''' - params = [gid] - return self.jsonrpc('getPeers', params) - - - - def getServers(self, gid): - ''' - aria2.getServers method - - gid: GID to query - - Returns a list of dictionaries. - ''' - params = [gid] - return self.jsonrpc('getServers', params) - - - - def tellActive(self, keys=None): - ''' - aria2.tellActive method - - keys: same as tellStatus - - Returns a list of dictionaries. The dictionaries are the same as those - returned by tellStatus. - ''' - if keys: - params = [keys] - else: - params = None - return self.jsonrpc('tellActive', params) - - - - def tellWaiting(self, offset, num, keys=None): - ''' - aria2.tellWaiting method - - offset: offset from start of waiting download queue - (negative values are counted from the end of the queue) - - num: number of downloads to return - - keys: same as tellStatus - - Returns a list of dictionaries. The dictionaries are the same as those - returned by tellStatus. - ''' - params = [offset, num] - if keys: - params.append(keys) - return self.jsonrpc('tellWaiting', params) - - - - def tellStopped(self, offset, num, keys=None): - ''' - aria2.tellStopped method - - offset: offset from oldest download (same semantics as tellWaiting) - - num: same as tellWaiting - - keys: same as tellStatus - - Returns a list of dictionaries. The dictionaries are the same as those - returned by tellStatus. - ''' - params = [offset, num] - if keys: - params.append(keys) - return self.jsonrpc('tellStopped', params) - - - - def changePosition(self, gid, pos, how): - ''' - aria2.changePosition method - - gid: GID to change - - pos: the position - - how: "POS_SET", "POS_CUR" or "POS_END" - ''' - params = [gid, pos, how] - return self.jsonrpc('changePosition', params) - - - - def changeUri(self, gid, fileIndex, delUris, addUris, position=None): - ''' - aria2.changePosition method - - gid: GID to change - - fileIndex: file to affect (1-based) - - delUris: URIs to remove - - addUris: URIs to add - - position: where URIs are inserted, after URIs have been removed - ''' - params = [gid, fileIndex, delUris, addUris] - if position: - params.append(position) - return self.jsonrpc('changePosition', params) - - - - def getOption(self, gid): - ''' - aria2.getOption method - - gid: GID to query - - Returns a dictionary of options. - ''' - params = [gid] - return self.jsonrpc('getOption', params) - - - - def changeOption(self, gid, options): - ''' - aria2.changeOption method - - gid: GID to change - - options: dictionary of new options - (not all options can be changed for active downloads) - ''' - params = [gid, options] - return self.jsonrpc('changeOption', params) - - - - def getGlobalOption(self): - ''' - aria2.getGlobalOption method - - Returns a dictionary. - ''' - return self.jsonrpc('getGlobalOption') - - - - def changeGlobalOption(self, options): - ''' - aria2.changeGlobalOption method - - options: dictionary of new options - ''' - params = [options] - return self.jsonrpc('changeGlobalOption', params) - - - - def getGlobalStat(self): - ''' - aria2.getGlobalStat method - - Returns a dictionary. - ''' - return self.jsonrpc('getGlobalStat') - - - - def purgeDownloadResult(self): - ''' - aria2.purgeDownloadResult method - ''' - self.jsonrpc('purgeDownloadResult') - - - - def removeDownloadResult(self, gid): - ''' - aria2.removeDownloadResult method - - gid: GID to remove - ''' - params = [gid] - return self.jsonrpc('removeDownloadResult', params) - - - - def getVersion(self): - ''' - aria2.getVersion method - - Returns a dictionary. - ''' - return self.jsonrpc('getVersion') - - - - def getSessionInfo(self): - ''' - aria2.getSessionInfo method - - Returns a dictionary. - ''' - return self.jsonrpc('getSessionInfo') - - - - def shutdown(self): - ''' - aria2.shutdown method - ''' - return self.jsonrpc('shutdown') - - - - def forceShutdown(self): - ''' - aria2.forceShutdown method - ''' - return self.jsonrpc('forceShutdown') - - - - def multicall(self, methods): - ''' - aria2.multicall method - - methods: list of dictionaries (keys: methodName, params) - - The method names must be those used by Aria2c, e.g. "aria2.tellStatus". - ''' - return self.jsonrpc('multicall', [methods], prefix='system.') - - - - -############################# Convenience Methods ############################## - - def add_torrent(self, path, uris=None, options=None, position=None): - ''' - A wrapper around addTorrent for loading files. - ''' - with open(path, 'r') as f: - torrent = base64.encode(f.read()) - return self.addTorrent(torrent, uris, options, position) - - - - def add_metalink(self, path, uris=None, options=None, position=None): - ''' - A wrapper around addMetalink for loading files. - ''' - with open(path, 'r') as f: - metalink = base64.encode(f.read()) - return self.addMetalink(metalink, uris, options, position) - - - - def get_status(self, gid): - ''' - Get the status of a single GID. - ''' - response = self.tellStatus(gid, ['status']) - return get_status(response) - - - - def wait_for_final_status(self, gid, interval=1): - ''' - Wait for a GID to complete or fail and return its status. - ''' - if not interval or interval < 0: - interval = 1 - while True: - status = self.get_status(gid) - if status in TEMPORARY_STATUS: - time.sleep(interval) - else: - return status - - - - def get_statuses(self, gids): - ''' - Get the status of multiple GIDs. The status of each is yielded in order. - ''' - methods = [ - { - 'methodName' : 'aria2.tellStatus', - 'params' : [gid, ['gid', 'status']] - } - for gid in gids - ] - results = self.multicall(methods) - if results: - status = dict((r[0]['gid'], r[0]['status']) for r in results) - for gid in gids: - try: - yield status[gid] - except KeyError: - logger.error('Aria2 RPC server returned no status for GID {}'.format(gid)) - yield 'error' - else: - logger.error('no response from Aria2 RPC server') - for gid in gids: - yield 'error' - - - - def wait_for_final_statuses(self, gids, interval=1): - ''' - Wait for multiple GIDs to complete or fail and return their statuses in - order. - - gids: - A flat list of GIDs. - ''' - if not interval or interval < 0: - interval = 1 - statusmap = dict((g, None) for g in gids) - remaining = list( - g for g,s in statusmap.items() if s is None - ) - while remaining: - for g, s in zip(remaining, self.get_statuses(remaining)): - if s in TEMPORARY_STATUS: - continue - else: - statusmap[g] = s - remaining = list( - g for g,s in statusmap.items() if s is None - ) - if remaining: - time.sleep(interval) - for g in gids: - yield statusmap[g] - - - - def print_global_status(self): - ''' - Print global status of the RPC server. - ''' - status = self.getGlobalStat() - if status: - numWaiting = int(status['numWaiting']) - numStopped = int(status['numStopped']) - keys = ['totalLength', 'completedLength'] - total = self.tellActive(keys) - waiting = self.tellWaiting(0, numWaiting, keys) - if waiting: - total += waiting - stopped = self.tellStopped(0, numStopped, keys) - if stopped: - total += stopped - - downloadSpeed = int(status['downloadSpeed']) - uploadSpeed = int(status['uploadSpeed']) - totalLength = sum(int(x['totalLength']) for x in total) - completedLength = sum(int(x['completedLength']) for x in total) - remaining = totalLength - completedLength - - status['downloadSpeed'] = format_bytes(downloadSpeed) + '/s' - status['uploadSpeed'] = format_bytes(uploadSpeed) + '/s' - - preordered = ('downloadSpeed', 'uploadSpeed') - - rows = list() - for k in sorted(status): - if k in preordered: - continue - rows.append((k, status[k])) - - rows.extend((x, status[x]) for x in preordered) - - if totalLength > 0: - rows.append(('total', format(format_bytes(totalLength)))) - rows.append(('completed', format(format_bytes(completedLength)))) - rows.append(('remaining', format(format_bytes(remaining)))) - if completedLength == totalLength: - eta = 'finished' - else: - try: - eta = format_seconds(remaining // downloadSpeed) - except ZeroDivisionError: - eta = 'never' - rows.append(('ETA', eta)) - - l = max(len(r[0]) for r in rows) - r = max(len(r[1]) for r in rows) - r = max(r, len(self.uri) - (l + 2)) - fmt = '{:<' + str(l) + 's} {:>' + str(r) + 's}' - - print(self.uri) - for k, v in rows: - print(fmt.format(k, v)) - - - - def queue_uris(self, uris, options, interval=None): - ''' - Enqueue URIs and wait for download to finish while printing status at - regular intervals. - ''' - gid = self.addUri(uris, options) - print('GID: {}'.format(gid)) - - if gid and interval is not None: - blanker = '' - while True: - response = self.tellStatus(gid, ['status']) - if response: - try: - status = response['status'] - except KeyError: - print('error: no status returned from Aria2 RPC server') - break - print('{}\rstatus: {}'.format(blanker, status)), - blanker = ' ' * len(status) - if status in TEMPORARY_STATUS: - time.sleep(interval) - else: - break - else: - print('error: no response from server') - break - - - -######################### Polymethod download handlers ######################### - - def polymethod_enqueue_many(self, downloads): - ''' - Enqueue downloads. - - downloads: Same as polymethod_download(). - ''' - methods = list( - { - 'methodName': 'aria2.{}'.format(d[0]), - 'params': list(d[1:]) - } for d in downloads - ) - return self.multicall(methods) - - - - def polymethod_wait_many(self, gids, interval=1): - ''' - Wait for the GIDs to complete or fail and return their statuses. - - gids: - A list of lists of GIDs. - ''' - # The flattened list of GIDs - gs = list(g for gs in gids for g in gs) - statusmap = dict(tuple(zip( - gs, - self.wait_for_final_statuses(gs, interval=interval) - ))) - for gs in gids: - yield list(statusmap.get(g, 'error') for g in gs) - - - - - def polymethod_enqueue_one(self, download): - ''' - Same as polymethod_enqueue_many but for one element. - ''' - return getattr(self, download[0])(*download[1:]) - - - - def polymethod_download(self, downloads, interval=1): - ''' - Enqueue a series of downloads and wait for them to finish. Iterate over the - status of each, in order. - - downloads: - An iterable over (, , ...) where indicates the "add" - method to use ('addUri', 'addTorrent', 'addMetalink') and everything that - follows are arguments to pass to that method. - - interval: - The status check interval while waiting. - - Iterates over the download status of finished downloads. "complete" - indicates success. Lists of statuses will be returned for downloads that - create multiple GIDs (e.g. metalinks). - ''' - gids = self.polymethod_enqueue_many(downloads) - return self.polymethod_wait_many(gids, interval=interval) - - - - def polymethod_download_bool(self, *args, **kwargs): - ''' - A wrapper around polymethod_download() which returns a boolean for each - download to indicate success (True) or failure (False). - ''' + gid: GID to change + + options: dictionary of new options + (not all options can be changed for active downloads) + ''' + params = [gid, options] + return self.jsonrpc('changeOption', params) + + def getGlobalOption(self): + ''' + aria2.getGlobalOption method + + Returns a dictionary. + ''' + return self.jsonrpc('getGlobalOption') + + def changeGlobalOption(self, options): + ''' + aria2.changeGlobalOption method + + options: dictionary of new options + ''' + params = [options] + return self.jsonrpc('changeGlobalOption', params) + + def getGlobalStat(self): + ''' + aria2.getGlobalStat method + + Returns a dictionary. + ''' + return self.jsonrpc('getGlobalStat') + + def purgeDownloadResult(self): + ''' + aria2.purgeDownloadResult method + ''' + self.jsonrpc('purgeDownloadResult') + + def removeDownloadResult(self, gid): + ''' + aria2.removeDownloadResult method + + gid: GID to remove + ''' + params = [gid] + return self.jsonrpc('removeDownloadResult', params) + + def getVersion(self): + ''' + aria2.getVersion method + + Returns a dictionary. + ''' + return self.jsonrpc('getVersion') + + def getSessionInfo(self): + ''' + aria2.getSessionInfo method + + Returns a dictionary. + ''' + return self.jsonrpc('getSessionInfo') + + def shutdown(self): + ''' + aria2.shutdown method + ''' + return self.jsonrpc('shutdown') + + def forceShutdown(self): + ''' + aria2.forceShutdown method + ''' + return self.jsonrpc('forceShutdown') + + def multicall(self, methods): + ''' + aria2.multicall method + + methods: list of dictionaries (keys: methodName, params) + + The method names must be those used by Aria2c, e.g. "aria2.tellStatus". + ''' + return self.jsonrpc('multicall', [methods], prefix='system.') + + +# ########################### Convenience Methods ############################## + def add_torrent(self, path, uris=None, options=None, position=None): + ''' + A wrapper around addTorrent for loading files. + ''' + with open(path, 'r') as f: + torrent = base64.encode(f.read()) + return self.addTorrent(torrent, uris, options, position) + + def add_metalink(self, path, uris=None, options=None, position=None): + ''' + A wrapper around addMetalink for loading files. + ''' + with open(path, 'r') as f: + metalink = base64.encode(f.read()) + return self.addMetalink(metalink, uris, options, position) + + def get_status(self, gid): + ''' + Get the status of a single GID. + ''' + response = self.tellStatus(gid, ['status']) + return get_status(response) + + def wait_for_final_status(self, gid, interval=1): + ''' + Wait for a GID to complete or fail and return its status. + ''' + if not interval or interval < 0: + interval = 1 + while True: + status = self.get_status(gid) + if status in TEMPORARY_STATUS: + time.sleep(interval) + else: + return status + + def get_statuses(self, gids): + ''' + Get the status of multiple GIDs. The status of each is yielded in order. + ''' + methods = [ + { + 'methodName': 'aria2.tellStatus', + 'params': [gid, ['gid', 'status']] + } + for gid in gids + ] + results = self.multicall(methods) + if results: + status = dict((r[0]['gid'], r[0]['status']) for r in results) + for gid in gids: + try: + yield status[gid] + except KeyError: + logger.error('Aria2 RPC server returned no status for GID {}'.format(gid)) + yield 'error' + else: + logger.error('no response from Aria2 RPC server') + for gid in gids: + yield 'error' + + def wait_for_final_statuses(self, gids, interval=1): + ''' + Wait for multiple GIDs to complete or fail and return their statuses in + order. + + gids: + A flat list of GIDs. + ''' + if not interval or interval < 0: + interval = 1 + statusmap = dict((g, None) for g in gids) + remaining = list( + g for g, s in statusmap.items() if s is None + ) + while remaining: + for g, s in zip(remaining, self.get_statuses(remaining)): + if s in TEMPORARY_STATUS: + continue + else: + statusmap[g] = s + remaining = list( + g for g, s in statusmap.items() if s is None + ) + if remaining: + time.sleep(interval) + for g in gids: + yield statusmap[g] + + def print_global_status(self): + ''' + Print global status of the RPC server. + ''' + status = self.getGlobalStat() + if status: + numWaiting = int(status['numWaiting']) + numStopped = int(status['numStopped']) + keys = ['totalLength', 'completedLength'] + total = self.tellActive(keys) + waiting = self.tellWaiting(0, numWaiting, keys) + if waiting: + total += waiting + stopped = self.tellStopped(0, numStopped, keys) + if stopped: + total += stopped + + downloadSpeed = int(status['downloadSpeed']) + uploadSpeed = int(status['uploadSpeed']) + totalLength = sum(int(x['totalLength']) for x in total) + completedLength = sum(int(x['completedLength']) for x in total) + remaining = totalLength - completedLength + + status['downloadSpeed'] = format_bytes(downloadSpeed) + '/s' + status['uploadSpeed'] = format_bytes(uploadSpeed) + '/s' + + preordered = ('downloadSpeed', 'uploadSpeed') + + rows = list() + for k in sorted(status): + if k in preordered: + continue + rows.append((k, status[k])) + + rows.extend((x, status[x]) for x in preordered) + + if totalLength > 0: + rows.append(('total', format(format_bytes(totalLength)))) + rows.append(('completed', format(format_bytes(completedLength)))) + rows.append(('remaining', format(format_bytes(remaining)))) + if completedLength == totalLength: + eta = 'finished' + else: + try: + eta = format_seconds(remaining // downloadSpeed) + except ZeroDivisionError: + eta = 'never' + rows.append(('ETA', eta)) + + l = max(len(r[0]) for r in rows) + r = max(len(r[1]) for r in rows) + r = max(r, len(self.uri) - (l + 2)) + fmt = '{:<' + str(l) + 's} {:>' + str(r) + 's}' + + print(self.uri) + for k, v in rows: + print(fmt.format(k, v)) + + def queue_uris(self, uris, options, interval=None): + ''' + Enqueue URIs and wait for download to finish while printing status at + regular intervals. + ''' + gid = self.addUri(uris, options) + print('GID: {}'.format(gid)) + + if gid and interval is not None: + blanker = '' + while True: + response = self.tellStatus(gid, ['status']) + if response: + try: + status = response['status'] + except KeyError: + print('error: no status returned from Aria2 RPC server') + break + print('{}\rstatus: {}'.format(blanker, status)), + blanker = ' ' * len(status) + if status in TEMPORARY_STATUS: + time.sleep(interval) + else: + break + else: + print('error: no response from server') + break + + +# ####################### Polymethod download handlers ######################### + def polymethod_enqueue_many(self, downloads): + ''' + Enqueue downloads. + + downloads: Same as polymethod_download(). + ''' + methods = list( + { + 'methodName': 'aria2.{}'.format(d[0]), + 'params': list(d[1:]) + } for d in downloads + ) + return self.multicall(methods) + + def polymethod_wait_many(self, gids, interval=1): + ''' + Wait for the GIDs to complete or fail and return their statuses. + + gids: + A list of lists of GIDs. + ''' + # The flattened list of GIDs + gs = list(g for gs in gids for g in gs) + statusmap = dict(tuple(zip( + gs, + self.wait_for_final_statuses(gs, interval=interval) + ))) + for gs in gids: + yield list(statusmap.get(g, 'error') for g in gs) + + def polymethod_enqueue_one(self, download): + ''' + Same as polymethod_enqueue_many but for one element. + ''' + return getattr(self, download[0])(*download[1:]) + + def polymethod_download(self, downloads, interval=1): + ''' + Enqueue a series of downloads and wait for them to finish. Iterate over the + status of each, in order. + + downloads: + An iterable over (, , ...) where indicates the "add" + method to use ('addUri', 'addTorrent', 'addMetalink') and everything that + follows are arguments to pass to that method. + + interval: + The status check interval while waiting. + + Iterates over the download status of finished downloads. "complete" + indicates success. Lists of statuses will be returned for downloads that + create multiple GIDs (e.g. metalinks). + ''' + gids = self.polymethod_enqueue_many(downloads) + return self.polymethod_wait_many(gids, interval=interval) + + def polymethod_download_bool(self, *args, **kwargs): + ''' + A wrapper around polymethod_download() which returns a boolean for each + download to indicate success (True) or failure (False). + ''' # for status in self.polymethod_download(*args, **kwargs): # yield all(s == 'complete' for s in status) - return list( - all(s == 'complete' for s in status) - for status in self.polymethod_download(*args, **kwargs) - ) - \ No newline at end of file + return list( + all(s == 'complete' for s in status) + for status in self.polymethod_download(*args, **kwargs) + ) diff --git a/headphones/deezloader.py b/headphones/deezloader.py index db14ae5d..528cded6 100644 --- a/headphones/deezloader.py +++ b/headphones/deezloader.py @@ -26,7 +26,6 @@ import binascii from beets.mediafile import MediaFile from headphones import logger, request, helpers import headphones -from twisted.conch.insults import helper # Public constants PROVIDER_NAME = 'Deezer' @@ -53,6 +52,7 @@ __cookies = None __tracks_cache = {} __albums_cache = {} + def __getApiToken(): global __cookies response = request.request_response(url="http://www.deezer.com/", headers=__HTTP_HEADERS) @@ -67,39 +67,42 @@ def __getApiToken(): if not token: logger.error(u"Deezloader: Unable to get api token") - + + def getAlbumByLink(album_link): """Returns deezer album infos using album link url - + :param str album_link: deezer album link url (eg: 'http://www.deezer.com/album/1234567/') """ matches = re.search(r"album\/([0-9]+)\/?$", album_link) if matches: return getAlbum(matches.group(1)) - + + def getAlbum(album_id): """Returns deezer album infos - + :param int album_id: deezer album id """ global __albums_cache - + if str(album_id) in __albums_cache: return __albums_cache[str(album_id)] - + url = __API_INFO_URL + "album/" + str(album_id) data = request.request_json(url=url, headers=__HTTP_HEADERS, cookies=__cookies) - + if data and 'error' not in data: __albums_cache[str(album_id)] = data return data else: logger.debug("Deezloader: Can't load album infos") return None - + + def searchAlbums(search_term): """Search for deezer albums using search term - + :param str search_term: search term to search album for """ logger.info(u'Searching Deezer using term: "%s"' % search_term) @@ -108,7 +111,7 @@ def searchAlbums(search_term): data = request.request_json(url=url, headers=__HTTP_HEADERS, cookies=__cookies) albums = [] - + # Process content if data and 'total' in data and data['total'] > 0 and 'data' in data: for item in data['data']: @@ -121,13 +124,14 @@ def searchAlbums(search_term): return albums + def __matchAlbums(albums, artist_name, album_title, album_length): resultlist = [] - + for album in albums: total_size = 0 tracks_found = 0 - + for track in album['tracks']['data']: t = getTrack(track['id']) if t: @@ -139,29 +143,29 @@ def __matchAlbums(albums, artist_name, album_title, album_length): size = t["FILESIZE_MP3_128"] else: size = t["FILESIZE_MP3_64"] - + size = int(size) total_size += size tracks_found += 1 logger.debug(u'Found song "%s". Size: %s' % (t['SNG_TITLE'], helpers.bytes_to_mb(size))) - + matched = True mismatch_reason = 'matched!' - + if album_length > 0 and abs(int(album['duration']) - album_length) > 240: matched = False mismatch_reason = (u'duration mismatch: %i not in [%i, %i]' % (int(album['duration']), (album_length - 240), (album_length + 240))) - elif (helpers.latinToAscii(re.sub(r"\W", "", album_title, 0, re.UNICODE)).lower() != + elif (helpers.latinToAscii(re.sub(r"\W", "", album_title, 0, re.UNICODE)).lower() != helpers.latinToAscii(re.sub(r"\W", "", album['title'], 0, re.UNICODE)).lower()): matched = False mismatch_reason = (u'album name mismatch: %s != %s' % (album['title'], album_title)) - + elif (helpers.latinToAscii(re.sub(r"\W", "", artist_name, 0, re.UNICODE)).lower() != helpers.latinToAscii(re.sub(r"\W", "", album['artist']['name'], 0, re.UNICODE)).lower()): matched = False mismatch_reason = (u'artist name mismatch: %s != %s' % (album['artist']['name'], artist_name)) - + resultlist.append( (album['artist']['name'] + ' - ' + album['title'] + ' [' + album['release_date'][:4] + '] (' + str(tracks_found) + '/' + str(album['nb_tracks']) + ')', total_size, album['link'], PROVIDER_NAME, "ddl", matched) @@ -170,6 +174,7 @@ def __matchAlbums(albums, artist_name, album_title, album_length): return resultlist + def searchAlbum(artist_name, album_title, user_search_term=None, album_length=None): """Search for deezer specific album. This will iterate over deezer albums and try to find best matches @@ -182,7 +187,7 @@ def searchAlbum(artist_name, album_title, user_search_term=None, album_length=No # User search term by-pass normal search if user_search_term: return __matchAlbums(searchAlbums(user_search_term), artist_name, album_title, album_length) - + resultlist = __matchAlbums(searchAlbums((artist_name + ' ' + album_title).strip()), artist_name, album_title, album_length) if resultlist: return resultlist @@ -190,35 +195,36 @@ def searchAlbum(artist_name, album_title, user_search_term=None, album_length=No # Deezer API supports unicode, so just remove non alphanumeric characters clean_artist_name = re.sub(r"[^\w\s]", " ", artist_name, 0, re.UNICODE).strip() clean_album_name = re.sub(r"[^\w\s]", " ", album_title, 0, re.UNICODE).strip() - + resultlist = __matchAlbums(searchAlbums((clean_artist_name + ' ' + clean_album_name).strip()), artist_name, album_title, album_length) if resultlist: return resultlist - + resultlist = __matchAlbums(searchAlbums(clean_artist_name), artist_name, album_title, album_length) if resultlist: return resultlist - + return resultlist + def getTrack(sng_id, try_reload_api=True): """Returns deezer track infos - + :param int sng_id: deezer song id - :param bool try_reload_api: whether or not try reloading API if session expired + :param bool try_reload_api: whether or not try reloading API if session expired """ global __tracks_cache - + if str(sng_id) in __tracks_cache: return __tracks_cache[str(sng_id)] - + data = "[{\"method\":\"song.getListData\",\"params\":{\"sng_ids\":[" + str(sng_id) + "]}}]" json = request.request_json(url=__API_URL, headers=__HTTP_HEADERS, method='post', params=__api_queries, data=data, cookies=__cookies) - + results = [] error = None invalid_token = False - + if json: # Check for errors if 'error' in json: @@ -230,7 +236,7 @@ def getTrack(sng_id, try_reload_api=True): error = json[0]['error'] if 'VALID_TOKEN_REQUIRED' in json[0]['error'] and json[0]['error']['VALID_TOKEN_REQUIRED'] == u"Invalid CSRF token": invalid_token = True - + # Got invalid token error if error: if invalid_token and try_reload_api: @@ -245,26 +251,27 @@ def getTrack(sng_id, try_reload_api=True): if 'token' in item: logger.error(u"An unknown error occurred in the Deezer parser: Uploaded Files are currently not supported") return - + sng_id = item["SNG_ID"] md5Origin = item["MD5_ORIGIN"] sng_format = 3 - + if item["FILESIZE_MP3_320"] <= 0: if item["FILESIZE_MP3_256"] > 0: sng_format = 5 else: sng_format = 1 - + mediaVersion = int(item["MEDIA_VERSION"]) item['downloadUrl'] = __getDownloadUrl(md5Origin, sng_id, sng_format, mediaVersion) - + __tracks_cache[sng_id] = item return item - + except Exception as e: logger.error(u"An unknown error occurred in the Deezer parser: %s" % e) + def __getDownloadUrl(md5Origin, sng_id, sng_format, mediaVersion): urlPart = md5Origin.encode('utf-8') + b'\xa4' + str(sng_format) + b'\xa4' + str(sng_id) + b'\xa4' + str(mediaVersion) md5val = md5(urlPart).hexdigest() @@ -273,6 +280,7 @@ def __getDownloadUrl(md5Origin, sng_id, sng_format, mediaVersion): ciphertext = cipher.encrypt(__pad(urlPart, AES.block_size)) return "http://e-cdn-proxy-" + md5Origin[:1] + ".deezer.com/mobile/1/" + binascii.hexlify(ciphertext).lower() + def __pad(raw, block_size): if (len(raw) % block_size == 0): return raw @@ -281,10 +289,11 @@ def __pad(raw, block_size): data = raw + padding_required * padChar return data + def __tagTrack(path, track): try: album = getAlbum(track['ALB_ID']) - + f = MediaFile(path) f.artist = track['ART_NAME'] f.album = track['ALB_TITLE'] @@ -295,22 +304,23 @@ def __tagTrack(path, track): f.bpm = track['BPM'] f.date = datetime.strptime(album['release_date'], '%Y-%m-%d').date() f.albumartist = album['artist']['name'] - if u'genres' in album and u'data' in album['genres']: + if u'genres' in album and u'data' in album['genres']: f.genres = [genre['name'] for genre in album['genres']['data']] - + f.save() except Exception as e: logger.error(u'Unable to tag deezer track "%s": %s' % (path, e)) + def decryptTracks(paths): """Decrypt downloaded deezer tracks. - + :param paths: list of path to deezer tracks (*.dzr files). """ # Note: tracks can be from different albums decrypted_tracks = {} - + # First pass: load tracks data for path in paths: try: @@ -322,15 +332,15 @@ def decryptTracks(paths): if album_folder not in decrypted_tracks: decrypted_tracks[album_folder] = {} - + if disk_number not in decrypted_tracks[album_folder]: decrypted_tracks[album_folder][disk_number] = {} decrypted_tracks[album_folder][disk_number][track_number] = track - + except Exception as e: logger.error(u'Unable to load deezer track infos "%s": %s' % (path, e)) - + # Second pass: decrypt tracks for album_folder in decrypted_tracks: multi_disks = len(decrypted_tracks[album_folder]) > 1 @@ -339,12 +349,12 @@ def decryptTracks(paths): try: filename = helpers.replace_illegal_chars(track['SNG_TITLE']).strip() filename = ('{:02d}'.format(track_number) + '_' + filename + '.mp3') - + # Add a 'cd x' sub-folder if album has more than one disk disk_folder = os.path.join(album_folder, 'cd ' + str(disk_number)) if multi_disks else album_folder - + dest = os.path.join(disk_folder, filename).encode(headphones.SYS_ENCODING, 'replace') - + # Decrypt track if not already done if not os.path.exists(dest): try: @@ -358,12 +368,13 @@ def decryptTracks(paths): continue decrypted_tracks[album_folder][disk_number][track_number]['path'] = dest - + except Exception as e: logger.error(u'Unable to decrypt deezer track "%s": %s' % (path, e)) - + return decrypted_tracks + def __decryptDownload(source, sng_id, dest): interval_chunk = 3 chunk_size = 2048 @@ -383,7 +394,7 @@ def __decryptDownload(source, sng_id, dest): if(i % interval_chunk == 0): cipher = Blowfish.new(blowFishKey, Blowfish.MODE_CBC, iv) chunk = cipher.decrypt(__pad(chunk, Blowfish.block_size)) - + fout.write(chunk) i += 1 chunk = f.read(chunk_size) @@ -391,6 +402,7 @@ def __decryptDownload(source, sng_id, dest): f.close() fout.close() + def __getBlowFishKey(encryptionKey): if encryptionKey < 1: encryptionKey *= -1 @@ -402,14 +414,15 @@ def __getBlowFishKey(encryptionKey): return __xorHex(parts) + def __xorHex(parts): data = "" for i in range(0, 16): character = ord(parts[0][i]) - + for j in range(1, len(parts)): character ^= ord(parts[j][i]) - + data += chr(character) - + return data diff --git a/headphones/helpers_test.py b/headphones/helpers_test.py index 14b8540d..54c9e9bf 100644 --- a/headphones/helpers_test.py +++ b/headphones/helpers_test.py @@ -14,9 +14,9 @@ class HelpersTest(TestCase): u'Symphonęy Nº9': 'Symphoney No.9', u'ÆæßðÞIJij': u'AeaessdThIJıj', u'Obsessió (Cerebral Apoplexy remix)': 'obsessio cerebral ' - 'apoplexy remix', + 'apoplexy remix', u'Doktór Hałabała i siedmiu zbojów': 'doktor halabala i siedmiu ' - 'zbojow', + 'zbojow', u'Arbetets Söner och Döttrar': 'arbetets soner och dottrar', u'Björk Guðmundsdóttir': 'bjork gudmundsdottir', u'L\'Arc~en~Ciel': 'larc en ciel', diff --git a/headphones/postprocessor.py b/headphones/postprocessor.py index 336030af..c9621506 100755 --- a/headphones/postprocessor.py +++ b/headphones/postprocessor.py @@ -261,7 +261,7 @@ def verify(albumid, albumpath, Kind=None, forced=False, keep_original_folder=Fal if downloaded_deezer_list: logger.info('Decrypting deezer tracks') decrypted_deezer_list = deezloader.decryptTracks(downloaded_deezer_list) - + # Check if album is complete based on album duration only # (total track numbers is not determinant enough due to hidden tracks for eg) db_track_duration = 0 @@ -271,7 +271,7 @@ def verify(albumid, albumpath, Kind=None, forced=False, keep_original_folder=Fal db_track_duration += track['TrackDuration'] / 1000 except: downloaded_track_duration = False - + try: for disk_number in decrypted_deezer_list[albumpath]: for track in decrypted_deezer_list[albumpath][disk_number].values(): @@ -279,14 +279,14 @@ def verify(albumid, albumpath, Kind=None, forced=False, keep_original_folder=Fal downloaded_track_duration += int(track['DURATION']) except: downloaded_track_duration = False - + if not downloaded_track_duration or not db_track_duration or abs(downloaded_track_duration - db_track_duration) > 240: - logger.info("Looks like " + + logger.info("Looks like " + os.path.basename(albumpath).decode(headphones.SYS_ENCODING, 'replace') + " isn't complete yet (duration mismatch). Will try again on the next run") return - downloaded_track_list = list(set(downloaded_track_list)) # Remove duplicates + downloaded_track_list = list(set(downloaded_track_list)) # Remove duplicates # test #1: metadata - usually works logger.debug('Verifying metadata...') diff --git a/headphones/searcher.py b/headphones/searcher.py index 2516bc31..f6554cb1 100644 --- a/headphones/searcher.py +++ b/headphones/searcher.py @@ -55,6 +55,7 @@ redobj = None # Persistent Aria2 RPC object __aria2rpc_obj = None + def getAria2RPC(): global __aria2rpc_obj if not __aria2rpc_obj: @@ -67,10 +68,12 @@ def getAria2RPC(): ) return __aria2rpc_obj + def reconfigure(): global __aria2rpc_obj __aria2rpc_obj = None + def fix_url(s, charset="utf-8"): """ Fix the URL so it is proper formatted and encoded. @@ -896,7 +899,7 @@ def send_to_downloader(data, bestqual, album): except Exception as e: logger.error(u'Error sending torrent to Aria2. Are you sure it\'s running? (%s)' % e) return - + else: folder_name = '%s - %s [%s]' % ( helpers.latinToAscii(album['ArtistName']).encode('UTF-8').replace('/', '_'), @@ -1274,6 +1277,7 @@ def verifyresult(title, artistterm, term, lossless): return True + def searchDdl(album, new=False, losslessOnly=False, albumlength=None, choose_specific_download=False): reldate = album['ReleaseDate'] @@ -1313,7 +1317,7 @@ def searchDdl(album, new=False, losslessOnly=False, albumlength=None, logger.debug(u'Using search term: "%s"' % helpers.latinToAscii(term)) resultlist = [] - + # Deezer only provides lossy if headphones.CONFIG.DEEZLOADER and not losslessOnly: resultlist += deezloader.searchAlbum(album['ArtistName'], album['AlbumTitle'], album['SearchTerm'], int(albumlength / 1000)) diff --git a/headphones/webserve.py b/headphones/webserve.py index 19e2ffb3..7e97cefb 100644 --- a/headphones/webserve.py +++ b/headphones/webserve.py @@ -1587,7 +1587,7 @@ class WebInterface(object): # Reconfigure musicbrainz database connection with the new values mb.startmb() - + # Reconfigure Aria2 searcher.reconfigure() From e8f436a5dbe40a12ceb0bc09fe7439ab8d8df006 Mon Sep 17 00:00:00 2001 From: Kallys Date: Wed, 19 Apr 2017 20:01:45 +0200 Subject: [PATCH 04/27] Fix travis errors bis --- headphones/helpers_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/headphones/helpers_test.py b/headphones/helpers_test.py index 54c9e9bf..14b8540d 100644 --- a/headphones/helpers_test.py +++ b/headphones/helpers_test.py @@ -14,9 +14,9 @@ class HelpersTest(TestCase): u'Symphonęy Nº9': 'Symphoney No.9', u'ÆæßðÞIJij': u'AeaessdThIJıj', u'Obsessió (Cerebral Apoplexy remix)': 'obsessio cerebral ' - 'apoplexy remix', + 'apoplexy remix', u'Doktór Hałabała i siedmiu zbojów': 'doktor halabala i siedmiu ' - 'zbojow', + 'zbojow', u'Arbetets Söner och Döttrar': 'arbetets soner och dottrar', u'Björk Guðmundsdóttir': 'bjork gudmundsdottir', u'L\'Arc~en~Ciel': 'larc en ciel', From 61a3abbf20373a1eb1e094fd53032db156c21079 Mon Sep 17 00:00:00 2001 From: Ade Date: Thu, 20 Apr 2017 20:10:34 +1200 Subject: [PATCH 05/27] last.fm series info --- headphones/cache.py | 33 +++++++++++++++++++++++++++------ headphones/helpers.py | 23 +++++++++++++++++++++++ headphones/mb.py | 24 ++++++++++++++++++++++++ headphones/metacritic.py | 3 ++- 4 files changed, 76 insertions(+), 7 deletions(-) diff --git a/headphones/cache.py b/headphones/cache.py index 202b7802..0e946677 100644 --- a/headphones/cache.py +++ b/headphones/cache.py @@ -16,7 +16,7 @@ import os import headphones -from headphones import db, helpers, logger, lastfm, request +from headphones import db, helpers, logger, lastfm, request, mb LASTFM_API_KEY = "690e1ed3bc00bc91804cd8f7fe5ed6d4" @@ -290,6 +290,14 @@ class Cache(object): data = lastfm.request_lastfm("artist.getinfo", mbid=self.id, api_key=LASTFM_API_KEY) + # Try with name if not found + if not data: + dbartist = myDB.action('SELECT ArtistName, Type FROM artists WHERE ArtistID=?', [self.id]).fetchone() + if dbartist: + data = lastfm.request_lastfm("artist.getinfo", + artist=helpers.clean_musicbrainz_name(dbartist['ArtistName']), + api_key=LASTFM_API_KEY) + if not data: return @@ -315,18 +323,31 @@ class Cache(object): else: dbalbum = myDB.action( - 'SELECT ArtistName, AlbumTitle, ReleaseID FROM albums WHERE AlbumID=?', + 'SELECT ArtistName, AlbumTitle, ReleaseID, Type FROM albums WHERE AlbumID=?', [self.id]).fetchone() if dbalbum['ReleaseID'] != self.id: data = lastfm.request_lastfm("album.getinfo", mbid=dbalbum['ReleaseID'], api_key=LASTFM_API_KEY) if not data: - data = lastfm.request_lastfm("album.getinfo", artist=dbalbum['ArtistName'], - album=dbalbum['AlbumTitle'], + data = lastfm.request_lastfm("album.getinfo", + artist=helpers.clean_musicbrainz_name(dbalbum['ArtistName']), + album=helpers.clean_musicbrainz_name(dbalbum['AlbumTitle']), api_key=LASTFM_API_KEY) else: - data = lastfm.request_lastfm("album.getinfo", artist=dbalbum['ArtistName'], - album=dbalbum['AlbumTitle'], api_key=LASTFM_API_KEY) + if dbalbum['Type'] != "part of": + data = lastfm.request_lastfm("album.getinfo", + artist=helpers.clean_musicbrainz_name(dbalbum['ArtistName']), + album=helpers.clean_musicbrainz_name(dbalbum['AlbumTitle']), + api_key=LASTFM_API_KEY) + else: + + # Series, use actual artist for the release-group + artist = mb.getArtistForReleaseGroup(self.id) + if artist: + data = lastfm.request_lastfm("album.getinfo", + artist=helpers.clean_musicbrainz_name(artist), + album=helpers.clean_musicbrainz_name(dbalbum['AlbumTitle']), + api_key=LASTFM_API_KEY) if not data: return diff --git a/headphones/helpers.py b/headphones/helpers.py index 2ba72101..86a60a10 100644 --- a/headphones/helpers.py +++ b/headphones/helpers.py @@ -257,6 +257,12 @@ _XLATE_SPECIAL = { u'&': ' and ', # expand & to ' and ' } +_XLATE_MUSICBRAINZ = { + # Translation table for Musicbrainz. + u"…": '...', # HORIZONTAL ELLIPSIS (U+2026) + u"’": "'", # APOSTROPHE (U+0027) + u"‐": "-", # EN DASH (U+2013) +} def _translate(s, dictionary): # type: (basestring,Mapping[basestring,basestring])->basestring @@ -325,6 +331,23 @@ def clean_name(s): return u +def clean_musicbrainz_name(s, return_as_string=True): + # type: (basestring)->unicode + """Substitute special Musicbrainz characters. + :param s: string to clean up, probably unicode. + :return: cleaned-up version of input string. + """ + if not isinstance(s, unicode): + u = unicode(s, 'ascii', 'replace') + else: + u = s + u = _translate(u, _XLATE_MUSICBRAINZ) + if return_as_string: + return u.encode('utf-8') + else: + return u + + def cleanTitle(title): title = re.sub('[\.\-\/\_]', ' ', title).lower() diff --git a/headphones/mb.py b/headphones/mb.py index faba6b8a..951675a2 100644 --- a/headphones/mb.py +++ b/headphones/mb.py @@ -770,3 +770,27 @@ def findAlbumID(artist=None, album=None): return False rgid = unicode(results[0]['id']) return rgid + + +def getArtistForReleaseGroup(rgid): + """ + Returns artist name for a release group + Used for series where we store the series instead of the artist + """ + releaseGroup = None + try: + with mb_lock: + releaseGroup = musicbrainzngs.get_release_group_by_id( + rgid, ["artists"]) + releaseGroup = releaseGroup['release-group'] + except musicbrainzngs.WebServiceError as e: + logger.warn( + 'Attempt to retrieve information from MusicBrainz for release group "%s" failed (%s)' % ( + rgid, str(e))) + mb_lock.snooze(5) + + if not releaseGroup: + return False + else: + return releaseGroup['artist-credit'][0]['artist']['name'] + diff --git a/headphones/metacritic.py b/headphones/metacritic.py index d482786f..4ff20140 100644 --- a/headphones/metacritic.py +++ b/headphones/metacritic.py @@ -27,8 +27,9 @@ def update(artistid, artist_name, release_groups): # cut down on api calls. If it's ineffective then we'll switch to search replacements = {" & ": " ", ".": ""} + mc_artist_name = helpers.clean_musicbrainz_name(artist_name, return_as_string=False) + mc_artist_name = mc_artist_name.replace("'", " ") mc_artist_name = helpers.replace_all(artist_name.lower(), replacements) - mc_artist_name = mc_artist_name.replace(" ", "-") headers = { From 50851a4953916b0ca6e0330c63ff8e919ed9a47a Mon Sep 17 00:00:00 2001 From: Ade Date: Thu, 20 Apr 2017 20:42:04 +1200 Subject: [PATCH 06/27] pep8 --- headphones/helpers.py | 1 + headphones/mb.py | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/headphones/helpers.py b/headphones/helpers.py index 86a60a10..4c815ad7 100644 --- a/headphones/helpers.py +++ b/headphones/helpers.py @@ -264,6 +264,7 @@ _XLATE_MUSICBRAINZ = { u"‐": "-", # EN DASH (U+2013) } + def _translate(s, dictionary): # type: (basestring,Mapping[basestring,basestring])->basestring return ''.join(dictionary.get(x, x) for x in s) diff --git a/headphones/mb.py b/headphones/mb.py index 951675a2..c087d6b9 100644 --- a/headphones/mb.py +++ b/headphones/mb.py @@ -793,4 +793,3 @@ def getArtistForReleaseGroup(rgid): return False else: return releaseGroup['artist-credit'][0]['artist']['name'] - From 7fec54289b03d88bb9d994e2b84ca7ed44398bdf Mon Sep 17 00:00:00 2001 From: AdeHub Date: Fri, 21 Apr 2017 19:45:37 +1200 Subject: [PATCH 07/27] Revert "Add deezloader provider and aria2 downloader" --- data/interfaces/default/config.html | 61 +- headphones/aria2.py | 979 ---------------------------- headphones/config.py | 7 - headphones/deezloader.py | 428 ------------ headphones/postprocessor.py | 40 +- headphones/searcher.py | 167 +---- headphones/webserve.py | 13 +- 7 files changed, 21 insertions(+), 1674 deletions(-) delete mode 100644 headphones/aria2.py delete mode 100644 headphones/deezloader.py diff --git a/data/interfaces/default/config.html b/data/interfaces/default/config.html index 43aead6a..841df0d1 100644 --- a/data/interfaces/default/config.html +++ b/data/interfaces/default/config.html @@ -304,45 +304,6 @@ - -
- Direct Download - Aria2 -
-
-
- - - usually http://localhost:6800/jsonrpc -
-
- - -
-
- - -
-
- - -
-
- - - Full path where ddl client downloads your music, e.g. /Users/name/Downloads/music -
-
@@ -500,7 +461,6 @@ NZBs Torrents - Direct Download No Preference
@@ -613,17 +573,6 @@ - -
- Direct Download - -
-
- -
-
- -
@@ -855,6 +804,7 @@
+ @@ -2455,11 +2405,6 @@ $("#deluge_options").show(); } - if ($("#ddl_downloader_aria").is(":checked")) - { - $("#ddl_aria_options").show(); - } - $('input[type=radio]').change(function(){ if ($("#preferred_bitrate").is(":checked")) { @@ -2513,9 +2458,6 @@ { $("#torrent_blackhole_options,#utorrent_options,#transmission_options,#qbittorrent_options").fadeOut("fast", function() { $("#deluge_options").fadeIn() }); } - if ($("#ddl_downloader_aria").is(":checked")) - { - } }); $("#mirror").change(handleNewServerSelection); @@ -2618,7 +2560,6 @@ initConfigCheckbox("#enable_https"); initConfigCheckbox("#customauth"); initConfigCheckbox("#use_tquattrecentonze"); - initConfigCheckbox("#use_deezloader"); $('#twitterStep1').click(function () { diff --git a/headphones/aria2.py b/headphones/aria2.py deleted file mode 100644 index 06d8e4e0..00000000 --- a/headphones/aria2.py +++ /dev/null @@ -1,979 +0,0 @@ -# -*- coding: utf8 -*- -# Copyright (C) 2012-2016 Xyne -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# (version 2) as published by the Free Software Foundation. -# -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -from __future__ import with_statement -import base64 -import json -import math -import os -import ssl -import string -import time -import httplib -import urllib2 - -from headphones import logger - -# ################################ Constants ################################### - -DEFAULT_PORT = 6800 -SERVER_URI_FORMAT = '{}://{}:{:d}/jsonrpc' - -# Status values for unfinished downloads. -TEMPORARY_STATUS = ('active', 'waiting', 'paused') -# Status values for finished downloads. -FINAL_STATUS = ('complete', 'error') - -ARIA2_CONTROL_FILE_EXT = '.aria2' - -# ########################## Convenience Functions ############################# - - -def to_json_list(objs): - ''' - Wrap strings in lists. Other iterables are converted to lists directly. - ''' - if isinstance(objs, str): - return [objs] - elif not isinstance(objs, list): - return list(objs) - else: - return objs - - -def add_options_and_position(params, options=None, position=None): - ''' - Convenience method for adding options and position to parameters. - ''' - if options: - params.append(options) - if position: - if not isinstance(position, int): - try: - position = int(position) - except ValueError: - position = -1 - if position >= 0: - params.append(position) - return params - - -def get_status(response): - ''' - Process a status response. - ''' - if response: - try: - return response['status'] - except KeyError: - logger.error('no status returned from Aria2 RPC server') - return 'error' - else: - logger.error('no response from server') - return 'error' - - -def random_token(length, valid_chars=None): - ''' - Get a random secret token for the Aria2 RPC server. - - length: - The length of the token - - valid_chars: - A list or other ordered and indexable iterable of valid characters. If not - given of None, asciinumberic characters with some punctuation characters - will be used. - ''' - if not valid_chars: - valid_chars = string.ascii_letters + string.digits + '!@#$%^&*()-_=+' - number_of_chars = len(valid_chars) - bytes_to_read = math.ceil(math.log(number_of_chars) / math.log(0x100)) - max_value = 0x100**bytes_to_read - max_index = number_of_chars - 1 - token = '' - for _ in range(length): - value = int.from_bytes(os.urandom(bytes_to_read), byteorder='little') - index = round((value * max_index) / max_value) - token += valid_chars[index] - return token - -# ################ From python3-aur's ThreadedServers.common ################### - - -def format_bytes(size): - '''Format bytes for inferior humans.''' - if size < 0x400: - return '{:d} B'.format(size) - else: - size = float(size) / 0x400 - for prefix in ('KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB'): - if size < 0x400: - return '{:0.02f} {}'.format(size, prefix) - else: - size /= 0x400 - return '{:0.02f} YiB'.format(size) - - -def format_seconds(s): - '''Format seconds for inferior humans.''' - string = '' - for base, char in ( - (60, 's'), - (60, 'm'), - (24, 'h') - ): - s, r = divmod(s, base) - if s == 0: - return '{:d}{}{}'.format(r, char, string) - elif r != 0: - string = '{:02d}{}{}'.format(r, char, string) - else: - return '{:d}d{}'.format(s, string) - -# ############################ Aria2JsonRpcError ############################### - - -class Aria2JsonRpcError(Exception): - def __init__(self, msg, connection_error=False): - super(self.__class__, self).__init__(self, msg) - self.connection_error = connection_error - -# ############################ Aria2JsonRpc Class ############################## - - -class Aria2JsonRpc(object): - ''' - Interface class for interacting with an Aria2 RPC server. - ''' - # TODO: certificate options, etc. - def __init__( - self, ID, uri, - mode='normal', - token=None, - http_user=None, http_passwd=None, - server_cert=None, client_cert=None, client_cert_password=None, - ssl_protocol=None, - setup_function=None - ): - ''' - ID: the ID to send to the RPC interface - - uri: the URI of the RPC interface - - mode: - normal - process requests immediately - batch - queue requests (run with "process_queue") - format - return RPC request objects - - token: - RPC method-level authorization token (set using `--rpc-secret`) - - http_user, http_password: - HTTP Basic authentication credentials (deprecated) - - server_cert: - server certificate for HTTPS connections - - client_cert: - client certificate for HTTPS connections - - client_cert_password: - prompt for client certificate password - - ssl_protocol: - SSL protocol from the ssl module - - setup_function: - A function to invoke prior to the first server call. This could be the - launch() method of an Aria2RpcServer instance, for example. This attribute - is set automatically in instances returned from Aria2RpcServer.get_a2jr() - ''' - self.id = ID - self.uri = uri - self.mode = mode - self.queue = [] - self.handlers = dict() - self.token = token - self.setup_function = setup_function - - if None not in (http_user, http_passwd): - self.add_HTTPBasicAuthHandler(http_user, http_passwd) - - if server_cert or client_cert: - self.add_HTTPSHandler( - server_cert=server_cert, - client_cert=client_cert, - client_cert_password=client_cert_password, - protocol=ssl_protocol - ) - - self.update_opener() - - def iter_handlers(self): - ''' - Iterate over handlers. - ''' - for name in ('HTTPS', 'HTTPBasicAuth'): - try: - yield self.handlers[name] - except KeyError: - pass - - def update_opener(self): - ''' - Build an opener from the current handlers. - ''' - self.opener = urllib2.build_opener(*self.iter_handlers()) - - def remove_handler(self, name): - ''' - Remove a handler. - ''' - try: - del self.handlers[name] - except KeyError: - pass - - def add_HTTPBasicAuthHandler(self, user, passwd): - ''' - Add a handler for HTTP Basic authentication. - - If either user or passwd are None, the handler is removed. - ''' - handler = urllib2.HTTPBasicAuthHandler() - handler.add_password( - realm='aria2', - uri=self.uri, - user=user, - passwd=passwd, - ) - self.handlers['HTTPBasicAuth'] = handler - - def remove_HTTPBasicAuthHandler(self): - self.remove_handler('HTTPBasicAuth') - - def add_HTTPSHandler( - self, - server_cert=None, - client_cert=None, - client_cert_password=None, - protocol=None, - ): - ''' - Add a handler for HTTPS connections with optional server and client - certificates. - ''' - if not protocol: - protocol = ssl.PROTOCOL_TLSv1 -# protocol = ssl.PROTOCOL_TLSv1_1 # openssl 1.0.1+ -# protocol = ssl.PROTOCOL_TLSv1_2 # Python 3.4+ - context = ssl.SSLContext(protocol) - - if server_cert: - context.verify_mode = ssl.CERT_REQUIRED - context.load_verify_locations(cafile=server_cert) - else: - context.verify_mode = ssl.CERT_OPTIONAL - - if client_cert: - context.load_cert_chain(client_cert, password=client_cert_password) - - self.handlers['HTTPS'] = urllib2.HTTPSHandler( - context=context, - check_hostname=False - ) - - def remove_HTTPSHandler(self): - self.remove_handler('HTTPS') - - def send_request(self, req_obj): - ''' - Send the request and return the response. - ''' - if self.setup_function: - self.setup_function() - self.setup_function = None - logger.debug("Aria req_obj: %s" % json.dumps(req_obj, indent=2, sort_keys=True)) - req = json.dumps(req_obj).encode('UTF-8') - try: - f = self.opener.open(self.uri, req) - obj = json.loads(f.read()) - try: - return obj['result'] - except KeyError: - raise Aria2JsonRpcError('unexpected result: {}'.format(obj)) - except (urllib2.URLError) as e: - # This should work but URLError does not set the errno attribute: - # e.errno == errno.ECONNREFUSED - raise Aria2JsonRpcError( - str(e), - connection_error=( - '111' in str(e) - ) - ) - except httplib.BadStatusLine as e: - raise Aria2JsonRpcError('{}: BadStatusLine: {} (HTTPS error?)'.format( - self.__class__.__name__, e - )) - - def jsonrpc(self, method, params=None, prefix='aria2.'): - ''' - POST a request to the RPC interface. - ''' - if not params: - params = [] - - if self.token is not None: - token_str = 'token:{}'.format(self.token) - if method == 'multicall': - for p in params[0]: - try: - p['params'].insert(0, token_str) - except KeyError: - p['params'] = [token_str] - else: - params.insert(0, token_str) - - req_obj = { - 'jsonrpc': '2.0', - 'id': self.id, - 'method': prefix + method, - 'params': params, - } - if self.mode == 'batch': - self.queue.append(req_obj) - return None - elif self.mode == 'format': - return req_obj - else: - return self.send_request(req_obj) - - def process_queue(self): - ''' - Processed queued requests. - ''' - req_obj = self.queue - self.queue = [] - return self.send_request(req_obj) - - -# ############################# Standard Methods ############################### - def addUri(self, uris, options=None, position=None): - ''' - aria2.addUri method - - uris: list of URIs - - options: dictionary of additional options - - position: position in queue - - Returns a GID - ''' - params = [uris] - params = add_options_and_position(params, options, position) - return self.jsonrpc('addUri', params) - - def addTorrent(self, torrent, uris=None, options=None, position=None): - ''' - aria2.addTorrent method - - torrent: base64-encoded torrent file - - uris: list of webseed URIs - - options: dictionary of additional options - - position: position in queue - - Returns a GID. - ''' - params = [torrent] - if uris: - params.append(uris) - params = add_options_and_position(params, options, position) - return self.jsonrpc('addTorrent', params) - - def addMetalink(self, metalink, options=None, position=None): - ''' - aria2.addMetalink method - - metalink: base64-encoded metalink file - - options: dictionary of additional options - - position: position in queue - - Returns an array of GIDs. - ''' - params = [metalink] - params = add_options_and_position(params, options, position) - return self.jsonrpc('addTorrent', params) - - def remove(self, gid): - ''' - aria2.remove method - - gid: GID to remove - ''' - params = [gid] - return self.jsonrpc('remove', params) - - def forceRemove(self, gid): - ''' - aria2.forceRemove method - - gid: GID to remove - ''' - params = [gid] - return self.jsonrpc('forceRemove', params) - - def pause(self, gid): - ''' - aria2.pause method - - gid: GID to pause - ''' - params = [gid] - return self.jsonrpc('pause', params) - - def pauseAll(self): - ''' - aria2.pauseAll method - ''' - return self.jsonrpc('pauseAll') - - def forcePause(self, gid): - ''' - aria2.forcePause method - - gid: GID to pause - ''' - params = [gid] - return self.jsonrpc('forcePause', params) - - def forcePauseAll(self): - ''' - aria2.forcePauseAll method - ''' - return self.jsonrpc('forcePauseAll') - - def unpause(self, gid): - ''' - aria2.unpause method - - gid: GID to unpause - ''' - params = [gid] - return self.jsonrpc('unpause', params) - - def unpauseAll(self): - ''' - aria2.unpauseAll method - ''' - return self.jsonrpc('unpauseAll') - - def tellStatus(self, gid, keys=None): - ''' - aria2.tellStatus method - - gid: GID to query - - keys: subset of status keys to return (all keys are returned otherwise) - - Returns a dictionary. - ''' - params = [gid] - if keys: - params.append(keys) - return self.jsonrpc('tellStatus', params) - - def getUris(self, gid): - ''' - aria2.getUris method - - gid: GID to query - - Returns a list of dictionaries. - ''' - params = [gid] - return self.jsonrpc('getUris', params) - - def getFiles(self, gid): - ''' - aria2.getFiles method - - gid: GID to query - - Returns a list of dictionaries. - ''' - params = [gid] - return self.jsonrpc('getFiles', params) - - def getPeers(self, gid): - ''' - aria2.getPeers method - - gid: GID to query - - Returns a list of dictionaries. - ''' - params = [gid] - return self.jsonrpc('getPeers', params) - - def getServers(self, gid): - ''' - aria2.getServers method - - gid: GID to query - - Returns a list of dictionaries. - ''' - params = [gid] - return self.jsonrpc('getServers', params) - - def tellActive(self, keys=None): - ''' - aria2.tellActive method - - keys: same as tellStatus - - Returns a list of dictionaries. The dictionaries are the same as those - returned by tellStatus. - ''' - if keys: - params = [keys] - else: - params = None - return self.jsonrpc('tellActive', params) - - def tellWaiting(self, offset, num, keys=None): - ''' - aria2.tellWaiting method - - offset: offset from start of waiting download queue - (negative values are counted from the end of the queue) - - num: number of downloads to return - - keys: same as tellStatus - - Returns a list of dictionaries. The dictionaries are the same as those - returned by tellStatus. - ''' - params = [offset, num] - if keys: - params.append(keys) - return self.jsonrpc('tellWaiting', params) - - def tellStopped(self, offset, num, keys=None): - ''' - aria2.tellStopped method - - offset: offset from oldest download (same semantics as tellWaiting) - - num: same as tellWaiting - - keys: same as tellStatus - - Returns a list of dictionaries. The dictionaries are the same as those - returned by tellStatus. - ''' - params = [offset, num] - if keys: - params.append(keys) - return self.jsonrpc('tellStopped', params) - - def changePosition(self, gid, pos, how): - ''' - aria2.changePosition method - - gid: GID to change - - pos: the position - - how: "POS_SET", "POS_CUR" or "POS_END" - ''' - params = [gid, pos, how] - return self.jsonrpc('changePosition', params) - - def changeUri(self, gid, fileIndex, delUris, addUris, position=None): - ''' - aria2.changePosition method - - gid: GID to change - - fileIndex: file to affect (1-based) - - delUris: URIs to remove - - addUris: URIs to add - - position: where URIs are inserted, after URIs have been removed - ''' - params = [gid, fileIndex, delUris, addUris] - if position: - params.append(position) - return self.jsonrpc('changePosition', params) - - def getOption(self, gid): - ''' - aria2.getOption method - - gid: GID to query - - Returns a dictionary of options. - ''' - params = [gid] - return self.jsonrpc('getOption', params) - - def changeOption(self, gid, options): - ''' - aria2.changeOption method - - gid: GID to change - - options: dictionary of new options - (not all options can be changed for active downloads) - ''' - params = [gid, options] - return self.jsonrpc('changeOption', params) - - def getGlobalOption(self): - ''' - aria2.getGlobalOption method - - Returns a dictionary. - ''' - return self.jsonrpc('getGlobalOption') - - def changeGlobalOption(self, options): - ''' - aria2.changeGlobalOption method - - options: dictionary of new options - ''' - params = [options] - return self.jsonrpc('changeGlobalOption', params) - - def getGlobalStat(self): - ''' - aria2.getGlobalStat method - - Returns a dictionary. - ''' - return self.jsonrpc('getGlobalStat') - - def purgeDownloadResult(self): - ''' - aria2.purgeDownloadResult method - ''' - self.jsonrpc('purgeDownloadResult') - - def removeDownloadResult(self, gid): - ''' - aria2.removeDownloadResult method - - gid: GID to remove - ''' - params = [gid] - return self.jsonrpc('removeDownloadResult', params) - - def getVersion(self): - ''' - aria2.getVersion method - - Returns a dictionary. - ''' - return self.jsonrpc('getVersion') - - def getSessionInfo(self): - ''' - aria2.getSessionInfo method - - Returns a dictionary. - ''' - return self.jsonrpc('getSessionInfo') - - def shutdown(self): - ''' - aria2.shutdown method - ''' - return self.jsonrpc('shutdown') - - def forceShutdown(self): - ''' - aria2.forceShutdown method - ''' - return self.jsonrpc('forceShutdown') - - def multicall(self, methods): - ''' - aria2.multicall method - - methods: list of dictionaries (keys: methodName, params) - - The method names must be those used by Aria2c, e.g. "aria2.tellStatus". - ''' - return self.jsonrpc('multicall', [methods], prefix='system.') - - -# ########################### Convenience Methods ############################## - def add_torrent(self, path, uris=None, options=None, position=None): - ''' - A wrapper around addTorrent for loading files. - ''' - with open(path, 'r') as f: - torrent = base64.encode(f.read()) - return self.addTorrent(torrent, uris, options, position) - - def add_metalink(self, path, uris=None, options=None, position=None): - ''' - A wrapper around addMetalink for loading files. - ''' - with open(path, 'r') as f: - metalink = base64.encode(f.read()) - return self.addMetalink(metalink, uris, options, position) - - def get_status(self, gid): - ''' - Get the status of a single GID. - ''' - response = self.tellStatus(gid, ['status']) - return get_status(response) - - def wait_for_final_status(self, gid, interval=1): - ''' - Wait for a GID to complete or fail and return its status. - ''' - if not interval or interval < 0: - interval = 1 - while True: - status = self.get_status(gid) - if status in TEMPORARY_STATUS: - time.sleep(interval) - else: - return status - - def get_statuses(self, gids): - ''' - Get the status of multiple GIDs. The status of each is yielded in order. - ''' - methods = [ - { - 'methodName': 'aria2.tellStatus', - 'params': [gid, ['gid', 'status']] - } - for gid in gids - ] - results = self.multicall(methods) - if results: - status = dict((r[0]['gid'], r[0]['status']) for r in results) - for gid in gids: - try: - yield status[gid] - except KeyError: - logger.error('Aria2 RPC server returned no status for GID {}'.format(gid)) - yield 'error' - else: - logger.error('no response from Aria2 RPC server') - for gid in gids: - yield 'error' - - def wait_for_final_statuses(self, gids, interval=1): - ''' - Wait for multiple GIDs to complete or fail and return their statuses in - order. - - gids: - A flat list of GIDs. - ''' - if not interval or interval < 0: - interval = 1 - statusmap = dict((g, None) for g in gids) - remaining = list( - g for g, s in statusmap.items() if s is None - ) - while remaining: - for g, s in zip(remaining, self.get_statuses(remaining)): - if s in TEMPORARY_STATUS: - continue - else: - statusmap[g] = s - remaining = list( - g for g, s in statusmap.items() if s is None - ) - if remaining: - time.sleep(interval) - for g in gids: - yield statusmap[g] - - def print_global_status(self): - ''' - Print global status of the RPC server. - ''' - status = self.getGlobalStat() - if status: - numWaiting = int(status['numWaiting']) - numStopped = int(status['numStopped']) - keys = ['totalLength', 'completedLength'] - total = self.tellActive(keys) - waiting = self.tellWaiting(0, numWaiting, keys) - if waiting: - total += waiting - stopped = self.tellStopped(0, numStopped, keys) - if stopped: - total += stopped - - downloadSpeed = int(status['downloadSpeed']) - uploadSpeed = int(status['uploadSpeed']) - totalLength = sum(int(x['totalLength']) for x in total) - completedLength = sum(int(x['completedLength']) for x in total) - remaining = totalLength - completedLength - - status['downloadSpeed'] = format_bytes(downloadSpeed) + '/s' - status['uploadSpeed'] = format_bytes(uploadSpeed) + '/s' - - preordered = ('downloadSpeed', 'uploadSpeed') - - rows = list() - for k in sorted(status): - if k in preordered: - continue - rows.append((k, status[k])) - - rows.extend((x, status[x]) for x in preordered) - - if totalLength > 0: - rows.append(('total', format(format_bytes(totalLength)))) - rows.append(('completed', format(format_bytes(completedLength)))) - rows.append(('remaining', format(format_bytes(remaining)))) - if completedLength == totalLength: - eta = 'finished' - else: - try: - eta = format_seconds(remaining // downloadSpeed) - except ZeroDivisionError: - eta = 'never' - rows.append(('ETA', eta)) - - l = max(len(r[0]) for r in rows) - r = max(len(r[1]) for r in rows) - r = max(r, len(self.uri) - (l + 2)) - fmt = '{:<' + str(l) + 's} {:>' + str(r) + 's}' - - print(self.uri) - for k, v in rows: - print(fmt.format(k, v)) - - def queue_uris(self, uris, options, interval=None): - ''' - Enqueue URIs and wait for download to finish while printing status at - regular intervals. - ''' - gid = self.addUri(uris, options) - print('GID: {}'.format(gid)) - - if gid and interval is not None: - blanker = '' - while True: - response = self.tellStatus(gid, ['status']) - if response: - try: - status = response['status'] - except KeyError: - print('error: no status returned from Aria2 RPC server') - break - print('{}\rstatus: {}'.format(blanker, status)), - blanker = ' ' * len(status) - if status in TEMPORARY_STATUS: - time.sleep(interval) - else: - break - else: - print('error: no response from server') - break - - -# ####################### Polymethod download handlers ######################### - def polymethod_enqueue_many(self, downloads): - ''' - Enqueue downloads. - - downloads: Same as polymethod_download(). - ''' - methods = list( - { - 'methodName': 'aria2.{}'.format(d[0]), - 'params': list(d[1:]) - } for d in downloads - ) - return self.multicall(methods) - - def polymethod_wait_many(self, gids, interval=1): - ''' - Wait for the GIDs to complete or fail and return their statuses. - - gids: - A list of lists of GIDs. - ''' - # The flattened list of GIDs - gs = list(g for gs in gids for g in gs) - statusmap = dict(tuple(zip( - gs, - self.wait_for_final_statuses(gs, interval=interval) - ))) - for gs in gids: - yield list(statusmap.get(g, 'error') for g in gs) - - def polymethod_enqueue_one(self, download): - ''' - Same as polymethod_enqueue_many but for one element. - ''' - return getattr(self, download[0])(*download[1:]) - - def polymethod_download(self, downloads, interval=1): - ''' - Enqueue a series of downloads and wait for them to finish. Iterate over the - status of each, in order. - - downloads: - An iterable over (, , ...) where indicates the "add" - method to use ('addUri', 'addTorrent', 'addMetalink') and everything that - follows are arguments to pass to that method. - - interval: - The status check interval while waiting. - - Iterates over the download status of finished downloads. "complete" - indicates success. Lists of statuses will be returned for downloads that - create multiple GIDs (e.g. metalinks). - ''' - gids = self.polymethod_enqueue_many(downloads) - return self.polymethod_wait_many(gids, interval=interval) - - def polymethod_download_bool(self, *args, **kwargs): - ''' - A wrapper around polymethod_download() which returns a boolean for each - download to indicate success (True) or failure (False). - ''' -# for status in self.polymethod_download(*args, **kwargs): -# yield all(s == 'complete' for s in status) - return list( - all(s == 'complete' for s in status) - for status in self.polymethod_download(*args, **kwargs) - ) diff --git a/headphones/config.py b/headphones/config.py index 86aa9216..57af4c16 100644 --- a/headphones/config.py +++ b/headphones/config.py @@ -46,10 +46,6 @@ _CONFIG_DEFINITIONS = { 'APOLLO_RATIO': (str, 'Apollo.rip', ''), 'APOLLO_USERNAME': (str, 'Apollo.rip', ''), 'APOLLO_URL': (str, 'Apollo.rip', 'https://apollo.rip'), - 'ARIA_HOST': (str, 'Aria2', ''), - 'ARIA_PASSWORD': (str, 'Aria2', ''), - 'ARIA_TOKEN': (str, 'Aria2', ''), - 'ARIA_USERNAME': (str, 'Aria2', ''), 'AUTOWANT_ALL': (int, 'General', 0), 'AUTOWANT_MANUALLY_ADDED': (int, 'General', 1), 'AUTOWANT_UPCOMING': (int, 'General', 1), @@ -77,8 +73,6 @@ _CONFIG_DEFINITIONS = { 'CUSTOMPORT': (int, 'General', 5000), 'CUSTOMSLEEP': (int, 'General', 1), 'CUSTOMUSER': (str, 'General', ''), - 'DDL_DOWNLOADER': (int, 'General', 0), - 'DEEZLOADER': (int, 'DeezLoader', 0), 'DELETE_LOSSLESS_FILES': (int, 'General', 1), 'DELUGE_HOST': (str, 'Deluge', ''), 'DELUGE_CERT': (str, 'Deluge', ''), @@ -92,7 +86,6 @@ _CONFIG_DEFINITIONS = { 'DOWNLOAD_DIR': (path, 'General', ''), 'DOWNLOAD_SCAN_INTERVAL': (int, 'General', 5), 'DOWNLOAD_TORRENT_DIR': (path, 'General', ''), - 'DOWNLOAD_DDL_DIR': (path, 'General', ''), 'DO_NOT_OVERRIDE_GIT_BRANCH': (int, 'General', 0), 'EMAIL_ENABLED': (int, 'Email', 0), 'EMAIL_FROM': (str, 'Email', ''), diff --git a/headphones/deezloader.py b/headphones/deezloader.py deleted file mode 100644 index 528cded6..00000000 --- a/headphones/deezloader.py +++ /dev/null @@ -1,428 +0,0 @@ -# -*- coding: utf-8 -*- -# Deezloader (c) 2016 by ParadoxalManiak -# -# Deezloader is licensed under a -# Creative Commons Attribution-NonCommercial-ShareAlike 3.0 Unported License. -# -# You should have received a copy of the license along with this -# work. If not, see . -# -# Version 2.1.0 -# Maintained by ParadoxalManiak -# Original work by ZzMTV -# -# Author's disclaimer: -# I am not responsible for the usage of this program by other people. -# I do not recommend you doing this illegally or against Deezer's terms of service. -# This project is licensed under CC BY-NC-SA 4.0 - -import re -import os -from datetime import datetime -from Crypto.Cipher import AES, Blowfish -from hashlib import md5 -import binascii - -from beets.mediafile import MediaFile -from headphones import logger, request, helpers -import headphones - -# Public constants -PROVIDER_NAME = 'Deezer' - -# Internal constants -__API_URL = "http://www.deezer.com/ajax/gw-light.php" -__API_INFO_URL = "http://api.deezer.com/" -__HTTP_HEADERS = { - "User-Agent": "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36", - "Content-Language": "en-US", - "Cache-Control": "max-age=0", - "Accept": "*/*", - "Accept-Charset": "utf-8,ISO-8859-1;q=0.7,*;q=0.3", - "Accept-Language": "de-DE,de;q=0.8,en-US;q=0.6,en;q=0.4" -} - -# Internal variables -__api_queries = { - 'api_version': "1.0", - 'api_token': "None", - 'input': "3" -} -__cookies = None -__tracks_cache = {} -__albums_cache = {} - - -def __getApiToken(): - global __cookies - response = request.request_response(url="http://www.deezer.com/", headers=__HTTP_HEADERS) - __cookies = response.cookies - data = response.content - if data: - matches = re.search(r"checkForm\s*=\s*['|\"](.*)[\"|'];", data) - if matches: - token = matches.group(1) - __api_queries['api_token'] = token - logger.debug(u"Deezloader : api token loeaded ('%s')" % token) - - if not token: - logger.error(u"Deezloader: Unable to get api token") - - -def getAlbumByLink(album_link): - """Returns deezer album infos using album link url - - :param str album_link: deezer album link url (eg: 'http://www.deezer.com/album/1234567/') - """ - matches = re.search(r"album\/([0-9]+)\/?$", album_link) - if matches: - return getAlbum(matches.group(1)) - - -def getAlbum(album_id): - """Returns deezer album infos - - :param int album_id: deezer album id - """ - global __albums_cache - - if str(album_id) in __albums_cache: - return __albums_cache[str(album_id)] - - url = __API_INFO_URL + "album/" + str(album_id) - data = request.request_json(url=url, headers=__HTTP_HEADERS, cookies=__cookies) - - if data and 'error' not in data: - __albums_cache[str(album_id)] = data - return data - else: - logger.debug("Deezloader: Can't load album infos") - return None - - -def searchAlbums(search_term): - """Search for deezer albums using search term - - :param str search_term: search term to search album for - """ - logger.info(u'Searching Deezer using term: "%s"' % search_term) - - url = __API_INFO_URL + "search/album?q=" + search_term - data = request.request_json(url=url, headers=__HTTP_HEADERS, cookies=__cookies) - - albums = [] - - # Process content - if data and 'total' in data and data['total'] > 0 and 'data' in data: - for item in data['data']: - try: - albums.append(getAlbum(item['id'])) - except Exception as e: - logger.error(u"An unknown error occurred in the Deezer parser: %s" % e) - else: - logger.info(u'No results found from Deezer using term: "%s"' % search_term) - - return albums - - -def __matchAlbums(albums, artist_name, album_title, album_length): - resultlist = [] - - for album in albums: - total_size = 0 - tracks_found = 0 - - for track in album['tracks']['data']: - t = getTrack(track['id']) - if t: - if t["FILESIZE_MP3_320"] > 0: - size = t["FILESIZE_MP3_320"] - elif t["FILESIZE_MP3_256"] > 0: - size = t["FILESIZE_MP3_256"] - elif t["FILESIZE_MP3_128"] > 0: - size = t["FILESIZE_MP3_128"] - else: - size = t["FILESIZE_MP3_64"] - - size = int(size) - total_size += size - tracks_found += 1 - logger.debug(u'Found song "%s". Size: %s' % (t['SNG_TITLE'], helpers.bytes_to_mb(size))) - - matched = True - mismatch_reason = 'matched!' - - if album_length > 0 and abs(int(album['duration']) - album_length) > 240: - matched = False - mismatch_reason = (u'duration mismatch: %i not in [%i, %i]' % (int(album['duration']), (album_length - 240), (album_length + 240))) - - elif (helpers.latinToAscii(re.sub(r"\W", "", album_title, 0, re.UNICODE)).lower() != - helpers.latinToAscii(re.sub(r"\W", "", album['title'], 0, re.UNICODE)).lower()): - matched = False - mismatch_reason = (u'album name mismatch: %s != %s' % (album['title'], album_title)) - - elif (helpers.latinToAscii(re.sub(r"\W", "", artist_name, 0, re.UNICODE)).lower() != - helpers.latinToAscii(re.sub(r"\W", "", album['artist']['name'], 0, re.UNICODE)).lower()): - matched = False - mismatch_reason = (u'artist name mismatch: %s != %s' % (album['artist']['name'], artist_name)) - - resultlist.append( - (album['artist']['name'] + ' - ' + album['title'] + ' [' + album['release_date'][:4] + '] (' + str(tracks_found) + '/' + str(album['nb_tracks']) + ')', - total_size, album['link'], PROVIDER_NAME, "ddl", matched) - ) - logger.info(u'Found "%s". Tracks %i/%i. Size: %s (%s)' % (album['title'], tracks_found, album['nb_tracks'], helpers.bytes_to_mb(total_size), mismatch_reason)) - - return resultlist - - -def searchAlbum(artist_name, album_title, user_search_term=None, album_length=None): - """Search for deezer specific album. - This will iterate over deezer albums and try to find best matches - - :param str artist_name: album artist name - :param str album_title: album title - :param str user_search_term: search terms provided by user - :param int album_length: targeted album duration in seconds - """ - # User search term by-pass normal search - if user_search_term: - return __matchAlbums(searchAlbums(user_search_term), artist_name, album_title, album_length) - - resultlist = __matchAlbums(searchAlbums((artist_name + ' ' + album_title).strip()), artist_name, album_title, album_length) - if resultlist: - return resultlist - - # Deezer API supports unicode, so just remove non alphanumeric characters - clean_artist_name = re.sub(r"[^\w\s]", " ", artist_name, 0, re.UNICODE).strip() - clean_album_name = re.sub(r"[^\w\s]", " ", album_title, 0, re.UNICODE).strip() - - resultlist = __matchAlbums(searchAlbums((clean_artist_name + ' ' + clean_album_name).strip()), artist_name, album_title, album_length) - if resultlist: - return resultlist - - resultlist = __matchAlbums(searchAlbums(clean_artist_name), artist_name, album_title, album_length) - if resultlist: - return resultlist - - return resultlist - - -def getTrack(sng_id, try_reload_api=True): - """Returns deezer track infos - - :param int sng_id: deezer song id - :param bool try_reload_api: whether or not try reloading API if session expired - """ - global __tracks_cache - - if str(sng_id) in __tracks_cache: - return __tracks_cache[str(sng_id)] - - data = "[{\"method\":\"song.getListData\",\"params\":{\"sng_ids\":[" + str(sng_id) + "]}}]" - json = request.request_json(url=__API_URL, headers=__HTTP_HEADERS, method='post', params=__api_queries, data=data, cookies=__cookies) - - results = [] - error = None - invalid_token = False - - if json: - # Check for errors - if 'error' in json: - error = json['error'] - if 'GATEWAY_ERROR' in json['error'] and json['error']['GATEWAY_ERROR'] == u"invalid api token": - invalid_token = True - - elif 'error' in json[0] and json[0]['error']: - error = json[0]['error'] - if 'VALID_TOKEN_REQUIRED' in json[0]['error'] and json[0]['error']['VALID_TOKEN_REQUIRED'] == u"Invalid CSRF token": - invalid_token = True - - # Got invalid token error - if error: - if invalid_token and try_reload_api: - __getApiToken() - return getTrack(sng_id, False) - else: - logger.error(u"An unknown error occurred in the Deezer parser: %s" % error) - else: - try: - results = json[0]['results'] - item = results['data'][0] - if 'token' in item: - logger.error(u"An unknown error occurred in the Deezer parser: Uploaded Files are currently not supported") - return - - sng_id = item["SNG_ID"] - md5Origin = item["MD5_ORIGIN"] - sng_format = 3 - - if item["FILESIZE_MP3_320"] <= 0: - if item["FILESIZE_MP3_256"] > 0: - sng_format = 5 - else: - sng_format = 1 - - mediaVersion = int(item["MEDIA_VERSION"]) - item['downloadUrl'] = __getDownloadUrl(md5Origin, sng_id, sng_format, mediaVersion) - - __tracks_cache[sng_id] = item - return item - - except Exception as e: - logger.error(u"An unknown error occurred in the Deezer parser: %s" % e) - - -def __getDownloadUrl(md5Origin, sng_id, sng_format, mediaVersion): - urlPart = md5Origin.encode('utf-8') + b'\xa4' + str(sng_format) + b'\xa4' + str(sng_id) + b'\xa4' + str(mediaVersion) - md5val = md5(urlPart).hexdigest() - urlPart = md5val + b'\xa4' + urlPart + b'\xa4' - cipher = AES.new('jo6aey6haid2Teih', AES.MODE_ECB) - ciphertext = cipher.encrypt(__pad(urlPart, AES.block_size)) - return "http://e-cdn-proxy-" + md5Origin[:1] + ".deezer.com/mobile/1/" + binascii.hexlify(ciphertext).lower() - - -def __pad(raw, block_size): - if (len(raw) % block_size == 0): - return raw - padding_required = block_size - (len(raw) % block_size) - padChar = b'\x00' - data = raw + padding_required * padChar - return data - - -def __tagTrack(path, track): - try: - album = getAlbum(track['ALB_ID']) - - f = MediaFile(path) - f.artist = track['ART_NAME'] - f.album = track['ALB_TITLE'] - f.title = track['SNG_TITLE'] - f.track = track['TRACK_NUMBER'] - f.tracktotal = album['nb_tracks'] - f.disc = track['DISK_NUMBER'] - f.bpm = track['BPM'] - f.date = datetime.strptime(album['release_date'], '%Y-%m-%d').date() - f.albumartist = album['artist']['name'] - if u'genres' in album and u'data' in album['genres']: - f.genres = [genre['name'] for genre in album['genres']['data']] - - f.save() - - except Exception as e: - logger.error(u'Unable to tag deezer track "%s": %s' % (path, e)) - - -def decryptTracks(paths): - """Decrypt downloaded deezer tracks. - - :param paths: list of path to deezer tracks (*.dzr files). - """ - # Note: tracks can be from different albums - decrypted_tracks = {} - - # First pass: load tracks data - for path in paths: - try: - album_folder = os.path.dirname(path) - sng_id = os.path.splitext(os.path.basename(path))[0] - track = getTrack(sng_id) - track_number = int(track['TRACK_NUMBER']) - disk_number = int(track['DISK_NUMBER']) - - if album_folder not in decrypted_tracks: - decrypted_tracks[album_folder] = {} - - if disk_number not in decrypted_tracks[album_folder]: - decrypted_tracks[album_folder][disk_number] = {} - - decrypted_tracks[album_folder][disk_number][track_number] = track - - except Exception as e: - logger.error(u'Unable to load deezer track infos "%s": %s' % (path, e)) - - # Second pass: decrypt tracks - for album_folder in decrypted_tracks: - multi_disks = len(decrypted_tracks[album_folder]) > 1 - for disk_number in decrypted_tracks[album_folder]: - for track_number, track in decrypted_tracks[album_folder][disk_number].items(): - try: - filename = helpers.replace_illegal_chars(track['SNG_TITLE']).strip() - filename = ('{:02d}'.format(track_number) + '_' + filename + '.mp3') - - # Add a 'cd x' sub-folder if album has more than one disk - disk_folder = os.path.join(album_folder, 'cd ' + str(disk_number)) if multi_disks else album_folder - - dest = os.path.join(disk_folder, filename).encode(headphones.SYS_ENCODING, 'replace') - - # Decrypt track if not already done - if not os.path.exists(dest): - try: - __decryptDownload(path, sng_id, dest) - __tagTrack(dest, track) - except Exception as e: - logger.error(u'Unable to decrypt deezer track "%s": %s' % (path, e)) - if os.path.exists(dest): - os.remove(dest) - decrypted_tracks[album_folder][disk_number].pop(track_number) - continue - - decrypted_tracks[album_folder][disk_number][track_number]['path'] = dest - - except Exception as e: - logger.error(u'Unable to decrypt deezer track "%s": %s' % (path, e)) - - return decrypted_tracks - - -def __decryptDownload(source, sng_id, dest): - interval_chunk = 3 - chunk_size = 2048 - blowFishKey = __getBlowFishKey(sng_id) - i = 0 - iv = "\x00\x01\x02\x03\x04\x05\x06\x07" - - dest_folder = os.path.dirname(dest) - if not os.path.exists(dest_folder): - os.makedirs(dest_folder) - - f = open(source, "rb") - fout = open(dest, "wb") - try: - chunk = f.read(chunk_size) - while chunk: - if(i % interval_chunk == 0): - cipher = Blowfish.new(blowFishKey, Blowfish.MODE_CBC, iv) - chunk = cipher.decrypt(__pad(chunk, Blowfish.block_size)) - - fout.write(chunk) - i += 1 - chunk = f.read(chunk_size) - finally: - f.close() - fout.close() - - -def __getBlowFishKey(encryptionKey): - if encryptionKey < 1: - encryptionKey *= -1 - - hashcode = md5(str(encryptionKey)).hexdigest() - hPart = hashcode[0:16] - lPart = hashcode[16:32] - parts = ['g4el58wc0zvf9na1', hPart, lPart] - - return __xorHex(parts) - - -def __xorHex(parts): - data = "" - for i in range(0, 16): - character = ord(parts[0][i]) - - for j in range(1, len(parts)): - character ^= ord(parts[j][i]) - - data += chr(character) - - return data diff --git a/headphones/postprocessor.py b/headphones/postprocessor.py index c9621506..e3deee23 100755 --- a/headphones/postprocessor.py +++ b/headphones/postprocessor.py @@ -29,7 +29,7 @@ from beetsplug import lyrics as beetslyrics from headphones import notifiers, utorrent, transmission, deluge, qbittorrent from headphones import db, albumart, librarysync from headphones import logger, helpers, mb, music_encoder -from headphones import metadata, deezloader +from headphones import metadata postprocessor_lock = threading.Lock() @@ -47,8 +47,6 @@ def checkFolder(): single = False if album['Kind'] == 'nzb': download_dir = headphones.CONFIG.DOWNLOAD_DIR - elif album['Kind'] == 'ddl': - download_dir = headphones.CONFIG.DOWNLOAD_DDL_DIR else: if headphones.CONFIG.DELUGE_DONE_DIRECTORY and headphones.CONFIG.TORRENT_DOWNLOADER == 3: download_dir = headphones.CONFIG.DELUGE_DONE_DIRECTORY @@ -206,7 +204,6 @@ def verify(albumid, albumpath, Kind=None, forced=False, keep_original_folder=Fal tracks = myDB.select('SELECT * from tracks WHERE AlbumID=?', [albumid]) downloaded_track_list = [] - downloaded_deezer_list = [] downloaded_cuecount = 0 for r, d, f in os.walk(albumpath): @@ -215,10 +212,8 @@ def verify(albumid, albumpath, Kind=None, forced=False, keep_original_folder=Fal downloaded_track_list.append(os.path.join(r, files)) elif files.lower().endswith('.cue'): downloaded_cuecount += 1 - elif files.lower().endswith('.dzr'): - downloaded_deezer_list.append(os.path.join(r, files)) # if any of the files end in *.part, we know the torrent isn't done yet. Process if forced, though - elif files.lower().endswith(('.part', '.utpart', '.aria2')) and not forced: + elif files.lower().endswith(('.part', '.utpart')) and not forced: logger.info( "Looks like " + os.path.basename(albumpath).decode(headphones.SYS_ENCODING, 'replace') + " isn't complete yet. Will try again on the next run") @@ -257,37 +252,6 @@ def verify(albumid, albumpath, Kind=None, forced=False, keep_original_folder=Fal downloaded_track_list = helpers.get_downloaded_track_list(albumpath) keep_original_folder = False - # Decrypt deezer tracks - if downloaded_deezer_list: - logger.info('Decrypting deezer tracks') - decrypted_deezer_list = deezloader.decryptTracks(downloaded_deezer_list) - - # Check if album is complete based on album duration only - # (total track numbers is not determinant enough due to hidden tracks for eg) - db_track_duration = 0 - downloaded_track_duration = 0 - try: - for track in tracks: - db_track_duration += track['TrackDuration'] / 1000 - except: - downloaded_track_duration = False - - try: - for disk_number in decrypted_deezer_list[albumpath]: - for track in decrypted_deezer_list[albumpath][disk_number].values(): - downloaded_track_list.append(track['path']) - downloaded_track_duration += int(track['DURATION']) - except: - downloaded_track_duration = False - - if not downloaded_track_duration or not db_track_duration or abs(downloaded_track_duration - db_track_duration) > 240: - logger.info("Looks like " + - os.path.basename(albumpath).decode(headphones.SYS_ENCODING, 'replace') + - " isn't complete yet (duration mismatch). Will try again on the next run") - return - - downloaded_track_list = list(set(downloaded_track_list)) # Remove duplicates - # test #1: metadata - usually works logger.debug('Verifying metadata...') diff --git a/headphones/searcher.py b/headphones/searcher.py index f6554cb1..b0652f7f 100644 --- a/headphones/searcher.py +++ b/headphones/searcher.py @@ -36,7 +36,6 @@ import headphones from headphones.common import USER_AGENT from headphones import logger, db, helpers, classes, sab, nzbget, request from headphones import utorrent, transmission, notifiers, rutracker, deluge, qbittorrent -from headphones import deezloader, aria2 from bencode import bencode, bdecode # Magnet to torrent services, for Black hole. Stolen from CouchPotato. @@ -52,27 +51,6 @@ ruobj = None # Persistent RED API object redobj = None -# Persistent Aria2 RPC object -__aria2rpc_obj = None - - -def getAria2RPC(): - global __aria2rpc_obj - if not __aria2rpc_obj: - __aria2rpc_obj = aria2.Aria2JsonRpc( - ID='headphones', - uri=headphones.CONFIG.ARIA_HOST, - token=headphones.CONFIG.ARIA_TOKEN if headphones.CONFIG.ARIA_TOKEN else None, - http_user=headphones.CONFIG.ARIA_USERNAME if headphones.CONFIG.ARIA_USERNAME else None, - http_passwd=headphones.CONFIG.ARIA_PASSWORD if headphones.CONFIG.ARIA_PASSWORD else None - ) - return __aria2rpc_obj - - -def reconfigure(): - global __aria2rpc_obj - __aria2rpc_obj = None - def fix_url(s, charset="utf-8"): """ @@ -303,53 +281,32 @@ def do_sorted_search(album, new, losslessOnly, choose_specific_download=False): headphones.CONFIG.STRIKE or headphones.CONFIG.TQUATTRECENTONZE) - DDL_PROVIDERS = (headphones.CONFIG.DEEZLOADER) - + results = [] myDB = db.DBConnection() albumlength = myDB.select('SELECT sum(TrackDuration) from tracks WHERE AlbumID=?', [album['AlbumID']])[0][0] - nzb_results = None - torrent_results = None - ddl_results = None - if headphones.CONFIG.PREFER_TORRENTS == 0 and not choose_specific_download: if NZB_PROVIDERS and NZB_DOWNLOADERS: - nzb_results = searchNZB(album, new, losslessOnly, albumlength) + results = searchNZB(album, new, losslessOnly, albumlength) - if not nzb_results: - if DDL_PROVIDERS: - ddl_results = searchDdl(album, new, losslessOnly, albumlength) - - if TORRENT_PROVIDERS: - torrent_results = searchTorrent(album, new, losslessOnly, albumlength) + if not results and TORRENT_PROVIDERS: + results = searchTorrent(album, new, losslessOnly, albumlength) elif headphones.CONFIG.PREFER_TORRENTS == 1 and not choose_specific_download: if TORRENT_PROVIDERS: - torrent_results = searchTorrent(album, new, losslessOnly, albumlength) + results = searchTorrent(album, new, losslessOnly, albumlength) - if not torrent_results: - if DDL_PROVIDERS: - ddl_results = searchDdl(album, new, losslessOnly, albumlength) - - if NZB_PROVIDERS and NZB_DOWNLOADERS: - nzb_results = searchNZB(album, new, losslessOnly, albumlength) - - elif headphones.CONFIG.PREFER_TORRENTS == 3 and not choose_specific_download: - - if DDL_PROVIDERS: - ddl_results = searchDdl(album, new, losslessOnly, albumlength) - - if not ddl_results: - if TORRENT_PROVIDERS: - torrent_results = searchTorrent(album, new, losslessOnly, albumlength) - - if NZB_PROVIDERS and NZB_DOWNLOADERS: - nzb_results = searchNZB(album, new, losslessOnly, albumlength) + if not results and NZB_PROVIDERS and NZB_DOWNLOADERS: + results = searchNZB(album, new, losslessOnly, albumlength) else: + + nzb_results = None + torrent_results = None + if NZB_PROVIDERS and NZB_DOWNLOADERS: nzb_results = searchNZB(album, new, losslessOnly, albumlength, choose_specific_download) @@ -357,19 +314,13 @@ def do_sorted_search(album, new, losslessOnly, choose_specific_download=False): torrent_results = searchTorrent(album, new, losslessOnly, albumlength, choose_specific_download) - if DDL_PROVIDERS: - ddl_results = searchDdl(album, new, losslessOnly, albumlength, choose_specific_download) + if not nzb_results: + nzb_results = [] - if not nzb_results: - nzb_results = [] + if not torrent_results: + torrent_results = [] - if not torrent_results: - torrent_results = [] - - if not ddl_results: - ddl_results = [] - - results = nzb_results + torrent_results + ddl_results + results = nzb_results + torrent_results if choose_specific_download: return results @@ -875,31 +826,6 @@ def send_to_downloader(data, bestqual, album): except Exception as e: logger.error('Couldn\'t write NZB file: %s', e) return - - elif kind == 'ddl': - folder_name = '%s - %s [%s]' % ( - helpers.latinToAscii(album['ArtistName']).encode('UTF-8').replace('/', '_'), - helpers.latinToAscii(album['AlbumTitle']).encode('UTF-8').replace('/', '_'), - get_year_from_release_date(album['ReleaseDate'])) - - # Aria2 downloader - if headphones.CONFIG.DDL_DOWNLOADER == 0: - logger.info("Sending download to Aria2") - - try: - deezer_album = deezloader.getAlbumByLink(bestqual[2]) - - for album_track in deezer_album['tracks']['data']: - track = deezloader.getTrack(album_track['id']) - if track: - filename = track['SNG_ID'] + '.dzr' - logger.debug(u'Sending song "%s" to Aria' % track['SNG_TITLE']) - getAria2RPC().addUri([track['downloadUrl']], {'out': filename, 'auto-file-renaming': 'false', 'continue': 'true', 'dir': folder_name}) - - except Exception as e: - logger.error(u'Error sending torrent to Aria2. Are you sure it\'s running? (%s)' % e) - return - else: folder_name = '%s - %s [%s]' % ( helpers.latinToAscii(album['ArtistName']).encode('UTF-8').replace('/', '_'), @@ -1278,62 +1204,6 @@ def verifyresult(title, artistterm, term, lossless): return True -def searchDdl(album, new=False, losslessOnly=False, albumlength=None, - choose_specific_download=False): - reldate = album['ReleaseDate'] - year = get_year_from_release_date(reldate) - - # MERGE THIS WITH THE TERM CLEANUP FROM searchNZB - dic = {'...': '', ' & ': ' ', ' = ': ' ', '?': '', '$': 's', ' + ': ' ', '"': '', ',': ' ', - '*': ''} - - semi_cleanalbum = helpers.replace_all(album['AlbumTitle'], dic) - cleanalbum = helpers.latinToAscii(semi_cleanalbum) - semi_cleanartist = helpers.replace_all(album['ArtistName'], dic) - cleanartist = helpers.latinToAscii(semi_cleanartist) - - # Use provided term if available, otherwise build our own (this code needs to be cleaned up since a lot - # of these torrent providers are just using cleanartist/cleanalbum terms - if album['SearchTerm']: - term = album['SearchTerm'] - elif album['Type'] == 'part of': - term = cleanalbum + " " + year - else: - # FLAC usually doesn't have a year for some reason so I'll leave it out - # Various Artist albums might be listed as VA, so I'll leave that out too - # Only use the year if the term could return a bunch of different albums, i.e. self-titled albums - if album['ArtistName'] in album['AlbumTitle'] or len(album['ArtistName']) < 4 or len( - album['AlbumTitle']) < 4: - term = cleanartist + ' ' + cleanalbum + ' ' + year - elif album['ArtistName'] == 'Various Artists': - term = cleanalbum + ' ' + year - else: - term = cleanartist + ' ' + cleanalbum - - # Replace bad characters in the term and unicode it - term = re.sub('[\.\-\/]', ' ', term).encode('utf-8') - artistterm = re.sub('[\.\-\/]', ' ', cleanartist).encode('utf-8', 'replace') - - logger.debug(u'Using search term: "%s"' % helpers.latinToAscii(term)) - - resultlist = [] - - # Deezer only provides lossy - if headphones.CONFIG.DEEZLOADER and not losslessOnly: - resultlist += deezloader.searchAlbum(album['ArtistName'], album['AlbumTitle'], album['SearchTerm'], int(albumlength / 1000)) - - # attempt to verify that this isn't a substring result - # when looking for "Foo - Foo" we don't want "Foobar" - # this should be less of an issue when it isn't a self-titled album so we'll only check vs artist - results = [result for result in resultlist if verifyresult(result[0], artistterm, term, losslessOnly)] - - # Additional filtering for size etc - if results and not choose_specific_download: - results = more_filtering(results, album, albumlength, new) - - return results - - def searchTorrent(album, new=False, losslessOnly=False, albumlength=None, choose_specific_download=False): global apolloobj # persistent apollo.rip api object to reduce number of login attempts @@ -2166,10 +2036,7 @@ def searchTorrent(album, new=False, losslessOnly=False, albumlength=None, def preprocess(resultlist): for result in resultlist: - if result[3] == deezloader.PROVIDER_NAME: - return True, result - - elif result[4] == 'torrent': + if result[4] == 'torrent': # rutracker always needs the torrent data if result[3] == 'rutracker.org': diff --git a/headphones/webserve.py b/headphones/webserve.py index 7e97cefb..855fed24 100644 --- a/headphones/webserve.py +++ b/headphones/webserve.py @@ -1174,10 +1174,6 @@ class WebInterface(object): "utorrent_username": headphones.CONFIG.UTORRENT_USERNAME, "utorrent_password": headphones.CONFIG.UTORRENT_PASSWORD, "utorrent_label": headphones.CONFIG.UTORRENT_LABEL, - "aria_host": headphones.CONFIG.ARIA_HOST, - "aria_password": headphones.CONFIG.ARIA_PASSWORD, - "aria_token": headphones.CONFIG.ARIA_TOKEN, - "aria_username": headphones.CONFIG.ARIA_USERNAME, "nzb_downloader_sabnzbd": radio(headphones.CONFIG.NZB_DOWNLOADER, 0), "nzb_downloader_nzbget": radio(headphones.CONFIG.NZB_DOWNLOADER, 1), "nzb_downloader_blackhole": radio(headphones.CONFIG.NZB_DOWNLOADER, 2), @@ -1186,7 +1182,6 @@ class WebInterface(object): "torrent_downloader_utorrent": radio(headphones.CONFIG.TORRENT_DOWNLOADER, 2), "torrent_downloader_deluge": radio(headphones.CONFIG.TORRENT_DOWNLOADER, 3), "torrent_downloader_qbittorrent": radio(headphones.CONFIG.TORRENT_DOWNLOADER, 4), - "ddl_downloader_aria": radio(headphones.CONFIG.DDL_DOWNLOADER, 0), "download_dir": headphones.CONFIG.DOWNLOAD_DIR, "use_blackhole": checked(headphones.CONFIG.BLACKHOLE), "blackhole_dir": headphones.CONFIG.BLACKHOLE_DIR, @@ -1248,8 +1243,6 @@ class WebInterface(object): "use_tquattrecentonze": checked(headphones.CONFIG.TQUATTRECENTONZE), "tquattrecentonze_user": headphones.CONFIG.TQUATTRECENTONZE_USER, "tquattrecentonze_password": headphones.CONFIG.TQUATTRECENTONZE_PASSWORD, - "download_ddl_dir": headphones.CONFIG.DOWNLOAD_DDL_DIR, - "use_deezloader": checked(headphones.CONFIG.DEEZLOADER), "pref_qual_0": radio(headphones.CONFIG.PREFERRED_QUALITY, 0), "pref_qual_1": radio(headphones.CONFIG.PREFERRED_QUALITY, 1), "pref_qual_2": radio(headphones.CONFIG.PREFERRED_QUALITY, 2), @@ -1295,7 +1288,6 @@ class WebInterface(object): "prefer_torrents_0": radio(headphones.CONFIG.PREFER_TORRENTS, 0), "prefer_torrents_1": radio(headphones.CONFIG.PREFER_TORRENTS, 1), "prefer_torrents_2": radio(headphones.CONFIG.PREFER_TORRENTS, 2), - "prefer_torrents_3": radio(headphones.CONFIG.PREFER_TORRENTS, 3), "magnet_links_0": radio(headphones.CONFIG.MAGNET_LINKS, 0), "magnet_links_1": radio(headphones.CONFIG.MAGNET_LINKS, 1), "magnet_links_2": radio(headphones.CONFIG.MAGNET_LINKS, 2), @@ -1454,7 +1446,7 @@ class WebInterface(object): "launch_browser", "enable_https", "api_enabled", "use_blackhole", "headphones_indexer", "use_newznab", "newznab_enabled", "use_torznab", "torznab_enabled", "use_nzbsorg", "use_omgwtfnzbs", "use_kat", "use_piratebay", "use_oldpiratebay", - "use_mininova", "use_waffles", "use_rutracker", "use_deezloader", + "use_mininova", "use_waffles", "use_rutracker", "use_apollo", "use_redacted", "use_strike", "use_tquattrecentonze", "preferred_bitrate_allow_lossless", "detect_bitrate", "ignore_clean_releases", "freeze_db", "cue_split", "move_files", "rename_files", "correct_metadata", "cleanup_files", "keep_nfo", "add_album_art", @@ -1588,9 +1580,6 @@ class WebInterface(object): # Reconfigure musicbrainz database connection with the new values mb.startmb() - # Reconfigure Aria2 - searcher.reconfigure() - raise cherrypy.HTTPRedirect("config") @cherrypy.expose From d6e1a16286b32fad91127e86b47469b60bf79ddf Mon Sep 17 00:00:00 2001 From: Ade Date: Sat, 22 Apr 2017 11:54:18 +1200 Subject: [PATCH 08/27] Pass full mb unicode names to beets Possibly fixes https://github.com/rembo10/headphones/issues/2919 --- headphones/postprocessor.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/headphones/postprocessor.py b/headphones/postprocessor.py index e3deee23..de68b9fa 100755 --- a/headphones/postprocessor.py +++ b/headphones/postprocessor.py @@ -955,10 +955,8 @@ def correctMetadata(albumid, release, downloaded_track_list): try: cur_artist, cur_album, prop = autotag.tag_album(items, - search_artist=helpers.latinToAscii( - release['ArtistName']), - search_album=helpers.latinToAscii( - release['AlbumTitle'])) + search_artist=release['ArtistName'], + search_album=release['AlbumTitle']) candidates = prop.candidates rec = prop.recommendation except Exception as e: From 1e3cbb64cafa5990a7ab76e16156ed45cbec7d79 Mon Sep 17 00:00:00 2001 From: Ade Date: Sun, 23 Apr 2017 20:08:28 +1200 Subject: [PATCH 09/27] beets recommendation logging --- headphones/helpers.py | 25 +++++++++++++++++++++++++ headphones/postprocessor.py | 19 ++++++++++++++----- 2 files changed, 39 insertions(+), 5 deletions(-) diff --git a/headphones/helpers.py b/headphones/helpers.py index 4c815ad7..10f3cbff 100644 --- a/headphones/helpers.py +++ b/headphones/helpers.py @@ -23,6 +23,10 @@ import sys import tempfile import glob +from beets import logging as beetslogging +import six +from contextlib import contextmanager + import fnmatch import re import os @@ -975,3 +979,24 @@ def create_https_certificates(ssl_cert, ssl_key): return False return True + + +class BeetsLogCapture(beetslogging.Handler): + + def __init__(self): + beetslogging.Handler.__init__(self) + self.messages = [] + + def emit(self, record): + self.messages.append(six.text_type(record.msg)) + + +@contextmanager +def capture_beets_log(logger='beets'): + capture = BeetsLogCapture() + log = beetslogging.getLogger(logger) + log.addHandler(capture) + try: + yield capture.messages + finally: + log.removeHandler(capture) \ No newline at end of file diff --git a/headphones/postprocessor.py b/headphones/postprocessor.py index de68b9fa..e8ffc602 100755 --- a/headphones/postprocessor.py +++ b/headphones/postprocessor.py @@ -24,6 +24,7 @@ import beets import headphones from beets import autotag from beets import config as beetsconfig +from beets import logging as beetslogging from beets.mediafile import MediaFile, FileTypeError, UnreadableFileError from beetsplug import lyrics as beetslyrics from headphones import notifiers, utorrent, transmission, deluge, qbittorrent @@ -954,11 +955,19 @@ def correctMetadata(albumid, release, downloaded_track_list): continue try: - cur_artist, cur_album, prop = autotag.tag_album(items, - search_artist=release['ArtistName'], - search_album=release['AlbumTitle']) - candidates = prop.candidates - rec = prop.recommendation + logger.debug('Getting recommendation from beets. Artist: %s. Album: %s. Tracks: %s', release['ArtistName'], + release['AlbumTitle'], len(items)) + beetslog = beetslogging.getLogger('beets') + beetslog.set_global_level(beetslogging.DEBUG) + with helpers.capture_beets_log() as logs: + cur_artist, cur_album, prop = autotag.tag_album(items, + search_artist=release['ArtistName'], + search_album=release['AlbumTitle']) + candidates = prop.candidates + rec = prop.recommendation + for log in logs: + logger.debug('Beets: %s', log) + beetslog.set_global_level(beetslogging.NOTSET) except Exception as e: logger.error('Error getting recommendation: %s. Not writing metadata', e) return False From 18d85d12dd8a2a530d5c13f6387713690ddde6cf Mon Sep 17 00:00:00 2001 From: Ade Date: Sun, 23 Apr 2017 20:26:04 +1200 Subject: [PATCH 10/27] pep --- headphones/helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/headphones/helpers.py b/headphones/helpers.py index 10f3cbff..d9b491f7 100644 --- a/headphones/helpers.py +++ b/headphones/helpers.py @@ -999,4 +999,4 @@ def capture_beets_log(logger='beets'): try: yield capture.messages finally: - log.removeHandler(capture) \ No newline at end of file + log.removeHandler(capture) From 2df13ad8234b382a3f1dbe376795f66fbcd80741 Mon Sep 17 00:00:00 2001 From: Ade Date: Mon, 24 Apr 2017 14:16:08 +1200 Subject: [PATCH 11/27] bit more beets logging --- headphones/postprocessor.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/headphones/postprocessor.py b/headphones/postprocessor.py index e8ffc602..ab00ab1a 100755 --- a/headphones/postprocessor.py +++ b/headphones/postprocessor.py @@ -958,7 +958,8 @@ def correctMetadata(albumid, release, downloaded_track_list): logger.debug('Getting recommendation from beets. Artist: %s. Album: %s. Tracks: %s', release['ArtistName'], release['AlbumTitle'], len(items)) beetslog = beetslogging.getLogger('beets') - beetslog.set_global_level(beetslogging.DEBUG) + beetslog.set_global_level(beetslogging.DEBUG) if headphones.VERBOSE else beetslog.set_global_level( + beetslogging.CRITICAL) with helpers.capture_beets_log() as logs: cur_artist, cur_album, prop = autotag.tag_album(items, search_artist=release['ArtistName'], From 04e8767ea99e804ccb909d3293f8d6e2561d3726 Mon Sep 17 00:00:00 2001 From: Kallys Date: Mon, 24 Apr 2017 14:32:51 +0200 Subject: [PATCH 12/27] Integration of deezloader provider by ParadoxalManiak under CC BY-NC-SA 4.0 Integration of aria2 downloader by Xyne under GPLv2 I'm not responsible of any kind for the usage of this programs by other people. These integrations come with no warranty. Please refer to your local country laws and respect Deezer's terms of service. --- data/interfaces/default/config.html | 64 ++ headphones/aria2.py | 979 ++++++++++++++++++++++++++++ headphones/classes.py | 31 + headphones/config.py | 7 + headphones/deezloader.py | 441 +++++++++++++ headphones/postprocessor.py | 40 +- headphones/searcher.py | 167 ++++- headphones/webserve.py | 13 +- 8 files changed, 1722 insertions(+), 20 deletions(-) create mode 100644 headphones/aria2.py create mode 100644 headphones/deezloader.py diff --git a/data/interfaces/default/config.html b/data/interfaces/default/config.html index 841df0d1..bd331f2c 100644 --- a/data/interfaces/default/config.html +++ b/data/interfaces/default/config.html @@ -304,6 +304,45 @@ + +
+ Direct Download + Aria2 +
+
+
+ + + usually http://localhost:6800/jsonrpc +
+
+ + +
+
+ + +
+
+ + +
+
+ + + Full path where ddl client downloads your music, e.g. /Users/name/Downloads/music +
+
@@ -461,6 +500,7 @@ NZBs Torrents + Direct Download No Preference
@@ -573,6 +613,21 @@ + +
+ Direct Download + +
+
+ + +
+
+ +
@@ -2405,6 +2460,11 @@ $("#deluge_options").show(); } + if ($("#ddl_downloader_aria").is(":checked")) + { + $("#ddl_aria_options").show(); + } + $('input[type=radio]').change(function(){ if ($("#preferred_bitrate").is(":checked")) { @@ -2458,6 +2518,9 @@ { $("#torrent_blackhole_options,#utorrent_options,#transmission_options,#qbittorrent_options").fadeOut("fast", function() { $("#deluge_options").fadeIn() }); } + if ($("#ddl_downloader_aria").is(":checked")) + { + } }); $("#mirror").change(handleNewServerSelection); @@ -2560,6 +2623,7 @@ initConfigCheckbox("#enable_https"); initConfigCheckbox("#customauth"); initConfigCheckbox("#use_tquattrecentonze"); + initConfigCheckbox("#use_deezloader"); $('#twitterStep1').click(function () { diff --git a/headphones/aria2.py b/headphones/aria2.py new file mode 100644 index 00000000..06d8e4e0 --- /dev/null +++ b/headphones/aria2.py @@ -0,0 +1,979 @@ +# -*- coding: utf8 -*- +# Copyright (C) 2012-2016 Xyne +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# (version 2) as published by the Free Software Foundation. +# +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +from __future__ import with_statement +import base64 +import json +import math +import os +import ssl +import string +import time +import httplib +import urllib2 + +from headphones import logger + +# ################################ Constants ################################### + +DEFAULT_PORT = 6800 +SERVER_URI_FORMAT = '{}://{}:{:d}/jsonrpc' + +# Status values for unfinished downloads. +TEMPORARY_STATUS = ('active', 'waiting', 'paused') +# Status values for finished downloads. +FINAL_STATUS = ('complete', 'error') + +ARIA2_CONTROL_FILE_EXT = '.aria2' + +# ########################## Convenience Functions ############################# + + +def to_json_list(objs): + ''' + Wrap strings in lists. Other iterables are converted to lists directly. + ''' + if isinstance(objs, str): + return [objs] + elif not isinstance(objs, list): + return list(objs) + else: + return objs + + +def add_options_and_position(params, options=None, position=None): + ''' + Convenience method for adding options and position to parameters. + ''' + if options: + params.append(options) + if position: + if not isinstance(position, int): + try: + position = int(position) + except ValueError: + position = -1 + if position >= 0: + params.append(position) + return params + + +def get_status(response): + ''' + Process a status response. + ''' + if response: + try: + return response['status'] + except KeyError: + logger.error('no status returned from Aria2 RPC server') + return 'error' + else: + logger.error('no response from server') + return 'error' + + +def random_token(length, valid_chars=None): + ''' + Get a random secret token for the Aria2 RPC server. + + length: + The length of the token + + valid_chars: + A list or other ordered and indexable iterable of valid characters. If not + given of None, asciinumberic characters with some punctuation characters + will be used. + ''' + if not valid_chars: + valid_chars = string.ascii_letters + string.digits + '!@#$%^&*()-_=+' + number_of_chars = len(valid_chars) + bytes_to_read = math.ceil(math.log(number_of_chars) / math.log(0x100)) + max_value = 0x100**bytes_to_read + max_index = number_of_chars - 1 + token = '' + for _ in range(length): + value = int.from_bytes(os.urandom(bytes_to_read), byteorder='little') + index = round((value * max_index) / max_value) + token += valid_chars[index] + return token + +# ################ From python3-aur's ThreadedServers.common ################### + + +def format_bytes(size): + '''Format bytes for inferior humans.''' + if size < 0x400: + return '{:d} B'.format(size) + else: + size = float(size) / 0x400 + for prefix in ('KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB'): + if size < 0x400: + return '{:0.02f} {}'.format(size, prefix) + else: + size /= 0x400 + return '{:0.02f} YiB'.format(size) + + +def format_seconds(s): + '''Format seconds for inferior humans.''' + string = '' + for base, char in ( + (60, 's'), + (60, 'm'), + (24, 'h') + ): + s, r = divmod(s, base) + if s == 0: + return '{:d}{}{}'.format(r, char, string) + elif r != 0: + string = '{:02d}{}{}'.format(r, char, string) + else: + return '{:d}d{}'.format(s, string) + +# ############################ Aria2JsonRpcError ############################### + + +class Aria2JsonRpcError(Exception): + def __init__(self, msg, connection_error=False): + super(self.__class__, self).__init__(self, msg) + self.connection_error = connection_error + +# ############################ Aria2JsonRpc Class ############################## + + +class Aria2JsonRpc(object): + ''' + Interface class for interacting with an Aria2 RPC server. + ''' + # TODO: certificate options, etc. + def __init__( + self, ID, uri, + mode='normal', + token=None, + http_user=None, http_passwd=None, + server_cert=None, client_cert=None, client_cert_password=None, + ssl_protocol=None, + setup_function=None + ): + ''' + ID: the ID to send to the RPC interface + + uri: the URI of the RPC interface + + mode: + normal - process requests immediately + batch - queue requests (run with "process_queue") + format - return RPC request objects + + token: + RPC method-level authorization token (set using `--rpc-secret`) + + http_user, http_password: + HTTP Basic authentication credentials (deprecated) + + server_cert: + server certificate for HTTPS connections + + client_cert: + client certificate for HTTPS connections + + client_cert_password: + prompt for client certificate password + + ssl_protocol: + SSL protocol from the ssl module + + setup_function: + A function to invoke prior to the first server call. This could be the + launch() method of an Aria2RpcServer instance, for example. This attribute + is set automatically in instances returned from Aria2RpcServer.get_a2jr() + ''' + self.id = ID + self.uri = uri + self.mode = mode + self.queue = [] + self.handlers = dict() + self.token = token + self.setup_function = setup_function + + if None not in (http_user, http_passwd): + self.add_HTTPBasicAuthHandler(http_user, http_passwd) + + if server_cert or client_cert: + self.add_HTTPSHandler( + server_cert=server_cert, + client_cert=client_cert, + client_cert_password=client_cert_password, + protocol=ssl_protocol + ) + + self.update_opener() + + def iter_handlers(self): + ''' + Iterate over handlers. + ''' + for name in ('HTTPS', 'HTTPBasicAuth'): + try: + yield self.handlers[name] + except KeyError: + pass + + def update_opener(self): + ''' + Build an opener from the current handlers. + ''' + self.opener = urllib2.build_opener(*self.iter_handlers()) + + def remove_handler(self, name): + ''' + Remove a handler. + ''' + try: + del self.handlers[name] + except KeyError: + pass + + def add_HTTPBasicAuthHandler(self, user, passwd): + ''' + Add a handler for HTTP Basic authentication. + + If either user or passwd are None, the handler is removed. + ''' + handler = urllib2.HTTPBasicAuthHandler() + handler.add_password( + realm='aria2', + uri=self.uri, + user=user, + passwd=passwd, + ) + self.handlers['HTTPBasicAuth'] = handler + + def remove_HTTPBasicAuthHandler(self): + self.remove_handler('HTTPBasicAuth') + + def add_HTTPSHandler( + self, + server_cert=None, + client_cert=None, + client_cert_password=None, + protocol=None, + ): + ''' + Add a handler for HTTPS connections with optional server and client + certificates. + ''' + if not protocol: + protocol = ssl.PROTOCOL_TLSv1 +# protocol = ssl.PROTOCOL_TLSv1_1 # openssl 1.0.1+ +# protocol = ssl.PROTOCOL_TLSv1_2 # Python 3.4+ + context = ssl.SSLContext(protocol) + + if server_cert: + context.verify_mode = ssl.CERT_REQUIRED + context.load_verify_locations(cafile=server_cert) + else: + context.verify_mode = ssl.CERT_OPTIONAL + + if client_cert: + context.load_cert_chain(client_cert, password=client_cert_password) + + self.handlers['HTTPS'] = urllib2.HTTPSHandler( + context=context, + check_hostname=False + ) + + def remove_HTTPSHandler(self): + self.remove_handler('HTTPS') + + def send_request(self, req_obj): + ''' + Send the request and return the response. + ''' + if self.setup_function: + self.setup_function() + self.setup_function = None + logger.debug("Aria req_obj: %s" % json.dumps(req_obj, indent=2, sort_keys=True)) + req = json.dumps(req_obj).encode('UTF-8') + try: + f = self.opener.open(self.uri, req) + obj = json.loads(f.read()) + try: + return obj['result'] + except KeyError: + raise Aria2JsonRpcError('unexpected result: {}'.format(obj)) + except (urllib2.URLError) as e: + # This should work but URLError does not set the errno attribute: + # e.errno == errno.ECONNREFUSED + raise Aria2JsonRpcError( + str(e), + connection_error=( + '111' in str(e) + ) + ) + except httplib.BadStatusLine as e: + raise Aria2JsonRpcError('{}: BadStatusLine: {} (HTTPS error?)'.format( + self.__class__.__name__, e + )) + + def jsonrpc(self, method, params=None, prefix='aria2.'): + ''' + POST a request to the RPC interface. + ''' + if not params: + params = [] + + if self.token is not None: + token_str = 'token:{}'.format(self.token) + if method == 'multicall': + for p in params[0]: + try: + p['params'].insert(0, token_str) + except KeyError: + p['params'] = [token_str] + else: + params.insert(0, token_str) + + req_obj = { + 'jsonrpc': '2.0', + 'id': self.id, + 'method': prefix + method, + 'params': params, + } + if self.mode == 'batch': + self.queue.append(req_obj) + return None + elif self.mode == 'format': + return req_obj + else: + return self.send_request(req_obj) + + def process_queue(self): + ''' + Processed queued requests. + ''' + req_obj = self.queue + self.queue = [] + return self.send_request(req_obj) + + +# ############################# Standard Methods ############################### + def addUri(self, uris, options=None, position=None): + ''' + aria2.addUri method + + uris: list of URIs + + options: dictionary of additional options + + position: position in queue + + Returns a GID + ''' + params = [uris] + params = add_options_and_position(params, options, position) + return self.jsonrpc('addUri', params) + + def addTorrent(self, torrent, uris=None, options=None, position=None): + ''' + aria2.addTorrent method + + torrent: base64-encoded torrent file + + uris: list of webseed URIs + + options: dictionary of additional options + + position: position in queue + + Returns a GID. + ''' + params = [torrent] + if uris: + params.append(uris) + params = add_options_and_position(params, options, position) + return self.jsonrpc('addTorrent', params) + + def addMetalink(self, metalink, options=None, position=None): + ''' + aria2.addMetalink method + + metalink: base64-encoded metalink file + + options: dictionary of additional options + + position: position in queue + + Returns an array of GIDs. + ''' + params = [metalink] + params = add_options_and_position(params, options, position) + return self.jsonrpc('addTorrent', params) + + def remove(self, gid): + ''' + aria2.remove method + + gid: GID to remove + ''' + params = [gid] + return self.jsonrpc('remove', params) + + def forceRemove(self, gid): + ''' + aria2.forceRemove method + + gid: GID to remove + ''' + params = [gid] + return self.jsonrpc('forceRemove', params) + + def pause(self, gid): + ''' + aria2.pause method + + gid: GID to pause + ''' + params = [gid] + return self.jsonrpc('pause', params) + + def pauseAll(self): + ''' + aria2.pauseAll method + ''' + return self.jsonrpc('pauseAll') + + def forcePause(self, gid): + ''' + aria2.forcePause method + + gid: GID to pause + ''' + params = [gid] + return self.jsonrpc('forcePause', params) + + def forcePauseAll(self): + ''' + aria2.forcePauseAll method + ''' + return self.jsonrpc('forcePauseAll') + + def unpause(self, gid): + ''' + aria2.unpause method + + gid: GID to unpause + ''' + params = [gid] + return self.jsonrpc('unpause', params) + + def unpauseAll(self): + ''' + aria2.unpauseAll method + ''' + return self.jsonrpc('unpauseAll') + + def tellStatus(self, gid, keys=None): + ''' + aria2.tellStatus method + + gid: GID to query + + keys: subset of status keys to return (all keys are returned otherwise) + + Returns a dictionary. + ''' + params = [gid] + if keys: + params.append(keys) + return self.jsonrpc('tellStatus', params) + + def getUris(self, gid): + ''' + aria2.getUris method + + gid: GID to query + + Returns a list of dictionaries. + ''' + params = [gid] + return self.jsonrpc('getUris', params) + + def getFiles(self, gid): + ''' + aria2.getFiles method + + gid: GID to query + + Returns a list of dictionaries. + ''' + params = [gid] + return self.jsonrpc('getFiles', params) + + def getPeers(self, gid): + ''' + aria2.getPeers method + + gid: GID to query + + Returns a list of dictionaries. + ''' + params = [gid] + return self.jsonrpc('getPeers', params) + + def getServers(self, gid): + ''' + aria2.getServers method + + gid: GID to query + + Returns a list of dictionaries. + ''' + params = [gid] + return self.jsonrpc('getServers', params) + + def tellActive(self, keys=None): + ''' + aria2.tellActive method + + keys: same as tellStatus + + Returns a list of dictionaries. The dictionaries are the same as those + returned by tellStatus. + ''' + if keys: + params = [keys] + else: + params = None + return self.jsonrpc('tellActive', params) + + def tellWaiting(self, offset, num, keys=None): + ''' + aria2.tellWaiting method + + offset: offset from start of waiting download queue + (negative values are counted from the end of the queue) + + num: number of downloads to return + + keys: same as tellStatus + + Returns a list of dictionaries. The dictionaries are the same as those + returned by tellStatus. + ''' + params = [offset, num] + if keys: + params.append(keys) + return self.jsonrpc('tellWaiting', params) + + def tellStopped(self, offset, num, keys=None): + ''' + aria2.tellStopped method + + offset: offset from oldest download (same semantics as tellWaiting) + + num: same as tellWaiting + + keys: same as tellStatus + + Returns a list of dictionaries. The dictionaries are the same as those + returned by tellStatus. + ''' + params = [offset, num] + if keys: + params.append(keys) + return self.jsonrpc('tellStopped', params) + + def changePosition(self, gid, pos, how): + ''' + aria2.changePosition method + + gid: GID to change + + pos: the position + + how: "POS_SET", "POS_CUR" or "POS_END" + ''' + params = [gid, pos, how] + return self.jsonrpc('changePosition', params) + + def changeUri(self, gid, fileIndex, delUris, addUris, position=None): + ''' + aria2.changePosition method + + gid: GID to change + + fileIndex: file to affect (1-based) + + delUris: URIs to remove + + addUris: URIs to add + + position: where URIs are inserted, after URIs have been removed + ''' + params = [gid, fileIndex, delUris, addUris] + if position: + params.append(position) + return self.jsonrpc('changePosition', params) + + def getOption(self, gid): + ''' + aria2.getOption method + + gid: GID to query + + Returns a dictionary of options. + ''' + params = [gid] + return self.jsonrpc('getOption', params) + + def changeOption(self, gid, options): + ''' + aria2.changeOption method + + gid: GID to change + + options: dictionary of new options + (not all options can be changed for active downloads) + ''' + params = [gid, options] + return self.jsonrpc('changeOption', params) + + def getGlobalOption(self): + ''' + aria2.getGlobalOption method + + Returns a dictionary. + ''' + return self.jsonrpc('getGlobalOption') + + def changeGlobalOption(self, options): + ''' + aria2.changeGlobalOption method + + options: dictionary of new options + ''' + params = [options] + return self.jsonrpc('changeGlobalOption', params) + + def getGlobalStat(self): + ''' + aria2.getGlobalStat method + + Returns a dictionary. + ''' + return self.jsonrpc('getGlobalStat') + + def purgeDownloadResult(self): + ''' + aria2.purgeDownloadResult method + ''' + self.jsonrpc('purgeDownloadResult') + + def removeDownloadResult(self, gid): + ''' + aria2.removeDownloadResult method + + gid: GID to remove + ''' + params = [gid] + return self.jsonrpc('removeDownloadResult', params) + + def getVersion(self): + ''' + aria2.getVersion method + + Returns a dictionary. + ''' + return self.jsonrpc('getVersion') + + def getSessionInfo(self): + ''' + aria2.getSessionInfo method + + Returns a dictionary. + ''' + return self.jsonrpc('getSessionInfo') + + def shutdown(self): + ''' + aria2.shutdown method + ''' + return self.jsonrpc('shutdown') + + def forceShutdown(self): + ''' + aria2.forceShutdown method + ''' + return self.jsonrpc('forceShutdown') + + def multicall(self, methods): + ''' + aria2.multicall method + + methods: list of dictionaries (keys: methodName, params) + + The method names must be those used by Aria2c, e.g. "aria2.tellStatus". + ''' + return self.jsonrpc('multicall', [methods], prefix='system.') + + +# ########################### Convenience Methods ############################## + def add_torrent(self, path, uris=None, options=None, position=None): + ''' + A wrapper around addTorrent for loading files. + ''' + with open(path, 'r') as f: + torrent = base64.encode(f.read()) + return self.addTorrent(torrent, uris, options, position) + + def add_metalink(self, path, uris=None, options=None, position=None): + ''' + A wrapper around addMetalink for loading files. + ''' + with open(path, 'r') as f: + metalink = base64.encode(f.read()) + return self.addMetalink(metalink, uris, options, position) + + def get_status(self, gid): + ''' + Get the status of a single GID. + ''' + response = self.tellStatus(gid, ['status']) + return get_status(response) + + def wait_for_final_status(self, gid, interval=1): + ''' + Wait for a GID to complete or fail and return its status. + ''' + if not interval or interval < 0: + interval = 1 + while True: + status = self.get_status(gid) + if status in TEMPORARY_STATUS: + time.sleep(interval) + else: + return status + + def get_statuses(self, gids): + ''' + Get the status of multiple GIDs. The status of each is yielded in order. + ''' + methods = [ + { + 'methodName': 'aria2.tellStatus', + 'params': [gid, ['gid', 'status']] + } + for gid in gids + ] + results = self.multicall(methods) + if results: + status = dict((r[0]['gid'], r[0]['status']) for r in results) + for gid in gids: + try: + yield status[gid] + except KeyError: + logger.error('Aria2 RPC server returned no status for GID {}'.format(gid)) + yield 'error' + else: + logger.error('no response from Aria2 RPC server') + for gid in gids: + yield 'error' + + def wait_for_final_statuses(self, gids, interval=1): + ''' + Wait for multiple GIDs to complete or fail and return their statuses in + order. + + gids: + A flat list of GIDs. + ''' + if not interval or interval < 0: + interval = 1 + statusmap = dict((g, None) for g in gids) + remaining = list( + g for g, s in statusmap.items() if s is None + ) + while remaining: + for g, s in zip(remaining, self.get_statuses(remaining)): + if s in TEMPORARY_STATUS: + continue + else: + statusmap[g] = s + remaining = list( + g for g, s in statusmap.items() if s is None + ) + if remaining: + time.sleep(interval) + for g in gids: + yield statusmap[g] + + def print_global_status(self): + ''' + Print global status of the RPC server. + ''' + status = self.getGlobalStat() + if status: + numWaiting = int(status['numWaiting']) + numStopped = int(status['numStopped']) + keys = ['totalLength', 'completedLength'] + total = self.tellActive(keys) + waiting = self.tellWaiting(0, numWaiting, keys) + if waiting: + total += waiting + stopped = self.tellStopped(0, numStopped, keys) + if stopped: + total += stopped + + downloadSpeed = int(status['downloadSpeed']) + uploadSpeed = int(status['uploadSpeed']) + totalLength = sum(int(x['totalLength']) for x in total) + completedLength = sum(int(x['completedLength']) for x in total) + remaining = totalLength - completedLength + + status['downloadSpeed'] = format_bytes(downloadSpeed) + '/s' + status['uploadSpeed'] = format_bytes(uploadSpeed) + '/s' + + preordered = ('downloadSpeed', 'uploadSpeed') + + rows = list() + for k in sorted(status): + if k in preordered: + continue + rows.append((k, status[k])) + + rows.extend((x, status[x]) for x in preordered) + + if totalLength > 0: + rows.append(('total', format(format_bytes(totalLength)))) + rows.append(('completed', format(format_bytes(completedLength)))) + rows.append(('remaining', format(format_bytes(remaining)))) + if completedLength == totalLength: + eta = 'finished' + else: + try: + eta = format_seconds(remaining // downloadSpeed) + except ZeroDivisionError: + eta = 'never' + rows.append(('ETA', eta)) + + l = max(len(r[0]) for r in rows) + r = max(len(r[1]) for r in rows) + r = max(r, len(self.uri) - (l + 2)) + fmt = '{:<' + str(l) + 's} {:>' + str(r) + 's}' + + print(self.uri) + for k, v in rows: + print(fmt.format(k, v)) + + def queue_uris(self, uris, options, interval=None): + ''' + Enqueue URIs and wait for download to finish while printing status at + regular intervals. + ''' + gid = self.addUri(uris, options) + print('GID: {}'.format(gid)) + + if gid and interval is not None: + blanker = '' + while True: + response = self.tellStatus(gid, ['status']) + if response: + try: + status = response['status'] + except KeyError: + print('error: no status returned from Aria2 RPC server') + break + print('{}\rstatus: {}'.format(blanker, status)), + blanker = ' ' * len(status) + if status in TEMPORARY_STATUS: + time.sleep(interval) + else: + break + else: + print('error: no response from server') + break + + +# ####################### Polymethod download handlers ######################### + def polymethod_enqueue_many(self, downloads): + ''' + Enqueue downloads. + + downloads: Same as polymethod_download(). + ''' + methods = list( + { + 'methodName': 'aria2.{}'.format(d[0]), + 'params': list(d[1:]) + } for d in downloads + ) + return self.multicall(methods) + + def polymethod_wait_many(self, gids, interval=1): + ''' + Wait for the GIDs to complete or fail and return their statuses. + + gids: + A list of lists of GIDs. + ''' + # The flattened list of GIDs + gs = list(g for gs in gids for g in gs) + statusmap = dict(tuple(zip( + gs, + self.wait_for_final_statuses(gs, interval=interval) + ))) + for gs in gids: + yield list(statusmap.get(g, 'error') for g in gs) + + def polymethod_enqueue_one(self, download): + ''' + Same as polymethod_enqueue_many but for one element. + ''' + return getattr(self, download[0])(*download[1:]) + + def polymethod_download(self, downloads, interval=1): + ''' + Enqueue a series of downloads and wait for them to finish. Iterate over the + status of each, in order. + + downloads: + An iterable over (, , ...) where indicates the "add" + method to use ('addUri', 'addTorrent', 'addMetalink') and everything that + follows are arguments to pass to that method. + + interval: + The status check interval while waiting. + + Iterates over the download status of finished downloads. "complete" + indicates success. Lists of statuses will be returned for downloads that + create multiple GIDs (e.g. metalinks). + ''' + gids = self.polymethod_enqueue_many(downloads) + return self.polymethod_wait_many(gids, interval=interval) + + def polymethod_download_bool(self, *args, **kwargs): + ''' + A wrapper around polymethod_download() which returns a boolean for each + download to indicate success (True) or failure (False). + ''' +# for status in self.polymethod_download(*args, **kwargs): +# yield all(s == 'complete' for s in status) + return list( + all(s == 'complete' for s in status) + for status in self.polymethod_download(*args, **kwargs) + ) diff --git a/headphones/classes.py b/headphones/classes.py index 6015a0f2..54a10a55 100644 --- a/headphones/classes.py +++ b/headphones/classes.py @@ -21,6 +21,7 @@ import urllib from common import USER_AGENT +from collections import OrderedDict class HeadphonesURLopener(urllib.FancyURLopener): @@ -135,3 +136,33 @@ class Proper: def __str__(self): return str(self.date) + " " + self.name + " " + str(self.season) + "x" + str( self.episode) + " of " + str(self.tvdbid) + + +class OptionalImport(object): + ''' + Dummy class for optional import (imports needed for optional features). + ''' + def __init__(self, name): + self.__name = name + + def __getattr__(self, attr): + raise ImportError('The following package is required to use this feature: {0}'.format(self.__name)) + + +class CacheDict(OrderedDict): + ''' + Ordered dictionary with fixed size, designed for caching. + ''' + def __init__(self, *args, **kwds): + self.size_limit = kwds.pop("size_limit", None) + OrderedDict.__init__(self, *args, **kwds) + self._check_size_limit() + + def __setitem__(self, key, value): + OrderedDict.__setitem__(self, key, value) + self._check_size_limit() + + def _check_size_limit(self): + if self.size_limit is not None: + while len(self) > self.size_limit: + self.popitem(last=False) diff --git a/headphones/config.py b/headphones/config.py index 57af4c16..86aa9216 100644 --- a/headphones/config.py +++ b/headphones/config.py @@ -46,6 +46,10 @@ _CONFIG_DEFINITIONS = { 'APOLLO_RATIO': (str, 'Apollo.rip', ''), 'APOLLO_USERNAME': (str, 'Apollo.rip', ''), 'APOLLO_URL': (str, 'Apollo.rip', 'https://apollo.rip'), + 'ARIA_HOST': (str, 'Aria2', ''), + 'ARIA_PASSWORD': (str, 'Aria2', ''), + 'ARIA_TOKEN': (str, 'Aria2', ''), + 'ARIA_USERNAME': (str, 'Aria2', ''), 'AUTOWANT_ALL': (int, 'General', 0), 'AUTOWANT_MANUALLY_ADDED': (int, 'General', 1), 'AUTOWANT_UPCOMING': (int, 'General', 1), @@ -73,6 +77,8 @@ _CONFIG_DEFINITIONS = { 'CUSTOMPORT': (int, 'General', 5000), 'CUSTOMSLEEP': (int, 'General', 1), 'CUSTOMUSER': (str, 'General', ''), + 'DDL_DOWNLOADER': (int, 'General', 0), + 'DEEZLOADER': (int, 'DeezLoader', 0), 'DELETE_LOSSLESS_FILES': (int, 'General', 1), 'DELUGE_HOST': (str, 'Deluge', ''), 'DELUGE_CERT': (str, 'Deluge', ''), @@ -86,6 +92,7 @@ _CONFIG_DEFINITIONS = { 'DOWNLOAD_DIR': (path, 'General', ''), 'DOWNLOAD_SCAN_INTERVAL': (int, 'General', 5), 'DOWNLOAD_TORRENT_DIR': (path, 'General', ''), + 'DOWNLOAD_DDL_DIR': (path, 'General', ''), 'DO_NOT_OVERRIDE_GIT_BRANCH': (int, 'General', 0), 'EMAIL_ENABLED': (int, 'Email', 0), 'EMAIL_FROM': (str, 'Email', ''), diff --git a/headphones/deezloader.py b/headphones/deezloader.py new file mode 100644 index 00000000..8de85df9 --- /dev/null +++ b/headphones/deezloader.py @@ -0,0 +1,441 @@ +# -*- coding: utf-8 -*- +# Deezloader (c) 2016 by ParadoxalManiak +# +# Deezloader is licensed under a +# Creative Commons Attribution-NonCommercial-ShareAlike 3.0 Unported License. +# +# You should have received a copy of the license along with this +# work. If not, see . +# +# Version 2.1.0 +# Maintained by ParadoxalManiak +# Original work by ZzMTV +# +# Author's disclaimer: +# I am not responsible for the usage of this program by other people. +# I do not recommend you doing this illegally or against Deezer's terms of service. +# This project is licensed under CC BY-NC-SA 4.0 + +import re +import os +from datetime import datetime +from hashlib import md5 +import binascii + +from beets.mediafile import MediaFile +from headphones import logger, request, helpers +from headphones.classes import OptionalImport, CacheDict +import headphones + + +# Try to import optional Crypto.Cipher packages +try: + from Crypto.Cipher import AES, Blowfish +except ImportError: + AES = OptionalImport('Crypto.Cipher.AES') + Blowfish = OptionalImport('Crypto.Cipher.Blowfish') + +# Public constants +PROVIDER_NAME = 'Deezer' + +# Internal constants +__API_URL = "http://www.deezer.com/ajax/gw-light.php" +__API_INFO_URL = "http://api.deezer.com/" +__HTTP_HEADERS = { + "User-Agent": "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36", + "Content-Language": "en-US", + "Cache-Control": "max-age=0", + "Accept": "*/*", + "Accept-Charset": "utf-8,ISO-8859-1;q=0.7,*;q=0.3", + "Accept-Language": "de-DE,de;q=0.8,en-US;q=0.6,en;q=0.4" +} + +# Internal variables +__api_queries = { + 'api_version': "1.0", + 'api_token': "None", + 'input': "3" +} +__cookies = None +__tracks_cache = CacheDict(size_limit=512) +__albums_cache = CacheDict(size_limit=64) + + +def __getApiToken(): + global __cookies + response = request.request_response(url="http://www.deezer.com/", headers=__HTTP_HEADERS) + __cookies = response.cookies + data = response.content + if data: + matches = re.search(r"checkForm\s*=\s*['|\"](.*)[\"|'];", data) + if matches: + token = matches.group(1) + __api_queries['api_token'] = token + logger.debug(u"Deezloader : api token loaded ('%s')" % token) + + if not token: + logger.error(u"Deezloader: Unable to get api token") + + +def getAlbumByLink(album_link): + """Returns deezer album infos using album link url + + :param str album_link: deezer album link url (eg: 'http://www.deezer.com/album/1234567/') + """ + matches = re.search(r"album\/([0-9]+)\/?$", album_link) + if matches: + return getAlbum(matches.group(1)) + + +def getAlbum(album_id): + """Returns deezer album infos + + :param int album_id: deezer album id + """ + global __albums_cache + + if str(album_id) in __albums_cache: + return __albums_cache[str(album_id)] + + url = __API_INFO_URL + "album/" + str(album_id) + data = request.request_json(url=url, headers=__HTTP_HEADERS, cookies=__cookies) + + if data and 'error' not in data: + __albums_cache[str(album_id)] = data + return data + else: + logger.debug("Deezloader: Can't load album infos") + return None + + +def searchAlbums(search_term): + """Search for deezer albums using search term + + :param str search_term: search term to search album for + """ + logger.info(u'Searching Deezer using term: "%s"' % search_term) + + url = __API_INFO_URL + "search/album?q=" + search_term + data = request.request_json(url=url, headers=__HTTP_HEADERS, cookies=__cookies) + + albums = [] + + # Process content + if data and 'total' in data and data['total'] > 0 and 'data' in data: + for item in data['data']: + try: + albums.append(getAlbum(item['id'])) + except Exception as e: + logger.error(u"An unknown error occurred in the Deezer search album parser: %s" % e) + else: + logger.info(u'No results found from Deezer using term: "%s"' % search_term) + + return albums + + +def __matchAlbums(albums, artist_name, album_title, album_length): + resultlist = [] + + for album in albums: + total_size = 0 + tracks_found = 0 + + for track in album['tracks']['data']: + t = getTrack(track['id']) + if t: + if t["FILESIZE_MP3_320"] > 0: + size = t["FILESIZE_MP3_320"] + elif t["FILESIZE_MP3_256"] > 0: + size = t["FILESIZE_MP3_256"] + elif t["FILESIZE_MP3_128"] > 0: + size = t["FILESIZE_MP3_128"] + else: + size = t["FILESIZE_MP3_64"] + + size = int(size) + total_size += size + tracks_found += 1 + logger.debug(u'Found song "%s". Size: %s' % (t['SNG_TITLE'], helpers.bytes_to_mb(size))) + + if tracks_found == 0: + logger.info(u'Ignoring album "%s" (no tracks to download)' % album['title']) + continue + + matched = True + mismatch_reason = 'matched!' + + if album_length > 0 and abs(int(album['duration']) - album_length) > 240: + matched = False + mismatch_reason = (u'duration mismatch: %i not in [%i, %i]' % (int(album['duration']), (album_length - 240), (album_length + 240))) + + elif (helpers.latinToAscii(re.sub(r"\W", "", album_title, 0, re.UNICODE)).lower() != + helpers.latinToAscii(re.sub(r"\W", "", album['title'], 0, re.UNICODE)).lower()): + matched = False + mismatch_reason = (u'album name mismatch: %s != %s' % (album['title'], album_title)) + + elif (helpers.latinToAscii(re.sub(r"\W", "", artist_name, 0, re.UNICODE)).lower() != + helpers.latinToAscii(re.sub(r"\W", "", album['artist']['name'], 0, re.UNICODE)).lower()): + matched = False + mismatch_reason = (u'artist name mismatch: %s != %s' % (album['artist']['name'], artist_name)) + + resultlist.append( + (album['artist']['name'] + ' - ' + album['title'] + ' [' + album['release_date'][:4] + '] (' + str(tracks_found) + '/' + str(album['nb_tracks']) + ')', + total_size, album['link'], PROVIDER_NAME, "ddl", matched) + ) + logger.info(u'Found "%s". Tracks %i/%i. Size: %s (%s)' % (album['title'], tracks_found, album['nb_tracks'], helpers.bytes_to_mb(total_size), mismatch_reason)) + + return resultlist + + +def searchAlbum(artist_name, album_title, user_search_term=None, album_length=None): + """Search for deezer specific album. + This will iterate over deezer albums and try to find best matches + + :param str artist_name: album artist name + :param str album_title: album title + :param str user_search_term: search terms provided by user + :param int album_length: targeted album duration in seconds + """ + # User search term by-pass normal search + if user_search_term: + return __matchAlbums(searchAlbums(user_search_term), artist_name, album_title, album_length) + + resultlist = __matchAlbums(searchAlbums((artist_name + ' ' + album_title).strip()), artist_name, album_title, album_length) + if resultlist: + return resultlist + + # Deezer API supports unicode, so just remove non alphanumeric characters + clean_artist_name = re.sub(r"[^\w\s]", " ", artist_name, 0, re.UNICODE).strip() + clean_album_name = re.sub(r"[^\w\s]", " ", album_title, 0, re.UNICODE).strip() + + resultlist = __matchAlbums(searchAlbums((clean_artist_name + ' ' + clean_album_name).strip()), artist_name, album_title, album_length) + if resultlist: + return resultlist + + resultlist = __matchAlbums(searchAlbums(clean_artist_name), artist_name, album_title, album_length) + if resultlist: + return resultlist + + return resultlist + + +def getTrack(sng_id, try_reload_api=True): + """Returns deezer track infos + + :param int sng_id: deezer song id + :param bool try_reload_api: whether or not try reloading API if session expired + """ + global __tracks_cache + + if str(sng_id) in __tracks_cache: + return __tracks_cache[str(sng_id)] + + data = "[{\"method\":\"song.getListData\",\"params\":{\"sng_ids\":[" + str(sng_id) + "]}}]" + json = request.request_json(url=__API_URL, headers=__HTTP_HEADERS, method='post', params=__api_queries, data=data, cookies=__cookies) + + results = [] + error = None + invalid_token = False + + if json: + # Check for errors + if 'error' in json: + error = json['error'] + if 'GATEWAY_ERROR' in json['error'] and json['error']['GATEWAY_ERROR'] == u"invalid api token": + invalid_token = True + + elif 'error' in json[0] and json[0]['error']: + error = json[0]['error'] + if 'VALID_TOKEN_REQUIRED' in json[0]['error'] and json[0]['error']['VALID_TOKEN_REQUIRED'] == u"Invalid CSRF token": + invalid_token = True + + # Got invalid token error + if error: + if invalid_token and try_reload_api: + __getApiToken() + return getTrack(sng_id, False) + else: + logger.error(u"An unknown error occurred in the Deezer track parser: %s" % error) + else: + try: + results = json[0]['results'] + item = results['data'][0] + if 'token' in item: + logger.error(u"An unknown error occurred in the Deezer parser: Uploaded Files are currently not supported") + return + + sng_id = item["SNG_ID"] + md5Origin = item["MD5_ORIGIN"] + sng_format = 3 + + if item["FILESIZE_MP3_320"] <= 0: + if item["FILESIZE_MP3_256"] > 0: + sng_format = 5 + else: + sng_format = 1 + + mediaVersion = int(item["MEDIA_VERSION"]) + item['downloadUrl'] = __getDownloadUrl(md5Origin, sng_id, sng_format, mediaVersion) + + __tracks_cache[sng_id] = item + return item + + except Exception as e: + logger.error(u"An unknown error occurred in the Deezer track parser: %s" % e) + + +def __getDownloadUrl(md5Origin, sng_id, sng_format, mediaVersion): + urlPart = md5Origin.encode('utf-8') + b'\xa4' + str(sng_format) + b'\xa4' + str(sng_id) + b'\xa4' + str(mediaVersion) + md5val = md5(urlPart).hexdigest() + urlPart = md5val + b'\xa4' + urlPart + b'\xa4' + cipher = AES.new('jo6aey6haid2Teih', AES.MODE_ECB) + ciphertext = cipher.encrypt(__pad(urlPart, AES.block_size)) + return "http://e-cdn-proxy-" + md5Origin[:1] + ".deezer.com/mobile/1/" + binascii.hexlify(ciphertext).lower() + + +def __pad(raw, block_size): + if (len(raw) % block_size == 0): + return raw + padding_required = block_size - (len(raw) % block_size) + padChar = b'\x00' + data = raw + padding_required * padChar + return data + + +def __tagTrack(path, track): + try: + album = getAlbum(track['ALB_ID']) + + f = MediaFile(path) + f.artist = track['ART_NAME'] + f.album = track['ALB_TITLE'] + f.title = track['SNG_TITLE'] + f.track = track['TRACK_NUMBER'] + f.tracktotal = album['nb_tracks'] + f.disc = track['DISK_NUMBER'] + f.bpm = track['BPM'] + f.date = datetime.strptime(album['release_date'], '%Y-%m-%d').date() + f.albumartist = album['artist']['name'] + if u'genres' in album and u'data' in album['genres']: + f.genres = [genre['name'] for genre in album['genres']['data']] + + f.save() + + except Exception as e: + logger.error(u'Unable to tag deezer track "%s": %s' % (path, e)) + + +def decryptTracks(paths): + """Decrypt downloaded deezer tracks. + + :param paths: list of path to deezer tracks (*.dzr files). + """ + # Note: tracks can be from different albums + decrypted_tracks = {} + + # First pass: load tracks data + for path in paths: + try: + album_folder = os.path.dirname(path) + sng_id = os.path.splitext(os.path.basename(path))[0] + track = getTrack(sng_id) + if track: + track_number = int(track['TRACK_NUMBER']) + disk_number = int(track['DISK_NUMBER']) + + if album_folder not in decrypted_tracks: + decrypted_tracks[album_folder] = {} + + if disk_number not in decrypted_tracks[album_folder]: + decrypted_tracks[album_folder][disk_number] = {} + + decrypted_tracks[album_folder][disk_number][track_number] = track + + except Exception as e: + logger.error(u'Unable to load deezer track infos "%s": %s' % (path, e)) + + # Second pass: decrypt tracks + for album_folder in decrypted_tracks: + multi_disks = len(decrypted_tracks[album_folder]) > 1 + for disk_number in decrypted_tracks[album_folder]: + for track_number, track in decrypted_tracks[album_folder][disk_number].items(): + try: + filename = helpers.replace_illegal_chars(track['SNG_TITLE']).strip() + filename = ('{:02d}'.format(track_number) + '_' + filename + '.mp3') + + # Add a 'cd x' sub-folder if album has more than one disk + disk_folder = os.path.join(album_folder, 'cd ' + str(disk_number)) if multi_disks else album_folder + + dest = os.path.join(disk_folder, filename).encode(headphones.SYS_ENCODING, 'replace') + + # Decrypt track if not already done + if not os.path.exists(dest): + try: + __decryptDownload(path, sng_id, dest) + __tagTrack(dest, track) + except Exception as e: + logger.error(u'Unable to decrypt deezer track "%s": %s' % (path, e)) + if os.path.exists(dest): + os.remove(dest) + decrypted_tracks[album_folder][disk_number].pop(track_number) + continue + + decrypted_tracks[album_folder][disk_number][track_number]['path'] = dest + + except Exception as e: + logger.error(u'Unable to decrypt deezer track "%s": %s' % (path, e)) + + return decrypted_tracks + + +def __decryptDownload(source, sng_id, dest): + interval_chunk = 3 + chunk_size = 2048 + blowFishKey = __getBlowFishKey(sng_id) + i = 0 + iv = "\x00\x01\x02\x03\x04\x05\x06\x07" + + dest_folder = os.path.dirname(dest) + if not os.path.exists(dest_folder): + os.makedirs(dest_folder) + + f = open(source, "rb") + fout = open(dest, "wb") + try: + chunk = f.read(chunk_size) + while chunk: + if(i % interval_chunk == 0): + cipher = Blowfish.new(blowFishKey, Blowfish.MODE_CBC, iv) + chunk = cipher.decrypt(__pad(chunk, Blowfish.block_size)) + + fout.write(chunk) + i += 1 + chunk = f.read(chunk_size) + finally: + f.close() + fout.close() + + +def __getBlowFishKey(encryptionKey): + if encryptionKey < 1: + encryptionKey *= -1 + + hashcode = md5(str(encryptionKey)).hexdigest() + hPart = hashcode[0:16] + lPart = hashcode[16:32] + parts = ['g4el58wc0zvf9na1', hPart, lPart] + + return __xorHex(parts) + + +def __xorHex(parts): + data = "" + for i in range(0, 16): + character = ord(parts[0][i]) + + for j in range(1, len(parts)): + character ^= ord(parts[j][i]) + + data += chr(character) + + return data diff --git a/headphones/postprocessor.py b/headphones/postprocessor.py index ab00ab1a..0b574cd2 100755 --- a/headphones/postprocessor.py +++ b/headphones/postprocessor.py @@ -30,7 +30,7 @@ from beetsplug import lyrics as beetslyrics from headphones import notifiers, utorrent, transmission, deluge, qbittorrent from headphones import db, albumart, librarysync from headphones import logger, helpers, mb, music_encoder -from headphones import metadata +from headphones import metadata, deezloader postprocessor_lock = threading.Lock() @@ -48,6 +48,8 @@ def checkFolder(): single = False if album['Kind'] == 'nzb': download_dir = headphones.CONFIG.DOWNLOAD_DIR + elif album['Kind'] == 'ddl': + download_dir = headphones.CONFIG.DOWNLOAD_DDL_DIR else: if headphones.CONFIG.DELUGE_DONE_DIRECTORY and headphones.CONFIG.TORRENT_DOWNLOADER == 3: download_dir = headphones.CONFIG.DELUGE_DONE_DIRECTORY @@ -205,6 +207,7 @@ def verify(albumid, albumpath, Kind=None, forced=False, keep_original_folder=Fal tracks = myDB.select('SELECT * from tracks WHERE AlbumID=?', [albumid]) downloaded_track_list = [] + downloaded_deezer_list = [] downloaded_cuecount = 0 for r, d, f in os.walk(albumpath): @@ -213,8 +216,10 @@ def verify(albumid, albumpath, Kind=None, forced=False, keep_original_folder=Fal downloaded_track_list.append(os.path.join(r, files)) elif files.lower().endswith('.cue'): downloaded_cuecount += 1 + elif files.lower().endswith('.dzr'): + downloaded_deezer_list.append(os.path.join(r, files)) # if any of the files end in *.part, we know the torrent isn't done yet. Process if forced, though - elif files.lower().endswith(('.part', '.utpart')) and not forced: + elif files.lower().endswith(('.part', '.utpart', '.aria2')) and not forced: logger.info( "Looks like " + os.path.basename(albumpath).decode(headphones.SYS_ENCODING, 'replace') + " isn't complete yet. Will try again on the next run") @@ -253,6 +258,37 @@ def verify(albumid, albumpath, Kind=None, forced=False, keep_original_folder=Fal downloaded_track_list = helpers.get_downloaded_track_list(albumpath) keep_original_folder = False + # Decrypt deezer tracks + if downloaded_deezer_list: + logger.info('Decrypting deezer tracks') + decrypted_deezer_list = deezloader.decryptTracks(downloaded_deezer_list) + + # Check if album is complete based on album duration only + # (total track numbers is not determinant enough due to hidden tracks for eg) + db_track_duration = 0 + downloaded_track_duration = 0 + try: + for track in tracks: + db_track_duration += track['TrackDuration'] / 1000 + except: + downloaded_track_duration = False + + try: + for disk_number in decrypted_deezer_list[albumpath]: + for track in decrypted_deezer_list[albumpath][disk_number].values(): + downloaded_track_list.append(track['path']) + downloaded_track_duration += int(track['DURATION']) + except: + downloaded_track_duration = False + + if not downloaded_track_duration or not db_track_duration or abs(downloaded_track_duration - db_track_duration) > 240: + logger.info("Looks like " + + os.path.basename(albumpath).decode(headphones.SYS_ENCODING, 'replace') + + " isn't complete yet (duration mismatch). Will try again on the next run") + return + + downloaded_track_list = list(set(downloaded_track_list)) # Remove duplicates + # test #1: metadata - usually works logger.debug('Verifying metadata...') diff --git a/headphones/searcher.py b/headphones/searcher.py index b0652f7f..f6554cb1 100644 --- a/headphones/searcher.py +++ b/headphones/searcher.py @@ -36,6 +36,7 @@ import headphones from headphones.common import USER_AGENT from headphones import logger, db, helpers, classes, sab, nzbget, request from headphones import utorrent, transmission, notifiers, rutracker, deluge, qbittorrent +from headphones import deezloader, aria2 from bencode import bencode, bdecode # Magnet to torrent services, for Black hole. Stolen from CouchPotato. @@ -51,6 +52,27 @@ ruobj = None # Persistent RED API object redobj = None +# Persistent Aria2 RPC object +__aria2rpc_obj = None + + +def getAria2RPC(): + global __aria2rpc_obj + if not __aria2rpc_obj: + __aria2rpc_obj = aria2.Aria2JsonRpc( + ID='headphones', + uri=headphones.CONFIG.ARIA_HOST, + token=headphones.CONFIG.ARIA_TOKEN if headphones.CONFIG.ARIA_TOKEN else None, + http_user=headphones.CONFIG.ARIA_USERNAME if headphones.CONFIG.ARIA_USERNAME else None, + http_passwd=headphones.CONFIG.ARIA_PASSWORD if headphones.CONFIG.ARIA_PASSWORD else None + ) + return __aria2rpc_obj + + +def reconfigure(): + global __aria2rpc_obj + __aria2rpc_obj = None + def fix_url(s, charset="utf-8"): """ @@ -281,32 +303,53 @@ def do_sorted_search(album, new, losslessOnly, choose_specific_download=False): headphones.CONFIG.STRIKE or headphones.CONFIG.TQUATTRECENTONZE) - results = [] + DDL_PROVIDERS = (headphones.CONFIG.DEEZLOADER) + myDB = db.DBConnection() albumlength = myDB.select('SELECT sum(TrackDuration) from tracks WHERE AlbumID=?', [album['AlbumID']])[0][0] + nzb_results = None + torrent_results = None + ddl_results = None + if headphones.CONFIG.PREFER_TORRENTS == 0 and not choose_specific_download: if NZB_PROVIDERS and NZB_DOWNLOADERS: - results = searchNZB(album, new, losslessOnly, albumlength) + nzb_results = searchNZB(album, new, losslessOnly, albumlength) - if not results and TORRENT_PROVIDERS: - results = searchTorrent(album, new, losslessOnly, albumlength) + if not nzb_results: + if DDL_PROVIDERS: + ddl_results = searchDdl(album, new, losslessOnly, albumlength) + + if TORRENT_PROVIDERS: + torrent_results = searchTorrent(album, new, losslessOnly, albumlength) elif headphones.CONFIG.PREFER_TORRENTS == 1 and not choose_specific_download: if TORRENT_PROVIDERS: - results = searchTorrent(album, new, losslessOnly, albumlength) + torrent_results = searchTorrent(album, new, losslessOnly, albumlength) - if not results and NZB_PROVIDERS and NZB_DOWNLOADERS: - results = searchNZB(album, new, losslessOnly, albumlength) + if not torrent_results: + if DDL_PROVIDERS: + ddl_results = searchDdl(album, new, losslessOnly, albumlength) + + if NZB_PROVIDERS and NZB_DOWNLOADERS: + nzb_results = searchNZB(album, new, losslessOnly, albumlength) + + elif headphones.CONFIG.PREFER_TORRENTS == 3 and not choose_specific_download: + + if DDL_PROVIDERS: + ddl_results = searchDdl(album, new, losslessOnly, albumlength) + + if not ddl_results: + if TORRENT_PROVIDERS: + torrent_results = searchTorrent(album, new, losslessOnly, albumlength) + + if NZB_PROVIDERS and NZB_DOWNLOADERS: + nzb_results = searchNZB(album, new, losslessOnly, albumlength) else: - - nzb_results = None - torrent_results = None - if NZB_PROVIDERS and NZB_DOWNLOADERS: nzb_results = searchNZB(album, new, losslessOnly, albumlength, choose_specific_download) @@ -314,13 +357,19 @@ def do_sorted_search(album, new, losslessOnly, choose_specific_download=False): torrent_results = searchTorrent(album, new, losslessOnly, albumlength, choose_specific_download) - if not nzb_results: - nzb_results = [] + if DDL_PROVIDERS: + ddl_results = searchDdl(album, new, losslessOnly, albumlength, choose_specific_download) - if not torrent_results: - torrent_results = [] + if not nzb_results: + nzb_results = [] - results = nzb_results + torrent_results + if not torrent_results: + torrent_results = [] + + if not ddl_results: + ddl_results = [] + + results = nzb_results + torrent_results + ddl_results if choose_specific_download: return results @@ -826,6 +875,31 @@ def send_to_downloader(data, bestqual, album): except Exception as e: logger.error('Couldn\'t write NZB file: %s', e) return + + elif kind == 'ddl': + folder_name = '%s - %s [%s]' % ( + helpers.latinToAscii(album['ArtistName']).encode('UTF-8').replace('/', '_'), + helpers.latinToAscii(album['AlbumTitle']).encode('UTF-8').replace('/', '_'), + get_year_from_release_date(album['ReleaseDate'])) + + # Aria2 downloader + if headphones.CONFIG.DDL_DOWNLOADER == 0: + logger.info("Sending download to Aria2") + + try: + deezer_album = deezloader.getAlbumByLink(bestqual[2]) + + for album_track in deezer_album['tracks']['data']: + track = deezloader.getTrack(album_track['id']) + if track: + filename = track['SNG_ID'] + '.dzr' + logger.debug(u'Sending song "%s" to Aria' % track['SNG_TITLE']) + getAria2RPC().addUri([track['downloadUrl']], {'out': filename, 'auto-file-renaming': 'false', 'continue': 'true', 'dir': folder_name}) + + except Exception as e: + logger.error(u'Error sending torrent to Aria2. Are you sure it\'s running? (%s)' % e) + return + else: folder_name = '%s - %s [%s]' % ( helpers.latinToAscii(album['ArtistName']).encode('UTF-8').replace('/', '_'), @@ -1204,6 +1278,62 @@ def verifyresult(title, artistterm, term, lossless): return True +def searchDdl(album, new=False, losslessOnly=False, albumlength=None, + choose_specific_download=False): + reldate = album['ReleaseDate'] + year = get_year_from_release_date(reldate) + + # MERGE THIS WITH THE TERM CLEANUP FROM searchNZB + dic = {'...': '', ' & ': ' ', ' = ': ' ', '?': '', '$': 's', ' + ': ' ', '"': '', ',': ' ', + '*': ''} + + semi_cleanalbum = helpers.replace_all(album['AlbumTitle'], dic) + cleanalbum = helpers.latinToAscii(semi_cleanalbum) + semi_cleanartist = helpers.replace_all(album['ArtistName'], dic) + cleanartist = helpers.latinToAscii(semi_cleanartist) + + # Use provided term if available, otherwise build our own (this code needs to be cleaned up since a lot + # of these torrent providers are just using cleanartist/cleanalbum terms + if album['SearchTerm']: + term = album['SearchTerm'] + elif album['Type'] == 'part of': + term = cleanalbum + " " + year + else: + # FLAC usually doesn't have a year for some reason so I'll leave it out + # Various Artist albums might be listed as VA, so I'll leave that out too + # Only use the year if the term could return a bunch of different albums, i.e. self-titled albums + if album['ArtistName'] in album['AlbumTitle'] or len(album['ArtistName']) < 4 or len( + album['AlbumTitle']) < 4: + term = cleanartist + ' ' + cleanalbum + ' ' + year + elif album['ArtistName'] == 'Various Artists': + term = cleanalbum + ' ' + year + else: + term = cleanartist + ' ' + cleanalbum + + # Replace bad characters in the term and unicode it + term = re.sub('[\.\-\/]', ' ', term).encode('utf-8') + artistterm = re.sub('[\.\-\/]', ' ', cleanartist).encode('utf-8', 'replace') + + logger.debug(u'Using search term: "%s"' % helpers.latinToAscii(term)) + + resultlist = [] + + # Deezer only provides lossy + if headphones.CONFIG.DEEZLOADER and not losslessOnly: + resultlist += deezloader.searchAlbum(album['ArtistName'], album['AlbumTitle'], album['SearchTerm'], int(albumlength / 1000)) + + # attempt to verify that this isn't a substring result + # when looking for "Foo - Foo" we don't want "Foobar" + # this should be less of an issue when it isn't a self-titled album so we'll only check vs artist + results = [result for result in resultlist if verifyresult(result[0], artistterm, term, losslessOnly)] + + # Additional filtering for size etc + if results and not choose_specific_download: + results = more_filtering(results, album, albumlength, new) + + return results + + def searchTorrent(album, new=False, losslessOnly=False, albumlength=None, choose_specific_download=False): global apolloobj # persistent apollo.rip api object to reduce number of login attempts @@ -2036,7 +2166,10 @@ def searchTorrent(album, new=False, losslessOnly=False, albumlength=None, def preprocess(resultlist): for result in resultlist: - if result[4] == 'torrent': + if result[3] == deezloader.PROVIDER_NAME: + return True, result + + elif result[4] == 'torrent': # rutracker always needs the torrent data if result[3] == 'rutracker.org': diff --git a/headphones/webserve.py b/headphones/webserve.py index 855fed24..7e97cefb 100644 --- a/headphones/webserve.py +++ b/headphones/webserve.py @@ -1174,6 +1174,10 @@ class WebInterface(object): "utorrent_username": headphones.CONFIG.UTORRENT_USERNAME, "utorrent_password": headphones.CONFIG.UTORRENT_PASSWORD, "utorrent_label": headphones.CONFIG.UTORRENT_LABEL, + "aria_host": headphones.CONFIG.ARIA_HOST, + "aria_password": headphones.CONFIG.ARIA_PASSWORD, + "aria_token": headphones.CONFIG.ARIA_TOKEN, + "aria_username": headphones.CONFIG.ARIA_USERNAME, "nzb_downloader_sabnzbd": radio(headphones.CONFIG.NZB_DOWNLOADER, 0), "nzb_downloader_nzbget": radio(headphones.CONFIG.NZB_DOWNLOADER, 1), "nzb_downloader_blackhole": radio(headphones.CONFIG.NZB_DOWNLOADER, 2), @@ -1182,6 +1186,7 @@ class WebInterface(object): "torrent_downloader_utorrent": radio(headphones.CONFIG.TORRENT_DOWNLOADER, 2), "torrent_downloader_deluge": radio(headphones.CONFIG.TORRENT_DOWNLOADER, 3), "torrent_downloader_qbittorrent": radio(headphones.CONFIG.TORRENT_DOWNLOADER, 4), + "ddl_downloader_aria": radio(headphones.CONFIG.DDL_DOWNLOADER, 0), "download_dir": headphones.CONFIG.DOWNLOAD_DIR, "use_blackhole": checked(headphones.CONFIG.BLACKHOLE), "blackhole_dir": headphones.CONFIG.BLACKHOLE_DIR, @@ -1243,6 +1248,8 @@ class WebInterface(object): "use_tquattrecentonze": checked(headphones.CONFIG.TQUATTRECENTONZE), "tquattrecentonze_user": headphones.CONFIG.TQUATTRECENTONZE_USER, "tquattrecentonze_password": headphones.CONFIG.TQUATTRECENTONZE_PASSWORD, + "download_ddl_dir": headphones.CONFIG.DOWNLOAD_DDL_DIR, + "use_deezloader": checked(headphones.CONFIG.DEEZLOADER), "pref_qual_0": radio(headphones.CONFIG.PREFERRED_QUALITY, 0), "pref_qual_1": radio(headphones.CONFIG.PREFERRED_QUALITY, 1), "pref_qual_2": radio(headphones.CONFIG.PREFERRED_QUALITY, 2), @@ -1288,6 +1295,7 @@ class WebInterface(object): "prefer_torrents_0": radio(headphones.CONFIG.PREFER_TORRENTS, 0), "prefer_torrents_1": radio(headphones.CONFIG.PREFER_TORRENTS, 1), "prefer_torrents_2": radio(headphones.CONFIG.PREFER_TORRENTS, 2), + "prefer_torrents_3": radio(headphones.CONFIG.PREFER_TORRENTS, 3), "magnet_links_0": radio(headphones.CONFIG.MAGNET_LINKS, 0), "magnet_links_1": radio(headphones.CONFIG.MAGNET_LINKS, 1), "magnet_links_2": radio(headphones.CONFIG.MAGNET_LINKS, 2), @@ -1446,7 +1454,7 @@ class WebInterface(object): "launch_browser", "enable_https", "api_enabled", "use_blackhole", "headphones_indexer", "use_newznab", "newznab_enabled", "use_torznab", "torznab_enabled", "use_nzbsorg", "use_omgwtfnzbs", "use_kat", "use_piratebay", "use_oldpiratebay", - "use_mininova", "use_waffles", "use_rutracker", + "use_mininova", "use_waffles", "use_rutracker", "use_deezloader", "use_apollo", "use_redacted", "use_strike", "use_tquattrecentonze", "preferred_bitrate_allow_lossless", "detect_bitrate", "ignore_clean_releases", "freeze_db", "cue_split", "move_files", "rename_files", "correct_metadata", "cleanup_files", "keep_nfo", "add_album_art", @@ -1580,6 +1588,9 @@ class WebInterface(object): # Reconfigure musicbrainz database connection with the new values mb.startmb() + # Reconfigure Aria2 + searcher.reconfigure() + raise cherrypy.HTTPRedirect("config") @cherrypy.expose From cef1563b4b7e517555d5f3cc9aa0a1f3c74f8cd7 Mon Sep 17 00:00:00 2001 From: Ade Date: Tue, 25 Apr 2017 17:03:59 +1200 Subject: [PATCH 13/27] Trap errors for Scan --- headphones/webserve.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/headphones/webserve.py b/headphones/webserve.py index 855fed24..f46b5254 100644 --- a/headphones/webserve.py +++ b/headphones/webserve.py @@ -871,11 +871,19 @@ class WebInterface(object): def musicScan(self, path, scan=0, redirect=None, autoadd=0, libraryscan=0): headphones.CONFIG.LIBRARYSCAN = libraryscan headphones.CONFIG.AUTO_ADD_ARTISTS = autoadd - headphones.CONFIG.MUSIC_DIR = path - headphones.CONFIG.write() + + try: + params = {} + headphones.CONFIG.MUSIC_DIR = path + headphones.CONFIG.write() + except Exception as e: + logger.warn("Cannot save scan directory to config: %s", e) + if scan: + params = {"dir": path} + if scan: try: - threading.Thread(target=librarysync.libraryScan).start() + threading.Thread(target=librarysync.libraryScan, kwargs=params).start() except Exception as e: logger.error('Unable to complete the scan: %s' % e) if redirect: From 39e589bc08df93d01468a57dd74087ab06debc46 Mon Sep 17 00:00:00 2001 From: Ade Date: Tue, 25 Apr 2017 18:16:33 +1200 Subject: [PATCH 14/27] Use Release Id for beets matching if supplied --- headphones/lastfm.py | 2 +- headphones/postprocessor.py | 14 +++++++++++--- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/headphones/lastfm.py b/headphones/lastfm.py index 89ab7448..8f906d5f 100644 --- a/headphones/lastfm.py +++ b/headphones/lastfm.py @@ -55,7 +55,7 @@ def request_lastfm(method, **kwargs): return if "error" in data: - logger.error("Last.FM returned an error: %s", data["message"]) + logger.debug("Last.FM returned an error: %s", data["message"]) return return data diff --git a/headphones/postprocessor.py b/headphones/postprocessor.py index ab00ab1a..38e4bba5 100755 --- a/headphones/postprocessor.py +++ b/headphones/postprocessor.py @@ -954,16 +954,24 @@ def correctMetadata(albumid, release, downloaded_track_list): if not items: continue + search_ids = [] + logger.debug('Getting recommendation from beets. Artist: %s. Album: %s. Tracks: %s', release['ArtistName'], + release['AlbumTitle'], len(items)) + + # Try with specific release, e.g. alternate release selected from albumPage + if release['ReleaseID'] != release['AlbumID']: + logger.debug('trying beets with specific Release ID: %s', release['ReleaseID']) + search_ids = [release['ReleaseID']] + try: - logger.debug('Getting recommendation from beets. Artist: %s. Album: %s. Tracks: %s', release['ArtistName'], - release['AlbumTitle'], len(items)) beetslog = beetslogging.getLogger('beets') beetslog.set_global_level(beetslogging.DEBUG) if headphones.VERBOSE else beetslog.set_global_level( beetslogging.CRITICAL) with helpers.capture_beets_log() as logs: cur_artist, cur_album, prop = autotag.tag_album(items, search_artist=release['ArtistName'], - search_album=release['AlbumTitle']) + search_album=release['AlbumTitle'], + search_ids=search_ids) candidates = prop.candidates rec = prop.recommendation for log in logs: From bc0ce99adfa7df8176d874581d6c83a19fcc251e Mon Sep 17 00:00:00 2001 From: Ade Date: Sat, 29 Apr 2017 18:05:10 +1200 Subject: [PATCH 15/27] Set clean name to lower --- headphones/__init__.py | 8 ++++++++ headphones/helpers.py | 1 + 2 files changed, 9 insertions(+) diff --git a/headphones/__init__.py b/headphones/__init__.py index 20476169..8247e1c2 100644 --- a/headphones/__init__.py +++ b/headphones/__init__.py @@ -620,6 +620,14 @@ def dbcheck(): c.execute('ALTER TABLE snatched ADD COLUMN TorrentHash TEXT') c.execute('UPDATE snatched SET TorrentHash = FolderName WHERE Status LIKE "Seed_%"') + # One off script to set CleanName to lower case + clean_name_mixed = c.execute('SELECT CleanName FROM have ORDER BY Date Desc').fetchone()[0] + if clean_name_mixed != clean_name_mixed.lower(): + logger.info("Updating track clean name, this could take some time...") + c.execute('UPDATE tracks SET CleanName = LOWER(CleanName) WHERE LOWER(CleanName) != CleanName') + c.execute('UPDATE alltracks SET CleanName = LOWER(CleanName) WHERE LOWER(CleanName) != CleanName') + c.execute('UPDATE have SET CleanName = LOWER(CleanName) WHERE LOWER(CleanName) != CleanName') + conn.commit() c.close() diff --git a/headphones/helpers.py b/headphones/helpers.py index d9b491f7..05f6108b 100644 --- a/headphones/helpers.py +++ b/headphones/helpers.py @@ -333,6 +333,7 @@ def clean_name(s): # 6. trim u = u.strip() # 7. lowercase + u = u.lower() return u From cd2860f4e373cdd7c358bc1bf5204d7944d8d0e6 Mon Sep 17 00:00:00 2001 From: Kallys Date: Sun, 30 Apr 2017 14:00:10 +0200 Subject: [PATCH 16/27] Fix #2930 Fix #2931 --- data/interfaces/default/config.html | 8 +++++++- headphones/deezloader.py | 3 ++- headphones/searcher.py | 5 ++++- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/data/interfaces/default/config.html b/data/interfaces/default/config.html index bd331f2c..18200892 100644 --- a/data/interfaces/default/config.html +++ b/data/interfaces/default/config.html @@ -335,12 +335,18 @@ +
+ Note: With Aria2, you can specify a different download directory for downloads sent from Headphones. + Set it in the Music Download Directory below +
+
+
- Full path where ddl client downloads your music, e.g. /Users/name/Downloads/music + Full path where your direct download client downloads your music, e.g. /Users/name/Downloads/music
diff --git a/headphones/deezloader.py b/headphones/deezloader.py index 8de85df9..09eb60d9 100644 --- a/headphones/deezloader.py +++ b/headphones/deezloader.py @@ -350,6 +350,7 @@ def decryptTracks(paths): decrypted_tracks[album_folder][disk_number] = {} decrypted_tracks[album_folder][disk_number][track_number] = track + decrypted_tracks[album_folder][disk_number][track_number]['path'] = path except Exception as e: logger.error(u'Unable to load deezer track infos "%s": %s' % (path, e)) @@ -371,7 +372,7 @@ def decryptTracks(paths): # Decrypt track if not already done if not os.path.exists(dest): try: - __decryptDownload(path, sng_id, dest) + __decryptDownload(track['path'], track['SNG_ID'], dest) __tagTrack(dest, track) except Exception as e: logger.error(u'Unable to decrypt deezer track "%s": %s' % (path, e)) diff --git a/headphones/searcher.py b/headphones/searcher.py index f6554cb1..fdef6153 100644 --- a/headphones/searcher.py +++ b/headphones/searcher.py @@ -894,7 +894,10 @@ def send_to_downloader(data, bestqual, album): if track: filename = track['SNG_ID'] + '.dzr' logger.debug(u'Sending song "%s" to Aria' % track['SNG_TITLE']) - getAria2RPC().addUri([track['downloadUrl']], {'out': filename, 'auto-file-renaming': 'false', 'continue': 'true', 'dir': folder_name}) + getAria2RPC().addUri( + [track['downloadUrl']], + {'out': filename, 'auto-file-renaming': 'false', 'continue': 'true', + 'dir': os.path.join(headphones.CONFIG.DOWNLOAD_DDL_DIR, folder_name)}) except Exception as e: logger.error(u'Error sending torrent to Aria2. Are you sure it\'s running? (%s)' % e) From 54edacdfe7e90116ceec932e2f585d4c1503a289 Mon Sep 17 00:00:00 2001 From: pratstercs Date: Mon, 1 May 2017 00:29:39 +0100 Subject: [PATCH 17/27] Fixed bug (#2933 at least) where an artist will null name errors when trying to remove from library as logger is trying to concat artist name (NoneType) with string --- headphones/webserve.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/headphones/webserve.py b/headphones/webserve.py index 267b78bc..a4063000 100644 --- a/headphones/webserve.py +++ b/headphones/webserve.py @@ -252,7 +252,10 @@ class WebInterface(object): namecheck = myDB.select('SELECT ArtistName from artists where ArtistID=?', [ArtistID]) for name in namecheck: artistname = name['ArtistName'] - logger.info(u"Deleting all traces of artist: " + artistname) + try: + logger.info(u"Deleting all traces of artist: " + artistname) + except TypeError: + logger.info(u"Deleting all traces of artist: null") myDB.action('DELETE from artists WHERE ArtistID=?', [ArtistID]) from headphones import cache From 74d5333bdf460d7e574b036b3734ce1712e82020 Mon Sep 17 00:00:00 2001 From: pratstercs Date: Mon, 1 May 2017 00:35:28 +0100 Subject: [PATCH 18/27] Fixing tabs/spaces --- headphones/webserve.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/headphones/webserve.py b/headphones/webserve.py index a4063000..648e293b 100644 --- a/headphones/webserve.py +++ b/headphones/webserve.py @@ -253,7 +253,7 @@ class WebInterface(object): for name in namecheck: artistname = name['ArtistName'] try: - logger.info(u"Deleting all traces of artist: " + artistname) + logger.info(u"Deleting all traces of artist: " + artistname) except TypeError: logger.info(u"Deleting all traces of artist: null") myDB.action('DELETE from artists WHERE ArtistID=?', [ArtistID]) From e3c77900391e72583262db9d4bc04bcd91cd043f Mon Sep 17 00:00:00 2001 From: Kallys Date: Tue, 2 May 2017 13:41:29 +0200 Subject: [PATCH 19/27] Revert "Fix #2930" This reverts commit cd2860f4e373cdd7c358bc1bf5204d7944d8d0e6. --- data/interfaces/default/config.html | 8 +------- headphones/deezloader.py | 3 +-- headphones/searcher.py | 5 +---- 3 files changed, 3 insertions(+), 13 deletions(-) diff --git a/data/interfaces/default/config.html b/data/interfaces/default/config.html index 18200892..bd331f2c 100644 --- a/data/interfaces/default/config.html +++ b/data/interfaces/default/config.html @@ -335,18 +335,12 @@ -
- Note: With Aria2, you can specify a different download directory for downloads sent from Headphones. - Set it in the Music Download Directory below -
- -
- Full path where your direct download client downloads your music, e.g. /Users/name/Downloads/music + Full path where ddl client downloads your music, e.g. /Users/name/Downloads/music
diff --git a/headphones/deezloader.py b/headphones/deezloader.py index 09eb60d9..8de85df9 100644 --- a/headphones/deezloader.py +++ b/headphones/deezloader.py @@ -350,7 +350,6 @@ def decryptTracks(paths): decrypted_tracks[album_folder][disk_number] = {} decrypted_tracks[album_folder][disk_number][track_number] = track - decrypted_tracks[album_folder][disk_number][track_number]['path'] = path except Exception as e: logger.error(u'Unable to load deezer track infos "%s": %s' % (path, e)) @@ -372,7 +371,7 @@ def decryptTracks(paths): # Decrypt track if not already done if not os.path.exists(dest): try: - __decryptDownload(track['path'], track['SNG_ID'], dest) + __decryptDownload(path, sng_id, dest) __tagTrack(dest, track) except Exception as e: logger.error(u'Unable to decrypt deezer track "%s": %s' % (path, e)) diff --git a/headphones/searcher.py b/headphones/searcher.py index fdef6153..f6554cb1 100644 --- a/headphones/searcher.py +++ b/headphones/searcher.py @@ -894,10 +894,7 @@ def send_to_downloader(data, bestqual, album): if track: filename = track['SNG_ID'] + '.dzr' logger.debug(u'Sending song "%s" to Aria' % track['SNG_TITLE']) - getAria2RPC().addUri( - [track['downloadUrl']], - {'out': filename, 'auto-file-renaming': 'false', 'continue': 'true', - 'dir': os.path.join(headphones.CONFIG.DOWNLOAD_DDL_DIR, folder_name)}) + getAria2RPC().addUri([track['downloadUrl']], {'out': filename, 'auto-file-renaming': 'false', 'continue': 'true', 'dir': folder_name}) except Exception as e: logger.error(u'Error sending torrent to Aria2. Are you sure it\'s running? (%s)' % e) From 0acb5a413ac30b26a1a964f428c0fa2b3d74aa73 Mon Sep 17 00:00:00 2001 From: Kallys Date: Tue, 2 May 2017 13:41:51 +0200 Subject: [PATCH 20/27] Revert "Integration of deezloader provider by ParadoxalManiak under CC BY-NC-SA 4.0" This reverts commit 04e8767ea99e804ccb909d3293f8d6e2561d3726. --- data/interfaces/default/config.html | 64 -- headphones/aria2.py | 979 ---------------------------- headphones/classes.py | 31 - headphones/config.py | 7 - headphones/deezloader.py | 441 ------------- headphones/postprocessor.py | 40 +- headphones/searcher.py | 167 +---- headphones/webserve.py | 13 +- 8 files changed, 20 insertions(+), 1722 deletions(-) delete mode 100644 headphones/aria2.py delete mode 100644 headphones/deezloader.py diff --git a/data/interfaces/default/config.html b/data/interfaces/default/config.html index bd331f2c..841df0d1 100644 --- a/data/interfaces/default/config.html +++ b/data/interfaces/default/config.html @@ -304,45 +304,6 @@ - -
- Direct Download - Aria2 -
-
-
- - - usually http://localhost:6800/jsonrpc -
-
- - -
-
- - -
-
- - -
-
- - - Full path where ddl client downloads your music, e.g. /Users/name/Downloads/music -
-
@@ -500,7 +461,6 @@ NZBs Torrents - Direct Download No Preference
@@ -613,21 +573,6 @@ - -
- Direct Download - -
-
- - -
-
- -
@@ -2460,11 +2405,6 @@ $("#deluge_options").show(); } - if ($("#ddl_downloader_aria").is(":checked")) - { - $("#ddl_aria_options").show(); - } - $('input[type=radio]').change(function(){ if ($("#preferred_bitrate").is(":checked")) { @@ -2518,9 +2458,6 @@ { $("#torrent_blackhole_options,#utorrent_options,#transmission_options,#qbittorrent_options").fadeOut("fast", function() { $("#deluge_options").fadeIn() }); } - if ($("#ddl_downloader_aria").is(":checked")) - { - } }); $("#mirror").change(handleNewServerSelection); @@ -2623,7 +2560,6 @@ initConfigCheckbox("#enable_https"); initConfigCheckbox("#customauth"); initConfigCheckbox("#use_tquattrecentonze"); - initConfigCheckbox("#use_deezloader"); $('#twitterStep1').click(function () { diff --git a/headphones/aria2.py b/headphones/aria2.py deleted file mode 100644 index 06d8e4e0..00000000 --- a/headphones/aria2.py +++ /dev/null @@ -1,979 +0,0 @@ -# -*- coding: utf8 -*- -# Copyright (C) 2012-2016 Xyne -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# (version 2) as published by the Free Software Foundation. -# -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -from __future__ import with_statement -import base64 -import json -import math -import os -import ssl -import string -import time -import httplib -import urllib2 - -from headphones import logger - -# ################################ Constants ################################### - -DEFAULT_PORT = 6800 -SERVER_URI_FORMAT = '{}://{}:{:d}/jsonrpc' - -# Status values for unfinished downloads. -TEMPORARY_STATUS = ('active', 'waiting', 'paused') -# Status values for finished downloads. -FINAL_STATUS = ('complete', 'error') - -ARIA2_CONTROL_FILE_EXT = '.aria2' - -# ########################## Convenience Functions ############################# - - -def to_json_list(objs): - ''' - Wrap strings in lists. Other iterables are converted to lists directly. - ''' - if isinstance(objs, str): - return [objs] - elif not isinstance(objs, list): - return list(objs) - else: - return objs - - -def add_options_and_position(params, options=None, position=None): - ''' - Convenience method for adding options and position to parameters. - ''' - if options: - params.append(options) - if position: - if not isinstance(position, int): - try: - position = int(position) - except ValueError: - position = -1 - if position >= 0: - params.append(position) - return params - - -def get_status(response): - ''' - Process a status response. - ''' - if response: - try: - return response['status'] - except KeyError: - logger.error('no status returned from Aria2 RPC server') - return 'error' - else: - logger.error('no response from server') - return 'error' - - -def random_token(length, valid_chars=None): - ''' - Get a random secret token for the Aria2 RPC server. - - length: - The length of the token - - valid_chars: - A list or other ordered and indexable iterable of valid characters. If not - given of None, asciinumberic characters with some punctuation characters - will be used. - ''' - if not valid_chars: - valid_chars = string.ascii_letters + string.digits + '!@#$%^&*()-_=+' - number_of_chars = len(valid_chars) - bytes_to_read = math.ceil(math.log(number_of_chars) / math.log(0x100)) - max_value = 0x100**bytes_to_read - max_index = number_of_chars - 1 - token = '' - for _ in range(length): - value = int.from_bytes(os.urandom(bytes_to_read), byteorder='little') - index = round((value * max_index) / max_value) - token += valid_chars[index] - return token - -# ################ From python3-aur's ThreadedServers.common ################### - - -def format_bytes(size): - '''Format bytes for inferior humans.''' - if size < 0x400: - return '{:d} B'.format(size) - else: - size = float(size) / 0x400 - for prefix in ('KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB'): - if size < 0x400: - return '{:0.02f} {}'.format(size, prefix) - else: - size /= 0x400 - return '{:0.02f} YiB'.format(size) - - -def format_seconds(s): - '''Format seconds for inferior humans.''' - string = '' - for base, char in ( - (60, 's'), - (60, 'm'), - (24, 'h') - ): - s, r = divmod(s, base) - if s == 0: - return '{:d}{}{}'.format(r, char, string) - elif r != 0: - string = '{:02d}{}{}'.format(r, char, string) - else: - return '{:d}d{}'.format(s, string) - -# ############################ Aria2JsonRpcError ############################### - - -class Aria2JsonRpcError(Exception): - def __init__(self, msg, connection_error=False): - super(self.__class__, self).__init__(self, msg) - self.connection_error = connection_error - -# ############################ Aria2JsonRpc Class ############################## - - -class Aria2JsonRpc(object): - ''' - Interface class for interacting with an Aria2 RPC server. - ''' - # TODO: certificate options, etc. - def __init__( - self, ID, uri, - mode='normal', - token=None, - http_user=None, http_passwd=None, - server_cert=None, client_cert=None, client_cert_password=None, - ssl_protocol=None, - setup_function=None - ): - ''' - ID: the ID to send to the RPC interface - - uri: the URI of the RPC interface - - mode: - normal - process requests immediately - batch - queue requests (run with "process_queue") - format - return RPC request objects - - token: - RPC method-level authorization token (set using `--rpc-secret`) - - http_user, http_password: - HTTP Basic authentication credentials (deprecated) - - server_cert: - server certificate for HTTPS connections - - client_cert: - client certificate for HTTPS connections - - client_cert_password: - prompt for client certificate password - - ssl_protocol: - SSL protocol from the ssl module - - setup_function: - A function to invoke prior to the first server call. This could be the - launch() method of an Aria2RpcServer instance, for example. This attribute - is set automatically in instances returned from Aria2RpcServer.get_a2jr() - ''' - self.id = ID - self.uri = uri - self.mode = mode - self.queue = [] - self.handlers = dict() - self.token = token - self.setup_function = setup_function - - if None not in (http_user, http_passwd): - self.add_HTTPBasicAuthHandler(http_user, http_passwd) - - if server_cert or client_cert: - self.add_HTTPSHandler( - server_cert=server_cert, - client_cert=client_cert, - client_cert_password=client_cert_password, - protocol=ssl_protocol - ) - - self.update_opener() - - def iter_handlers(self): - ''' - Iterate over handlers. - ''' - for name in ('HTTPS', 'HTTPBasicAuth'): - try: - yield self.handlers[name] - except KeyError: - pass - - def update_opener(self): - ''' - Build an opener from the current handlers. - ''' - self.opener = urllib2.build_opener(*self.iter_handlers()) - - def remove_handler(self, name): - ''' - Remove a handler. - ''' - try: - del self.handlers[name] - except KeyError: - pass - - def add_HTTPBasicAuthHandler(self, user, passwd): - ''' - Add a handler for HTTP Basic authentication. - - If either user or passwd are None, the handler is removed. - ''' - handler = urllib2.HTTPBasicAuthHandler() - handler.add_password( - realm='aria2', - uri=self.uri, - user=user, - passwd=passwd, - ) - self.handlers['HTTPBasicAuth'] = handler - - def remove_HTTPBasicAuthHandler(self): - self.remove_handler('HTTPBasicAuth') - - def add_HTTPSHandler( - self, - server_cert=None, - client_cert=None, - client_cert_password=None, - protocol=None, - ): - ''' - Add a handler for HTTPS connections with optional server and client - certificates. - ''' - if not protocol: - protocol = ssl.PROTOCOL_TLSv1 -# protocol = ssl.PROTOCOL_TLSv1_1 # openssl 1.0.1+ -# protocol = ssl.PROTOCOL_TLSv1_2 # Python 3.4+ - context = ssl.SSLContext(protocol) - - if server_cert: - context.verify_mode = ssl.CERT_REQUIRED - context.load_verify_locations(cafile=server_cert) - else: - context.verify_mode = ssl.CERT_OPTIONAL - - if client_cert: - context.load_cert_chain(client_cert, password=client_cert_password) - - self.handlers['HTTPS'] = urllib2.HTTPSHandler( - context=context, - check_hostname=False - ) - - def remove_HTTPSHandler(self): - self.remove_handler('HTTPS') - - def send_request(self, req_obj): - ''' - Send the request and return the response. - ''' - if self.setup_function: - self.setup_function() - self.setup_function = None - logger.debug("Aria req_obj: %s" % json.dumps(req_obj, indent=2, sort_keys=True)) - req = json.dumps(req_obj).encode('UTF-8') - try: - f = self.opener.open(self.uri, req) - obj = json.loads(f.read()) - try: - return obj['result'] - except KeyError: - raise Aria2JsonRpcError('unexpected result: {}'.format(obj)) - except (urllib2.URLError) as e: - # This should work but URLError does not set the errno attribute: - # e.errno == errno.ECONNREFUSED - raise Aria2JsonRpcError( - str(e), - connection_error=( - '111' in str(e) - ) - ) - except httplib.BadStatusLine as e: - raise Aria2JsonRpcError('{}: BadStatusLine: {} (HTTPS error?)'.format( - self.__class__.__name__, e - )) - - def jsonrpc(self, method, params=None, prefix='aria2.'): - ''' - POST a request to the RPC interface. - ''' - if not params: - params = [] - - if self.token is not None: - token_str = 'token:{}'.format(self.token) - if method == 'multicall': - for p in params[0]: - try: - p['params'].insert(0, token_str) - except KeyError: - p['params'] = [token_str] - else: - params.insert(0, token_str) - - req_obj = { - 'jsonrpc': '2.0', - 'id': self.id, - 'method': prefix + method, - 'params': params, - } - if self.mode == 'batch': - self.queue.append(req_obj) - return None - elif self.mode == 'format': - return req_obj - else: - return self.send_request(req_obj) - - def process_queue(self): - ''' - Processed queued requests. - ''' - req_obj = self.queue - self.queue = [] - return self.send_request(req_obj) - - -# ############################# Standard Methods ############################### - def addUri(self, uris, options=None, position=None): - ''' - aria2.addUri method - - uris: list of URIs - - options: dictionary of additional options - - position: position in queue - - Returns a GID - ''' - params = [uris] - params = add_options_and_position(params, options, position) - return self.jsonrpc('addUri', params) - - def addTorrent(self, torrent, uris=None, options=None, position=None): - ''' - aria2.addTorrent method - - torrent: base64-encoded torrent file - - uris: list of webseed URIs - - options: dictionary of additional options - - position: position in queue - - Returns a GID. - ''' - params = [torrent] - if uris: - params.append(uris) - params = add_options_and_position(params, options, position) - return self.jsonrpc('addTorrent', params) - - def addMetalink(self, metalink, options=None, position=None): - ''' - aria2.addMetalink method - - metalink: base64-encoded metalink file - - options: dictionary of additional options - - position: position in queue - - Returns an array of GIDs. - ''' - params = [metalink] - params = add_options_and_position(params, options, position) - return self.jsonrpc('addTorrent', params) - - def remove(self, gid): - ''' - aria2.remove method - - gid: GID to remove - ''' - params = [gid] - return self.jsonrpc('remove', params) - - def forceRemove(self, gid): - ''' - aria2.forceRemove method - - gid: GID to remove - ''' - params = [gid] - return self.jsonrpc('forceRemove', params) - - def pause(self, gid): - ''' - aria2.pause method - - gid: GID to pause - ''' - params = [gid] - return self.jsonrpc('pause', params) - - def pauseAll(self): - ''' - aria2.pauseAll method - ''' - return self.jsonrpc('pauseAll') - - def forcePause(self, gid): - ''' - aria2.forcePause method - - gid: GID to pause - ''' - params = [gid] - return self.jsonrpc('forcePause', params) - - def forcePauseAll(self): - ''' - aria2.forcePauseAll method - ''' - return self.jsonrpc('forcePauseAll') - - def unpause(self, gid): - ''' - aria2.unpause method - - gid: GID to unpause - ''' - params = [gid] - return self.jsonrpc('unpause', params) - - def unpauseAll(self): - ''' - aria2.unpauseAll method - ''' - return self.jsonrpc('unpauseAll') - - def tellStatus(self, gid, keys=None): - ''' - aria2.tellStatus method - - gid: GID to query - - keys: subset of status keys to return (all keys are returned otherwise) - - Returns a dictionary. - ''' - params = [gid] - if keys: - params.append(keys) - return self.jsonrpc('tellStatus', params) - - def getUris(self, gid): - ''' - aria2.getUris method - - gid: GID to query - - Returns a list of dictionaries. - ''' - params = [gid] - return self.jsonrpc('getUris', params) - - def getFiles(self, gid): - ''' - aria2.getFiles method - - gid: GID to query - - Returns a list of dictionaries. - ''' - params = [gid] - return self.jsonrpc('getFiles', params) - - def getPeers(self, gid): - ''' - aria2.getPeers method - - gid: GID to query - - Returns a list of dictionaries. - ''' - params = [gid] - return self.jsonrpc('getPeers', params) - - def getServers(self, gid): - ''' - aria2.getServers method - - gid: GID to query - - Returns a list of dictionaries. - ''' - params = [gid] - return self.jsonrpc('getServers', params) - - def tellActive(self, keys=None): - ''' - aria2.tellActive method - - keys: same as tellStatus - - Returns a list of dictionaries. The dictionaries are the same as those - returned by tellStatus. - ''' - if keys: - params = [keys] - else: - params = None - return self.jsonrpc('tellActive', params) - - def tellWaiting(self, offset, num, keys=None): - ''' - aria2.tellWaiting method - - offset: offset from start of waiting download queue - (negative values are counted from the end of the queue) - - num: number of downloads to return - - keys: same as tellStatus - - Returns a list of dictionaries. The dictionaries are the same as those - returned by tellStatus. - ''' - params = [offset, num] - if keys: - params.append(keys) - return self.jsonrpc('tellWaiting', params) - - def tellStopped(self, offset, num, keys=None): - ''' - aria2.tellStopped method - - offset: offset from oldest download (same semantics as tellWaiting) - - num: same as tellWaiting - - keys: same as tellStatus - - Returns a list of dictionaries. The dictionaries are the same as those - returned by tellStatus. - ''' - params = [offset, num] - if keys: - params.append(keys) - return self.jsonrpc('tellStopped', params) - - def changePosition(self, gid, pos, how): - ''' - aria2.changePosition method - - gid: GID to change - - pos: the position - - how: "POS_SET", "POS_CUR" or "POS_END" - ''' - params = [gid, pos, how] - return self.jsonrpc('changePosition', params) - - def changeUri(self, gid, fileIndex, delUris, addUris, position=None): - ''' - aria2.changePosition method - - gid: GID to change - - fileIndex: file to affect (1-based) - - delUris: URIs to remove - - addUris: URIs to add - - position: where URIs are inserted, after URIs have been removed - ''' - params = [gid, fileIndex, delUris, addUris] - if position: - params.append(position) - return self.jsonrpc('changePosition', params) - - def getOption(self, gid): - ''' - aria2.getOption method - - gid: GID to query - - Returns a dictionary of options. - ''' - params = [gid] - return self.jsonrpc('getOption', params) - - def changeOption(self, gid, options): - ''' - aria2.changeOption method - - gid: GID to change - - options: dictionary of new options - (not all options can be changed for active downloads) - ''' - params = [gid, options] - return self.jsonrpc('changeOption', params) - - def getGlobalOption(self): - ''' - aria2.getGlobalOption method - - Returns a dictionary. - ''' - return self.jsonrpc('getGlobalOption') - - def changeGlobalOption(self, options): - ''' - aria2.changeGlobalOption method - - options: dictionary of new options - ''' - params = [options] - return self.jsonrpc('changeGlobalOption', params) - - def getGlobalStat(self): - ''' - aria2.getGlobalStat method - - Returns a dictionary. - ''' - return self.jsonrpc('getGlobalStat') - - def purgeDownloadResult(self): - ''' - aria2.purgeDownloadResult method - ''' - self.jsonrpc('purgeDownloadResult') - - def removeDownloadResult(self, gid): - ''' - aria2.removeDownloadResult method - - gid: GID to remove - ''' - params = [gid] - return self.jsonrpc('removeDownloadResult', params) - - def getVersion(self): - ''' - aria2.getVersion method - - Returns a dictionary. - ''' - return self.jsonrpc('getVersion') - - def getSessionInfo(self): - ''' - aria2.getSessionInfo method - - Returns a dictionary. - ''' - return self.jsonrpc('getSessionInfo') - - def shutdown(self): - ''' - aria2.shutdown method - ''' - return self.jsonrpc('shutdown') - - def forceShutdown(self): - ''' - aria2.forceShutdown method - ''' - return self.jsonrpc('forceShutdown') - - def multicall(self, methods): - ''' - aria2.multicall method - - methods: list of dictionaries (keys: methodName, params) - - The method names must be those used by Aria2c, e.g. "aria2.tellStatus". - ''' - return self.jsonrpc('multicall', [methods], prefix='system.') - - -# ########################### Convenience Methods ############################## - def add_torrent(self, path, uris=None, options=None, position=None): - ''' - A wrapper around addTorrent for loading files. - ''' - with open(path, 'r') as f: - torrent = base64.encode(f.read()) - return self.addTorrent(torrent, uris, options, position) - - def add_metalink(self, path, uris=None, options=None, position=None): - ''' - A wrapper around addMetalink for loading files. - ''' - with open(path, 'r') as f: - metalink = base64.encode(f.read()) - return self.addMetalink(metalink, uris, options, position) - - def get_status(self, gid): - ''' - Get the status of a single GID. - ''' - response = self.tellStatus(gid, ['status']) - return get_status(response) - - def wait_for_final_status(self, gid, interval=1): - ''' - Wait for a GID to complete or fail and return its status. - ''' - if not interval or interval < 0: - interval = 1 - while True: - status = self.get_status(gid) - if status in TEMPORARY_STATUS: - time.sleep(interval) - else: - return status - - def get_statuses(self, gids): - ''' - Get the status of multiple GIDs. The status of each is yielded in order. - ''' - methods = [ - { - 'methodName': 'aria2.tellStatus', - 'params': [gid, ['gid', 'status']] - } - for gid in gids - ] - results = self.multicall(methods) - if results: - status = dict((r[0]['gid'], r[0]['status']) for r in results) - for gid in gids: - try: - yield status[gid] - except KeyError: - logger.error('Aria2 RPC server returned no status for GID {}'.format(gid)) - yield 'error' - else: - logger.error('no response from Aria2 RPC server') - for gid in gids: - yield 'error' - - def wait_for_final_statuses(self, gids, interval=1): - ''' - Wait for multiple GIDs to complete or fail and return their statuses in - order. - - gids: - A flat list of GIDs. - ''' - if not interval or interval < 0: - interval = 1 - statusmap = dict((g, None) for g in gids) - remaining = list( - g for g, s in statusmap.items() if s is None - ) - while remaining: - for g, s in zip(remaining, self.get_statuses(remaining)): - if s in TEMPORARY_STATUS: - continue - else: - statusmap[g] = s - remaining = list( - g for g, s in statusmap.items() if s is None - ) - if remaining: - time.sleep(interval) - for g in gids: - yield statusmap[g] - - def print_global_status(self): - ''' - Print global status of the RPC server. - ''' - status = self.getGlobalStat() - if status: - numWaiting = int(status['numWaiting']) - numStopped = int(status['numStopped']) - keys = ['totalLength', 'completedLength'] - total = self.tellActive(keys) - waiting = self.tellWaiting(0, numWaiting, keys) - if waiting: - total += waiting - stopped = self.tellStopped(0, numStopped, keys) - if stopped: - total += stopped - - downloadSpeed = int(status['downloadSpeed']) - uploadSpeed = int(status['uploadSpeed']) - totalLength = sum(int(x['totalLength']) for x in total) - completedLength = sum(int(x['completedLength']) for x in total) - remaining = totalLength - completedLength - - status['downloadSpeed'] = format_bytes(downloadSpeed) + '/s' - status['uploadSpeed'] = format_bytes(uploadSpeed) + '/s' - - preordered = ('downloadSpeed', 'uploadSpeed') - - rows = list() - for k in sorted(status): - if k in preordered: - continue - rows.append((k, status[k])) - - rows.extend((x, status[x]) for x in preordered) - - if totalLength > 0: - rows.append(('total', format(format_bytes(totalLength)))) - rows.append(('completed', format(format_bytes(completedLength)))) - rows.append(('remaining', format(format_bytes(remaining)))) - if completedLength == totalLength: - eta = 'finished' - else: - try: - eta = format_seconds(remaining // downloadSpeed) - except ZeroDivisionError: - eta = 'never' - rows.append(('ETA', eta)) - - l = max(len(r[0]) for r in rows) - r = max(len(r[1]) for r in rows) - r = max(r, len(self.uri) - (l + 2)) - fmt = '{:<' + str(l) + 's} {:>' + str(r) + 's}' - - print(self.uri) - for k, v in rows: - print(fmt.format(k, v)) - - def queue_uris(self, uris, options, interval=None): - ''' - Enqueue URIs and wait for download to finish while printing status at - regular intervals. - ''' - gid = self.addUri(uris, options) - print('GID: {}'.format(gid)) - - if gid and interval is not None: - blanker = '' - while True: - response = self.tellStatus(gid, ['status']) - if response: - try: - status = response['status'] - except KeyError: - print('error: no status returned from Aria2 RPC server') - break - print('{}\rstatus: {}'.format(blanker, status)), - blanker = ' ' * len(status) - if status in TEMPORARY_STATUS: - time.sleep(interval) - else: - break - else: - print('error: no response from server') - break - - -# ####################### Polymethod download handlers ######################### - def polymethod_enqueue_many(self, downloads): - ''' - Enqueue downloads. - - downloads: Same as polymethod_download(). - ''' - methods = list( - { - 'methodName': 'aria2.{}'.format(d[0]), - 'params': list(d[1:]) - } for d in downloads - ) - return self.multicall(methods) - - def polymethod_wait_many(self, gids, interval=1): - ''' - Wait for the GIDs to complete or fail and return their statuses. - - gids: - A list of lists of GIDs. - ''' - # The flattened list of GIDs - gs = list(g for gs in gids for g in gs) - statusmap = dict(tuple(zip( - gs, - self.wait_for_final_statuses(gs, interval=interval) - ))) - for gs in gids: - yield list(statusmap.get(g, 'error') for g in gs) - - def polymethod_enqueue_one(self, download): - ''' - Same as polymethod_enqueue_many but for one element. - ''' - return getattr(self, download[0])(*download[1:]) - - def polymethod_download(self, downloads, interval=1): - ''' - Enqueue a series of downloads and wait for them to finish. Iterate over the - status of each, in order. - - downloads: - An iterable over (, , ...) where indicates the "add" - method to use ('addUri', 'addTorrent', 'addMetalink') and everything that - follows are arguments to pass to that method. - - interval: - The status check interval while waiting. - - Iterates over the download status of finished downloads. "complete" - indicates success. Lists of statuses will be returned for downloads that - create multiple GIDs (e.g. metalinks). - ''' - gids = self.polymethod_enqueue_many(downloads) - return self.polymethod_wait_many(gids, interval=interval) - - def polymethod_download_bool(self, *args, **kwargs): - ''' - A wrapper around polymethod_download() which returns a boolean for each - download to indicate success (True) or failure (False). - ''' -# for status in self.polymethod_download(*args, **kwargs): -# yield all(s == 'complete' for s in status) - return list( - all(s == 'complete' for s in status) - for status in self.polymethod_download(*args, **kwargs) - ) diff --git a/headphones/classes.py b/headphones/classes.py index 54a10a55..6015a0f2 100644 --- a/headphones/classes.py +++ b/headphones/classes.py @@ -21,7 +21,6 @@ import urllib from common import USER_AGENT -from collections import OrderedDict class HeadphonesURLopener(urllib.FancyURLopener): @@ -136,33 +135,3 @@ class Proper: def __str__(self): return str(self.date) + " " + self.name + " " + str(self.season) + "x" + str( self.episode) + " of " + str(self.tvdbid) - - -class OptionalImport(object): - ''' - Dummy class for optional import (imports needed for optional features). - ''' - def __init__(self, name): - self.__name = name - - def __getattr__(self, attr): - raise ImportError('The following package is required to use this feature: {0}'.format(self.__name)) - - -class CacheDict(OrderedDict): - ''' - Ordered dictionary with fixed size, designed for caching. - ''' - def __init__(self, *args, **kwds): - self.size_limit = kwds.pop("size_limit", None) - OrderedDict.__init__(self, *args, **kwds) - self._check_size_limit() - - def __setitem__(self, key, value): - OrderedDict.__setitem__(self, key, value) - self._check_size_limit() - - def _check_size_limit(self): - if self.size_limit is not None: - while len(self) > self.size_limit: - self.popitem(last=False) diff --git a/headphones/config.py b/headphones/config.py index 86aa9216..57af4c16 100644 --- a/headphones/config.py +++ b/headphones/config.py @@ -46,10 +46,6 @@ _CONFIG_DEFINITIONS = { 'APOLLO_RATIO': (str, 'Apollo.rip', ''), 'APOLLO_USERNAME': (str, 'Apollo.rip', ''), 'APOLLO_URL': (str, 'Apollo.rip', 'https://apollo.rip'), - 'ARIA_HOST': (str, 'Aria2', ''), - 'ARIA_PASSWORD': (str, 'Aria2', ''), - 'ARIA_TOKEN': (str, 'Aria2', ''), - 'ARIA_USERNAME': (str, 'Aria2', ''), 'AUTOWANT_ALL': (int, 'General', 0), 'AUTOWANT_MANUALLY_ADDED': (int, 'General', 1), 'AUTOWANT_UPCOMING': (int, 'General', 1), @@ -77,8 +73,6 @@ _CONFIG_DEFINITIONS = { 'CUSTOMPORT': (int, 'General', 5000), 'CUSTOMSLEEP': (int, 'General', 1), 'CUSTOMUSER': (str, 'General', ''), - 'DDL_DOWNLOADER': (int, 'General', 0), - 'DEEZLOADER': (int, 'DeezLoader', 0), 'DELETE_LOSSLESS_FILES': (int, 'General', 1), 'DELUGE_HOST': (str, 'Deluge', ''), 'DELUGE_CERT': (str, 'Deluge', ''), @@ -92,7 +86,6 @@ _CONFIG_DEFINITIONS = { 'DOWNLOAD_DIR': (path, 'General', ''), 'DOWNLOAD_SCAN_INTERVAL': (int, 'General', 5), 'DOWNLOAD_TORRENT_DIR': (path, 'General', ''), - 'DOWNLOAD_DDL_DIR': (path, 'General', ''), 'DO_NOT_OVERRIDE_GIT_BRANCH': (int, 'General', 0), 'EMAIL_ENABLED': (int, 'Email', 0), 'EMAIL_FROM': (str, 'Email', ''), diff --git a/headphones/deezloader.py b/headphones/deezloader.py deleted file mode 100644 index 8de85df9..00000000 --- a/headphones/deezloader.py +++ /dev/null @@ -1,441 +0,0 @@ -# -*- coding: utf-8 -*- -# Deezloader (c) 2016 by ParadoxalManiak -# -# Deezloader is licensed under a -# Creative Commons Attribution-NonCommercial-ShareAlike 3.0 Unported License. -# -# You should have received a copy of the license along with this -# work. If not, see . -# -# Version 2.1.0 -# Maintained by ParadoxalManiak -# Original work by ZzMTV -# -# Author's disclaimer: -# I am not responsible for the usage of this program by other people. -# I do not recommend you doing this illegally or against Deezer's terms of service. -# This project is licensed under CC BY-NC-SA 4.0 - -import re -import os -from datetime import datetime -from hashlib import md5 -import binascii - -from beets.mediafile import MediaFile -from headphones import logger, request, helpers -from headphones.classes import OptionalImport, CacheDict -import headphones - - -# Try to import optional Crypto.Cipher packages -try: - from Crypto.Cipher import AES, Blowfish -except ImportError: - AES = OptionalImport('Crypto.Cipher.AES') - Blowfish = OptionalImport('Crypto.Cipher.Blowfish') - -# Public constants -PROVIDER_NAME = 'Deezer' - -# Internal constants -__API_URL = "http://www.deezer.com/ajax/gw-light.php" -__API_INFO_URL = "http://api.deezer.com/" -__HTTP_HEADERS = { - "User-Agent": "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36", - "Content-Language": "en-US", - "Cache-Control": "max-age=0", - "Accept": "*/*", - "Accept-Charset": "utf-8,ISO-8859-1;q=0.7,*;q=0.3", - "Accept-Language": "de-DE,de;q=0.8,en-US;q=0.6,en;q=0.4" -} - -# Internal variables -__api_queries = { - 'api_version': "1.0", - 'api_token': "None", - 'input': "3" -} -__cookies = None -__tracks_cache = CacheDict(size_limit=512) -__albums_cache = CacheDict(size_limit=64) - - -def __getApiToken(): - global __cookies - response = request.request_response(url="http://www.deezer.com/", headers=__HTTP_HEADERS) - __cookies = response.cookies - data = response.content - if data: - matches = re.search(r"checkForm\s*=\s*['|\"](.*)[\"|'];", data) - if matches: - token = matches.group(1) - __api_queries['api_token'] = token - logger.debug(u"Deezloader : api token loaded ('%s')" % token) - - if not token: - logger.error(u"Deezloader: Unable to get api token") - - -def getAlbumByLink(album_link): - """Returns deezer album infos using album link url - - :param str album_link: deezer album link url (eg: 'http://www.deezer.com/album/1234567/') - """ - matches = re.search(r"album\/([0-9]+)\/?$", album_link) - if matches: - return getAlbum(matches.group(1)) - - -def getAlbum(album_id): - """Returns deezer album infos - - :param int album_id: deezer album id - """ - global __albums_cache - - if str(album_id) in __albums_cache: - return __albums_cache[str(album_id)] - - url = __API_INFO_URL + "album/" + str(album_id) - data = request.request_json(url=url, headers=__HTTP_HEADERS, cookies=__cookies) - - if data and 'error' not in data: - __albums_cache[str(album_id)] = data - return data - else: - logger.debug("Deezloader: Can't load album infos") - return None - - -def searchAlbums(search_term): - """Search for deezer albums using search term - - :param str search_term: search term to search album for - """ - logger.info(u'Searching Deezer using term: "%s"' % search_term) - - url = __API_INFO_URL + "search/album?q=" + search_term - data = request.request_json(url=url, headers=__HTTP_HEADERS, cookies=__cookies) - - albums = [] - - # Process content - if data and 'total' in data and data['total'] > 0 and 'data' in data: - for item in data['data']: - try: - albums.append(getAlbum(item['id'])) - except Exception as e: - logger.error(u"An unknown error occurred in the Deezer search album parser: %s" % e) - else: - logger.info(u'No results found from Deezer using term: "%s"' % search_term) - - return albums - - -def __matchAlbums(albums, artist_name, album_title, album_length): - resultlist = [] - - for album in albums: - total_size = 0 - tracks_found = 0 - - for track in album['tracks']['data']: - t = getTrack(track['id']) - if t: - if t["FILESIZE_MP3_320"] > 0: - size = t["FILESIZE_MP3_320"] - elif t["FILESIZE_MP3_256"] > 0: - size = t["FILESIZE_MP3_256"] - elif t["FILESIZE_MP3_128"] > 0: - size = t["FILESIZE_MP3_128"] - else: - size = t["FILESIZE_MP3_64"] - - size = int(size) - total_size += size - tracks_found += 1 - logger.debug(u'Found song "%s". Size: %s' % (t['SNG_TITLE'], helpers.bytes_to_mb(size))) - - if tracks_found == 0: - logger.info(u'Ignoring album "%s" (no tracks to download)' % album['title']) - continue - - matched = True - mismatch_reason = 'matched!' - - if album_length > 0 and abs(int(album['duration']) - album_length) > 240: - matched = False - mismatch_reason = (u'duration mismatch: %i not in [%i, %i]' % (int(album['duration']), (album_length - 240), (album_length + 240))) - - elif (helpers.latinToAscii(re.sub(r"\W", "", album_title, 0, re.UNICODE)).lower() != - helpers.latinToAscii(re.sub(r"\W", "", album['title'], 0, re.UNICODE)).lower()): - matched = False - mismatch_reason = (u'album name mismatch: %s != %s' % (album['title'], album_title)) - - elif (helpers.latinToAscii(re.sub(r"\W", "", artist_name, 0, re.UNICODE)).lower() != - helpers.latinToAscii(re.sub(r"\W", "", album['artist']['name'], 0, re.UNICODE)).lower()): - matched = False - mismatch_reason = (u'artist name mismatch: %s != %s' % (album['artist']['name'], artist_name)) - - resultlist.append( - (album['artist']['name'] + ' - ' + album['title'] + ' [' + album['release_date'][:4] + '] (' + str(tracks_found) + '/' + str(album['nb_tracks']) + ')', - total_size, album['link'], PROVIDER_NAME, "ddl", matched) - ) - logger.info(u'Found "%s". Tracks %i/%i. Size: %s (%s)' % (album['title'], tracks_found, album['nb_tracks'], helpers.bytes_to_mb(total_size), mismatch_reason)) - - return resultlist - - -def searchAlbum(artist_name, album_title, user_search_term=None, album_length=None): - """Search for deezer specific album. - This will iterate over deezer albums and try to find best matches - - :param str artist_name: album artist name - :param str album_title: album title - :param str user_search_term: search terms provided by user - :param int album_length: targeted album duration in seconds - """ - # User search term by-pass normal search - if user_search_term: - return __matchAlbums(searchAlbums(user_search_term), artist_name, album_title, album_length) - - resultlist = __matchAlbums(searchAlbums((artist_name + ' ' + album_title).strip()), artist_name, album_title, album_length) - if resultlist: - return resultlist - - # Deezer API supports unicode, so just remove non alphanumeric characters - clean_artist_name = re.sub(r"[^\w\s]", " ", artist_name, 0, re.UNICODE).strip() - clean_album_name = re.sub(r"[^\w\s]", " ", album_title, 0, re.UNICODE).strip() - - resultlist = __matchAlbums(searchAlbums((clean_artist_name + ' ' + clean_album_name).strip()), artist_name, album_title, album_length) - if resultlist: - return resultlist - - resultlist = __matchAlbums(searchAlbums(clean_artist_name), artist_name, album_title, album_length) - if resultlist: - return resultlist - - return resultlist - - -def getTrack(sng_id, try_reload_api=True): - """Returns deezer track infos - - :param int sng_id: deezer song id - :param bool try_reload_api: whether or not try reloading API if session expired - """ - global __tracks_cache - - if str(sng_id) in __tracks_cache: - return __tracks_cache[str(sng_id)] - - data = "[{\"method\":\"song.getListData\",\"params\":{\"sng_ids\":[" + str(sng_id) + "]}}]" - json = request.request_json(url=__API_URL, headers=__HTTP_HEADERS, method='post', params=__api_queries, data=data, cookies=__cookies) - - results = [] - error = None - invalid_token = False - - if json: - # Check for errors - if 'error' in json: - error = json['error'] - if 'GATEWAY_ERROR' in json['error'] and json['error']['GATEWAY_ERROR'] == u"invalid api token": - invalid_token = True - - elif 'error' in json[0] and json[0]['error']: - error = json[0]['error'] - if 'VALID_TOKEN_REQUIRED' in json[0]['error'] and json[0]['error']['VALID_TOKEN_REQUIRED'] == u"Invalid CSRF token": - invalid_token = True - - # Got invalid token error - if error: - if invalid_token and try_reload_api: - __getApiToken() - return getTrack(sng_id, False) - else: - logger.error(u"An unknown error occurred in the Deezer track parser: %s" % error) - else: - try: - results = json[0]['results'] - item = results['data'][0] - if 'token' in item: - logger.error(u"An unknown error occurred in the Deezer parser: Uploaded Files are currently not supported") - return - - sng_id = item["SNG_ID"] - md5Origin = item["MD5_ORIGIN"] - sng_format = 3 - - if item["FILESIZE_MP3_320"] <= 0: - if item["FILESIZE_MP3_256"] > 0: - sng_format = 5 - else: - sng_format = 1 - - mediaVersion = int(item["MEDIA_VERSION"]) - item['downloadUrl'] = __getDownloadUrl(md5Origin, sng_id, sng_format, mediaVersion) - - __tracks_cache[sng_id] = item - return item - - except Exception as e: - logger.error(u"An unknown error occurred in the Deezer track parser: %s" % e) - - -def __getDownloadUrl(md5Origin, sng_id, sng_format, mediaVersion): - urlPart = md5Origin.encode('utf-8') + b'\xa4' + str(sng_format) + b'\xa4' + str(sng_id) + b'\xa4' + str(mediaVersion) - md5val = md5(urlPart).hexdigest() - urlPart = md5val + b'\xa4' + urlPart + b'\xa4' - cipher = AES.new('jo6aey6haid2Teih', AES.MODE_ECB) - ciphertext = cipher.encrypt(__pad(urlPart, AES.block_size)) - return "http://e-cdn-proxy-" + md5Origin[:1] + ".deezer.com/mobile/1/" + binascii.hexlify(ciphertext).lower() - - -def __pad(raw, block_size): - if (len(raw) % block_size == 0): - return raw - padding_required = block_size - (len(raw) % block_size) - padChar = b'\x00' - data = raw + padding_required * padChar - return data - - -def __tagTrack(path, track): - try: - album = getAlbum(track['ALB_ID']) - - f = MediaFile(path) - f.artist = track['ART_NAME'] - f.album = track['ALB_TITLE'] - f.title = track['SNG_TITLE'] - f.track = track['TRACK_NUMBER'] - f.tracktotal = album['nb_tracks'] - f.disc = track['DISK_NUMBER'] - f.bpm = track['BPM'] - f.date = datetime.strptime(album['release_date'], '%Y-%m-%d').date() - f.albumartist = album['artist']['name'] - if u'genres' in album and u'data' in album['genres']: - f.genres = [genre['name'] for genre in album['genres']['data']] - - f.save() - - except Exception as e: - logger.error(u'Unable to tag deezer track "%s": %s' % (path, e)) - - -def decryptTracks(paths): - """Decrypt downloaded deezer tracks. - - :param paths: list of path to deezer tracks (*.dzr files). - """ - # Note: tracks can be from different albums - decrypted_tracks = {} - - # First pass: load tracks data - for path in paths: - try: - album_folder = os.path.dirname(path) - sng_id = os.path.splitext(os.path.basename(path))[0] - track = getTrack(sng_id) - if track: - track_number = int(track['TRACK_NUMBER']) - disk_number = int(track['DISK_NUMBER']) - - if album_folder not in decrypted_tracks: - decrypted_tracks[album_folder] = {} - - if disk_number not in decrypted_tracks[album_folder]: - decrypted_tracks[album_folder][disk_number] = {} - - decrypted_tracks[album_folder][disk_number][track_number] = track - - except Exception as e: - logger.error(u'Unable to load deezer track infos "%s": %s' % (path, e)) - - # Second pass: decrypt tracks - for album_folder in decrypted_tracks: - multi_disks = len(decrypted_tracks[album_folder]) > 1 - for disk_number in decrypted_tracks[album_folder]: - for track_number, track in decrypted_tracks[album_folder][disk_number].items(): - try: - filename = helpers.replace_illegal_chars(track['SNG_TITLE']).strip() - filename = ('{:02d}'.format(track_number) + '_' + filename + '.mp3') - - # Add a 'cd x' sub-folder if album has more than one disk - disk_folder = os.path.join(album_folder, 'cd ' + str(disk_number)) if multi_disks else album_folder - - dest = os.path.join(disk_folder, filename).encode(headphones.SYS_ENCODING, 'replace') - - # Decrypt track if not already done - if not os.path.exists(dest): - try: - __decryptDownload(path, sng_id, dest) - __tagTrack(dest, track) - except Exception as e: - logger.error(u'Unable to decrypt deezer track "%s": %s' % (path, e)) - if os.path.exists(dest): - os.remove(dest) - decrypted_tracks[album_folder][disk_number].pop(track_number) - continue - - decrypted_tracks[album_folder][disk_number][track_number]['path'] = dest - - except Exception as e: - logger.error(u'Unable to decrypt deezer track "%s": %s' % (path, e)) - - return decrypted_tracks - - -def __decryptDownload(source, sng_id, dest): - interval_chunk = 3 - chunk_size = 2048 - blowFishKey = __getBlowFishKey(sng_id) - i = 0 - iv = "\x00\x01\x02\x03\x04\x05\x06\x07" - - dest_folder = os.path.dirname(dest) - if not os.path.exists(dest_folder): - os.makedirs(dest_folder) - - f = open(source, "rb") - fout = open(dest, "wb") - try: - chunk = f.read(chunk_size) - while chunk: - if(i % interval_chunk == 0): - cipher = Blowfish.new(blowFishKey, Blowfish.MODE_CBC, iv) - chunk = cipher.decrypt(__pad(chunk, Blowfish.block_size)) - - fout.write(chunk) - i += 1 - chunk = f.read(chunk_size) - finally: - f.close() - fout.close() - - -def __getBlowFishKey(encryptionKey): - if encryptionKey < 1: - encryptionKey *= -1 - - hashcode = md5(str(encryptionKey)).hexdigest() - hPart = hashcode[0:16] - lPart = hashcode[16:32] - parts = ['g4el58wc0zvf9na1', hPart, lPart] - - return __xorHex(parts) - - -def __xorHex(parts): - data = "" - for i in range(0, 16): - character = ord(parts[0][i]) - - for j in range(1, len(parts)): - character ^= ord(parts[j][i]) - - data += chr(character) - - return data diff --git a/headphones/postprocessor.py b/headphones/postprocessor.py index 3b6f493f..38e4bba5 100755 --- a/headphones/postprocessor.py +++ b/headphones/postprocessor.py @@ -30,7 +30,7 @@ from beetsplug import lyrics as beetslyrics from headphones import notifiers, utorrent, transmission, deluge, qbittorrent from headphones import db, albumart, librarysync from headphones import logger, helpers, mb, music_encoder -from headphones import metadata, deezloader +from headphones import metadata postprocessor_lock = threading.Lock() @@ -48,8 +48,6 @@ def checkFolder(): single = False if album['Kind'] == 'nzb': download_dir = headphones.CONFIG.DOWNLOAD_DIR - elif album['Kind'] == 'ddl': - download_dir = headphones.CONFIG.DOWNLOAD_DDL_DIR else: if headphones.CONFIG.DELUGE_DONE_DIRECTORY and headphones.CONFIG.TORRENT_DOWNLOADER == 3: download_dir = headphones.CONFIG.DELUGE_DONE_DIRECTORY @@ -207,7 +205,6 @@ def verify(albumid, albumpath, Kind=None, forced=False, keep_original_folder=Fal tracks = myDB.select('SELECT * from tracks WHERE AlbumID=?', [albumid]) downloaded_track_list = [] - downloaded_deezer_list = [] downloaded_cuecount = 0 for r, d, f in os.walk(albumpath): @@ -216,10 +213,8 @@ def verify(albumid, albumpath, Kind=None, forced=False, keep_original_folder=Fal downloaded_track_list.append(os.path.join(r, files)) elif files.lower().endswith('.cue'): downloaded_cuecount += 1 - elif files.lower().endswith('.dzr'): - downloaded_deezer_list.append(os.path.join(r, files)) # if any of the files end in *.part, we know the torrent isn't done yet. Process if forced, though - elif files.lower().endswith(('.part', '.utpart', '.aria2')) and not forced: + elif files.lower().endswith(('.part', '.utpart')) and not forced: logger.info( "Looks like " + os.path.basename(albumpath).decode(headphones.SYS_ENCODING, 'replace') + " isn't complete yet. Will try again on the next run") @@ -258,37 +253,6 @@ def verify(albumid, albumpath, Kind=None, forced=False, keep_original_folder=Fal downloaded_track_list = helpers.get_downloaded_track_list(albumpath) keep_original_folder = False - # Decrypt deezer tracks - if downloaded_deezer_list: - logger.info('Decrypting deezer tracks') - decrypted_deezer_list = deezloader.decryptTracks(downloaded_deezer_list) - - # Check if album is complete based on album duration only - # (total track numbers is not determinant enough due to hidden tracks for eg) - db_track_duration = 0 - downloaded_track_duration = 0 - try: - for track in tracks: - db_track_duration += track['TrackDuration'] / 1000 - except: - downloaded_track_duration = False - - try: - for disk_number in decrypted_deezer_list[albumpath]: - for track in decrypted_deezer_list[albumpath][disk_number].values(): - downloaded_track_list.append(track['path']) - downloaded_track_duration += int(track['DURATION']) - except: - downloaded_track_duration = False - - if not downloaded_track_duration or not db_track_duration or abs(downloaded_track_duration - db_track_duration) > 240: - logger.info("Looks like " + - os.path.basename(albumpath).decode(headphones.SYS_ENCODING, 'replace') + - " isn't complete yet (duration mismatch). Will try again on the next run") - return - - downloaded_track_list = list(set(downloaded_track_list)) # Remove duplicates - # test #1: metadata - usually works logger.debug('Verifying metadata...') diff --git a/headphones/searcher.py b/headphones/searcher.py index f6554cb1..b0652f7f 100644 --- a/headphones/searcher.py +++ b/headphones/searcher.py @@ -36,7 +36,6 @@ import headphones from headphones.common import USER_AGENT from headphones import logger, db, helpers, classes, sab, nzbget, request from headphones import utorrent, transmission, notifiers, rutracker, deluge, qbittorrent -from headphones import deezloader, aria2 from bencode import bencode, bdecode # Magnet to torrent services, for Black hole. Stolen from CouchPotato. @@ -52,27 +51,6 @@ ruobj = None # Persistent RED API object redobj = None -# Persistent Aria2 RPC object -__aria2rpc_obj = None - - -def getAria2RPC(): - global __aria2rpc_obj - if not __aria2rpc_obj: - __aria2rpc_obj = aria2.Aria2JsonRpc( - ID='headphones', - uri=headphones.CONFIG.ARIA_HOST, - token=headphones.CONFIG.ARIA_TOKEN if headphones.CONFIG.ARIA_TOKEN else None, - http_user=headphones.CONFIG.ARIA_USERNAME if headphones.CONFIG.ARIA_USERNAME else None, - http_passwd=headphones.CONFIG.ARIA_PASSWORD if headphones.CONFIG.ARIA_PASSWORD else None - ) - return __aria2rpc_obj - - -def reconfigure(): - global __aria2rpc_obj - __aria2rpc_obj = None - def fix_url(s, charset="utf-8"): """ @@ -303,53 +281,32 @@ def do_sorted_search(album, new, losslessOnly, choose_specific_download=False): headphones.CONFIG.STRIKE or headphones.CONFIG.TQUATTRECENTONZE) - DDL_PROVIDERS = (headphones.CONFIG.DEEZLOADER) - + results = [] myDB = db.DBConnection() albumlength = myDB.select('SELECT sum(TrackDuration) from tracks WHERE AlbumID=?', [album['AlbumID']])[0][0] - nzb_results = None - torrent_results = None - ddl_results = None - if headphones.CONFIG.PREFER_TORRENTS == 0 and not choose_specific_download: if NZB_PROVIDERS and NZB_DOWNLOADERS: - nzb_results = searchNZB(album, new, losslessOnly, albumlength) + results = searchNZB(album, new, losslessOnly, albumlength) - if not nzb_results: - if DDL_PROVIDERS: - ddl_results = searchDdl(album, new, losslessOnly, albumlength) - - if TORRENT_PROVIDERS: - torrent_results = searchTorrent(album, new, losslessOnly, albumlength) + if not results and TORRENT_PROVIDERS: + results = searchTorrent(album, new, losslessOnly, albumlength) elif headphones.CONFIG.PREFER_TORRENTS == 1 and not choose_specific_download: if TORRENT_PROVIDERS: - torrent_results = searchTorrent(album, new, losslessOnly, albumlength) + results = searchTorrent(album, new, losslessOnly, albumlength) - if not torrent_results: - if DDL_PROVIDERS: - ddl_results = searchDdl(album, new, losslessOnly, albumlength) - - if NZB_PROVIDERS and NZB_DOWNLOADERS: - nzb_results = searchNZB(album, new, losslessOnly, albumlength) - - elif headphones.CONFIG.PREFER_TORRENTS == 3 and not choose_specific_download: - - if DDL_PROVIDERS: - ddl_results = searchDdl(album, new, losslessOnly, albumlength) - - if not ddl_results: - if TORRENT_PROVIDERS: - torrent_results = searchTorrent(album, new, losslessOnly, albumlength) - - if NZB_PROVIDERS and NZB_DOWNLOADERS: - nzb_results = searchNZB(album, new, losslessOnly, albumlength) + if not results and NZB_PROVIDERS and NZB_DOWNLOADERS: + results = searchNZB(album, new, losslessOnly, albumlength) else: + + nzb_results = None + torrent_results = None + if NZB_PROVIDERS and NZB_DOWNLOADERS: nzb_results = searchNZB(album, new, losslessOnly, albumlength, choose_specific_download) @@ -357,19 +314,13 @@ def do_sorted_search(album, new, losslessOnly, choose_specific_download=False): torrent_results = searchTorrent(album, new, losslessOnly, albumlength, choose_specific_download) - if DDL_PROVIDERS: - ddl_results = searchDdl(album, new, losslessOnly, albumlength, choose_specific_download) + if not nzb_results: + nzb_results = [] - if not nzb_results: - nzb_results = [] + if not torrent_results: + torrent_results = [] - if not torrent_results: - torrent_results = [] - - if not ddl_results: - ddl_results = [] - - results = nzb_results + torrent_results + ddl_results + results = nzb_results + torrent_results if choose_specific_download: return results @@ -875,31 +826,6 @@ def send_to_downloader(data, bestqual, album): except Exception as e: logger.error('Couldn\'t write NZB file: %s', e) return - - elif kind == 'ddl': - folder_name = '%s - %s [%s]' % ( - helpers.latinToAscii(album['ArtistName']).encode('UTF-8').replace('/', '_'), - helpers.latinToAscii(album['AlbumTitle']).encode('UTF-8').replace('/', '_'), - get_year_from_release_date(album['ReleaseDate'])) - - # Aria2 downloader - if headphones.CONFIG.DDL_DOWNLOADER == 0: - logger.info("Sending download to Aria2") - - try: - deezer_album = deezloader.getAlbumByLink(bestqual[2]) - - for album_track in deezer_album['tracks']['data']: - track = deezloader.getTrack(album_track['id']) - if track: - filename = track['SNG_ID'] + '.dzr' - logger.debug(u'Sending song "%s" to Aria' % track['SNG_TITLE']) - getAria2RPC().addUri([track['downloadUrl']], {'out': filename, 'auto-file-renaming': 'false', 'continue': 'true', 'dir': folder_name}) - - except Exception as e: - logger.error(u'Error sending torrent to Aria2. Are you sure it\'s running? (%s)' % e) - return - else: folder_name = '%s - %s [%s]' % ( helpers.latinToAscii(album['ArtistName']).encode('UTF-8').replace('/', '_'), @@ -1278,62 +1204,6 @@ def verifyresult(title, artistterm, term, lossless): return True -def searchDdl(album, new=False, losslessOnly=False, albumlength=None, - choose_specific_download=False): - reldate = album['ReleaseDate'] - year = get_year_from_release_date(reldate) - - # MERGE THIS WITH THE TERM CLEANUP FROM searchNZB - dic = {'...': '', ' & ': ' ', ' = ': ' ', '?': '', '$': 's', ' + ': ' ', '"': '', ',': ' ', - '*': ''} - - semi_cleanalbum = helpers.replace_all(album['AlbumTitle'], dic) - cleanalbum = helpers.latinToAscii(semi_cleanalbum) - semi_cleanartist = helpers.replace_all(album['ArtistName'], dic) - cleanartist = helpers.latinToAscii(semi_cleanartist) - - # Use provided term if available, otherwise build our own (this code needs to be cleaned up since a lot - # of these torrent providers are just using cleanartist/cleanalbum terms - if album['SearchTerm']: - term = album['SearchTerm'] - elif album['Type'] == 'part of': - term = cleanalbum + " " + year - else: - # FLAC usually doesn't have a year for some reason so I'll leave it out - # Various Artist albums might be listed as VA, so I'll leave that out too - # Only use the year if the term could return a bunch of different albums, i.e. self-titled albums - if album['ArtistName'] in album['AlbumTitle'] or len(album['ArtistName']) < 4 or len( - album['AlbumTitle']) < 4: - term = cleanartist + ' ' + cleanalbum + ' ' + year - elif album['ArtistName'] == 'Various Artists': - term = cleanalbum + ' ' + year - else: - term = cleanartist + ' ' + cleanalbum - - # Replace bad characters in the term and unicode it - term = re.sub('[\.\-\/]', ' ', term).encode('utf-8') - artistterm = re.sub('[\.\-\/]', ' ', cleanartist).encode('utf-8', 'replace') - - logger.debug(u'Using search term: "%s"' % helpers.latinToAscii(term)) - - resultlist = [] - - # Deezer only provides lossy - if headphones.CONFIG.DEEZLOADER and not losslessOnly: - resultlist += deezloader.searchAlbum(album['ArtistName'], album['AlbumTitle'], album['SearchTerm'], int(albumlength / 1000)) - - # attempt to verify that this isn't a substring result - # when looking for "Foo - Foo" we don't want "Foobar" - # this should be less of an issue when it isn't a self-titled album so we'll only check vs artist - results = [result for result in resultlist if verifyresult(result[0], artistterm, term, losslessOnly)] - - # Additional filtering for size etc - if results and not choose_specific_download: - results = more_filtering(results, album, albumlength, new) - - return results - - def searchTorrent(album, new=False, losslessOnly=False, albumlength=None, choose_specific_download=False): global apolloobj # persistent apollo.rip api object to reduce number of login attempts @@ -2166,10 +2036,7 @@ def searchTorrent(album, new=False, losslessOnly=False, albumlength=None, def preprocess(resultlist): for result in resultlist: - if result[3] == deezloader.PROVIDER_NAME: - return True, result - - elif result[4] == 'torrent': + if result[4] == 'torrent': # rutracker always needs the torrent data if result[3] == 'rutracker.org': diff --git a/headphones/webserve.py b/headphones/webserve.py index 648e293b..2b0f2699 100644 --- a/headphones/webserve.py +++ b/headphones/webserve.py @@ -1185,10 +1185,6 @@ class WebInterface(object): "utorrent_username": headphones.CONFIG.UTORRENT_USERNAME, "utorrent_password": headphones.CONFIG.UTORRENT_PASSWORD, "utorrent_label": headphones.CONFIG.UTORRENT_LABEL, - "aria_host": headphones.CONFIG.ARIA_HOST, - "aria_password": headphones.CONFIG.ARIA_PASSWORD, - "aria_token": headphones.CONFIG.ARIA_TOKEN, - "aria_username": headphones.CONFIG.ARIA_USERNAME, "nzb_downloader_sabnzbd": radio(headphones.CONFIG.NZB_DOWNLOADER, 0), "nzb_downloader_nzbget": radio(headphones.CONFIG.NZB_DOWNLOADER, 1), "nzb_downloader_blackhole": radio(headphones.CONFIG.NZB_DOWNLOADER, 2), @@ -1197,7 +1193,6 @@ class WebInterface(object): "torrent_downloader_utorrent": radio(headphones.CONFIG.TORRENT_DOWNLOADER, 2), "torrent_downloader_deluge": radio(headphones.CONFIG.TORRENT_DOWNLOADER, 3), "torrent_downloader_qbittorrent": radio(headphones.CONFIG.TORRENT_DOWNLOADER, 4), - "ddl_downloader_aria": radio(headphones.CONFIG.DDL_DOWNLOADER, 0), "download_dir": headphones.CONFIG.DOWNLOAD_DIR, "use_blackhole": checked(headphones.CONFIG.BLACKHOLE), "blackhole_dir": headphones.CONFIG.BLACKHOLE_DIR, @@ -1259,8 +1254,6 @@ class WebInterface(object): "use_tquattrecentonze": checked(headphones.CONFIG.TQUATTRECENTONZE), "tquattrecentonze_user": headphones.CONFIG.TQUATTRECENTONZE_USER, "tquattrecentonze_password": headphones.CONFIG.TQUATTRECENTONZE_PASSWORD, - "download_ddl_dir": headphones.CONFIG.DOWNLOAD_DDL_DIR, - "use_deezloader": checked(headphones.CONFIG.DEEZLOADER), "pref_qual_0": radio(headphones.CONFIG.PREFERRED_QUALITY, 0), "pref_qual_1": radio(headphones.CONFIG.PREFERRED_QUALITY, 1), "pref_qual_2": radio(headphones.CONFIG.PREFERRED_QUALITY, 2), @@ -1306,7 +1299,6 @@ class WebInterface(object): "prefer_torrents_0": radio(headphones.CONFIG.PREFER_TORRENTS, 0), "prefer_torrents_1": radio(headphones.CONFIG.PREFER_TORRENTS, 1), "prefer_torrents_2": radio(headphones.CONFIG.PREFER_TORRENTS, 2), - "prefer_torrents_3": radio(headphones.CONFIG.PREFER_TORRENTS, 3), "magnet_links_0": radio(headphones.CONFIG.MAGNET_LINKS, 0), "magnet_links_1": radio(headphones.CONFIG.MAGNET_LINKS, 1), "magnet_links_2": radio(headphones.CONFIG.MAGNET_LINKS, 2), @@ -1465,7 +1457,7 @@ class WebInterface(object): "launch_browser", "enable_https", "api_enabled", "use_blackhole", "headphones_indexer", "use_newznab", "newznab_enabled", "use_torznab", "torznab_enabled", "use_nzbsorg", "use_omgwtfnzbs", "use_kat", "use_piratebay", "use_oldpiratebay", - "use_mininova", "use_waffles", "use_rutracker", "use_deezloader", + "use_mininova", "use_waffles", "use_rutracker", "use_apollo", "use_redacted", "use_strike", "use_tquattrecentonze", "preferred_bitrate_allow_lossless", "detect_bitrate", "ignore_clean_releases", "freeze_db", "cue_split", "move_files", "rename_files", "correct_metadata", "cleanup_files", "keep_nfo", "add_album_art", @@ -1599,9 +1591,6 @@ class WebInterface(object): # Reconfigure musicbrainz database connection with the new values mb.startmb() - # Reconfigure Aria2 - searcher.reconfigure() - raise cherrypy.HTTPRedirect("config") @cherrypy.expose From 6c0b4518828b230ded4ab91e4f79b56f48ef50c6 Mon Sep 17 00:00:00 2001 From: Noam Date: Thu, 18 May 2017 17:02:53 +0300 Subject: [PATCH 21/27] rutracker optional session cookie --- data/interfaces/default/config.html | 4 ++++ headphones/config.py | 1 + headphones/rutracker.py | 5 ++++- headphones/webserve.py | 1 + 4 files changed, 10 insertions(+), 1 deletion(-) diff --git a/data/interfaces/default/config.html b/data/interfaces/default/config.html index 841df0d1..02f40718 100644 --- a/data/interfaces/default/config.html +++ b/data/interfaces/default/config.html @@ -649,6 +649,10 @@ +
+ + +
diff --git a/headphones/config.py b/headphones/config.py index 57af4c16..98b23785 100644 --- a/headphones/config.py +++ b/headphones/config.py @@ -248,6 +248,7 @@ _CONFIG_DEFINITIONS = { 'RUTRACKER_PASSWORD': (str, 'Rutracker', ''), 'RUTRACKER_RATIO': (str, 'Rutracker', ''), 'RUTRACKER_USER': (str, 'Rutracker', ''), + 'RUTRACKER_COOKIE': (str, 'Rutracker', ''), 'SAB_APIKEY': (str, 'SABnzbd', ''), 'SAB_CATEGORY': (str, 'SABnzbd', ''), 'SAB_HOST': (str, 'SABnzbd', ''), diff --git a/headphones/rutracker.py b/headphones/rutracker.py index 658d490e..7d094906 100644 --- a/headphones/rutracker.py +++ b/headphones/rutracker.py @@ -49,7 +49,10 @@ class Rutracker(object): # try again if not self.has_bb_session_cookie(r): time.sleep(10) - r = self.session.post(loginpage, data=post_params, timeout=self.timeout, allow_redirects=False) + if headphones.CONFIG.RUTRACKER_COOKIE: + r = self.session.post(loginpage, data=post_params, timeout=self.timeout, allow_redirects=False, cookies={'bb_session':headphones.CONFIG.RUTRACKER_COOKIE}) + else: + r = self.session.post(loginpage, data=post_params, timeout=self.timeout, allow_redirects=False) if self.has_bb_session_cookie(r): self.loggedin = True logger.info("Successfully logged in to rutracker") diff --git a/headphones/webserve.py b/headphones/webserve.py index 2b0f2699..ee353fe2 100644 --- a/headphones/webserve.py +++ b/headphones/webserve.py @@ -1240,6 +1240,7 @@ class WebInterface(object): "rutracker_user": headphones.CONFIG.RUTRACKER_USER, "rutracker_password": headphones.CONFIG.RUTRACKER_PASSWORD, "rutracker_ratio": headphones.CONFIG.RUTRACKER_RATIO, + "rutracker_cookie": headphones.CONFIG.RUTRACKER_COOKIE, "use_apollo": checked(headphones.CONFIG.APOLLO), "apollo_username": headphones.CONFIG.APOLLO_USERNAME, "apollo_password": headphones.CONFIG.APOLLO_PASSWORD, From ecadf8cdabdd01820edebb01e4e5045fab832744 Mon Sep 17 00:00:00 2001 From: Noam Date: Thu, 18 May 2017 18:26:16 +0300 Subject: [PATCH 22/27] log cookie attempt --- headphones/rutracker.py | 1 + 1 file changed, 1 insertion(+) diff --git a/headphones/rutracker.py b/headphones/rutracker.py index 7d094906..1c689504 100644 --- a/headphones/rutracker.py +++ b/headphones/rutracker.py @@ -50,6 +50,7 @@ class Rutracker(object): if not self.has_bb_session_cookie(r): time.sleep(10) if headphones.CONFIG.RUTRACKER_COOKIE: + logger.info("Attempting to log in using predefined cookie...") r = self.session.post(loginpage, data=post_params, timeout=self.timeout, allow_redirects=False, cookies={'bb_session':headphones.CONFIG.RUTRACKER_COOKIE}) else: r = self.session.post(loginpage, data=post_params, timeout=self.timeout, allow_redirects=False) From 3110659633de1d3cada334bf4714f24b1c0b781c Mon Sep 17 00:00:00 2001 From: Noam Date: Thu, 18 May 2017 18:28:01 +0300 Subject: [PATCH 23/27] travis space --- headphones/rutracker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/headphones/rutracker.py b/headphones/rutracker.py index 1c689504..7d18ca1e 100644 --- a/headphones/rutracker.py +++ b/headphones/rutracker.py @@ -51,7 +51,7 @@ class Rutracker(object): time.sleep(10) if headphones.CONFIG.RUTRACKER_COOKIE: logger.info("Attempting to log in using predefined cookie...") - r = self.session.post(loginpage, data=post_params, timeout=self.timeout, allow_redirects=False, cookies={'bb_session':headphones.CONFIG.RUTRACKER_COOKIE}) + r = self.session.post(loginpage, data=post_params, timeout=self.timeout, allow_redirects=False, cookies={'bb_session': headphones.CONFIG.RUTRACKER_COOKIE}) else: r = self.session.post(loginpage, data=post_params, timeout=self.timeout, allow_redirects=False) if self.has_bb_session_cookie(r): From 4d674689b1a2e255a10c662dc801354b10a8996c Mon Sep 17 00:00:00 2001 From: Ade Date: Fri, 16 Jun 2017 13:02:13 +1200 Subject: [PATCH 24/27] rutracker search term fix --- headphones/rutracker.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/headphones/rutracker.py b/headphones/rutracker.py index 7d18ca1e..7185984d 100644 --- a/headphones/rutracker.py +++ b/headphones/rutracker.py @@ -98,7 +98,11 @@ class Rutracker(object): # sort by size, descending. sort = '&o=7&s=2' - searchurl = "%s?nm=%s%s%s" % (self.search_referer, urllib.quote(searchterm), format, sort) + try: + searchurl = "%s?nm=%s%s%s" % (self.search_referer, urllib.quote(searchterm), format, sort) + except: + searchterm = searchterm.encode('utf-8') + searchurl = "%s?nm=%s%s%s" % (self.search_referer, urllib.quote(searchterm), format, sort) logger.info("Searching rutracker using term: %s", searchterm) return searchurl From 3e44baaded1f826607caa803a5e6c1f4fa957d54 Mon Sep 17 00:00:00 2001 From: Ade Date: Sat, 17 Jun 2017 21:36:17 +1200 Subject: [PATCH 25/27] Boxcar icon url --- headphones/notifiers.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/headphones/notifiers.py b/headphones/notifiers.py index f8f1b3f8..54f4f4b5 100644 --- a/headphones/notifiers.py +++ b/headphones/notifiers.py @@ -790,7 +790,9 @@ class BOXCAR(object): 'user_credentials': headphones.CONFIG.BOXCAR_TOKEN, 'notification[title]': title.encode('utf-8'), 'notification[long_message]': message.encode('utf-8'), - 'notification[sound]': "done" + 'notification[sound]': "done", + 'notification[icon_url]': "https://raw.githubusercontent.com/rembo10/headphones/master/data/images" + "/headphoneslogo.png" }) req = urllib2.Request(self.url) From ee7254ddbee1d40ddcd33b7c2c92580fb1a850c7 Mon Sep 17 00:00:00 2001 From: Ade Date: Sat, 24 Jun 2017 09:12:17 +1200 Subject: [PATCH 26/27] Version check logging --- headphones/versioncheck.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/headphones/versioncheck.py b/headphones/versioncheck.py index 341f7926..11c4e225 100644 --- a/headphones/versioncheck.py +++ b/headphones/versioncheck.py @@ -39,14 +39,15 @@ def runGit(args): try: logger.debug('Trying to execute: "' + cmd + '" with shell in ' + headphones.PROG_DIR) - p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True, + p = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + shell=True, cwd=headphones.PROG_DIR) output, err = p.communicate() output = output.strip() logger.debug('Git output: ' + output) - except OSError: - logger.debug('Command failed: %s', cmd) + except OSError as e: + logger.debug('Command failed: %s. Error: %s' % (cmd, e)) continue if 'not found' in output or "not recognized as an internal or external command" in output: From e96a6fc7c8503546ff1dc7f06a6bccc2ec8e85dc Mon Sep 17 00:00:00 2001 From: Ade Date: Sat, 24 Jun 2017 09:27:51 +1200 Subject: [PATCH 27/27] Transmission magnet issue Fixes #2972 --- headphones/transmission.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/headphones/transmission.py b/headphones/transmission.py index 54241751..47de2c18 100644 --- a/headphones/transmission.py +++ b/headphones/transmission.py @@ -34,7 +34,7 @@ _session_id = None def addTorrent(link, data=None): method = 'torrent-add' - if link.endswith('.torrent') and not link.startswith('http') or data: + if link.endswith('.torrent') and not link.startswith(('http', 'magnet')) or data: if data: metainfo = str(base64.b64encode(data)) else: