From 04e8767ea99e804ccb909d3293f8d6e2561d3726 Mon Sep 17 00:00:00 2001 From: Kallys Date: Mon, 24 Apr 2017 14:32:51 +0200 Subject: [PATCH] Integration of deezloader provider by ParadoxalManiak under CC BY-NC-SA 4.0 Integration of aria2 downloader by Xyne under GPLv2 I'm not responsible of any kind for the usage of this programs by other people. These integrations come with no warranty. Please refer to your local country laws and respect Deezer's terms of service. --- data/interfaces/default/config.html | 64 ++ headphones/aria2.py | 979 ++++++++++++++++++++++++++++ headphones/classes.py | 31 + headphones/config.py | 7 + headphones/deezloader.py | 441 +++++++++++++ headphones/postprocessor.py | 40 +- headphones/searcher.py | 167 ++++- headphones/webserve.py | 13 +- 8 files changed, 1722 insertions(+), 20 deletions(-) create mode 100644 headphones/aria2.py create mode 100644 headphones/deezloader.py diff --git a/data/interfaces/default/config.html b/data/interfaces/default/config.html index 841df0d1..bd331f2c 100644 --- a/data/interfaces/default/config.html +++ b/data/interfaces/default/config.html @@ -304,6 +304,45 @@ + +
+ Direct Download + Aria2 +
+
+
+ + + usually http://localhost:6800/jsonrpc +
+
+ + +
+
+ + +
+
+ + +
+
+ + + Full path where ddl client downloads your music, e.g. /Users/name/Downloads/music +
+
@@ -461,6 +500,7 @@ NZBs Torrents + Direct Download No Preference
@@ -573,6 +613,21 @@ + +
+ Direct Download + +
+
+ + +
+
+ +
@@ -2405,6 +2460,11 @@ $("#deluge_options").show(); } + if ($("#ddl_downloader_aria").is(":checked")) + { + $("#ddl_aria_options").show(); + } + $('input[type=radio]').change(function(){ if ($("#preferred_bitrate").is(":checked")) { @@ -2458,6 +2518,9 @@ { $("#torrent_blackhole_options,#utorrent_options,#transmission_options,#qbittorrent_options").fadeOut("fast", function() { $("#deluge_options").fadeIn() }); } + if ($("#ddl_downloader_aria").is(":checked")) + { + } }); $("#mirror").change(handleNewServerSelection); @@ -2560,6 +2623,7 @@ initConfigCheckbox("#enable_https"); initConfigCheckbox("#customauth"); initConfigCheckbox("#use_tquattrecentonze"); + initConfigCheckbox("#use_deezloader"); $('#twitterStep1').click(function () { diff --git a/headphones/aria2.py b/headphones/aria2.py new file mode 100644 index 00000000..06d8e4e0 --- /dev/null +++ b/headphones/aria2.py @@ -0,0 +1,979 @@ +# -*- coding: utf8 -*- +# Copyright (C) 2012-2016 Xyne +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# (version 2) as published by the Free Software Foundation. +# +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +from __future__ import with_statement +import base64 +import json +import math +import os +import ssl +import string +import time +import httplib +import urllib2 + +from headphones import logger + +# ################################ Constants ################################### + +DEFAULT_PORT = 6800 +SERVER_URI_FORMAT = '{}://{}:{:d}/jsonrpc' + +# Status values for unfinished downloads. +TEMPORARY_STATUS = ('active', 'waiting', 'paused') +# Status values for finished downloads. +FINAL_STATUS = ('complete', 'error') + +ARIA2_CONTROL_FILE_EXT = '.aria2' + +# ########################## Convenience Functions ############################# + + +def to_json_list(objs): + ''' + Wrap strings in lists. Other iterables are converted to lists directly. + ''' + if isinstance(objs, str): + return [objs] + elif not isinstance(objs, list): + return list(objs) + else: + return objs + + +def add_options_and_position(params, options=None, position=None): + ''' + Convenience method for adding options and position to parameters. + ''' + if options: + params.append(options) + if position: + if not isinstance(position, int): + try: + position = int(position) + except ValueError: + position = -1 + if position >= 0: + params.append(position) + return params + + +def get_status(response): + ''' + Process a status response. + ''' + if response: + try: + return response['status'] + except KeyError: + logger.error('no status returned from Aria2 RPC server') + return 'error' + else: + logger.error('no response from server') + return 'error' + + +def random_token(length, valid_chars=None): + ''' + Get a random secret token for the Aria2 RPC server. + + length: + The length of the token + + valid_chars: + A list or other ordered and indexable iterable of valid characters. If not + given of None, asciinumberic characters with some punctuation characters + will be used. + ''' + if not valid_chars: + valid_chars = string.ascii_letters + string.digits + '!@#$%^&*()-_=+' + number_of_chars = len(valid_chars) + bytes_to_read = math.ceil(math.log(number_of_chars) / math.log(0x100)) + max_value = 0x100**bytes_to_read + max_index = number_of_chars - 1 + token = '' + for _ in range(length): + value = int.from_bytes(os.urandom(bytes_to_read), byteorder='little') + index = round((value * max_index) / max_value) + token += valid_chars[index] + return token + +# ################ From python3-aur's ThreadedServers.common ################### + + +def format_bytes(size): + '''Format bytes for inferior humans.''' + if size < 0x400: + return '{:d} B'.format(size) + else: + size = float(size) / 0x400 + for prefix in ('KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB'): + if size < 0x400: + return '{:0.02f} {}'.format(size, prefix) + else: + size /= 0x400 + return '{:0.02f} YiB'.format(size) + + +def format_seconds(s): + '''Format seconds for inferior humans.''' + string = '' + for base, char in ( + (60, 's'), + (60, 'm'), + (24, 'h') + ): + s, r = divmod(s, base) + if s == 0: + return '{:d}{}{}'.format(r, char, string) + elif r != 0: + string = '{:02d}{}{}'.format(r, char, string) + else: + return '{:d}d{}'.format(s, string) + +# ############################ Aria2JsonRpcError ############################### + + +class Aria2JsonRpcError(Exception): + def __init__(self, msg, connection_error=False): + super(self.__class__, self).__init__(self, msg) + self.connection_error = connection_error + +# ############################ Aria2JsonRpc Class ############################## + + +class Aria2JsonRpc(object): + ''' + Interface class for interacting with an Aria2 RPC server. + ''' + # TODO: certificate options, etc. + def __init__( + self, ID, uri, + mode='normal', + token=None, + http_user=None, http_passwd=None, + server_cert=None, client_cert=None, client_cert_password=None, + ssl_protocol=None, + setup_function=None + ): + ''' + ID: the ID to send to the RPC interface + + uri: the URI of the RPC interface + + mode: + normal - process requests immediately + batch - queue requests (run with "process_queue") + format - return RPC request objects + + token: + RPC method-level authorization token (set using `--rpc-secret`) + + http_user, http_password: + HTTP Basic authentication credentials (deprecated) + + server_cert: + server certificate for HTTPS connections + + client_cert: + client certificate for HTTPS connections + + client_cert_password: + prompt for client certificate password + + ssl_protocol: + SSL protocol from the ssl module + + setup_function: + A function to invoke prior to the first server call. This could be the + launch() method of an Aria2RpcServer instance, for example. This attribute + is set automatically in instances returned from Aria2RpcServer.get_a2jr() + ''' + self.id = ID + self.uri = uri + self.mode = mode + self.queue = [] + self.handlers = dict() + self.token = token + self.setup_function = setup_function + + if None not in (http_user, http_passwd): + self.add_HTTPBasicAuthHandler(http_user, http_passwd) + + if server_cert or client_cert: + self.add_HTTPSHandler( + server_cert=server_cert, + client_cert=client_cert, + client_cert_password=client_cert_password, + protocol=ssl_protocol + ) + + self.update_opener() + + def iter_handlers(self): + ''' + Iterate over handlers. + ''' + for name in ('HTTPS', 'HTTPBasicAuth'): + try: + yield self.handlers[name] + except KeyError: + pass + + def update_opener(self): + ''' + Build an opener from the current handlers. + ''' + self.opener = urllib2.build_opener(*self.iter_handlers()) + + def remove_handler(self, name): + ''' + Remove a handler. + ''' + try: + del self.handlers[name] + except KeyError: + pass + + def add_HTTPBasicAuthHandler(self, user, passwd): + ''' + Add a handler for HTTP Basic authentication. + + If either user or passwd are None, the handler is removed. + ''' + handler = urllib2.HTTPBasicAuthHandler() + handler.add_password( + realm='aria2', + uri=self.uri, + user=user, + passwd=passwd, + ) + self.handlers['HTTPBasicAuth'] = handler + + def remove_HTTPBasicAuthHandler(self): + self.remove_handler('HTTPBasicAuth') + + def add_HTTPSHandler( + self, + server_cert=None, + client_cert=None, + client_cert_password=None, + protocol=None, + ): + ''' + Add a handler for HTTPS connections with optional server and client + certificates. + ''' + if not protocol: + protocol = ssl.PROTOCOL_TLSv1 +# protocol = ssl.PROTOCOL_TLSv1_1 # openssl 1.0.1+ +# protocol = ssl.PROTOCOL_TLSv1_2 # Python 3.4+ + context = ssl.SSLContext(protocol) + + if server_cert: + context.verify_mode = ssl.CERT_REQUIRED + context.load_verify_locations(cafile=server_cert) + else: + context.verify_mode = ssl.CERT_OPTIONAL + + if client_cert: + context.load_cert_chain(client_cert, password=client_cert_password) + + self.handlers['HTTPS'] = urllib2.HTTPSHandler( + context=context, + check_hostname=False + ) + + def remove_HTTPSHandler(self): + self.remove_handler('HTTPS') + + def send_request(self, req_obj): + ''' + Send the request and return the response. + ''' + if self.setup_function: + self.setup_function() + self.setup_function = None + logger.debug("Aria req_obj: %s" % json.dumps(req_obj, indent=2, sort_keys=True)) + req = json.dumps(req_obj).encode('UTF-8') + try: + f = self.opener.open(self.uri, req) + obj = json.loads(f.read()) + try: + return obj['result'] + except KeyError: + raise Aria2JsonRpcError('unexpected result: {}'.format(obj)) + except (urllib2.URLError) as e: + # This should work but URLError does not set the errno attribute: + # e.errno == errno.ECONNREFUSED + raise Aria2JsonRpcError( + str(e), + connection_error=( + '111' in str(e) + ) + ) + except httplib.BadStatusLine as e: + raise Aria2JsonRpcError('{}: BadStatusLine: {} (HTTPS error?)'.format( + self.__class__.__name__, e + )) + + def jsonrpc(self, method, params=None, prefix='aria2.'): + ''' + POST a request to the RPC interface. + ''' + if not params: + params = [] + + if self.token is not None: + token_str = 'token:{}'.format(self.token) + if method == 'multicall': + for p in params[0]: + try: + p['params'].insert(0, token_str) + except KeyError: + p['params'] = [token_str] + else: + params.insert(0, token_str) + + req_obj = { + 'jsonrpc': '2.0', + 'id': self.id, + 'method': prefix + method, + 'params': params, + } + if self.mode == 'batch': + self.queue.append(req_obj) + return None + elif self.mode == 'format': + return req_obj + else: + return self.send_request(req_obj) + + def process_queue(self): + ''' + Processed queued requests. + ''' + req_obj = self.queue + self.queue = [] + return self.send_request(req_obj) + + +# ############################# Standard Methods ############################### + def addUri(self, uris, options=None, position=None): + ''' + aria2.addUri method + + uris: list of URIs + + options: dictionary of additional options + + position: position in queue + + Returns a GID + ''' + params = [uris] + params = add_options_and_position(params, options, position) + return self.jsonrpc('addUri', params) + + def addTorrent(self, torrent, uris=None, options=None, position=None): + ''' + aria2.addTorrent method + + torrent: base64-encoded torrent file + + uris: list of webseed URIs + + options: dictionary of additional options + + position: position in queue + + Returns a GID. + ''' + params = [torrent] + if uris: + params.append(uris) + params = add_options_and_position(params, options, position) + return self.jsonrpc('addTorrent', params) + + def addMetalink(self, metalink, options=None, position=None): + ''' + aria2.addMetalink method + + metalink: base64-encoded metalink file + + options: dictionary of additional options + + position: position in queue + + Returns an array of GIDs. + ''' + params = [metalink] + params = add_options_and_position(params, options, position) + return self.jsonrpc('addTorrent', params) + + def remove(self, gid): + ''' + aria2.remove method + + gid: GID to remove + ''' + params = [gid] + return self.jsonrpc('remove', params) + + def forceRemove(self, gid): + ''' + aria2.forceRemove method + + gid: GID to remove + ''' + params = [gid] + return self.jsonrpc('forceRemove', params) + + def pause(self, gid): + ''' + aria2.pause method + + gid: GID to pause + ''' + params = [gid] + return self.jsonrpc('pause', params) + + def pauseAll(self): + ''' + aria2.pauseAll method + ''' + return self.jsonrpc('pauseAll') + + def forcePause(self, gid): + ''' + aria2.forcePause method + + gid: GID to pause + ''' + params = [gid] + return self.jsonrpc('forcePause', params) + + def forcePauseAll(self): + ''' + aria2.forcePauseAll method + ''' + return self.jsonrpc('forcePauseAll') + + def unpause(self, gid): + ''' + aria2.unpause method + + gid: GID to unpause + ''' + params = [gid] + return self.jsonrpc('unpause', params) + + def unpauseAll(self): + ''' + aria2.unpauseAll method + ''' + return self.jsonrpc('unpauseAll') + + def tellStatus(self, gid, keys=None): + ''' + aria2.tellStatus method + + gid: GID to query + + keys: subset of status keys to return (all keys are returned otherwise) + + Returns a dictionary. + ''' + params = [gid] + if keys: + params.append(keys) + return self.jsonrpc('tellStatus', params) + + def getUris(self, gid): + ''' + aria2.getUris method + + gid: GID to query + + Returns a list of dictionaries. + ''' + params = [gid] + return self.jsonrpc('getUris', params) + + def getFiles(self, gid): + ''' + aria2.getFiles method + + gid: GID to query + + Returns a list of dictionaries. + ''' + params = [gid] + return self.jsonrpc('getFiles', params) + + def getPeers(self, gid): + ''' + aria2.getPeers method + + gid: GID to query + + Returns a list of dictionaries. + ''' + params = [gid] + return self.jsonrpc('getPeers', params) + + def getServers(self, gid): + ''' + aria2.getServers method + + gid: GID to query + + Returns a list of dictionaries. + ''' + params = [gid] + return self.jsonrpc('getServers', params) + + def tellActive(self, keys=None): + ''' + aria2.tellActive method + + keys: same as tellStatus + + Returns a list of dictionaries. The dictionaries are the same as those + returned by tellStatus. + ''' + if keys: + params = [keys] + else: + params = None + return self.jsonrpc('tellActive', params) + + def tellWaiting(self, offset, num, keys=None): + ''' + aria2.tellWaiting method + + offset: offset from start of waiting download queue + (negative values are counted from the end of the queue) + + num: number of downloads to return + + keys: same as tellStatus + + Returns a list of dictionaries. The dictionaries are the same as those + returned by tellStatus. + ''' + params = [offset, num] + if keys: + params.append(keys) + return self.jsonrpc('tellWaiting', params) + + def tellStopped(self, offset, num, keys=None): + ''' + aria2.tellStopped method + + offset: offset from oldest download (same semantics as tellWaiting) + + num: same as tellWaiting + + keys: same as tellStatus + + Returns a list of dictionaries. The dictionaries are the same as those + returned by tellStatus. + ''' + params = [offset, num] + if keys: + params.append(keys) + return self.jsonrpc('tellStopped', params) + + def changePosition(self, gid, pos, how): + ''' + aria2.changePosition method + + gid: GID to change + + pos: the position + + how: "POS_SET", "POS_CUR" or "POS_END" + ''' + params = [gid, pos, how] + return self.jsonrpc('changePosition', params) + + def changeUri(self, gid, fileIndex, delUris, addUris, position=None): + ''' + aria2.changePosition method + + gid: GID to change + + fileIndex: file to affect (1-based) + + delUris: URIs to remove + + addUris: URIs to add + + position: where URIs are inserted, after URIs have been removed + ''' + params = [gid, fileIndex, delUris, addUris] + if position: + params.append(position) + return self.jsonrpc('changePosition', params) + + def getOption(self, gid): + ''' + aria2.getOption method + + gid: GID to query + + Returns a dictionary of options. + ''' + params = [gid] + return self.jsonrpc('getOption', params) + + def changeOption(self, gid, options): + ''' + aria2.changeOption method + + gid: GID to change + + options: dictionary of new options + (not all options can be changed for active downloads) + ''' + params = [gid, options] + return self.jsonrpc('changeOption', params) + + def getGlobalOption(self): + ''' + aria2.getGlobalOption method + + Returns a dictionary. + ''' + return self.jsonrpc('getGlobalOption') + + def changeGlobalOption(self, options): + ''' + aria2.changeGlobalOption method + + options: dictionary of new options + ''' + params = [options] + return self.jsonrpc('changeGlobalOption', params) + + def getGlobalStat(self): + ''' + aria2.getGlobalStat method + + Returns a dictionary. + ''' + return self.jsonrpc('getGlobalStat') + + def purgeDownloadResult(self): + ''' + aria2.purgeDownloadResult method + ''' + self.jsonrpc('purgeDownloadResult') + + def removeDownloadResult(self, gid): + ''' + aria2.removeDownloadResult method + + gid: GID to remove + ''' + params = [gid] + return self.jsonrpc('removeDownloadResult', params) + + def getVersion(self): + ''' + aria2.getVersion method + + Returns a dictionary. + ''' + return self.jsonrpc('getVersion') + + def getSessionInfo(self): + ''' + aria2.getSessionInfo method + + Returns a dictionary. + ''' + return self.jsonrpc('getSessionInfo') + + def shutdown(self): + ''' + aria2.shutdown method + ''' + return self.jsonrpc('shutdown') + + def forceShutdown(self): + ''' + aria2.forceShutdown method + ''' + return self.jsonrpc('forceShutdown') + + def multicall(self, methods): + ''' + aria2.multicall method + + methods: list of dictionaries (keys: methodName, params) + + The method names must be those used by Aria2c, e.g. "aria2.tellStatus". + ''' + return self.jsonrpc('multicall', [methods], prefix='system.') + + +# ########################### Convenience Methods ############################## + def add_torrent(self, path, uris=None, options=None, position=None): + ''' + A wrapper around addTorrent for loading files. + ''' + with open(path, 'r') as f: + torrent = base64.encode(f.read()) + return self.addTorrent(torrent, uris, options, position) + + def add_metalink(self, path, uris=None, options=None, position=None): + ''' + A wrapper around addMetalink for loading files. + ''' + with open(path, 'r') as f: + metalink = base64.encode(f.read()) + return self.addMetalink(metalink, uris, options, position) + + def get_status(self, gid): + ''' + Get the status of a single GID. + ''' + response = self.tellStatus(gid, ['status']) + return get_status(response) + + def wait_for_final_status(self, gid, interval=1): + ''' + Wait for a GID to complete or fail and return its status. + ''' + if not interval or interval < 0: + interval = 1 + while True: + status = self.get_status(gid) + if status in TEMPORARY_STATUS: + time.sleep(interval) + else: + return status + + def get_statuses(self, gids): + ''' + Get the status of multiple GIDs. The status of each is yielded in order. + ''' + methods = [ + { + 'methodName': 'aria2.tellStatus', + 'params': [gid, ['gid', 'status']] + } + for gid in gids + ] + results = self.multicall(methods) + if results: + status = dict((r[0]['gid'], r[0]['status']) for r in results) + for gid in gids: + try: + yield status[gid] + except KeyError: + logger.error('Aria2 RPC server returned no status for GID {}'.format(gid)) + yield 'error' + else: + logger.error('no response from Aria2 RPC server') + for gid in gids: + yield 'error' + + def wait_for_final_statuses(self, gids, interval=1): + ''' + Wait for multiple GIDs to complete or fail and return their statuses in + order. + + gids: + A flat list of GIDs. + ''' + if not interval or interval < 0: + interval = 1 + statusmap = dict((g, None) for g in gids) + remaining = list( + g for g, s in statusmap.items() if s is None + ) + while remaining: + for g, s in zip(remaining, self.get_statuses(remaining)): + if s in TEMPORARY_STATUS: + continue + else: + statusmap[g] = s + remaining = list( + g for g, s in statusmap.items() if s is None + ) + if remaining: + time.sleep(interval) + for g in gids: + yield statusmap[g] + + def print_global_status(self): + ''' + Print global status of the RPC server. + ''' + status = self.getGlobalStat() + if status: + numWaiting = int(status['numWaiting']) + numStopped = int(status['numStopped']) + keys = ['totalLength', 'completedLength'] + total = self.tellActive(keys) + waiting = self.tellWaiting(0, numWaiting, keys) + if waiting: + total += waiting + stopped = self.tellStopped(0, numStopped, keys) + if stopped: + total += stopped + + downloadSpeed = int(status['downloadSpeed']) + uploadSpeed = int(status['uploadSpeed']) + totalLength = sum(int(x['totalLength']) for x in total) + completedLength = sum(int(x['completedLength']) for x in total) + remaining = totalLength - completedLength + + status['downloadSpeed'] = format_bytes(downloadSpeed) + '/s' + status['uploadSpeed'] = format_bytes(uploadSpeed) + '/s' + + preordered = ('downloadSpeed', 'uploadSpeed') + + rows = list() + for k in sorted(status): + if k in preordered: + continue + rows.append((k, status[k])) + + rows.extend((x, status[x]) for x in preordered) + + if totalLength > 0: + rows.append(('total', format(format_bytes(totalLength)))) + rows.append(('completed', format(format_bytes(completedLength)))) + rows.append(('remaining', format(format_bytes(remaining)))) + if completedLength == totalLength: + eta = 'finished' + else: + try: + eta = format_seconds(remaining // downloadSpeed) + except ZeroDivisionError: + eta = 'never' + rows.append(('ETA', eta)) + + l = max(len(r[0]) for r in rows) + r = max(len(r[1]) for r in rows) + r = max(r, len(self.uri) - (l + 2)) + fmt = '{:<' + str(l) + 's} {:>' + str(r) + 's}' + + print(self.uri) + for k, v in rows: + print(fmt.format(k, v)) + + def queue_uris(self, uris, options, interval=None): + ''' + Enqueue URIs and wait for download to finish while printing status at + regular intervals. + ''' + gid = self.addUri(uris, options) + print('GID: {}'.format(gid)) + + if gid and interval is not None: + blanker = '' + while True: + response = self.tellStatus(gid, ['status']) + if response: + try: + status = response['status'] + except KeyError: + print('error: no status returned from Aria2 RPC server') + break + print('{}\rstatus: {}'.format(blanker, status)), + blanker = ' ' * len(status) + if status in TEMPORARY_STATUS: + time.sleep(interval) + else: + break + else: + print('error: no response from server') + break + + +# ####################### Polymethod download handlers ######################### + def polymethod_enqueue_many(self, downloads): + ''' + Enqueue downloads. + + downloads: Same as polymethod_download(). + ''' + methods = list( + { + 'methodName': 'aria2.{}'.format(d[0]), + 'params': list(d[1:]) + } for d in downloads + ) + return self.multicall(methods) + + def polymethod_wait_many(self, gids, interval=1): + ''' + Wait for the GIDs to complete or fail and return their statuses. + + gids: + A list of lists of GIDs. + ''' + # The flattened list of GIDs + gs = list(g for gs in gids for g in gs) + statusmap = dict(tuple(zip( + gs, + self.wait_for_final_statuses(gs, interval=interval) + ))) + for gs in gids: + yield list(statusmap.get(g, 'error') for g in gs) + + def polymethod_enqueue_one(self, download): + ''' + Same as polymethod_enqueue_many but for one element. + ''' + return getattr(self, download[0])(*download[1:]) + + def polymethod_download(self, downloads, interval=1): + ''' + Enqueue a series of downloads and wait for them to finish. Iterate over the + status of each, in order. + + downloads: + An iterable over (, , ...) where indicates the "add" + method to use ('addUri', 'addTorrent', 'addMetalink') and everything that + follows are arguments to pass to that method. + + interval: + The status check interval while waiting. + + Iterates over the download status of finished downloads. "complete" + indicates success. Lists of statuses will be returned for downloads that + create multiple GIDs (e.g. metalinks). + ''' + gids = self.polymethod_enqueue_many(downloads) + return self.polymethod_wait_many(gids, interval=interval) + + def polymethod_download_bool(self, *args, **kwargs): + ''' + A wrapper around polymethod_download() which returns a boolean for each + download to indicate success (True) or failure (False). + ''' +# for status in self.polymethod_download(*args, **kwargs): +# yield all(s == 'complete' for s in status) + return list( + all(s == 'complete' for s in status) + for status in self.polymethod_download(*args, **kwargs) + ) diff --git a/headphones/classes.py b/headphones/classes.py index 6015a0f2..54a10a55 100644 --- a/headphones/classes.py +++ b/headphones/classes.py @@ -21,6 +21,7 @@ import urllib from common import USER_AGENT +from collections import OrderedDict class HeadphonesURLopener(urllib.FancyURLopener): @@ -135,3 +136,33 @@ class Proper: def __str__(self): return str(self.date) + " " + self.name + " " + str(self.season) + "x" + str( self.episode) + " of " + str(self.tvdbid) + + +class OptionalImport(object): + ''' + Dummy class for optional import (imports needed for optional features). + ''' + def __init__(self, name): + self.__name = name + + def __getattr__(self, attr): + raise ImportError('The following package is required to use this feature: {0}'.format(self.__name)) + + +class CacheDict(OrderedDict): + ''' + Ordered dictionary with fixed size, designed for caching. + ''' + def __init__(self, *args, **kwds): + self.size_limit = kwds.pop("size_limit", None) + OrderedDict.__init__(self, *args, **kwds) + self._check_size_limit() + + def __setitem__(self, key, value): + OrderedDict.__setitem__(self, key, value) + self._check_size_limit() + + def _check_size_limit(self): + if self.size_limit is not None: + while len(self) > self.size_limit: + self.popitem(last=False) diff --git a/headphones/config.py b/headphones/config.py index 57af4c16..86aa9216 100644 --- a/headphones/config.py +++ b/headphones/config.py @@ -46,6 +46,10 @@ _CONFIG_DEFINITIONS = { 'APOLLO_RATIO': (str, 'Apollo.rip', ''), 'APOLLO_USERNAME': (str, 'Apollo.rip', ''), 'APOLLO_URL': (str, 'Apollo.rip', 'https://apollo.rip'), + 'ARIA_HOST': (str, 'Aria2', ''), + 'ARIA_PASSWORD': (str, 'Aria2', ''), + 'ARIA_TOKEN': (str, 'Aria2', ''), + 'ARIA_USERNAME': (str, 'Aria2', ''), 'AUTOWANT_ALL': (int, 'General', 0), 'AUTOWANT_MANUALLY_ADDED': (int, 'General', 1), 'AUTOWANT_UPCOMING': (int, 'General', 1), @@ -73,6 +77,8 @@ _CONFIG_DEFINITIONS = { 'CUSTOMPORT': (int, 'General', 5000), 'CUSTOMSLEEP': (int, 'General', 1), 'CUSTOMUSER': (str, 'General', ''), + 'DDL_DOWNLOADER': (int, 'General', 0), + 'DEEZLOADER': (int, 'DeezLoader', 0), 'DELETE_LOSSLESS_FILES': (int, 'General', 1), 'DELUGE_HOST': (str, 'Deluge', ''), 'DELUGE_CERT': (str, 'Deluge', ''), @@ -86,6 +92,7 @@ _CONFIG_DEFINITIONS = { 'DOWNLOAD_DIR': (path, 'General', ''), 'DOWNLOAD_SCAN_INTERVAL': (int, 'General', 5), 'DOWNLOAD_TORRENT_DIR': (path, 'General', ''), + 'DOWNLOAD_DDL_DIR': (path, 'General', ''), 'DO_NOT_OVERRIDE_GIT_BRANCH': (int, 'General', 0), 'EMAIL_ENABLED': (int, 'Email', 0), 'EMAIL_FROM': (str, 'Email', ''), diff --git a/headphones/deezloader.py b/headphones/deezloader.py new file mode 100644 index 00000000..8de85df9 --- /dev/null +++ b/headphones/deezloader.py @@ -0,0 +1,441 @@ +# -*- coding: utf-8 -*- +# Deezloader (c) 2016 by ParadoxalManiak +# +# Deezloader is licensed under a +# Creative Commons Attribution-NonCommercial-ShareAlike 3.0 Unported License. +# +# You should have received a copy of the license along with this +# work. If not, see . +# +# Version 2.1.0 +# Maintained by ParadoxalManiak +# Original work by ZzMTV +# +# Author's disclaimer: +# I am not responsible for the usage of this program by other people. +# I do not recommend you doing this illegally or against Deezer's terms of service. +# This project is licensed under CC BY-NC-SA 4.0 + +import re +import os +from datetime import datetime +from hashlib import md5 +import binascii + +from beets.mediafile import MediaFile +from headphones import logger, request, helpers +from headphones.classes import OptionalImport, CacheDict +import headphones + + +# Try to import optional Crypto.Cipher packages +try: + from Crypto.Cipher import AES, Blowfish +except ImportError: + AES = OptionalImport('Crypto.Cipher.AES') + Blowfish = OptionalImport('Crypto.Cipher.Blowfish') + +# Public constants +PROVIDER_NAME = 'Deezer' + +# Internal constants +__API_URL = "http://www.deezer.com/ajax/gw-light.php" +__API_INFO_URL = "http://api.deezer.com/" +__HTTP_HEADERS = { + "User-Agent": "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36", + "Content-Language": "en-US", + "Cache-Control": "max-age=0", + "Accept": "*/*", + "Accept-Charset": "utf-8,ISO-8859-1;q=0.7,*;q=0.3", + "Accept-Language": "de-DE,de;q=0.8,en-US;q=0.6,en;q=0.4" +} + +# Internal variables +__api_queries = { + 'api_version': "1.0", + 'api_token': "None", + 'input': "3" +} +__cookies = None +__tracks_cache = CacheDict(size_limit=512) +__albums_cache = CacheDict(size_limit=64) + + +def __getApiToken(): + global __cookies + response = request.request_response(url="http://www.deezer.com/", headers=__HTTP_HEADERS) + __cookies = response.cookies + data = response.content + if data: + matches = re.search(r"checkForm\s*=\s*['|\"](.*)[\"|'];", data) + if matches: + token = matches.group(1) + __api_queries['api_token'] = token + logger.debug(u"Deezloader : api token loaded ('%s')" % token) + + if not token: + logger.error(u"Deezloader: Unable to get api token") + + +def getAlbumByLink(album_link): + """Returns deezer album infos using album link url + + :param str album_link: deezer album link url (eg: 'http://www.deezer.com/album/1234567/') + """ + matches = re.search(r"album\/([0-9]+)\/?$", album_link) + if matches: + return getAlbum(matches.group(1)) + + +def getAlbum(album_id): + """Returns deezer album infos + + :param int album_id: deezer album id + """ + global __albums_cache + + if str(album_id) in __albums_cache: + return __albums_cache[str(album_id)] + + url = __API_INFO_URL + "album/" + str(album_id) + data = request.request_json(url=url, headers=__HTTP_HEADERS, cookies=__cookies) + + if data and 'error' not in data: + __albums_cache[str(album_id)] = data + return data + else: + logger.debug("Deezloader: Can't load album infos") + return None + + +def searchAlbums(search_term): + """Search for deezer albums using search term + + :param str search_term: search term to search album for + """ + logger.info(u'Searching Deezer using term: "%s"' % search_term) + + url = __API_INFO_URL + "search/album?q=" + search_term + data = request.request_json(url=url, headers=__HTTP_HEADERS, cookies=__cookies) + + albums = [] + + # Process content + if data and 'total' in data and data['total'] > 0 and 'data' in data: + for item in data['data']: + try: + albums.append(getAlbum(item['id'])) + except Exception as e: + logger.error(u"An unknown error occurred in the Deezer search album parser: %s" % e) + else: + logger.info(u'No results found from Deezer using term: "%s"' % search_term) + + return albums + + +def __matchAlbums(albums, artist_name, album_title, album_length): + resultlist = [] + + for album in albums: + total_size = 0 + tracks_found = 0 + + for track in album['tracks']['data']: + t = getTrack(track['id']) + if t: + if t["FILESIZE_MP3_320"] > 0: + size = t["FILESIZE_MP3_320"] + elif t["FILESIZE_MP3_256"] > 0: + size = t["FILESIZE_MP3_256"] + elif t["FILESIZE_MP3_128"] > 0: + size = t["FILESIZE_MP3_128"] + else: + size = t["FILESIZE_MP3_64"] + + size = int(size) + total_size += size + tracks_found += 1 + logger.debug(u'Found song "%s". Size: %s' % (t['SNG_TITLE'], helpers.bytes_to_mb(size))) + + if tracks_found == 0: + logger.info(u'Ignoring album "%s" (no tracks to download)' % album['title']) + continue + + matched = True + mismatch_reason = 'matched!' + + if album_length > 0 and abs(int(album['duration']) - album_length) > 240: + matched = False + mismatch_reason = (u'duration mismatch: %i not in [%i, %i]' % (int(album['duration']), (album_length - 240), (album_length + 240))) + + elif (helpers.latinToAscii(re.sub(r"\W", "", album_title, 0, re.UNICODE)).lower() != + helpers.latinToAscii(re.sub(r"\W", "", album['title'], 0, re.UNICODE)).lower()): + matched = False + mismatch_reason = (u'album name mismatch: %s != %s' % (album['title'], album_title)) + + elif (helpers.latinToAscii(re.sub(r"\W", "", artist_name, 0, re.UNICODE)).lower() != + helpers.latinToAscii(re.sub(r"\W", "", album['artist']['name'], 0, re.UNICODE)).lower()): + matched = False + mismatch_reason = (u'artist name mismatch: %s != %s' % (album['artist']['name'], artist_name)) + + resultlist.append( + (album['artist']['name'] + ' - ' + album['title'] + ' [' + album['release_date'][:4] + '] (' + str(tracks_found) + '/' + str(album['nb_tracks']) + ')', + total_size, album['link'], PROVIDER_NAME, "ddl", matched) + ) + logger.info(u'Found "%s". Tracks %i/%i. Size: %s (%s)' % (album['title'], tracks_found, album['nb_tracks'], helpers.bytes_to_mb(total_size), mismatch_reason)) + + return resultlist + + +def searchAlbum(artist_name, album_title, user_search_term=None, album_length=None): + """Search for deezer specific album. + This will iterate over deezer albums and try to find best matches + + :param str artist_name: album artist name + :param str album_title: album title + :param str user_search_term: search terms provided by user + :param int album_length: targeted album duration in seconds + """ + # User search term by-pass normal search + if user_search_term: + return __matchAlbums(searchAlbums(user_search_term), artist_name, album_title, album_length) + + resultlist = __matchAlbums(searchAlbums((artist_name + ' ' + album_title).strip()), artist_name, album_title, album_length) + if resultlist: + return resultlist + + # Deezer API supports unicode, so just remove non alphanumeric characters + clean_artist_name = re.sub(r"[^\w\s]", " ", artist_name, 0, re.UNICODE).strip() + clean_album_name = re.sub(r"[^\w\s]", " ", album_title, 0, re.UNICODE).strip() + + resultlist = __matchAlbums(searchAlbums((clean_artist_name + ' ' + clean_album_name).strip()), artist_name, album_title, album_length) + if resultlist: + return resultlist + + resultlist = __matchAlbums(searchAlbums(clean_artist_name), artist_name, album_title, album_length) + if resultlist: + return resultlist + + return resultlist + + +def getTrack(sng_id, try_reload_api=True): + """Returns deezer track infos + + :param int sng_id: deezer song id + :param bool try_reload_api: whether or not try reloading API if session expired + """ + global __tracks_cache + + if str(sng_id) in __tracks_cache: + return __tracks_cache[str(sng_id)] + + data = "[{\"method\":\"song.getListData\",\"params\":{\"sng_ids\":[" + str(sng_id) + "]}}]" + json = request.request_json(url=__API_URL, headers=__HTTP_HEADERS, method='post', params=__api_queries, data=data, cookies=__cookies) + + results = [] + error = None + invalid_token = False + + if json: + # Check for errors + if 'error' in json: + error = json['error'] + if 'GATEWAY_ERROR' in json['error'] and json['error']['GATEWAY_ERROR'] == u"invalid api token": + invalid_token = True + + elif 'error' in json[0] and json[0]['error']: + error = json[0]['error'] + if 'VALID_TOKEN_REQUIRED' in json[0]['error'] and json[0]['error']['VALID_TOKEN_REQUIRED'] == u"Invalid CSRF token": + invalid_token = True + + # Got invalid token error + if error: + if invalid_token and try_reload_api: + __getApiToken() + return getTrack(sng_id, False) + else: + logger.error(u"An unknown error occurred in the Deezer track parser: %s" % error) + else: + try: + results = json[0]['results'] + item = results['data'][0] + if 'token' in item: + logger.error(u"An unknown error occurred in the Deezer parser: Uploaded Files are currently not supported") + return + + sng_id = item["SNG_ID"] + md5Origin = item["MD5_ORIGIN"] + sng_format = 3 + + if item["FILESIZE_MP3_320"] <= 0: + if item["FILESIZE_MP3_256"] > 0: + sng_format = 5 + else: + sng_format = 1 + + mediaVersion = int(item["MEDIA_VERSION"]) + item['downloadUrl'] = __getDownloadUrl(md5Origin, sng_id, sng_format, mediaVersion) + + __tracks_cache[sng_id] = item + return item + + except Exception as e: + logger.error(u"An unknown error occurred in the Deezer track parser: %s" % e) + + +def __getDownloadUrl(md5Origin, sng_id, sng_format, mediaVersion): + urlPart = md5Origin.encode('utf-8') + b'\xa4' + str(sng_format) + b'\xa4' + str(sng_id) + b'\xa4' + str(mediaVersion) + md5val = md5(urlPart).hexdigest() + urlPart = md5val + b'\xa4' + urlPart + b'\xa4' + cipher = AES.new('jo6aey6haid2Teih', AES.MODE_ECB) + ciphertext = cipher.encrypt(__pad(urlPart, AES.block_size)) + return "http://e-cdn-proxy-" + md5Origin[:1] + ".deezer.com/mobile/1/" + binascii.hexlify(ciphertext).lower() + + +def __pad(raw, block_size): + if (len(raw) % block_size == 0): + return raw + padding_required = block_size - (len(raw) % block_size) + padChar = b'\x00' + data = raw + padding_required * padChar + return data + + +def __tagTrack(path, track): + try: + album = getAlbum(track['ALB_ID']) + + f = MediaFile(path) + f.artist = track['ART_NAME'] + f.album = track['ALB_TITLE'] + f.title = track['SNG_TITLE'] + f.track = track['TRACK_NUMBER'] + f.tracktotal = album['nb_tracks'] + f.disc = track['DISK_NUMBER'] + f.bpm = track['BPM'] + f.date = datetime.strptime(album['release_date'], '%Y-%m-%d').date() + f.albumartist = album['artist']['name'] + if u'genres' in album and u'data' in album['genres']: + f.genres = [genre['name'] for genre in album['genres']['data']] + + f.save() + + except Exception as e: + logger.error(u'Unable to tag deezer track "%s": %s' % (path, e)) + + +def decryptTracks(paths): + """Decrypt downloaded deezer tracks. + + :param paths: list of path to deezer tracks (*.dzr files). + """ + # Note: tracks can be from different albums + decrypted_tracks = {} + + # First pass: load tracks data + for path in paths: + try: + album_folder = os.path.dirname(path) + sng_id = os.path.splitext(os.path.basename(path))[0] + track = getTrack(sng_id) + if track: + track_number = int(track['TRACK_NUMBER']) + disk_number = int(track['DISK_NUMBER']) + + if album_folder not in decrypted_tracks: + decrypted_tracks[album_folder] = {} + + if disk_number not in decrypted_tracks[album_folder]: + decrypted_tracks[album_folder][disk_number] = {} + + decrypted_tracks[album_folder][disk_number][track_number] = track + + except Exception as e: + logger.error(u'Unable to load deezer track infos "%s": %s' % (path, e)) + + # Second pass: decrypt tracks + for album_folder in decrypted_tracks: + multi_disks = len(decrypted_tracks[album_folder]) > 1 + for disk_number in decrypted_tracks[album_folder]: + for track_number, track in decrypted_tracks[album_folder][disk_number].items(): + try: + filename = helpers.replace_illegal_chars(track['SNG_TITLE']).strip() + filename = ('{:02d}'.format(track_number) + '_' + filename + '.mp3') + + # Add a 'cd x' sub-folder if album has more than one disk + disk_folder = os.path.join(album_folder, 'cd ' + str(disk_number)) if multi_disks else album_folder + + dest = os.path.join(disk_folder, filename).encode(headphones.SYS_ENCODING, 'replace') + + # Decrypt track if not already done + if not os.path.exists(dest): + try: + __decryptDownload(path, sng_id, dest) + __tagTrack(dest, track) + except Exception as e: + logger.error(u'Unable to decrypt deezer track "%s": %s' % (path, e)) + if os.path.exists(dest): + os.remove(dest) + decrypted_tracks[album_folder][disk_number].pop(track_number) + continue + + decrypted_tracks[album_folder][disk_number][track_number]['path'] = dest + + except Exception as e: + logger.error(u'Unable to decrypt deezer track "%s": %s' % (path, e)) + + return decrypted_tracks + + +def __decryptDownload(source, sng_id, dest): + interval_chunk = 3 + chunk_size = 2048 + blowFishKey = __getBlowFishKey(sng_id) + i = 0 + iv = "\x00\x01\x02\x03\x04\x05\x06\x07" + + dest_folder = os.path.dirname(dest) + if not os.path.exists(dest_folder): + os.makedirs(dest_folder) + + f = open(source, "rb") + fout = open(dest, "wb") + try: + chunk = f.read(chunk_size) + while chunk: + if(i % interval_chunk == 0): + cipher = Blowfish.new(blowFishKey, Blowfish.MODE_CBC, iv) + chunk = cipher.decrypt(__pad(chunk, Blowfish.block_size)) + + fout.write(chunk) + i += 1 + chunk = f.read(chunk_size) + finally: + f.close() + fout.close() + + +def __getBlowFishKey(encryptionKey): + if encryptionKey < 1: + encryptionKey *= -1 + + hashcode = md5(str(encryptionKey)).hexdigest() + hPart = hashcode[0:16] + lPart = hashcode[16:32] + parts = ['g4el58wc0zvf9na1', hPart, lPart] + + return __xorHex(parts) + + +def __xorHex(parts): + data = "" + for i in range(0, 16): + character = ord(parts[0][i]) + + for j in range(1, len(parts)): + character ^= ord(parts[j][i]) + + data += chr(character) + + return data diff --git a/headphones/postprocessor.py b/headphones/postprocessor.py index ab00ab1a..0b574cd2 100755 --- a/headphones/postprocessor.py +++ b/headphones/postprocessor.py @@ -30,7 +30,7 @@ from beetsplug import lyrics as beetslyrics from headphones import notifiers, utorrent, transmission, deluge, qbittorrent from headphones import db, albumart, librarysync from headphones import logger, helpers, mb, music_encoder -from headphones import metadata +from headphones import metadata, deezloader postprocessor_lock = threading.Lock() @@ -48,6 +48,8 @@ def checkFolder(): single = False if album['Kind'] == 'nzb': download_dir = headphones.CONFIG.DOWNLOAD_DIR + elif album['Kind'] == 'ddl': + download_dir = headphones.CONFIG.DOWNLOAD_DDL_DIR else: if headphones.CONFIG.DELUGE_DONE_DIRECTORY and headphones.CONFIG.TORRENT_DOWNLOADER == 3: download_dir = headphones.CONFIG.DELUGE_DONE_DIRECTORY @@ -205,6 +207,7 @@ def verify(albumid, albumpath, Kind=None, forced=False, keep_original_folder=Fal tracks = myDB.select('SELECT * from tracks WHERE AlbumID=?', [albumid]) downloaded_track_list = [] + downloaded_deezer_list = [] downloaded_cuecount = 0 for r, d, f in os.walk(albumpath): @@ -213,8 +216,10 @@ def verify(albumid, albumpath, Kind=None, forced=False, keep_original_folder=Fal downloaded_track_list.append(os.path.join(r, files)) elif files.lower().endswith('.cue'): downloaded_cuecount += 1 + elif files.lower().endswith('.dzr'): + downloaded_deezer_list.append(os.path.join(r, files)) # if any of the files end in *.part, we know the torrent isn't done yet. Process if forced, though - elif files.lower().endswith(('.part', '.utpart')) and not forced: + elif files.lower().endswith(('.part', '.utpart', '.aria2')) and not forced: logger.info( "Looks like " + os.path.basename(albumpath).decode(headphones.SYS_ENCODING, 'replace') + " isn't complete yet. Will try again on the next run") @@ -253,6 +258,37 @@ def verify(albumid, albumpath, Kind=None, forced=False, keep_original_folder=Fal downloaded_track_list = helpers.get_downloaded_track_list(albumpath) keep_original_folder = False + # Decrypt deezer tracks + if downloaded_deezer_list: + logger.info('Decrypting deezer tracks') + decrypted_deezer_list = deezloader.decryptTracks(downloaded_deezer_list) + + # Check if album is complete based on album duration only + # (total track numbers is not determinant enough due to hidden tracks for eg) + db_track_duration = 0 + downloaded_track_duration = 0 + try: + for track in tracks: + db_track_duration += track['TrackDuration'] / 1000 + except: + downloaded_track_duration = False + + try: + for disk_number in decrypted_deezer_list[albumpath]: + for track in decrypted_deezer_list[albumpath][disk_number].values(): + downloaded_track_list.append(track['path']) + downloaded_track_duration += int(track['DURATION']) + except: + downloaded_track_duration = False + + if not downloaded_track_duration or not db_track_duration or abs(downloaded_track_duration - db_track_duration) > 240: + logger.info("Looks like " + + os.path.basename(albumpath).decode(headphones.SYS_ENCODING, 'replace') + + " isn't complete yet (duration mismatch). Will try again on the next run") + return + + downloaded_track_list = list(set(downloaded_track_list)) # Remove duplicates + # test #1: metadata - usually works logger.debug('Verifying metadata...') diff --git a/headphones/searcher.py b/headphones/searcher.py index b0652f7f..f6554cb1 100644 --- a/headphones/searcher.py +++ b/headphones/searcher.py @@ -36,6 +36,7 @@ import headphones from headphones.common import USER_AGENT from headphones import logger, db, helpers, classes, sab, nzbget, request from headphones import utorrent, transmission, notifiers, rutracker, deluge, qbittorrent +from headphones import deezloader, aria2 from bencode import bencode, bdecode # Magnet to torrent services, for Black hole. Stolen from CouchPotato. @@ -51,6 +52,27 @@ ruobj = None # Persistent RED API object redobj = None +# Persistent Aria2 RPC object +__aria2rpc_obj = None + + +def getAria2RPC(): + global __aria2rpc_obj + if not __aria2rpc_obj: + __aria2rpc_obj = aria2.Aria2JsonRpc( + ID='headphones', + uri=headphones.CONFIG.ARIA_HOST, + token=headphones.CONFIG.ARIA_TOKEN if headphones.CONFIG.ARIA_TOKEN else None, + http_user=headphones.CONFIG.ARIA_USERNAME if headphones.CONFIG.ARIA_USERNAME else None, + http_passwd=headphones.CONFIG.ARIA_PASSWORD if headphones.CONFIG.ARIA_PASSWORD else None + ) + return __aria2rpc_obj + + +def reconfigure(): + global __aria2rpc_obj + __aria2rpc_obj = None + def fix_url(s, charset="utf-8"): """ @@ -281,32 +303,53 @@ def do_sorted_search(album, new, losslessOnly, choose_specific_download=False): headphones.CONFIG.STRIKE or headphones.CONFIG.TQUATTRECENTONZE) - results = [] + DDL_PROVIDERS = (headphones.CONFIG.DEEZLOADER) + myDB = db.DBConnection() albumlength = myDB.select('SELECT sum(TrackDuration) from tracks WHERE AlbumID=?', [album['AlbumID']])[0][0] + nzb_results = None + torrent_results = None + ddl_results = None + if headphones.CONFIG.PREFER_TORRENTS == 0 and not choose_specific_download: if NZB_PROVIDERS and NZB_DOWNLOADERS: - results = searchNZB(album, new, losslessOnly, albumlength) + nzb_results = searchNZB(album, new, losslessOnly, albumlength) - if not results and TORRENT_PROVIDERS: - results = searchTorrent(album, new, losslessOnly, albumlength) + if not nzb_results: + if DDL_PROVIDERS: + ddl_results = searchDdl(album, new, losslessOnly, albumlength) + + if TORRENT_PROVIDERS: + torrent_results = searchTorrent(album, new, losslessOnly, albumlength) elif headphones.CONFIG.PREFER_TORRENTS == 1 and not choose_specific_download: if TORRENT_PROVIDERS: - results = searchTorrent(album, new, losslessOnly, albumlength) + torrent_results = searchTorrent(album, new, losslessOnly, albumlength) - if not results and NZB_PROVIDERS and NZB_DOWNLOADERS: - results = searchNZB(album, new, losslessOnly, albumlength) + if not torrent_results: + if DDL_PROVIDERS: + ddl_results = searchDdl(album, new, losslessOnly, albumlength) + + if NZB_PROVIDERS and NZB_DOWNLOADERS: + nzb_results = searchNZB(album, new, losslessOnly, albumlength) + + elif headphones.CONFIG.PREFER_TORRENTS == 3 and not choose_specific_download: + + if DDL_PROVIDERS: + ddl_results = searchDdl(album, new, losslessOnly, albumlength) + + if not ddl_results: + if TORRENT_PROVIDERS: + torrent_results = searchTorrent(album, new, losslessOnly, albumlength) + + if NZB_PROVIDERS and NZB_DOWNLOADERS: + nzb_results = searchNZB(album, new, losslessOnly, albumlength) else: - - nzb_results = None - torrent_results = None - if NZB_PROVIDERS and NZB_DOWNLOADERS: nzb_results = searchNZB(album, new, losslessOnly, albumlength, choose_specific_download) @@ -314,13 +357,19 @@ def do_sorted_search(album, new, losslessOnly, choose_specific_download=False): torrent_results = searchTorrent(album, new, losslessOnly, albumlength, choose_specific_download) - if not nzb_results: - nzb_results = [] + if DDL_PROVIDERS: + ddl_results = searchDdl(album, new, losslessOnly, albumlength, choose_specific_download) - if not torrent_results: - torrent_results = [] + if not nzb_results: + nzb_results = [] - results = nzb_results + torrent_results + if not torrent_results: + torrent_results = [] + + if not ddl_results: + ddl_results = [] + + results = nzb_results + torrent_results + ddl_results if choose_specific_download: return results @@ -826,6 +875,31 @@ def send_to_downloader(data, bestqual, album): except Exception as e: logger.error('Couldn\'t write NZB file: %s', e) return + + elif kind == 'ddl': + folder_name = '%s - %s [%s]' % ( + helpers.latinToAscii(album['ArtistName']).encode('UTF-8').replace('/', '_'), + helpers.latinToAscii(album['AlbumTitle']).encode('UTF-8').replace('/', '_'), + get_year_from_release_date(album['ReleaseDate'])) + + # Aria2 downloader + if headphones.CONFIG.DDL_DOWNLOADER == 0: + logger.info("Sending download to Aria2") + + try: + deezer_album = deezloader.getAlbumByLink(bestqual[2]) + + for album_track in deezer_album['tracks']['data']: + track = deezloader.getTrack(album_track['id']) + if track: + filename = track['SNG_ID'] + '.dzr' + logger.debug(u'Sending song "%s" to Aria' % track['SNG_TITLE']) + getAria2RPC().addUri([track['downloadUrl']], {'out': filename, 'auto-file-renaming': 'false', 'continue': 'true', 'dir': folder_name}) + + except Exception as e: + logger.error(u'Error sending torrent to Aria2. Are you sure it\'s running? (%s)' % e) + return + else: folder_name = '%s - %s [%s]' % ( helpers.latinToAscii(album['ArtistName']).encode('UTF-8').replace('/', '_'), @@ -1204,6 +1278,62 @@ def verifyresult(title, artistterm, term, lossless): return True +def searchDdl(album, new=False, losslessOnly=False, albumlength=None, + choose_specific_download=False): + reldate = album['ReleaseDate'] + year = get_year_from_release_date(reldate) + + # MERGE THIS WITH THE TERM CLEANUP FROM searchNZB + dic = {'...': '', ' & ': ' ', ' = ': ' ', '?': '', '$': 's', ' + ': ' ', '"': '', ',': ' ', + '*': ''} + + semi_cleanalbum = helpers.replace_all(album['AlbumTitle'], dic) + cleanalbum = helpers.latinToAscii(semi_cleanalbum) + semi_cleanartist = helpers.replace_all(album['ArtistName'], dic) + cleanartist = helpers.latinToAscii(semi_cleanartist) + + # Use provided term if available, otherwise build our own (this code needs to be cleaned up since a lot + # of these torrent providers are just using cleanartist/cleanalbum terms + if album['SearchTerm']: + term = album['SearchTerm'] + elif album['Type'] == 'part of': + term = cleanalbum + " " + year + else: + # FLAC usually doesn't have a year for some reason so I'll leave it out + # Various Artist albums might be listed as VA, so I'll leave that out too + # Only use the year if the term could return a bunch of different albums, i.e. self-titled albums + if album['ArtistName'] in album['AlbumTitle'] or len(album['ArtistName']) < 4 or len( + album['AlbumTitle']) < 4: + term = cleanartist + ' ' + cleanalbum + ' ' + year + elif album['ArtistName'] == 'Various Artists': + term = cleanalbum + ' ' + year + else: + term = cleanartist + ' ' + cleanalbum + + # Replace bad characters in the term and unicode it + term = re.sub('[\.\-\/]', ' ', term).encode('utf-8') + artistterm = re.sub('[\.\-\/]', ' ', cleanartist).encode('utf-8', 'replace') + + logger.debug(u'Using search term: "%s"' % helpers.latinToAscii(term)) + + resultlist = [] + + # Deezer only provides lossy + if headphones.CONFIG.DEEZLOADER and not losslessOnly: + resultlist += deezloader.searchAlbum(album['ArtistName'], album['AlbumTitle'], album['SearchTerm'], int(albumlength / 1000)) + + # attempt to verify that this isn't a substring result + # when looking for "Foo - Foo" we don't want "Foobar" + # this should be less of an issue when it isn't a self-titled album so we'll only check vs artist + results = [result for result in resultlist if verifyresult(result[0], artistterm, term, losslessOnly)] + + # Additional filtering for size etc + if results and not choose_specific_download: + results = more_filtering(results, album, albumlength, new) + + return results + + def searchTorrent(album, new=False, losslessOnly=False, albumlength=None, choose_specific_download=False): global apolloobj # persistent apollo.rip api object to reduce number of login attempts @@ -2036,7 +2166,10 @@ def searchTorrent(album, new=False, losslessOnly=False, albumlength=None, def preprocess(resultlist): for result in resultlist: - if result[4] == 'torrent': + if result[3] == deezloader.PROVIDER_NAME: + return True, result + + elif result[4] == 'torrent': # rutracker always needs the torrent data if result[3] == 'rutracker.org': diff --git a/headphones/webserve.py b/headphones/webserve.py index 855fed24..7e97cefb 100644 --- a/headphones/webserve.py +++ b/headphones/webserve.py @@ -1174,6 +1174,10 @@ class WebInterface(object): "utorrent_username": headphones.CONFIG.UTORRENT_USERNAME, "utorrent_password": headphones.CONFIG.UTORRENT_PASSWORD, "utorrent_label": headphones.CONFIG.UTORRENT_LABEL, + "aria_host": headphones.CONFIG.ARIA_HOST, + "aria_password": headphones.CONFIG.ARIA_PASSWORD, + "aria_token": headphones.CONFIG.ARIA_TOKEN, + "aria_username": headphones.CONFIG.ARIA_USERNAME, "nzb_downloader_sabnzbd": radio(headphones.CONFIG.NZB_DOWNLOADER, 0), "nzb_downloader_nzbget": radio(headphones.CONFIG.NZB_DOWNLOADER, 1), "nzb_downloader_blackhole": radio(headphones.CONFIG.NZB_DOWNLOADER, 2), @@ -1182,6 +1186,7 @@ class WebInterface(object): "torrent_downloader_utorrent": radio(headphones.CONFIG.TORRENT_DOWNLOADER, 2), "torrent_downloader_deluge": radio(headphones.CONFIG.TORRENT_DOWNLOADER, 3), "torrent_downloader_qbittorrent": radio(headphones.CONFIG.TORRENT_DOWNLOADER, 4), + "ddl_downloader_aria": radio(headphones.CONFIG.DDL_DOWNLOADER, 0), "download_dir": headphones.CONFIG.DOWNLOAD_DIR, "use_blackhole": checked(headphones.CONFIG.BLACKHOLE), "blackhole_dir": headphones.CONFIG.BLACKHOLE_DIR, @@ -1243,6 +1248,8 @@ class WebInterface(object): "use_tquattrecentonze": checked(headphones.CONFIG.TQUATTRECENTONZE), "tquattrecentonze_user": headphones.CONFIG.TQUATTRECENTONZE_USER, "tquattrecentonze_password": headphones.CONFIG.TQUATTRECENTONZE_PASSWORD, + "download_ddl_dir": headphones.CONFIG.DOWNLOAD_DDL_DIR, + "use_deezloader": checked(headphones.CONFIG.DEEZLOADER), "pref_qual_0": radio(headphones.CONFIG.PREFERRED_QUALITY, 0), "pref_qual_1": radio(headphones.CONFIG.PREFERRED_QUALITY, 1), "pref_qual_2": radio(headphones.CONFIG.PREFERRED_QUALITY, 2), @@ -1288,6 +1295,7 @@ class WebInterface(object): "prefer_torrents_0": radio(headphones.CONFIG.PREFER_TORRENTS, 0), "prefer_torrents_1": radio(headphones.CONFIG.PREFER_TORRENTS, 1), "prefer_torrents_2": radio(headphones.CONFIG.PREFER_TORRENTS, 2), + "prefer_torrents_3": radio(headphones.CONFIG.PREFER_TORRENTS, 3), "magnet_links_0": radio(headphones.CONFIG.MAGNET_LINKS, 0), "magnet_links_1": radio(headphones.CONFIG.MAGNET_LINKS, 1), "magnet_links_2": radio(headphones.CONFIG.MAGNET_LINKS, 2), @@ -1446,7 +1454,7 @@ class WebInterface(object): "launch_browser", "enable_https", "api_enabled", "use_blackhole", "headphones_indexer", "use_newznab", "newznab_enabled", "use_torznab", "torznab_enabled", "use_nzbsorg", "use_omgwtfnzbs", "use_kat", "use_piratebay", "use_oldpiratebay", - "use_mininova", "use_waffles", "use_rutracker", + "use_mininova", "use_waffles", "use_rutracker", "use_deezloader", "use_apollo", "use_redacted", "use_strike", "use_tquattrecentonze", "preferred_bitrate_allow_lossless", "detect_bitrate", "ignore_clean_releases", "freeze_db", "cue_split", "move_files", "rename_files", "correct_metadata", "cleanup_files", "keep_nfo", "add_album_art", @@ -1580,6 +1588,9 @@ class WebInterface(object): # Reconfigure musicbrainz database connection with the new values mb.startmb() + # Reconfigure Aria2 + searcher.reconfigure() + raise cherrypy.HTTPRedirect("config") @cherrypy.expose