diff --git a/data/interfaces/default/config.html b/data/interfaces/default/config.html index 841df0d1..bd331f2c 100644 --- a/data/interfaces/default/config.html +++ b/data/interfaces/default/config.html @@ -304,6 +304,45 @@ + +
+ Direct Download + Aria2 +
+
+
+ + + usually http://localhost:6800/jsonrpc +
+
+ + +
+
+ + +
+
+ + +
+
+ + + Full path where ddl client downloads your music, e.g. /Users/name/Downloads/music +
+
@@ -461,6 +500,7 @@ NZBs Torrents + Direct Download No Preference
@@ -573,6 +613,21 @@ + +
+ Direct Download + +
+
+ + +
+
+ +
@@ -2405,6 +2460,11 @@ $("#deluge_options").show(); } + if ($("#ddl_downloader_aria").is(":checked")) + { + $("#ddl_aria_options").show(); + } + $('input[type=radio]').change(function(){ if ($("#preferred_bitrate").is(":checked")) { @@ -2458,6 +2518,9 @@ { $("#torrent_blackhole_options,#utorrent_options,#transmission_options,#qbittorrent_options").fadeOut("fast", function() { $("#deluge_options").fadeIn() }); } + if ($("#ddl_downloader_aria").is(":checked")) + { + } }); $("#mirror").change(handleNewServerSelection); @@ -2560,6 +2623,7 @@ initConfigCheckbox("#enable_https"); initConfigCheckbox("#customauth"); initConfigCheckbox("#use_tquattrecentonze"); + initConfigCheckbox("#use_deezloader"); $('#twitterStep1').click(function () { diff --git a/headphones/aria2.py b/headphones/aria2.py new file mode 100644 index 00000000..06d8e4e0 --- /dev/null +++ b/headphones/aria2.py @@ -0,0 +1,979 @@ +# -*- coding: utf8 -*- +# Copyright (C) 2012-2016 Xyne +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# (version 2) as published by the Free Software Foundation. +# +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +from __future__ import with_statement +import base64 +import json +import math +import os +import ssl +import string +import time +import httplib +import urllib2 + +from headphones import logger + +# ################################ Constants ################################### + +DEFAULT_PORT = 6800 +SERVER_URI_FORMAT = '{}://{}:{:d}/jsonrpc' + +# Status values for unfinished downloads. +TEMPORARY_STATUS = ('active', 'waiting', 'paused') +# Status values for finished downloads. +FINAL_STATUS = ('complete', 'error') + +ARIA2_CONTROL_FILE_EXT = '.aria2' + +# ########################## Convenience Functions ############################# + + +def to_json_list(objs): + ''' + Wrap strings in lists. Other iterables are converted to lists directly. + ''' + if isinstance(objs, str): + return [objs] + elif not isinstance(objs, list): + return list(objs) + else: + return objs + + +def add_options_and_position(params, options=None, position=None): + ''' + Convenience method for adding options and position to parameters. + ''' + if options: + params.append(options) + if position: + if not isinstance(position, int): + try: + position = int(position) + except ValueError: + position = -1 + if position >= 0: + params.append(position) + return params + + +def get_status(response): + ''' + Process a status response. + ''' + if response: + try: + return response['status'] + except KeyError: + logger.error('no status returned from Aria2 RPC server') + return 'error' + else: + logger.error('no response from server') + return 'error' + + +def random_token(length, valid_chars=None): + ''' + Get a random secret token for the Aria2 RPC server. + + length: + The length of the token + + valid_chars: + A list or other ordered and indexable iterable of valid characters. If not + given of None, asciinumberic characters with some punctuation characters + will be used. + ''' + if not valid_chars: + valid_chars = string.ascii_letters + string.digits + '!@#$%^&*()-_=+' + number_of_chars = len(valid_chars) + bytes_to_read = math.ceil(math.log(number_of_chars) / math.log(0x100)) + max_value = 0x100**bytes_to_read + max_index = number_of_chars - 1 + token = '' + for _ in range(length): + value = int.from_bytes(os.urandom(bytes_to_read), byteorder='little') + index = round((value * max_index) / max_value) + token += valid_chars[index] + return token + +# ################ From python3-aur's ThreadedServers.common ################### + + +def format_bytes(size): + '''Format bytes for inferior humans.''' + if size < 0x400: + return '{:d} B'.format(size) + else: + size = float(size) / 0x400 + for prefix in ('KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB'): + if size < 0x400: + return '{:0.02f} {}'.format(size, prefix) + else: + size /= 0x400 + return '{:0.02f} YiB'.format(size) + + +def format_seconds(s): + '''Format seconds for inferior humans.''' + string = '' + for base, char in ( + (60, 's'), + (60, 'm'), + (24, 'h') + ): + s, r = divmod(s, base) + if s == 0: + return '{:d}{}{}'.format(r, char, string) + elif r != 0: + string = '{:02d}{}{}'.format(r, char, string) + else: + return '{:d}d{}'.format(s, string) + +# ############################ Aria2JsonRpcError ############################### + + +class Aria2JsonRpcError(Exception): + def __init__(self, msg, connection_error=False): + super(self.__class__, self).__init__(self, msg) + self.connection_error = connection_error + +# ############################ Aria2JsonRpc Class ############################## + + +class Aria2JsonRpc(object): + ''' + Interface class for interacting with an Aria2 RPC server. + ''' + # TODO: certificate options, etc. + def __init__( + self, ID, uri, + mode='normal', + token=None, + http_user=None, http_passwd=None, + server_cert=None, client_cert=None, client_cert_password=None, + ssl_protocol=None, + setup_function=None + ): + ''' + ID: the ID to send to the RPC interface + + uri: the URI of the RPC interface + + mode: + normal - process requests immediately + batch - queue requests (run with "process_queue") + format - return RPC request objects + + token: + RPC method-level authorization token (set using `--rpc-secret`) + + http_user, http_password: + HTTP Basic authentication credentials (deprecated) + + server_cert: + server certificate for HTTPS connections + + client_cert: + client certificate for HTTPS connections + + client_cert_password: + prompt for client certificate password + + ssl_protocol: + SSL protocol from the ssl module + + setup_function: + A function to invoke prior to the first server call. This could be the + launch() method of an Aria2RpcServer instance, for example. This attribute + is set automatically in instances returned from Aria2RpcServer.get_a2jr() + ''' + self.id = ID + self.uri = uri + self.mode = mode + self.queue = [] + self.handlers = dict() + self.token = token + self.setup_function = setup_function + + if None not in (http_user, http_passwd): + self.add_HTTPBasicAuthHandler(http_user, http_passwd) + + if server_cert or client_cert: + self.add_HTTPSHandler( + server_cert=server_cert, + client_cert=client_cert, + client_cert_password=client_cert_password, + protocol=ssl_protocol + ) + + self.update_opener() + + def iter_handlers(self): + ''' + Iterate over handlers. + ''' + for name in ('HTTPS', 'HTTPBasicAuth'): + try: + yield self.handlers[name] + except KeyError: + pass + + def update_opener(self): + ''' + Build an opener from the current handlers. + ''' + self.opener = urllib2.build_opener(*self.iter_handlers()) + + def remove_handler(self, name): + ''' + Remove a handler. + ''' + try: + del self.handlers[name] + except KeyError: + pass + + def add_HTTPBasicAuthHandler(self, user, passwd): + ''' + Add a handler for HTTP Basic authentication. + + If either user or passwd are None, the handler is removed. + ''' + handler = urllib2.HTTPBasicAuthHandler() + handler.add_password( + realm='aria2', + uri=self.uri, + user=user, + passwd=passwd, + ) + self.handlers['HTTPBasicAuth'] = handler + + def remove_HTTPBasicAuthHandler(self): + self.remove_handler('HTTPBasicAuth') + + def add_HTTPSHandler( + self, + server_cert=None, + client_cert=None, + client_cert_password=None, + protocol=None, + ): + ''' + Add a handler for HTTPS connections with optional server and client + certificates. + ''' + if not protocol: + protocol = ssl.PROTOCOL_TLSv1 +# protocol = ssl.PROTOCOL_TLSv1_1 # openssl 1.0.1+ +# protocol = ssl.PROTOCOL_TLSv1_2 # Python 3.4+ + context = ssl.SSLContext(protocol) + + if server_cert: + context.verify_mode = ssl.CERT_REQUIRED + context.load_verify_locations(cafile=server_cert) + else: + context.verify_mode = ssl.CERT_OPTIONAL + + if client_cert: + context.load_cert_chain(client_cert, password=client_cert_password) + + self.handlers['HTTPS'] = urllib2.HTTPSHandler( + context=context, + check_hostname=False + ) + + def remove_HTTPSHandler(self): + self.remove_handler('HTTPS') + + def send_request(self, req_obj): + ''' + Send the request and return the response. + ''' + if self.setup_function: + self.setup_function() + self.setup_function = None + logger.debug("Aria req_obj: %s" % json.dumps(req_obj, indent=2, sort_keys=True)) + req = json.dumps(req_obj).encode('UTF-8') + try: + f = self.opener.open(self.uri, req) + obj = json.loads(f.read()) + try: + return obj['result'] + except KeyError: + raise Aria2JsonRpcError('unexpected result: {}'.format(obj)) + except (urllib2.URLError) as e: + # This should work but URLError does not set the errno attribute: + # e.errno == errno.ECONNREFUSED + raise Aria2JsonRpcError( + str(e), + connection_error=( + '111' in str(e) + ) + ) + except httplib.BadStatusLine as e: + raise Aria2JsonRpcError('{}: BadStatusLine: {} (HTTPS error?)'.format( + self.__class__.__name__, e + )) + + def jsonrpc(self, method, params=None, prefix='aria2.'): + ''' + POST a request to the RPC interface. + ''' + if not params: + params = [] + + if self.token is not None: + token_str = 'token:{}'.format(self.token) + if method == 'multicall': + for p in params[0]: + try: + p['params'].insert(0, token_str) + except KeyError: + p['params'] = [token_str] + else: + params.insert(0, token_str) + + req_obj = { + 'jsonrpc': '2.0', + 'id': self.id, + 'method': prefix + method, + 'params': params, + } + if self.mode == 'batch': + self.queue.append(req_obj) + return None + elif self.mode == 'format': + return req_obj + else: + return self.send_request(req_obj) + + def process_queue(self): + ''' + Processed queued requests. + ''' + req_obj = self.queue + self.queue = [] + return self.send_request(req_obj) + + +# ############################# Standard Methods ############################### + def addUri(self, uris, options=None, position=None): + ''' + aria2.addUri method + + uris: list of URIs + + options: dictionary of additional options + + position: position in queue + + Returns a GID + ''' + params = [uris] + params = add_options_and_position(params, options, position) + return self.jsonrpc('addUri', params) + + def addTorrent(self, torrent, uris=None, options=None, position=None): + ''' + aria2.addTorrent method + + torrent: base64-encoded torrent file + + uris: list of webseed URIs + + options: dictionary of additional options + + position: position in queue + + Returns a GID. + ''' + params = [torrent] + if uris: + params.append(uris) + params = add_options_and_position(params, options, position) + return self.jsonrpc('addTorrent', params) + + def addMetalink(self, metalink, options=None, position=None): + ''' + aria2.addMetalink method + + metalink: base64-encoded metalink file + + options: dictionary of additional options + + position: position in queue + + Returns an array of GIDs. + ''' + params = [metalink] + params = add_options_and_position(params, options, position) + return self.jsonrpc('addTorrent', params) + + def remove(self, gid): + ''' + aria2.remove method + + gid: GID to remove + ''' + params = [gid] + return self.jsonrpc('remove', params) + + def forceRemove(self, gid): + ''' + aria2.forceRemove method + + gid: GID to remove + ''' + params = [gid] + return self.jsonrpc('forceRemove', params) + + def pause(self, gid): + ''' + aria2.pause method + + gid: GID to pause + ''' + params = [gid] + return self.jsonrpc('pause', params) + + def pauseAll(self): + ''' + aria2.pauseAll method + ''' + return self.jsonrpc('pauseAll') + + def forcePause(self, gid): + ''' + aria2.forcePause method + + gid: GID to pause + ''' + params = [gid] + return self.jsonrpc('forcePause', params) + + def forcePauseAll(self): + ''' + aria2.forcePauseAll method + ''' + return self.jsonrpc('forcePauseAll') + + def unpause(self, gid): + ''' + aria2.unpause method + + gid: GID to unpause + ''' + params = [gid] + return self.jsonrpc('unpause', params) + + def unpauseAll(self): + ''' + aria2.unpauseAll method + ''' + return self.jsonrpc('unpauseAll') + + def tellStatus(self, gid, keys=None): + ''' + aria2.tellStatus method + + gid: GID to query + + keys: subset of status keys to return (all keys are returned otherwise) + + Returns a dictionary. + ''' + params = [gid] + if keys: + params.append(keys) + return self.jsonrpc('tellStatus', params) + + def getUris(self, gid): + ''' + aria2.getUris method + + gid: GID to query + + Returns a list of dictionaries. + ''' + params = [gid] + return self.jsonrpc('getUris', params) + + def getFiles(self, gid): + ''' + aria2.getFiles method + + gid: GID to query + + Returns a list of dictionaries. + ''' + params = [gid] + return self.jsonrpc('getFiles', params) + + def getPeers(self, gid): + ''' + aria2.getPeers method + + gid: GID to query + + Returns a list of dictionaries. + ''' + params = [gid] + return self.jsonrpc('getPeers', params) + + def getServers(self, gid): + ''' + aria2.getServers method + + gid: GID to query + + Returns a list of dictionaries. + ''' + params = [gid] + return self.jsonrpc('getServers', params) + + def tellActive(self, keys=None): + ''' + aria2.tellActive method + + keys: same as tellStatus + + Returns a list of dictionaries. The dictionaries are the same as those + returned by tellStatus. + ''' + if keys: + params = [keys] + else: + params = None + return self.jsonrpc('tellActive', params) + + def tellWaiting(self, offset, num, keys=None): + ''' + aria2.tellWaiting method + + offset: offset from start of waiting download queue + (negative values are counted from the end of the queue) + + num: number of downloads to return + + keys: same as tellStatus + + Returns a list of dictionaries. The dictionaries are the same as those + returned by tellStatus. + ''' + params = [offset, num] + if keys: + params.append(keys) + return self.jsonrpc('tellWaiting', params) + + def tellStopped(self, offset, num, keys=None): + ''' + aria2.tellStopped method + + offset: offset from oldest download (same semantics as tellWaiting) + + num: same as tellWaiting + + keys: same as tellStatus + + Returns a list of dictionaries. The dictionaries are the same as those + returned by tellStatus. + ''' + params = [offset, num] + if keys: + params.append(keys) + return self.jsonrpc('tellStopped', params) + + def changePosition(self, gid, pos, how): + ''' + aria2.changePosition method + + gid: GID to change + + pos: the position + + how: "POS_SET", "POS_CUR" or "POS_END" + ''' + params = [gid, pos, how] + return self.jsonrpc('changePosition', params) + + def changeUri(self, gid, fileIndex, delUris, addUris, position=None): + ''' + aria2.changePosition method + + gid: GID to change + + fileIndex: file to affect (1-based) + + delUris: URIs to remove + + addUris: URIs to add + + position: where URIs are inserted, after URIs have been removed + ''' + params = [gid, fileIndex, delUris, addUris] + if position: + params.append(position) + return self.jsonrpc('changePosition', params) + + def getOption(self, gid): + ''' + aria2.getOption method + + gid: GID to query + + Returns a dictionary of options. + ''' + params = [gid] + return self.jsonrpc('getOption', params) + + def changeOption(self, gid, options): + ''' + aria2.changeOption method + + gid: GID to change + + options: dictionary of new options + (not all options can be changed for active downloads) + ''' + params = [gid, options] + return self.jsonrpc('changeOption', params) + + def getGlobalOption(self): + ''' + aria2.getGlobalOption method + + Returns a dictionary. + ''' + return self.jsonrpc('getGlobalOption') + + def changeGlobalOption(self, options): + ''' + aria2.changeGlobalOption method + + options: dictionary of new options + ''' + params = [options] + return self.jsonrpc('changeGlobalOption', params) + + def getGlobalStat(self): + ''' + aria2.getGlobalStat method + + Returns a dictionary. + ''' + return self.jsonrpc('getGlobalStat') + + def purgeDownloadResult(self): + ''' + aria2.purgeDownloadResult method + ''' + self.jsonrpc('purgeDownloadResult') + + def removeDownloadResult(self, gid): + ''' + aria2.removeDownloadResult method + + gid: GID to remove + ''' + params = [gid] + return self.jsonrpc('removeDownloadResult', params) + + def getVersion(self): + ''' + aria2.getVersion method + + Returns a dictionary. + ''' + return self.jsonrpc('getVersion') + + def getSessionInfo(self): + ''' + aria2.getSessionInfo method + + Returns a dictionary. + ''' + return self.jsonrpc('getSessionInfo') + + def shutdown(self): + ''' + aria2.shutdown method + ''' + return self.jsonrpc('shutdown') + + def forceShutdown(self): + ''' + aria2.forceShutdown method + ''' + return self.jsonrpc('forceShutdown') + + def multicall(self, methods): + ''' + aria2.multicall method + + methods: list of dictionaries (keys: methodName, params) + + The method names must be those used by Aria2c, e.g. "aria2.tellStatus". + ''' + return self.jsonrpc('multicall', [methods], prefix='system.') + + +# ########################### Convenience Methods ############################## + def add_torrent(self, path, uris=None, options=None, position=None): + ''' + A wrapper around addTorrent for loading files. + ''' + with open(path, 'r') as f: + torrent = base64.encode(f.read()) + return self.addTorrent(torrent, uris, options, position) + + def add_metalink(self, path, uris=None, options=None, position=None): + ''' + A wrapper around addMetalink for loading files. + ''' + with open(path, 'r') as f: + metalink = base64.encode(f.read()) + return self.addMetalink(metalink, uris, options, position) + + def get_status(self, gid): + ''' + Get the status of a single GID. + ''' + response = self.tellStatus(gid, ['status']) + return get_status(response) + + def wait_for_final_status(self, gid, interval=1): + ''' + Wait for a GID to complete or fail and return its status. + ''' + if not interval or interval < 0: + interval = 1 + while True: + status = self.get_status(gid) + if status in TEMPORARY_STATUS: + time.sleep(interval) + else: + return status + + def get_statuses(self, gids): + ''' + Get the status of multiple GIDs. The status of each is yielded in order. + ''' + methods = [ + { + 'methodName': 'aria2.tellStatus', + 'params': [gid, ['gid', 'status']] + } + for gid in gids + ] + results = self.multicall(methods) + if results: + status = dict((r[0]['gid'], r[0]['status']) for r in results) + for gid in gids: + try: + yield status[gid] + except KeyError: + logger.error('Aria2 RPC server returned no status for GID {}'.format(gid)) + yield 'error' + else: + logger.error('no response from Aria2 RPC server') + for gid in gids: + yield 'error' + + def wait_for_final_statuses(self, gids, interval=1): + ''' + Wait for multiple GIDs to complete or fail and return their statuses in + order. + + gids: + A flat list of GIDs. + ''' + if not interval or interval < 0: + interval = 1 + statusmap = dict((g, None) for g in gids) + remaining = list( + g for g, s in statusmap.items() if s is None + ) + while remaining: + for g, s in zip(remaining, self.get_statuses(remaining)): + if s in TEMPORARY_STATUS: + continue + else: + statusmap[g] = s + remaining = list( + g for g, s in statusmap.items() if s is None + ) + if remaining: + time.sleep(interval) + for g in gids: + yield statusmap[g] + + def print_global_status(self): + ''' + Print global status of the RPC server. + ''' + status = self.getGlobalStat() + if status: + numWaiting = int(status['numWaiting']) + numStopped = int(status['numStopped']) + keys = ['totalLength', 'completedLength'] + total = self.tellActive(keys) + waiting = self.tellWaiting(0, numWaiting, keys) + if waiting: + total += waiting + stopped = self.tellStopped(0, numStopped, keys) + if stopped: + total += stopped + + downloadSpeed = int(status['downloadSpeed']) + uploadSpeed = int(status['uploadSpeed']) + totalLength = sum(int(x['totalLength']) for x in total) + completedLength = sum(int(x['completedLength']) for x in total) + remaining = totalLength - completedLength + + status['downloadSpeed'] = format_bytes(downloadSpeed) + '/s' + status['uploadSpeed'] = format_bytes(uploadSpeed) + '/s' + + preordered = ('downloadSpeed', 'uploadSpeed') + + rows = list() + for k in sorted(status): + if k in preordered: + continue + rows.append((k, status[k])) + + rows.extend((x, status[x]) for x in preordered) + + if totalLength > 0: + rows.append(('total', format(format_bytes(totalLength)))) + rows.append(('completed', format(format_bytes(completedLength)))) + rows.append(('remaining', format(format_bytes(remaining)))) + if completedLength == totalLength: + eta = 'finished' + else: + try: + eta = format_seconds(remaining // downloadSpeed) + except ZeroDivisionError: + eta = 'never' + rows.append(('ETA', eta)) + + l = max(len(r[0]) for r in rows) + r = max(len(r[1]) for r in rows) + r = max(r, len(self.uri) - (l + 2)) + fmt = '{:<' + str(l) + 's} {:>' + str(r) + 's}' + + print(self.uri) + for k, v in rows: + print(fmt.format(k, v)) + + def queue_uris(self, uris, options, interval=None): + ''' + Enqueue URIs and wait for download to finish while printing status at + regular intervals. + ''' + gid = self.addUri(uris, options) + print('GID: {}'.format(gid)) + + if gid and interval is not None: + blanker = '' + while True: + response = self.tellStatus(gid, ['status']) + if response: + try: + status = response['status'] + except KeyError: + print('error: no status returned from Aria2 RPC server') + break + print('{}\rstatus: {}'.format(blanker, status)), + blanker = ' ' * len(status) + if status in TEMPORARY_STATUS: + time.sleep(interval) + else: + break + else: + print('error: no response from server') + break + + +# ####################### Polymethod download handlers ######################### + def polymethod_enqueue_many(self, downloads): + ''' + Enqueue downloads. + + downloads: Same as polymethod_download(). + ''' + methods = list( + { + 'methodName': 'aria2.{}'.format(d[0]), + 'params': list(d[1:]) + } for d in downloads + ) + return self.multicall(methods) + + def polymethod_wait_many(self, gids, interval=1): + ''' + Wait for the GIDs to complete or fail and return their statuses. + + gids: + A list of lists of GIDs. + ''' + # The flattened list of GIDs + gs = list(g for gs in gids for g in gs) + statusmap = dict(tuple(zip( + gs, + self.wait_for_final_statuses(gs, interval=interval) + ))) + for gs in gids: + yield list(statusmap.get(g, 'error') for g in gs) + + def polymethod_enqueue_one(self, download): + ''' + Same as polymethod_enqueue_many but for one element. + ''' + return getattr(self, download[0])(*download[1:]) + + def polymethod_download(self, downloads, interval=1): + ''' + Enqueue a series of downloads and wait for them to finish. Iterate over the + status of each, in order. + + downloads: + An iterable over (, , ...) where indicates the "add" + method to use ('addUri', 'addTorrent', 'addMetalink') and everything that + follows are arguments to pass to that method. + + interval: + The status check interval while waiting. + + Iterates over the download status of finished downloads. "complete" + indicates success. Lists of statuses will be returned for downloads that + create multiple GIDs (e.g. metalinks). + ''' + gids = self.polymethod_enqueue_many(downloads) + return self.polymethod_wait_many(gids, interval=interval) + + def polymethod_download_bool(self, *args, **kwargs): + ''' + A wrapper around polymethod_download() which returns a boolean for each + download to indicate success (True) or failure (False). + ''' +# for status in self.polymethod_download(*args, **kwargs): +# yield all(s == 'complete' for s in status) + return list( + all(s == 'complete' for s in status) + for status in self.polymethod_download(*args, **kwargs) + ) diff --git a/headphones/classes.py b/headphones/classes.py index 6015a0f2..54a10a55 100644 --- a/headphones/classes.py +++ b/headphones/classes.py @@ -21,6 +21,7 @@ import urllib from common import USER_AGENT +from collections import OrderedDict class HeadphonesURLopener(urllib.FancyURLopener): @@ -135,3 +136,33 @@ class Proper: def __str__(self): return str(self.date) + " " + self.name + " " + str(self.season) + "x" + str( self.episode) + " of " + str(self.tvdbid) + + +class OptionalImport(object): + ''' + Dummy class for optional import (imports needed for optional features). + ''' + def __init__(self, name): + self.__name = name + + def __getattr__(self, attr): + raise ImportError('The following package is required to use this feature: {0}'.format(self.__name)) + + +class CacheDict(OrderedDict): + ''' + Ordered dictionary with fixed size, designed for caching. + ''' + def __init__(self, *args, **kwds): + self.size_limit = kwds.pop("size_limit", None) + OrderedDict.__init__(self, *args, **kwds) + self._check_size_limit() + + def __setitem__(self, key, value): + OrderedDict.__setitem__(self, key, value) + self._check_size_limit() + + def _check_size_limit(self): + if self.size_limit is not None: + while len(self) > self.size_limit: + self.popitem(last=False) diff --git a/headphones/config.py b/headphones/config.py index 57af4c16..86aa9216 100644 --- a/headphones/config.py +++ b/headphones/config.py @@ -46,6 +46,10 @@ _CONFIG_DEFINITIONS = { 'APOLLO_RATIO': (str, 'Apollo.rip', ''), 'APOLLO_USERNAME': (str, 'Apollo.rip', ''), 'APOLLO_URL': (str, 'Apollo.rip', 'https://apollo.rip'), + 'ARIA_HOST': (str, 'Aria2', ''), + 'ARIA_PASSWORD': (str, 'Aria2', ''), + 'ARIA_TOKEN': (str, 'Aria2', ''), + 'ARIA_USERNAME': (str, 'Aria2', ''), 'AUTOWANT_ALL': (int, 'General', 0), 'AUTOWANT_MANUALLY_ADDED': (int, 'General', 1), 'AUTOWANT_UPCOMING': (int, 'General', 1), @@ -73,6 +77,8 @@ _CONFIG_DEFINITIONS = { 'CUSTOMPORT': (int, 'General', 5000), 'CUSTOMSLEEP': (int, 'General', 1), 'CUSTOMUSER': (str, 'General', ''), + 'DDL_DOWNLOADER': (int, 'General', 0), + 'DEEZLOADER': (int, 'DeezLoader', 0), 'DELETE_LOSSLESS_FILES': (int, 'General', 1), 'DELUGE_HOST': (str, 'Deluge', ''), 'DELUGE_CERT': (str, 'Deluge', ''), @@ -86,6 +92,7 @@ _CONFIG_DEFINITIONS = { 'DOWNLOAD_DIR': (path, 'General', ''), 'DOWNLOAD_SCAN_INTERVAL': (int, 'General', 5), 'DOWNLOAD_TORRENT_DIR': (path, 'General', ''), + 'DOWNLOAD_DDL_DIR': (path, 'General', ''), 'DO_NOT_OVERRIDE_GIT_BRANCH': (int, 'General', 0), 'EMAIL_ENABLED': (int, 'Email', 0), 'EMAIL_FROM': (str, 'Email', ''), diff --git a/headphones/deezloader.py b/headphones/deezloader.py new file mode 100644 index 00000000..8de85df9 --- /dev/null +++ b/headphones/deezloader.py @@ -0,0 +1,441 @@ +# -*- coding: utf-8 -*- +# Deezloader (c) 2016 by ParadoxalManiak +# +# Deezloader is licensed under a +# Creative Commons Attribution-NonCommercial-ShareAlike 3.0 Unported License. +# +# You should have received a copy of the license along with this +# work. If not, see . +# +# Version 2.1.0 +# Maintained by ParadoxalManiak +# Original work by ZzMTV +# +# Author's disclaimer: +# I am not responsible for the usage of this program by other people. +# I do not recommend you doing this illegally or against Deezer's terms of service. +# This project is licensed under CC BY-NC-SA 4.0 + +import re +import os +from datetime import datetime +from hashlib import md5 +import binascii + +from beets.mediafile import MediaFile +from headphones import logger, request, helpers +from headphones.classes import OptionalImport, CacheDict +import headphones + + +# Try to import optional Crypto.Cipher packages +try: + from Crypto.Cipher import AES, Blowfish +except ImportError: + AES = OptionalImport('Crypto.Cipher.AES') + Blowfish = OptionalImport('Crypto.Cipher.Blowfish') + +# Public constants +PROVIDER_NAME = 'Deezer' + +# Internal constants +__API_URL = "http://www.deezer.com/ajax/gw-light.php" +__API_INFO_URL = "http://api.deezer.com/" +__HTTP_HEADERS = { + "User-Agent": "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36", + "Content-Language": "en-US", + "Cache-Control": "max-age=0", + "Accept": "*/*", + "Accept-Charset": "utf-8,ISO-8859-1;q=0.7,*;q=0.3", + "Accept-Language": "de-DE,de;q=0.8,en-US;q=0.6,en;q=0.4" +} + +# Internal variables +__api_queries = { + 'api_version': "1.0", + 'api_token': "None", + 'input': "3" +} +__cookies = None +__tracks_cache = CacheDict(size_limit=512) +__albums_cache = CacheDict(size_limit=64) + + +def __getApiToken(): + global __cookies + response = request.request_response(url="http://www.deezer.com/", headers=__HTTP_HEADERS) + __cookies = response.cookies + data = response.content + if data: + matches = re.search(r"checkForm\s*=\s*['|\"](.*)[\"|'];", data) + if matches: + token = matches.group(1) + __api_queries['api_token'] = token + logger.debug(u"Deezloader : api token loaded ('%s')" % token) + + if not token: + logger.error(u"Deezloader: Unable to get api token") + + +def getAlbumByLink(album_link): + """Returns deezer album infos using album link url + + :param str album_link: deezer album link url (eg: 'http://www.deezer.com/album/1234567/') + """ + matches = re.search(r"album\/([0-9]+)\/?$", album_link) + if matches: + return getAlbum(matches.group(1)) + + +def getAlbum(album_id): + """Returns deezer album infos + + :param int album_id: deezer album id + """ + global __albums_cache + + if str(album_id) in __albums_cache: + return __albums_cache[str(album_id)] + + url = __API_INFO_URL + "album/" + str(album_id) + data = request.request_json(url=url, headers=__HTTP_HEADERS, cookies=__cookies) + + if data and 'error' not in data: + __albums_cache[str(album_id)] = data + return data + else: + logger.debug("Deezloader: Can't load album infos") + return None + + +def searchAlbums(search_term): + """Search for deezer albums using search term + + :param str search_term: search term to search album for + """ + logger.info(u'Searching Deezer using term: "%s"' % search_term) + + url = __API_INFO_URL + "search/album?q=" + search_term + data = request.request_json(url=url, headers=__HTTP_HEADERS, cookies=__cookies) + + albums = [] + + # Process content + if data and 'total' in data and data['total'] > 0 and 'data' in data: + for item in data['data']: + try: + albums.append(getAlbum(item['id'])) + except Exception as e: + logger.error(u"An unknown error occurred in the Deezer search album parser: %s" % e) + else: + logger.info(u'No results found from Deezer using term: "%s"' % search_term) + + return albums + + +def __matchAlbums(albums, artist_name, album_title, album_length): + resultlist = [] + + for album in albums: + total_size = 0 + tracks_found = 0 + + for track in album['tracks']['data']: + t = getTrack(track['id']) + if t: + if t["FILESIZE_MP3_320"] > 0: + size = t["FILESIZE_MP3_320"] + elif t["FILESIZE_MP3_256"] > 0: + size = t["FILESIZE_MP3_256"] + elif t["FILESIZE_MP3_128"] > 0: + size = t["FILESIZE_MP3_128"] + else: + size = t["FILESIZE_MP3_64"] + + size = int(size) + total_size += size + tracks_found += 1 + logger.debug(u'Found song "%s". Size: %s' % (t['SNG_TITLE'], helpers.bytes_to_mb(size))) + + if tracks_found == 0: + logger.info(u'Ignoring album "%s" (no tracks to download)' % album['title']) + continue + + matched = True + mismatch_reason = 'matched!' + + if album_length > 0 and abs(int(album['duration']) - album_length) > 240: + matched = False + mismatch_reason = (u'duration mismatch: %i not in [%i, %i]' % (int(album['duration']), (album_length - 240), (album_length + 240))) + + elif (helpers.latinToAscii(re.sub(r"\W", "", album_title, 0, re.UNICODE)).lower() != + helpers.latinToAscii(re.sub(r"\W", "", album['title'], 0, re.UNICODE)).lower()): + matched = False + mismatch_reason = (u'album name mismatch: %s != %s' % (album['title'], album_title)) + + elif (helpers.latinToAscii(re.sub(r"\W", "", artist_name, 0, re.UNICODE)).lower() != + helpers.latinToAscii(re.sub(r"\W", "", album['artist']['name'], 0, re.UNICODE)).lower()): + matched = False + mismatch_reason = (u'artist name mismatch: %s != %s' % (album['artist']['name'], artist_name)) + + resultlist.append( + (album['artist']['name'] + ' - ' + album['title'] + ' [' + album['release_date'][:4] + '] (' + str(tracks_found) + '/' + str(album['nb_tracks']) + ')', + total_size, album['link'], PROVIDER_NAME, "ddl", matched) + ) + logger.info(u'Found "%s". Tracks %i/%i. Size: %s (%s)' % (album['title'], tracks_found, album['nb_tracks'], helpers.bytes_to_mb(total_size), mismatch_reason)) + + return resultlist + + +def searchAlbum(artist_name, album_title, user_search_term=None, album_length=None): + """Search for deezer specific album. + This will iterate over deezer albums and try to find best matches + + :param str artist_name: album artist name + :param str album_title: album title + :param str user_search_term: search terms provided by user + :param int album_length: targeted album duration in seconds + """ + # User search term by-pass normal search + if user_search_term: + return __matchAlbums(searchAlbums(user_search_term), artist_name, album_title, album_length) + + resultlist = __matchAlbums(searchAlbums((artist_name + ' ' + album_title).strip()), artist_name, album_title, album_length) + if resultlist: + return resultlist + + # Deezer API supports unicode, so just remove non alphanumeric characters + clean_artist_name = re.sub(r"[^\w\s]", " ", artist_name, 0, re.UNICODE).strip() + clean_album_name = re.sub(r"[^\w\s]", " ", album_title, 0, re.UNICODE).strip() + + resultlist = __matchAlbums(searchAlbums((clean_artist_name + ' ' + clean_album_name).strip()), artist_name, album_title, album_length) + if resultlist: + return resultlist + + resultlist = __matchAlbums(searchAlbums(clean_artist_name), artist_name, album_title, album_length) + if resultlist: + return resultlist + + return resultlist + + +def getTrack(sng_id, try_reload_api=True): + """Returns deezer track infos + + :param int sng_id: deezer song id + :param bool try_reload_api: whether or not try reloading API if session expired + """ + global __tracks_cache + + if str(sng_id) in __tracks_cache: + return __tracks_cache[str(sng_id)] + + data = "[{\"method\":\"song.getListData\",\"params\":{\"sng_ids\":[" + str(sng_id) + "]}}]" + json = request.request_json(url=__API_URL, headers=__HTTP_HEADERS, method='post', params=__api_queries, data=data, cookies=__cookies) + + results = [] + error = None + invalid_token = False + + if json: + # Check for errors + if 'error' in json: + error = json['error'] + if 'GATEWAY_ERROR' in json['error'] and json['error']['GATEWAY_ERROR'] == u"invalid api token": + invalid_token = True + + elif 'error' in json[0] and json[0]['error']: + error = json[0]['error'] + if 'VALID_TOKEN_REQUIRED' in json[0]['error'] and json[0]['error']['VALID_TOKEN_REQUIRED'] == u"Invalid CSRF token": + invalid_token = True + + # Got invalid token error + if error: + if invalid_token and try_reload_api: + __getApiToken() + return getTrack(sng_id, False) + else: + logger.error(u"An unknown error occurred in the Deezer track parser: %s" % error) + else: + try: + results = json[0]['results'] + item = results['data'][0] + if 'token' in item: + logger.error(u"An unknown error occurred in the Deezer parser: Uploaded Files are currently not supported") + return + + sng_id = item["SNG_ID"] + md5Origin = item["MD5_ORIGIN"] + sng_format = 3 + + if item["FILESIZE_MP3_320"] <= 0: + if item["FILESIZE_MP3_256"] > 0: + sng_format = 5 + else: + sng_format = 1 + + mediaVersion = int(item["MEDIA_VERSION"]) + item['downloadUrl'] = __getDownloadUrl(md5Origin, sng_id, sng_format, mediaVersion) + + __tracks_cache[sng_id] = item + return item + + except Exception as e: + logger.error(u"An unknown error occurred in the Deezer track parser: %s" % e) + + +def __getDownloadUrl(md5Origin, sng_id, sng_format, mediaVersion): + urlPart = md5Origin.encode('utf-8') + b'\xa4' + str(sng_format) + b'\xa4' + str(sng_id) + b'\xa4' + str(mediaVersion) + md5val = md5(urlPart).hexdigest() + urlPart = md5val + b'\xa4' + urlPart + b'\xa4' + cipher = AES.new('jo6aey6haid2Teih', AES.MODE_ECB) + ciphertext = cipher.encrypt(__pad(urlPart, AES.block_size)) + return "http://e-cdn-proxy-" + md5Origin[:1] + ".deezer.com/mobile/1/" + binascii.hexlify(ciphertext).lower() + + +def __pad(raw, block_size): + if (len(raw) % block_size == 0): + return raw + padding_required = block_size - (len(raw) % block_size) + padChar = b'\x00' + data = raw + padding_required * padChar + return data + + +def __tagTrack(path, track): + try: + album = getAlbum(track['ALB_ID']) + + f = MediaFile(path) + f.artist = track['ART_NAME'] + f.album = track['ALB_TITLE'] + f.title = track['SNG_TITLE'] + f.track = track['TRACK_NUMBER'] + f.tracktotal = album['nb_tracks'] + f.disc = track['DISK_NUMBER'] + f.bpm = track['BPM'] + f.date = datetime.strptime(album['release_date'], '%Y-%m-%d').date() + f.albumartist = album['artist']['name'] + if u'genres' in album and u'data' in album['genres']: + f.genres = [genre['name'] for genre in album['genres']['data']] + + f.save() + + except Exception as e: + logger.error(u'Unable to tag deezer track "%s": %s' % (path, e)) + + +def decryptTracks(paths): + """Decrypt downloaded deezer tracks. + + :param paths: list of path to deezer tracks (*.dzr files). + """ + # Note: tracks can be from different albums + decrypted_tracks = {} + + # First pass: load tracks data + for path in paths: + try: + album_folder = os.path.dirname(path) + sng_id = os.path.splitext(os.path.basename(path))[0] + track = getTrack(sng_id) + if track: + track_number = int(track['TRACK_NUMBER']) + disk_number = int(track['DISK_NUMBER']) + + if album_folder not in decrypted_tracks: + decrypted_tracks[album_folder] = {} + + if disk_number not in decrypted_tracks[album_folder]: + decrypted_tracks[album_folder][disk_number] = {} + + decrypted_tracks[album_folder][disk_number][track_number] = track + + except Exception as e: + logger.error(u'Unable to load deezer track infos "%s": %s' % (path, e)) + + # Second pass: decrypt tracks + for album_folder in decrypted_tracks: + multi_disks = len(decrypted_tracks[album_folder]) > 1 + for disk_number in decrypted_tracks[album_folder]: + for track_number, track in decrypted_tracks[album_folder][disk_number].items(): + try: + filename = helpers.replace_illegal_chars(track['SNG_TITLE']).strip() + filename = ('{:02d}'.format(track_number) + '_' + filename + '.mp3') + + # Add a 'cd x' sub-folder if album has more than one disk + disk_folder = os.path.join(album_folder, 'cd ' + str(disk_number)) if multi_disks else album_folder + + dest = os.path.join(disk_folder, filename).encode(headphones.SYS_ENCODING, 'replace') + + # Decrypt track if not already done + if not os.path.exists(dest): + try: + __decryptDownload(path, sng_id, dest) + __tagTrack(dest, track) + except Exception as e: + logger.error(u'Unable to decrypt deezer track "%s": %s' % (path, e)) + if os.path.exists(dest): + os.remove(dest) + decrypted_tracks[album_folder][disk_number].pop(track_number) + continue + + decrypted_tracks[album_folder][disk_number][track_number]['path'] = dest + + except Exception as e: + logger.error(u'Unable to decrypt deezer track "%s": %s' % (path, e)) + + return decrypted_tracks + + +def __decryptDownload(source, sng_id, dest): + interval_chunk = 3 + chunk_size = 2048 + blowFishKey = __getBlowFishKey(sng_id) + i = 0 + iv = "\x00\x01\x02\x03\x04\x05\x06\x07" + + dest_folder = os.path.dirname(dest) + if not os.path.exists(dest_folder): + os.makedirs(dest_folder) + + f = open(source, "rb") + fout = open(dest, "wb") + try: + chunk = f.read(chunk_size) + while chunk: + if(i % interval_chunk == 0): + cipher = Blowfish.new(blowFishKey, Blowfish.MODE_CBC, iv) + chunk = cipher.decrypt(__pad(chunk, Blowfish.block_size)) + + fout.write(chunk) + i += 1 + chunk = f.read(chunk_size) + finally: + f.close() + fout.close() + + +def __getBlowFishKey(encryptionKey): + if encryptionKey < 1: + encryptionKey *= -1 + + hashcode = md5(str(encryptionKey)).hexdigest() + hPart = hashcode[0:16] + lPart = hashcode[16:32] + parts = ['g4el58wc0zvf9na1', hPart, lPart] + + return __xorHex(parts) + + +def __xorHex(parts): + data = "" + for i in range(0, 16): + character = ord(parts[0][i]) + + for j in range(1, len(parts)): + character ^= ord(parts[j][i]) + + data += chr(character) + + return data diff --git a/headphones/postprocessor.py b/headphones/postprocessor.py index 38e4bba5..3b6f493f 100755 --- a/headphones/postprocessor.py +++ b/headphones/postprocessor.py @@ -30,7 +30,7 @@ from beetsplug import lyrics as beetslyrics from headphones import notifiers, utorrent, transmission, deluge, qbittorrent from headphones import db, albumart, librarysync from headphones import logger, helpers, mb, music_encoder -from headphones import metadata +from headphones import metadata, deezloader postprocessor_lock = threading.Lock() @@ -48,6 +48,8 @@ def checkFolder(): single = False if album['Kind'] == 'nzb': download_dir = headphones.CONFIG.DOWNLOAD_DIR + elif album['Kind'] == 'ddl': + download_dir = headphones.CONFIG.DOWNLOAD_DDL_DIR else: if headphones.CONFIG.DELUGE_DONE_DIRECTORY and headphones.CONFIG.TORRENT_DOWNLOADER == 3: download_dir = headphones.CONFIG.DELUGE_DONE_DIRECTORY @@ -205,6 +207,7 @@ def verify(albumid, albumpath, Kind=None, forced=False, keep_original_folder=Fal tracks = myDB.select('SELECT * from tracks WHERE AlbumID=?', [albumid]) downloaded_track_list = [] + downloaded_deezer_list = [] downloaded_cuecount = 0 for r, d, f in os.walk(albumpath): @@ -213,8 +216,10 @@ def verify(albumid, albumpath, Kind=None, forced=False, keep_original_folder=Fal downloaded_track_list.append(os.path.join(r, files)) elif files.lower().endswith('.cue'): downloaded_cuecount += 1 + elif files.lower().endswith('.dzr'): + downloaded_deezer_list.append(os.path.join(r, files)) # if any of the files end in *.part, we know the torrent isn't done yet. Process if forced, though - elif files.lower().endswith(('.part', '.utpart')) and not forced: + elif files.lower().endswith(('.part', '.utpart', '.aria2')) and not forced: logger.info( "Looks like " + os.path.basename(albumpath).decode(headphones.SYS_ENCODING, 'replace') + " isn't complete yet. Will try again on the next run") @@ -253,6 +258,37 @@ def verify(albumid, albumpath, Kind=None, forced=False, keep_original_folder=Fal downloaded_track_list = helpers.get_downloaded_track_list(albumpath) keep_original_folder = False + # Decrypt deezer tracks + if downloaded_deezer_list: + logger.info('Decrypting deezer tracks') + decrypted_deezer_list = deezloader.decryptTracks(downloaded_deezer_list) + + # Check if album is complete based on album duration only + # (total track numbers is not determinant enough due to hidden tracks for eg) + db_track_duration = 0 + downloaded_track_duration = 0 + try: + for track in tracks: + db_track_duration += track['TrackDuration'] / 1000 + except: + downloaded_track_duration = False + + try: + for disk_number in decrypted_deezer_list[albumpath]: + for track in decrypted_deezer_list[albumpath][disk_number].values(): + downloaded_track_list.append(track['path']) + downloaded_track_duration += int(track['DURATION']) + except: + downloaded_track_duration = False + + if not downloaded_track_duration or not db_track_duration or abs(downloaded_track_duration - db_track_duration) > 240: + logger.info("Looks like " + + os.path.basename(albumpath).decode(headphones.SYS_ENCODING, 'replace') + + " isn't complete yet (duration mismatch). Will try again on the next run") + return + + downloaded_track_list = list(set(downloaded_track_list)) # Remove duplicates + # test #1: metadata - usually works logger.debug('Verifying metadata...') diff --git a/headphones/searcher.py b/headphones/searcher.py index b0652f7f..f6554cb1 100644 --- a/headphones/searcher.py +++ b/headphones/searcher.py @@ -36,6 +36,7 @@ import headphones from headphones.common import USER_AGENT from headphones import logger, db, helpers, classes, sab, nzbget, request from headphones import utorrent, transmission, notifiers, rutracker, deluge, qbittorrent +from headphones import deezloader, aria2 from bencode import bencode, bdecode # Magnet to torrent services, for Black hole. Stolen from CouchPotato. @@ -51,6 +52,27 @@ ruobj = None # Persistent RED API object redobj = None +# Persistent Aria2 RPC object +__aria2rpc_obj = None + + +def getAria2RPC(): + global __aria2rpc_obj + if not __aria2rpc_obj: + __aria2rpc_obj = aria2.Aria2JsonRpc( + ID='headphones', + uri=headphones.CONFIG.ARIA_HOST, + token=headphones.CONFIG.ARIA_TOKEN if headphones.CONFIG.ARIA_TOKEN else None, + http_user=headphones.CONFIG.ARIA_USERNAME if headphones.CONFIG.ARIA_USERNAME else None, + http_passwd=headphones.CONFIG.ARIA_PASSWORD if headphones.CONFIG.ARIA_PASSWORD else None + ) + return __aria2rpc_obj + + +def reconfigure(): + global __aria2rpc_obj + __aria2rpc_obj = None + def fix_url(s, charset="utf-8"): """ @@ -281,32 +303,53 @@ def do_sorted_search(album, new, losslessOnly, choose_specific_download=False): headphones.CONFIG.STRIKE or headphones.CONFIG.TQUATTRECENTONZE) - results = [] + DDL_PROVIDERS = (headphones.CONFIG.DEEZLOADER) + myDB = db.DBConnection() albumlength = myDB.select('SELECT sum(TrackDuration) from tracks WHERE AlbumID=?', [album['AlbumID']])[0][0] + nzb_results = None + torrent_results = None + ddl_results = None + if headphones.CONFIG.PREFER_TORRENTS == 0 and not choose_specific_download: if NZB_PROVIDERS and NZB_DOWNLOADERS: - results = searchNZB(album, new, losslessOnly, albumlength) + nzb_results = searchNZB(album, new, losslessOnly, albumlength) - if not results and TORRENT_PROVIDERS: - results = searchTorrent(album, new, losslessOnly, albumlength) + if not nzb_results: + if DDL_PROVIDERS: + ddl_results = searchDdl(album, new, losslessOnly, albumlength) + + if TORRENT_PROVIDERS: + torrent_results = searchTorrent(album, new, losslessOnly, albumlength) elif headphones.CONFIG.PREFER_TORRENTS == 1 and not choose_specific_download: if TORRENT_PROVIDERS: - results = searchTorrent(album, new, losslessOnly, albumlength) + torrent_results = searchTorrent(album, new, losslessOnly, albumlength) - if not results and NZB_PROVIDERS and NZB_DOWNLOADERS: - results = searchNZB(album, new, losslessOnly, albumlength) + if not torrent_results: + if DDL_PROVIDERS: + ddl_results = searchDdl(album, new, losslessOnly, albumlength) + + if NZB_PROVIDERS and NZB_DOWNLOADERS: + nzb_results = searchNZB(album, new, losslessOnly, albumlength) + + elif headphones.CONFIG.PREFER_TORRENTS == 3 and not choose_specific_download: + + if DDL_PROVIDERS: + ddl_results = searchDdl(album, new, losslessOnly, albumlength) + + if not ddl_results: + if TORRENT_PROVIDERS: + torrent_results = searchTorrent(album, new, losslessOnly, albumlength) + + if NZB_PROVIDERS and NZB_DOWNLOADERS: + nzb_results = searchNZB(album, new, losslessOnly, albumlength) else: - - nzb_results = None - torrent_results = None - if NZB_PROVIDERS and NZB_DOWNLOADERS: nzb_results = searchNZB(album, new, losslessOnly, albumlength, choose_specific_download) @@ -314,13 +357,19 @@ def do_sorted_search(album, new, losslessOnly, choose_specific_download=False): torrent_results = searchTorrent(album, new, losslessOnly, albumlength, choose_specific_download) - if not nzb_results: - nzb_results = [] + if DDL_PROVIDERS: + ddl_results = searchDdl(album, new, losslessOnly, albumlength, choose_specific_download) - if not torrent_results: - torrent_results = [] + if not nzb_results: + nzb_results = [] - results = nzb_results + torrent_results + if not torrent_results: + torrent_results = [] + + if not ddl_results: + ddl_results = [] + + results = nzb_results + torrent_results + ddl_results if choose_specific_download: return results @@ -826,6 +875,31 @@ def send_to_downloader(data, bestqual, album): except Exception as e: logger.error('Couldn\'t write NZB file: %s', e) return + + elif kind == 'ddl': + folder_name = '%s - %s [%s]' % ( + helpers.latinToAscii(album['ArtistName']).encode('UTF-8').replace('/', '_'), + helpers.latinToAscii(album['AlbumTitle']).encode('UTF-8').replace('/', '_'), + get_year_from_release_date(album['ReleaseDate'])) + + # Aria2 downloader + if headphones.CONFIG.DDL_DOWNLOADER == 0: + logger.info("Sending download to Aria2") + + try: + deezer_album = deezloader.getAlbumByLink(bestqual[2]) + + for album_track in deezer_album['tracks']['data']: + track = deezloader.getTrack(album_track['id']) + if track: + filename = track['SNG_ID'] + '.dzr' + logger.debug(u'Sending song "%s" to Aria' % track['SNG_TITLE']) + getAria2RPC().addUri([track['downloadUrl']], {'out': filename, 'auto-file-renaming': 'false', 'continue': 'true', 'dir': folder_name}) + + except Exception as e: + logger.error(u'Error sending torrent to Aria2. Are you sure it\'s running? (%s)' % e) + return + else: folder_name = '%s - %s [%s]' % ( helpers.latinToAscii(album['ArtistName']).encode('UTF-8').replace('/', '_'), @@ -1204,6 +1278,62 @@ def verifyresult(title, artistterm, term, lossless): return True +def searchDdl(album, new=False, losslessOnly=False, albumlength=None, + choose_specific_download=False): + reldate = album['ReleaseDate'] + year = get_year_from_release_date(reldate) + + # MERGE THIS WITH THE TERM CLEANUP FROM searchNZB + dic = {'...': '', ' & ': ' ', ' = ': ' ', '?': '', '$': 's', ' + ': ' ', '"': '', ',': ' ', + '*': ''} + + semi_cleanalbum = helpers.replace_all(album['AlbumTitle'], dic) + cleanalbum = helpers.latinToAscii(semi_cleanalbum) + semi_cleanartist = helpers.replace_all(album['ArtistName'], dic) + cleanartist = helpers.latinToAscii(semi_cleanartist) + + # Use provided term if available, otherwise build our own (this code needs to be cleaned up since a lot + # of these torrent providers are just using cleanartist/cleanalbum terms + if album['SearchTerm']: + term = album['SearchTerm'] + elif album['Type'] == 'part of': + term = cleanalbum + " " + year + else: + # FLAC usually doesn't have a year for some reason so I'll leave it out + # Various Artist albums might be listed as VA, so I'll leave that out too + # Only use the year if the term could return a bunch of different albums, i.e. self-titled albums + if album['ArtistName'] in album['AlbumTitle'] or len(album['ArtistName']) < 4 or len( + album['AlbumTitle']) < 4: + term = cleanartist + ' ' + cleanalbum + ' ' + year + elif album['ArtistName'] == 'Various Artists': + term = cleanalbum + ' ' + year + else: + term = cleanartist + ' ' + cleanalbum + + # Replace bad characters in the term and unicode it + term = re.sub('[\.\-\/]', ' ', term).encode('utf-8') + artistterm = re.sub('[\.\-\/]', ' ', cleanartist).encode('utf-8', 'replace') + + logger.debug(u'Using search term: "%s"' % helpers.latinToAscii(term)) + + resultlist = [] + + # Deezer only provides lossy + if headphones.CONFIG.DEEZLOADER and not losslessOnly: + resultlist += deezloader.searchAlbum(album['ArtistName'], album['AlbumTitle'], album['SearchTerm'], int(albumlength / 1000)) + + # attempt to verify that this isn't a substring result + # when looking for "Foo - Foo" we don't want "Foobar" + # this should be less of an issue when it isn't a self-titled album so we'll only check vs artist + results = [result for result in resultlist if verifyresult(result[0], artistterm, term, losslessOnly)] + + # Additional filtering for size etc + if results and not choose_specific_download: + results = more_filtering(results, album, albumlength, new) + + return results + + def searchTorrent(album, new=False, losslessOnly=False, albumlength=None, choose_specific_download=False): global apolloobj # persistent apollo.rip api object to reduce number of login attempts @@ -2036,7 +2166,10 @@ def searchTorrent(album, new=False, losslessOnly=False, albumlength=None, def preprocess(resultlist): for result in resultlist: - if result[4] == 'torrent': + if result[3] == deezloader.PROVIDER_NAME: + return True, result + + elif result[4] == 'torrent': # rutracker always needs the torrent data if result[3] == 'rutracker.org': diff --git a/headphones/webserve.py b/headphones/webserve.py index f46b5254..267b78bc 100644 --- a/headphones/webserve.py +++ b/headphones/webserve.py @@ -1182,6 +1182,10 @@ class WebInterface(object): "utorrent_username": headphones.CONFIG.UTORRENT_USERNAME, "utorrent_password": headphones.CONFIG.UTORRENT_PASSWORD, "utorrent_label": headphones.CONFIG.UTORRENT_LABEL, + "aria_host": headphones.CONFIG.ARIA_HOST, + "aria_password": headphones.CONFIG.ARIA_PASSWORD, + "aria_token": headphones.CONFIG.ARIA_TOKEN, + "aria_username": headphones.CONFIG.ARIA_USERNAME, "nzb_downloader_sabnzbd": radio(headphones.CONFIG.NZB_DOWNLOADER, 0), "nzb_downloader_nzbget": radio(headphones.CONFIG.NZB_DOWNLOADER, 1), "nzb_downloader_blackhole": radio(headphones.CONFIG.NZB_DOWNLOADER, 2), @@ -1190,6 +1194,7 @@ class WebInterface(object): "torrent_downloader_utorrent": radio(headphones.CONFIG.TORRENT_DOWNLOADER, 2), "torrent_downloader_deluge": radio(headphones.CONFIG.TORRENT_DOWNLOADER, 3), "torrent_downloader_qbittorrent": radio(headphones.CONFIG.TORRENT_DOWNLOADER, 4), + "ddl_downloader_aria": radio(headphones.CONFIG.DDL_DOWNLOADER, 0), "download_dir": headphones.CONFIG.DOWNLOAD_DIR, "use_blackhole": checked(headphones.CONFIG.BLACKHOLE), "blackhole_dir": headphones.CONFIG.BLACKHOLE_DIR, @@ -1251,6 +1256,8 @@ class WebInterface(object): "use_tquattrecentonze": checked(headphones.CONFIG.TQUATTRECENTONZE), "tquattrecentonze_user": headphones.CONFIG.TQUATTRECENTONZE_USER, "tquattrecentonze_password": headphones.CONFIG.TQUATTRECENTONZE_PASSWORD, + "download_ddl_dir": headphones.CONFIG.DOWNLOAD_DDL_DIR, + "use_deezloader": checked(headphones.CONFIG.DEEZLOADER), "pref_qual_0": radio(headphones.CONFIG.PREFERRED_QUALITY, 0), "pref_qual_1": radio(headphones.CONFIG.PREFERRED_QUALITY, 1), "pref_qual_2": radio(headphones.CONFIG.PREFERRED_QUALITY, 2), @@ -1296,6 +1303,7 @@ class WebInterface(object): "prefer_torrents_0": radio(headphones.CONFIG.PREFER_TORRENTS, 0), "prefer_torrents_1": radio(headphones.CONFIG.PREFER_TORRENTS, 1), "prefer_torrents_2": radio(headphones.CONFIG.PREFER_TORRENTS, 2), + "prefer_torrents_3": radio(headphones.CONFIG.PREFER_TORRENTS, 3), "magnet_links_0": radio(headphones.CONFIG.MAGNET_LINKS, 0), "magnet_links_1": radio(headphones.CONFIG.MAGNET_LINKS, 1), "magnet_links_2": radio(headphones.CONFIG.MAGNET_LINKS, 2), @@ -1454,7 +1462,7 @@ class WebInterface(object): "launch_browser", "enable_https", "api_enabled", "use_blackhole", "headphones_indexer", "use_newznab", "newznab_enabled", "use_torznab", "torznab_enabled", "use_nzbsorg", "use_omgwtfnzbs", "use_kat", "use_piratebay", "use_oldpiratebay", - "use_mininova", "use_waffles", "use_rutracker", + "use_mininova", "use_waffles", "use_rutracker", "use_deezloader", "use_apollo", "use_redacted", "use_strike", "use_tquattrecentonze", "preferred_bitrate_allow_lossless", "detect_bitrate", "ignore_clean_releases", "freeze_db", "cue_split", "move_files", "rename_files", "correct_metadata", "cleanup_files", "keep_nfo", "add_album_art", @@ -1588,6 +1596,9 @@ class WebInterface(object): # Reconfigure musicbrainz database connection with the new values mb.startmb() + # Reconfigure Aria2 + searcher.reconfigure() + raise cherrypy.HTTPRedirect("config") @cherrypy.expose