From 34b8a11145e7d8150f0008da2cd79bae601b165d Mon Sep 17 00:00:00 2001 From: Kallys Date: Wed, 19 Apr 2017 19:55:41 +0200 Subject: [PATCH] Fix travis errors --- headphones/aria2.py | 1798 ++++++++++++++++------------------- headphones/deezloader.py | 117 ++- headphones/helpers_test.py | 4 +- headphones/postprocessor.py | 10 +- headphones/searcher.py | 8 +- headphones/webserve.py | 2 +- 6 files changed, 920 insertions(+), 1019 deletions(-) diff --git a/headphones/aria2.py b/headphones/aria2.py index 68c4cb0a..06d8e4e0 100644 --- a/headphones/aria2.py +++ b/headphones/aria2.py @@ -27,7 +27,7 @@ import urllib2 from headphones import logger -################################## Constants ################################### +# ################################ Constants ################################### DEFAULT_PORT = 6800 SERVER_URI_FORMAT = '{}://{}:{:d}/jsonrpc' @@ -39,1057 +39,941 @@ FINAL_STATUS = ('complete', 'error') ARIA2_CONTROL_FILE_EXT = '.aria2' -############################ Convenience Functions ############################# +# ########################## Convenience Functions ############################# + def to_json_list(objs): - ''' - Wrap strings in lists. Other iterables are converted to lists directly. - ''' - if isinstance(objs, str): - return [objs] - elif not isinstance(objs, list): - return list(objs) - else: - return objs - + ''' + Wrap strings in lists. Other iterables are converted to lists directly. + ''' + if isinstance(objs, str): + return [objs] + elif not isinstance(objs, list): + return list(objs) + else: + return objs def add_options_and_position(params, options=None, position=None): - ''' - Convenience method for adding options and position to parameters. - ''' - if options: - params.append(options) - if position: - if not isinstance(position, int): - try: - position = int(position) - except ValueError: - position = -1 - if position >= 0: - params.append(position) - return params - + ''' + Convenience method for adding options and position to parameters. + ''' + if options: + params.append(options) + if position: + if not isinstance(position, int): + try: + position = int(position) + except ValueError: + position = -1 + if position >= 0: + params.append(position) + return params def get_status(response): - ''' - Process a status response. - ''' - if response: - try: - return response['status'] - except KeyError: - logger.error('no status returned from Aria2 RPC server') - return 'error' - else: - logger.error('no response from server') - return 'error' - + ''' + Process a status response. + ''' + if response: + try: + return response['status'] + except KeyError: + logger.error('no status returned from Aria2 RPC server') + return 'error' + else: + logger.error('no response from server') + return 'error' def random_token(length, valid_chars=None): - ''' - Get a random secret token for the Aria2 RPC server. + ''' + Get a random secret token for the Aria2 RPC server. - length: - The length of the token + length: + The length of the token - valid_chars: - A list or other ordered and indexable iterable of valid characters. If not - given of None, asciinumberic characters with some punctuation characters - will be used. - ''' - if not valid_chars: - valid_chars = string.ascii_letters + string.digits + '!@#$%^&*()-_=+' - number_of_chars = len(valid_chars) - bytes_to_read = math.ceil(math.log(number_of_chars) / math.log(0x100)) - max_value = 0x100**bytes_to_read - max_index = number_of_chars - 1 - token = '' - for _ in range(length): - value = int.from_bytes(os.urandom(bytes_to_read), byteorder='little') - index = round((value * max_index)/max_value) - token += valid_chars[index] - return token + valid_chars: + A list or other ordered and indexable iterable of valid characters. If not + given of None, asciinumberic characters with some punctuation characters + will be used. + ''' + if not valid_chars: + valid_chars = string.ascii_letters + string.digits + '!@#$%^&*()-_=+' + number_of_chars = len(valid_chars) + bytes_to_read = math.ceil(math.log(number_of_chars) / math.log(0x100)) + max_value = 0x100**bytes_to_read + max_index = number_of_chars - 1 + token = '' + for _ in range(length): + value = int.from_bytes(os.urandom(bytes_to_read), byteorder='little') + index = round((value * max_index) / max_value) + token += valid_chars[index] + return token +# ################ From python3-aur's ThreadedServers.common ################### -################## From python3-aur's ThreadedServers.common ################### def format_bytes(size): - '''Format bytes for inferior humans.''' - if size < 0x400: - return '{:d} B'.format(size) - else: - size = float(size) / 0x400 - for prefix in ('KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB'): - if size < 0x400: - return '{:0.02f} {}'.format(size, prefix) - else: - size /= 0x400 - return '{:0.02f} YiB'.format(size) - + '''Format bytes for inferior humans.''' + if size < 0x400: + return '{:d} B'.format(size) + else: + size = float(size) / 0x400 + for prefix in ('KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB'): + if size < 0x400: + return '{:0.02f} {}'.format(size, prefix) + else: + size /= 0x400 + return '{:0.02f} YiB'.format(size) def format_seconds(s): - '''Format seconds for inferior humans.''' - string = '' - for base, char in ( - (60, 's'), - (60, 'm'), - (24, 'h') - ): - s, r = divmod(s, base) - if s == 0: - return '{:d}{}{}'.format(r, char, string) - elif r != 0: - string = '{:02d}{}{}'.format(r, char, string) - else: - return '{:d}d{}'.format(s, string) + '''Format seconds for inferior humans.''' + string = '' + for base, char in ( + (60, 's'), + (60, 'm'), + (24, 'h') + ): + s, r = divmod(s, base) + if s == 0: + return '{:d}{}{}'.format(r, char, string) + elif r != 0: + string = '{:02d}{}{}'.format(r, char, string) + else: + return '{:d}d{}'.format(s, string) + +# ############################ Aria2JsonRpcError ############################### -############################## Aria2JsonRpcError ############################### class Aria2JsonRpcError(Exception): - def __init__(self, msg, connection_error=False): - super(self.__class__, self).__init__(self, msg) - self.connection_error = connection_error + def __init__(self, msg, connection_error=False): + super(self.__class__, self).__init__(self, msg) + self.connection_error = connection_error + +# ############################ Aria2JsonRpc Class ############################## -############################## Aria2JsonRpc Class ############################## class Aria2JsonRpc(object): - ''' - Interface class for interacting with an Aria2 RPC server. - ''' - # TODO: certificate options, etc. - def __init__( - self, ID, uri, - mode='normal', - token=None, - http_user=None, http_passwd=None, - server_cert=None, client_cert=None, client_cert_password=None, - ssl_protocol=None, - setup_function=None - ): - ''' - ID: the ID to send to the RPC interface + ''' + Interface class for interacting with an Aria2 RPC server. + ''' + # TODO: certificate options, etc. + def __init__( + self, ID, uri, + mode='normal', + token=None, + http_user=None, http_passwd=None, + server_cert=None, client_cert=None, client_cert_password=None, + ssl_protocol=None, + setup_function=None + ): + ''' + ID: the ID to send to the RPC interface - uri: the URI of the RPC interface + uri: the URI of the RPC interface - mode: - normal - process requests immediately - batch - queue requests (run with "process_queue") - format - return RPC request objects + mode: + normal - process requests immediately + batch - queue requests (run with "process_queue") + format - return RPC request objects - token: - RPC method-level authorization token (set using `--rpc-secret`) + token: + RPC method-level authorization token (set using `--rpc-secret`) - http_user, http_password: - HTTP Basic authentication credentials (deprecated) + http_user, http_password: + HTTP Basic authentication credentials (deprecated) - server_cert: - server certificate for HTTPS connections + server_cert: + server certificate for HTTPS connections - client_cert: - client certificate for HTTPS connections + client_cert: + client certificate for HTTPS connections - client_cert_password: - prompt for client certificate password + client_cert_password: + prompt for client certificate password - ssl_protocol: - SSL protocol from the ssl module + ssl_protocol: + SSL protocol from the ssl module - setup_function: - A function to invoke prior to the first server call. This could be the - launch() method of an Aria2RpcServer instance, for example. This attribute - is set automatically in instances returned from Aria2RpcServer.get_a2jr() - ''' - self.id = ID - self.uri = uri - self.mode = mode - self.queue = [] - self.handlers = dict() - self.token = token - self.setup_function = setup_function + setup_function: + A function to invoke prior to the first server call. This could be the + launch() method of an Aria2RpcServer instance, for example. This attribute + is set automatically in instances returned from Aria2RpcServer.get_a2jr() + ''' + self.id = ID + self.uri = uri + self.mode = mode + self.queue = [] + self.handlers = dict() + self.token = token + self.setup_function = setup_function - if None not in (http_user, http_passwd): - self.add_HTTPBasicAuthHandler(http_user, http_passwd) + if None not in (http_user, http_passwd): + self.add_HTTPBasicAuthHandler(http_user, http_passwd) - if server_cert or client_cert: - self.add_HTTPSHandler( - server_cert=server_cert, - client_cert=client_cert, - client_cert_password=client_cert_password, - protocol=ssl_protocol - ) + if server_cert or client_cert: + self.add_HTTPSHandler( + server_cert=server_cert, + client_cert=client_cert, + client_cert_password=client_cert_password, + protocol=ssl_protocol + ) - self.update_opener() + self.update_opener() + def iter_handlers(self): + ''' + Iterate over handlers. + ''' + for name in ('HTTPS', 'HTTPBasicAuth'): + try: + yield self.handlers[name] + except KeyError: + pass + def update_opener(self): + ''' + Build an opener from the current handlers. + ''' + self.opener = urllib2.build_opener(*self.iter_handlers()) - def iter_handlers(self): - ''' - Iterate over handlers. - ''' - for name in ('HTTPS', 'HTTPBasicAuth'): - try: - yield self.handlers[name] - except KeyError: - pass + def remove_handler(self, name): + ''' + Remove a handler. + ''' + try: + del self.handlers[name] + except KeyError: + pass + def add_HTTPBasicAuthHandler(self, user, passwd): + ''' + Add a handler for HTTP Basic authentication. + If either user or passwd are None, the handler is removed. + ''' + handler = urllib2.HTTPBasicAuthHandler() + handler.add_password( + realm='aria2', + uri=self.uri, + user=user, + passwd=passwd, + ) + self.handlers['HTTPBasicAuth'] = handler - def update_opener(self): - ''' - Build an opener from the current handlers. - ''' - self.opener = urllib2.build_opener(*self.iter_handlers()) + def remove_HTTPBasicAuthHandler(self): + self.remove_handler('HTTPBasicAuth') - - - def remove_handler(self, name): - ''' - Remove a handler. - ''' - try: - del self.handlers[name] - except KeyError: - pass - - - - def add_HTTPBasicAuthHandler(self, user, passwd): - ''' - Add a handler for HTTP Basic authentication. - - If either user or passwd are None, the handler is removed. - ''' - handler = urllib2.HTTPBasicAuthHandler() - handler.add_password( - realm='aria2', - uri=self.uri, - user=user, - passwd=passwd, - ) - self.handlers['HTTPBasicAuth'] = handler - - - - def remove_HTTPBasicAuthHandler(self): - self.remove_handler('HTTPBasicAuth') - - - - def add_HTTPSHandler( - self, - server_cert=None, - client_cert=None, - client_cert_password=None, - protocol=None, - ): - ''' - Add a handler for HTTPS connections with optional server and client - certificates. - ''' - if not protocol: - protocol = ssl.PROTOCOL_TLSv1 + def add_HTTPSHandler( + self, + server_cert=None, + client_cert=None, + client_cert_password=None, + protocol=None, + ): + ''' + Add a handler for HTTPS connections with optional server and client + certificates. + ''' + if not protocol: + protocol = ssl.PROTOCOL_TLSv1 # protocol = ssl.PROTOCOL_TLSv1_1 # openssl 1.0.1+ # protocol = ssl.PROTOCOL_TLSv1_2 # Python 3.4+ - context = ssl.SSLContext(protocol) - - if server_cert: - context.verify_mode = ssl.CERT_REQUIRED - context.load_verify_locations(cafile=server_cert) - else: - context.verify_mode = ssl.CERT_OPTIONAL - - if client_cert: - context.load_cert_chain(client_cert, password=client_cert_password) - - self.handlers['HTTPS'] = urllib2.HTTPSHandler( - context=context, - check_hostname=False - ) - - - - def remove_HTTPSHandler(self): - self.remove_handler('HTTPS') - - - - def send_request(self, req_obj): - ''' - Send the request and return the response. - ''' - if self.setup_function: - self.setup_function() - self.setup_function = None - logger.debug("Aria req_obj: %s" % json.dumps(req_obj, indent=2, sort_keys=True)) - req = json.dumps(req_obj).encode('UTF-8') - try: - f = self.opener.open(self.uri, req) - obj = json.loads(f.read()) - try: - return obj['result'] - except KeyError: - raise Aria2JsonRpcError('unexpected result: {}'.format(obj)) - except (urllib2.URLError) as e: - # This should work but URLError does not set the errno attribute: - # e.errno == errno.ECONNREFUSED - raise Aria2JsonRpcError( - str(e), - connection_error=( - '111' in str(e) - ) - ) - except httplib.BadStatusLine as e: - raise Aria2JsonRpcError('{}: BadStatusLine: {} (HTTPS error?)'.format( - self.__class__.__name__, e - )) + context = ssl.SSLContext(protocol) + + if server_cert: + context.verify_mode = ssl.CERT_REQUIRED + context.load_verify_locations(cafile=server_cert) + else: + context.verify_mode = ssl.CERT_OPTIONAL + + if client_cert: + context.load_cert_chain(client_cert, password=client_cert_password) + + self.handlers['HTTPS'] = urllib2.HTTPSHandler( + context=context, + check_hostname=False + ) + + def remove_HTTPSHandler(self): + self.remove_handler('HTTPS') + + def send_request(self, req_obj): + ''' + Send the request and return the response. + ''' + if self.setup_function: + self.setup_function() + self.setup_function = None + logger.debug("Aria req_obj: %s" % json.dumps(req_obj, indent=2, sort_keys=True)) + req = json.dumps(req_obj).encode('UTF-8') + try: + f = self.opener.open(self.uri, req) + obj = json.loads(f.read()) + try: + return obj['result'] + except KeyError: + raise Aria2JsonRpcError('unexpected result: {}'.format(obj)) + except (urllib2.URLError) as e: + # This should work but URLError does not set the errno attribute: + # e.errno == errno.ECONNREFUSED + raise Aria2JsonRpcError( + str(e), + connection_error=( + '111' in str(e) + ) + ) + except httplib.BadStatusLine as e: + raise Aria2JsonRpcError('{}: BadStatusLine: {} (HTTPS error?)'.format( + self.__class__.__name__, e + )) + + def jsonrpc(self, method, params=None, prefix='aria2.'): + ''' + POST a request to the RPC interface. + ''' + if not params: + params = [] + + if self.token is not None: + token_str = 'token:{}'.format(self.token) + if method == 'multicall': + for p in params[0]: + try: + p['params'].insert(0, token_str) + except KeyError: + p['params'] = [token_str] + else: + params.insert(0, token_str) + + req_obj = { + 'jsonrpc': '2.0', + 'id': self.id, + 'method': prefix + method, + 'params': params, + } + if self.mode == 'batch': + self.queue.append(req_obj) + return None + elif self.mode == 'format': + return req_obj + else: + return self.send_request(req_obj) + + def process_queue(self): + ''' + Processed queued requests. + ''' + req_obj = self.queue + self.queue = [] + return self.send_request(req_obj) + + +# ############################# Standard Methods ############################### + def addUri(self, uris, options=None, position=None): + ''' + aria2.addUri method + + uris: list of URIs + + options: dictionary of additional options + + position: position in queue + + Returns a GID + ''' + params = [uris] + params = add_options_and_position(params, options, position) + return self.jsonrpc('addUri', params) + + def addTorrent(self, torrent, uris=None, options=None, position=None): + ''' + aria2.addTorrent method + + torrent: base64-encoded torrent file + + uris: list of webseed URIs + + options: dictionary of additional options + + position: position in queue + + Returns a GID. + ''' + params = [torrent] + if uris: + params.append(uris) + params = add_options_and_position(params, options, position) + return self.jsonrpc('addTorrent', params) + + def addMetalink(self, metalink, options=None, position=None): + ''' + aria2.addMetalink method + + metalink: base64-encoded metalink file + + options: dictionary of additional options + + position: position in queue + + Returns an array of GIDs. + ''' + params = [metalink] + params = add_options_and_position(params, options, position) + return self.jsonrpc('addTorrent', params) + + def remove(self, gid): + ''' + aria2.remove method + + gid: GID to remove + ''' + params = [gid] + return self.jsonrpc('remove', params) + + def forceRemove(self, gid): + ''' + aria2.forceRemove method + + gid: GID to remove + ''' + params = [gid] + return self.jsonrpc('forceRemove', params) + + def pause(self, gid): + ''' + aria2.pause method + + gid: GID to pause + ''' + params = [gid] + return self.jsonrpc('pause', params) + + def pauseAll(self): + ''' + aria2.pauseAll method + ''' + return self.jsonrpc('pauseAll') + def forcePause(self, gid): + ''' + aria2.forcePause method + gid: GID to pause + ''' + params = [gid] + return self.jsonrpc('forcePause', params) - def jsonrpc(self, method, params=None, prefix='aria2.'): - ''' - POST a request to the RPC interface. - ''' - if not params: - params = [] + def forcePauseAll(self): + ''' + aria2.forcePauseAll method + ''' + return self.jsonrpc('forcePauseAll') - if self.token is not None: - token_str = 'token:{}'.format(self.token) - if method == 'multicall': - for p in params[0]: - try: - p['params'].insert(0, token_str) - except KeyError: - p['params'] = [token_str] - else: - params.insert(0, token_str) + def unpause(self, gid): + ''' + aria2.unpause method - req_obj = { - 'jsonrpc' : '2.0', - 'id' : self.id, - 'method' : prefix + method, - 'params' : params, - } - if self.mode == 'batch': - self.queue.append(req_obj) - return None - elif self.mode == 'format': - return req_obj - else: - return self.send_request(req_obj) + gid: GID to unpause + ''' + params = [gid] + return self.jsonrpc('unpause', params) + def unpauseAll(self): + ''' + aria2.unpauseAll method + ''' + return self.jsonrpc('unpauseAll') + def tellStatus(self, gid, keys=None): + ''' + aria2.tellStatus method - def process_queue(self): - ''' - Processed queued requests. - ''' - req_obj = self.queue - self.queue = [] - return self.send_request(req_obj) + gid: GID to query + keys: subset of status keys to return (all keys are returned otherwise) + Returns a dictionary. + ''' + params = [gid] + if keys: + params.append(keys) + return self.jsonrpc('tellStatus', params) + + def getUris(self, gid): + ''' + aria2.getUris method -############################### Standard Methods ############################### + gid: GID to query - def addUri(self, uris, options=None, position=None): - ''' - aria2.addUri method + Returns a list of dictionaries. + ''' + params = [gid] + return self.jsonrpc('getUris', params) - uris: list of URIs + def getFiles(self, gid): + ''' + aria2.getFiles method - options: dictionary of additional options + gid: GID to query - position: position in queue + Returns a list of dictionaries. + ''' + params = [gid] + return self.jsonrpc('getFiles', params) - Returns a GID - ''' - params = [uris] - params = add_options_and_position(params, options, position) - return self.jsonrpc('addUri', params) + def getPeers(self, gid): + ''' + aria2.getPeers method + gid: GID to query + Returns a list of dictionaries. + ''' + params = [gid] + return self.jsonrpc('getPeers', params) - def addTorrent(self, torrent, uris=None, options=None, position=None): - ''' - aria2.addTorrent method + def getServers(self, gid): + ''' + aria2.getServers method - torrent: base64-encoded torrent file + gid: GID to query - uris: list of webseed URIs + Returns a list of dictionaries. + ''' + params = [gid] + return self.jsonrpc('getServers', params) - options: dictionary of additional options + def tellActive(self, keys=None): + ''' + aria2.tellActive method - position: position in queue + keys: same as tellStatus - Returns a GID. - ''' - params = [torrent] - if uris: - params.append(uris) - params = add_options_and_position(params, options, position) - return self.jsonrpc('addTorrent', params) + Returns a list of dictionaries. The dictionaries are the same as those + returned by tellStatus. + ''' + if keys: + params = [keys] + else: + params = None + return self.jsonrpc('tellActive', params) + def tellWaiting(self, offset, num, keys=None): + ''' + aria2.tellWaiting method + offset: offset from start of waiting download queue + (negative values are counted from the end of the queue) - def addMetalink(self, metalink, options=None, position=None): - ''' - aria2.addMetalink method + num: number of downloads to return - metalink: base64-encoded metalink file + keys: same as tellStatus - options: dictionary of additional options + Returns a list of dictionaries. The dictionaries are the same as those + returned by tellStatus. + ''' + params = [offset, num] + if keys: + params.append(keys) + return self.jsonrpc('tellWaiting', params) - position: position in queue + def tellStopped(self, offset, num, keys=None): + ''' + aria2.tellStopped method - Returns an array of GIDs. - ''' - params = [metalink] - params = add_options_and_position(params, options, position) - return self.jsonrpc('addTorrent', params) + offset: offset from oldest download (same semantics as tellWaiting) + num: same as tellWaiting + keys: same as tellStatus - def remove(self, gid): - ''' - aria2.remove method + Returns a list of dictionaries. The dictionaries are the same as those + returned by tellStatus. + ''' + params = [offset, num] + if keys: + params.append(keys) + return self.jsonrpc('tellStopped', params) - gid: GID to remove - ''' - params = [gid] - return self.jsonrpc('remove', params) + def changePosition(self, gid, pos, how): + ''' + aria2.changePosition method + gid: GID to change + pos: the position - def forceRemove(self, gid): - ''' - aria2.forceRemove method + how: "POS_SET", "POS_CUR" or "POS_END" + ''' + params = [gid, pos, how] + return self.jsonrpc('changePosition', params) - gid: GID to remove - ''' - params = [gid] - return self.jsonrpc('forceRemove', params) + def changeUri(self, gid, fileIndex, delUris, addUris, position=None): + ''' + aria2.changePosition method + gid: GID to change + fileIndex: file to affect (1-based) - def pause(self, gid): - ''' - aria2.pause method + delUris: URIs to remove - gid: GID to pause - ''' - params = [gid] - return self.jsonrpc('pause', params) + addUris: URIs to add + position: where URIs are inserted, after URIs have been removed + ''' + params = [gid, fileIndex, delUris, addUris] + if position: + params.append(position) + return self.jsonrpc('changePosition', params) + def getOption(self, gid): + ''' + aria2.getOption method - def pauseAll(self): - ''' - aria2.pauseAll method - ''' - return self.jsonrpc('pauseAll') + gid: GID to query + Returns a dictionary of options. + ''' + params = [gid] + return self.jsonrpc('getOption', params) + def changeOption(self, gid, options): + ''' + aria2.changeOption method - def forcePause(self, gid): - ''' - aria2.forcePause method - - gid: GID to pause - ''' - params = [gid] - return self.jsonrpc('forcePause', params) - - - - def forcePauseAll(self): - ''' - aria2.forcePauseAll method - ''' - return self.jsonrpc('forcePauseAll') - - - - def unpause(self, gid): - ''' - aria2.unpause method - - gid: GID to unpause - ''' - params = [gid] - return self.jsonrpc('unpause', params) - - - - def unpauseAll(self): - ''' - aria2.unpauseAll method - ''' - return self.jsonrpc('unpauseAll') - - - - def tellStatus(self, gid, keys=None): - ''' - aria2.tellStatus method - - gid: GID to query - - keys: subset of status keys to return (all keys are returned otherwise) - - Returns a dictionary. - ''' - params = [gid] - if keys: - params.append(keys) - return self.jsonrpc('tellStatus', params) - - - - def getUris(self, gid): - ''' - aria2.getUris method - - gid: GID to query - - Returns a list of dictionaries. - ''' - params = [gid] - return self.jsonrpc('getUris', params) - - - - def getFiles(self, gid): - ''' - aria2.getFiles method - - gid: GID to query - - Returns a list of dictionaries. - ''' - params = [gid] - return self.jsonrpc('getFiles', params) - - - - def getPeers(self, gid): - ''' - aria2.getPeers method - - gid: GID to query - - Returns a list of dictionaries. - ''' - params = [gid] - return self.jsonrpc('getPeers', params) - - - - def getServers(self, gid): - ''' - aria2.getServers method - - gid: GID to query - - Returns a list of dictionaries. - ''' - params = [gid] - return self.jsonrpc('getServers', params) - - - - def tellActive(self, keys=None): - ''' - aria2.tellActive method - - keys: same as tellStatus - - Returns a list of dictionaries. The dictionaries are the same as those - returned by tellStatus. - ''' - if keys: - params = [keys] - else: - params = None - return self.jsonrpc('tellActive', params) - - - - def tellWaiting(self, offset, num, keys=None): - ''' - aria2.tellWaiting method - - offset: offset from start of waiting download queue - (negative values are counted from the end of the queue) - - num: number of downloads to return - - keys: same as tellStatus - - Returns a list of dictionaries. The dictionaries are the same as those - returned by tellStatus. - ''' - params = [offset, num] - if keys: - params.append(keys) - return self.jsonrpc('tellWaiting', params) - - - - def tellStopped(self, offset, num, keys=None): - ''' - aria2.tellStopped method - - offset: offset from oldest download (same semantics as tellWaiting) - - num: same as tellWaiting - - keys: same as tellStatus - - Returns a list of dictionaries. The dictionaries are the same as those - returned by tellStatus. - ''' - params = [offset, num] - if keys: - params.append(keys) - return self.jsonrpc('tellStopped', params) - - - - def changePosition(self, gid, pos, how): - ''' - aria2.changePosition method - - gid: GID to change - - pos: the position - - how: "POS_SET", "POS_CUR" or "POS_END" - ''' - params = [gid, pos, how] - return self.jsonrpc('changePosition', params) - - - - def changeUri(self, gid, fileIndex, delUris, addUris, position=None): - ''' - aria2.changePosition method - - gid: GID to change - - fileIndex: file to affect (1-based) - - delUris: URIs to remove - - addUris: URIs to add - - position: where URIs are inserted, after URIs have been removed - ''' - params = [gid, fileIndex, delUris, addUris] - if position: - params.append(position) - return self.jsonrpc('changePosition', params) - - - - def getOption(self, gid): - ''' - aria2.getOption method - - gid: GID to query - - Returns a dictionary of options. - ''' - params = [gid] - return self.jsonrpc('getOption', params) - - - - def changeOption(self, gid, options): - ''' - aria2.changeOption method - - gid: GID to change - - options: dictionary of new options - (not all options can be changed for active downloads) - ''' - params = [gid, options] - return self.jsonrpc('changeOption', params) - - - - def getGlobalOption(self): - ''' - aria2.getGlobalOption method - - Returns a dictionary. - ''' - return self.jsonrpc('getGlobalOption') - - - - def changeGlobalOption(self, options): - ''' - aria2.changeGlobalOption method - - options: dictionary of new options - ''' - params = [options] - return self.jsonrpc('changeGlobalOption', params) - - - - def getGlobalStat(self): - ''' - aria2.getGlobalStat method - - Returns a dictionary. - ''' - return self.jsonrpc('getGlobalStat') - - - - def purgeDownloadResult(self): - ''' - aria2.purgeDownloadResult method - ''' - self.jsonrpc('purgeDownloadResult') - - - - def removeDownloadResult(self, gid): - ''' - aria2.removeDownloadResult method - - gid: GID to remove - ''' - params = [gid] - return self.jsonrpc('removeDownloadResult', params) - - - - def getVersion(self): - ''' - aria2.getVersion method - - Returns a dictionary. - ''' - return self.jsonrpc('getVersion') - - - - def getSessionInfo(self): - ''' - aria2.getSessionInfo method - - Returns a dictionary. - ''' - return self.jsonrpc('getSessionInfo') - - - - def shutdown(self): - ''' - aria2.shutdown method - ''' - return self.jsonrpc('shutdown') - - - - def forceShutdown(self): - ''' - aria2.forceShutdown method - ''' - return self.jsonrpc('forceShutdown') - - - - def multicall(self, methods): - ''' - aria2.multicall method - - methods: list of dictionaries (keys: methodName, params) - - The method names must be those used by Aria2c, e.g. "aria2.tellStatus". - ''' - return self.jsonrpc('multicall', [methods], prefix='system.') - - - - -############################# Convenience Methods ############################## - - def add_torrent(self, path, uris=None, options=None, position=None): - ''' - A wrapper around addTorrent for loading files. - ''' - with open(path, 'r') as f: - torrent = base64.encode(f.read()) - return self.addTorrent(torrent, uris, options, position) - - - - def add_metalink(self, path, uris=None, options=None, position=None): - ''' - A wrapper around addMetalink for loading files. - ''' - with open(path, 'r') as f: - metalink = base64.encode(f.read()) - return self.addMetalink(metalink, uris, options, position) - - - - def get_status(self, gid): - ''' - Get the status of a single GID. - ''' - response = self.tellStatus(gid, ['status']) - return get_status(response) - - - - def wait_for_final_status(self, gid, interval=1): - ''' - Wait for a GID to complete or fail and return its status. - ''' - if not interval or interval < 0: - interval = 1 - while True: - status = self.get_status(gid) - if status in TEMPORARY_STATUS: - time.sleep(interval) - else: - return status - - - - def get_statuses(self, gids): - ''' - Get the status of multiple GIDs. The status of each is yielded in order. - ''' - methods = [ - { - 'methodName' : 'aria2.tellStatus', - 'params' : [gid, ['gid', 'status']] - } - for gid in gids - ] - results = self.multicall(methods) - if results: - status = dict((r[0]['gid'], r[0]['status']) for r in results) - for gid in gids: - try: - yield status[gid] - except KeyError: - logger.error('Aria2 RPC server returned no status for GID {}'.format(gid)) - yield 'error' - else: - logger.error('no response from Aria2 RPC server') - for gid in gids: - yield 'error' - - - - def wait_for_final_statuses(self, gids, interval=1): - ''' - Wait for multiple GIDs to complete or fail and return their statuses in - order. - - gids: - A flat list of GIDs. - ''' - if not interval or interval < 0: - interval = 1 - statusmap = dict((g, None) for g in gids) - remaining = list( - g for g,s in statusmap.items() if s is None - ) - while remaining: - for g, s in zip(remaining, self.get_statuses(remaining)): - if s in TEMPORARY_STATUS: - continue - else: - statusmap[g] = s - remaining = list( - g for g,s in statusmap.items() if s is None - ) - if remaining: - time.sleep(interval) - for g in gids: - yield statusmap[g] - - - - def print_global_status(self): - ''' - Print global status of the RPC server. - ''' - status = self.getGlobalStat() - if status: - numWaiting = int(status['numWaiting']) - numStopped = int(status['numStopped']) - keys = ['totalLength', 'completedLength'] - total = self.tellActive(keys) - waiting = self.tellWaiting(0, numWaiting, keys) - if waiting: - total += waiting - stopped = self.tellStopped(0, numStopped, keys) - if stopped: - total += stopped - - downloadSpeed = int(status['downloadSpeed']) - uploadSpeed = int(status['uploadSpeed']) - totalLength = sum(int(x['totalLength']) for x in total) - completedLength = sum(int(x['completedLength']) for x in total) - remaining = totalLength - completedLength - - status['downloadSpeed'] = format_bytes(downloadSpeed) + '/s' - status['uploadSpeed'] = format_bytes(uploadSpeed) + '/s' - - preordered = ('downloadSpeed', 'uploadSpeed') - - rows = list() - for k in sorted(status): - if k in preordered: - continue - rows.append((k, status[k])) - - rows.extend((x, status[x]) for x in preordered) - - if totalLength > 0: - rows.append(('total', format(format_bytes(totalLength)))) - rows.append(('completed', format(format_bytes(completedLength)))) - rows.append(('remaining', format(format_bytes(remaining)))) - if completedLength == totalLength: - eta = 'finished' - else: - try: - eta = format_seconds(remaining // downloadSpeed) - except ZeroDivisionError: - eta = 'never' - rows.append(('ETA', eta)) - - l = max(len(r[0]) for r in rows) - r = max(len(r[1]) for r in rows) - r = max(r, len(self.uri) - (l + 2)) - fmt = '{:<' + str(l) + 's} {:>' + str(r) + 's}' - - print(self.uri) - for k, v in rows: - print(fmt.format(k, v)) - - - - def queue_uris(self, uris, options, interval=None): - ''' - Enqueue URIs and wait for download to finish while printing status at - regular intervals. - ''' - gid = self.addUri(uris, options) - print('GID: {}'.format(gid)) - - if gid and interval is not None: - blanker = '' - while True: - response = self.tellStatus(gid, ['status']) - if response: - try: - status = response['status'] - except KeyError: - print('error: no status returned from Aria2 RPC server') - break - print('{}\rstatus: {}'.format(blanker, status)), - blanker = ' ' * len(status) - if status in TEMPORARY_STATUS: - time.sleep(interval) - else: - break - else: - print('error: no response from server') - break - - - -######################### Polymethod download handlers ######################### - - def polymethod_enqueue_many(self, downloads): - ''' - Enqueue downloads. - - downloads: Same as polymethod_download(). - ''' - methods = list( - { - 'methodName': 'aria2.{}'.format(d[0]), - 'params': list(d[1:]) - } for d in downloads - ) - return self.multicall(methods) - - - - def polymethod_wait_many(self, gids, interval=1): - ''' - Wait for the GIDs to complete or fail and return their statuses. - - gids: - A list of lists of GIDs. - ''' - # The flattened list of GIDs - gs = list(g for gs in gids for g in gs) - statusmap = dict(tuple(zip( - gs, - self.wait_for_final_statuses(gs, interval=interval) - ))) - for gs in gids: - yield list(statusmap.get(g, 'error') for g in gs) - - - - - def polymethod_enqueue_one(self, download): - ''' - Same as polymethod_enqueue_many but for one element. - ''' - return getattr(self, download[0])(*download[1:]) - - - - def polymethod_download(self, downloads, interval=1): - ''' - Enqueue a series of downloads and wait for them to finish. Iterate over the - status of each, in order. - - downloads: - An iterable over (, , ...) where indicates the "add" - method to use ('addUri', 'addTorrent', 'addMetalink') and everything that - follows are arguments to pass to that method. - - interval: - The status check interval while waiting. - - Iterates over the download status of finished downloads. "complete" - indicates success. Lists of statuses will be returned for downloads that - create multiple GIDs (e.g. metalinks). - ''' - gids = self.polymethod_enqueue_many(downloads) - return self.polymethod_wait_many(gids, interval=interval) - - - - def polymethod_download_bool(self, *args, **kwargs): - ''' - A wrapper around polymethod_download() which returns a boolean for each - download to indicate success (True) or failure (False). - ''' + gid: GID to change + + options: dictionary of new options + (not all options can be changed for active downloads) + ''' + params = [gid, options] + return self.jsonrpc('changeOption', params) + + def getGlobalOption(self): + ''' + aria2.getGlobalOption method + + Returns a dictionary. + ''' + return self.jsonrpc('getGlobalOption') + + def changeGlobalOption(self, options): + ''' + aria2.changeGlobalOption method + + options: dictionary of new options + ''' + params = [options] + return self.jsonrpc('changeGlobalOption', params) + + def getGlobalStat(self): + ''' + aria2.getGlobalStat method + + Returns a dictionary. + ''' + return self.jsonrpc('getGlobalStat') + + def purgeDownloadResult(self): + ''' + aria2.purgeDownloadResult method + ''' + self.jsonrpc('purgeDownloadResult') + + def removeDownloadResult(self, gid): + ''' + aria2.removeDownloadResult method + + gid: GID to remove + ''' + params = [gid] + return self.jsonrpc('removeDownloadResult', params) + + def getVersion(self): + ''' + aria2.getVersion method + + Returns a dictionary. + ''' + return self.jsonrpc('getVersion') + + def getSessionInfo(self): + ''' + aria2.getSessionInfo method + + Returns a dictionary. + ''' + return self.jsonrpc('getSessionInfo') + + def shutdown(self): + ''' + aria2.shutdown method + ''' + return self.jsonrpc('shutdown') + + def forceShutdown(self): + ''' + aria2.forceShutdown method + ''' + return self.jsonrpc('forceShutdown') + + def multicall(self, methods): + ''' + aria2.multicall method + + methods: list of dictionaries (keys: methodName, params) + + The method names must be those used by Aria2c, e.g. "aria2.tellStatus". + ''' + return self.jsonrpc('multicall', [methods], prefix='system.') + + +# ########################### Convenience Methods ############################## + def add_torrent(self, path, uris=None, options=None, position=None): + ''' + A wrapper around addTorrent for loading files. + ''' + with open(path, 'r') as f: + torrent = base64.encode(f.read()) + return self.addTorrent(torrent, uris, options, position) + + def add_metalink(self, path, uris=None, options=None, position=None): + ''' + A wrapper around addMetalink for loading files. + ''' + with open(path, 'r') as f: + metalink = base64.encode(f.read()) + return self.addMetalink(metalink, uris, options, position) + + def get_status(self, gid): + ''' + Get the status of a single GID. + ''' + response = self.tellStatus(gid, ['status']) + return get_status(response) + + def wait_for_final_status(self, gid, interval=1): + ''' + Wait for a GID to complete or fail and return its status. + ''' + if not interval or interval < 0: + interval = 1 + while True: + status = self.get_status(gid) + if status in TEMPORARY_STATUS: + time.sleep(interval) + else: + return status + + def get_statuses(self, gids): + ''' + Get the status of multiple GIDs. The status of each is yielded in order. + ''' + methods = [ + { + 'methodName': 'aria2.tellStatus', + 'params': [gid, ['gid', 'status']] + } + for gid in gids + ] + results = self.multicall(methods) + if results: + status = dict((r[0]['gid'], r[0]['status']) for r in results) + for gid in gids: + try: + yield status[gid] + except KeyError: + logger.error('Aria2 RPC server returned no status for GID {}'.format(gid)) + yield 'error' + else: + logger.error('no response from Aria2 RPC server') + for gid in gids: + yield 'error' + + def wait_for_final_statuses(self, gids, interval=1): + ''' + Wait for multiple GIDs to complete or fail and return their statuses in + order. + + gids: + A flat list of GIDs. + ''' + if not interval or interval < 0: + interval = 1 + statusmap = dict((g, None) for g in gids) + remaining = list( + g for g, s in statusmap.items() if s is None + ) + while remaining: + for g, s in zip(remaining, self.get_statuses(remaining)): + if s in TEMPORARY_STATUS: + continue + else: + statusmap[g] = s + remaining = list( + g for g, s in statusmap.items() if s is None + ) + if remaining: + time.sleep(interval) + for g in gids: + yield statusmap[g] + + def print_global_status(self): + ''' + Print global status of the RPC server. + ''' + status = self.getGlobalStat() + if status: + numWaiting = int(status['numWaiting']) + numStopped = int(status['numStopped']) + keys = ['totalLength', 'completedLength'] + total = self.tellActive(keys) + waiting = self.tellWaiting(0, numWaiting, keys) + if waiting: + total += waiting + stopped = self.tellStopped(0, numStopped, keys) + if stopped: + total += stopped + + downloadSpeed = int(status['downloadSpeed']) + uploadSpeed = int(status['uploadSpeed']) + totalLength = sum(int(x['totalLength']) for x in total) + completedLength = sum(int(x['completedLength']) for x in total) + remaining = totalLength - completedLength + + status['downloadSpeed'] = format_bytes(downloadSpeed) + '/s' + status['uploadSpeed'] = format_bytes(uploadSpeed) + '/s' + + preordered = ('downloadSpeed', 'uploadSpeed') + + rows = list() + for k in sorted(status): + if k in preordered: + continue + rows.append((k, status[k])) + + rows.extend((x, status[x]) for x in preordered) + + if totalLength > 0: + rows.append(('total', format(format_bytes(totalLength)))) + rows.append(('completed', format(format_bytes(completedLength)))) + rows.append(('remaining', format(format_bytes(remaining)))) + if completedLength == totalLength: + eta = 'finished' + else: + try: + eta = format_seconds(remaining // downloadSpeed) + except ZeroDivisionError: + eta = 'never' + rows.append(('ETA', eta)) + + l = max(len(r[0]) for r in rows) + r = max(len(r[1]) for r in rows) + r = max(r, len(self.uri) - (l + 2)) + fmt = '{:<' + str(l) + 's} {:>' + str(r) + 's}' + + print(self.uri) + for k, v in rows: + print(fmt.format(k, v)) + + def queue_uris(self, uris, options, interval=None): + ''' + Enqueue URIs and wait for download to finish while printing status at + regular intervals. + ''' + gid = self.addUri(uris, options) + print('GID: {}'.format(gid)) + + if gid and interval is not None: + blanker = '' + while True: + response = self.tellStatus(gid, ['status']) + if response: + try: + status = response['status'] + except KeyError: + print('error: no status returned from Aria2 RPC server') + break + print('{}\rstatus: {}'.format(blanker, status)), + blanker = ' ' * len(status) + if status in TEMPORARY_STATUS: + time.sleep(interval) + else: + break + else: + print('error: no response from server') + break + + +# ####################### Polymethod download handlers ######################### + def polymethod_enqueue_many(self, downloads): + ''' + Enqueue downloads. + + downloads: Same as polymethod_download(). + ''' + methods = list( + { + 'methodName': 'aria2.{}'.format(d[0]), + 'params': list(d[1:]) + } for d in downloads + ) + return self.multicall(methods) + + def polymethod_wait_many(self, gids, interval=1): + ''' + Wait for the GIDs to complete or fail and return their statuses. + + gids: + A list of lists of GIDs. + ''' + # The flattened list of GIDs + gs = list(g for gs in gids for g in gs) + statusmap = dict(tuple(zip( + gs, + self.wait_for_final_statuses(gs, interval=interval) + ))) + for gs in gids: + yield list(statusmap.get(g, 'error') for g in gs) + + def polymethod_enqueue_one(self, download): + ''' + Same as polymethod_enqueue_many but for one element. + ''' + return getattr(self, download[0])(*download[1:]) + + def polymethod_download(self, downloads, interval=1): + ''' + Enqueue a series of downloads and wait for them to finish. Iterate over the + status of each, in order. + + downloads: + An iterable over (, , ...) where indicates the "add" + method to use ('addUri', 'addTorrent', 'addMetalink') and everything that + follows are arguments to pass to that method. + + interval: + The status check interval while waiting. + + Iterates over the download status of finished downloads. "complete" + indicates success. Lists of statuses will be returned for downloads that + create multiple GIDs (e.g. metalinks). + ''' + gids = self.polymethod_enqueue_many(downloads) + return self.polymethod_wait_many(gids, interval=interval) + + def polymethod_download_bool(self, *args, **kwargs): + ''' + A wrapper around polymethod_download() which returns a boolean for each + download to indicate success (True) or failure (False). + ''' # for status in self.polymethod_download(*args, **kwargs): # yield all(s == 'complete' for s in status) - return list( - all(s == 'complete' for s in status) - for status in self.polymethod_download(*args, **kwargs) - ) - \ No newline at end of file + return list( + all(s == 'complete' for s in status) + for status in self.polymethod_download(*args, **kwargs) + ) diff --git a/headphones/deezloader.py b/headphones/deezloader.py index db14ae5d..528cded6 100644 --- a/headphones/deezloader.py +++ b/headphones/deezloader.py @@ -26,7 +26,6 @@ import binascii from beets.mediafile import MediaFile from headphones import logger, request, helpers import headphones -from twisted.conch.insults import helper # Public constants PROVIDER_NAME = 'Deezer' @@ -53,6 +52,7 @@ __cookies = None __tracks_cache = {} __albums_cache = {} + def __getApiToken(): global __cookies response = request.request_response(url="http://www.deezer.com/", headers=__HTTP_HEADERS) @@ -67,39 +67,42 @@ def __getApiToken(): if not token: logger.error(u"Deezloader: Unable to get api token") - + + def getAlbumByLink(album_link): """Returns deezer album infos using album link url - + :param str album_link: deezer album link url (eg: 'http://www.deezer.com/album/1234567/') """ matches = re.search(r"album\/([0-9]+)\/?$", album_link) if matches: return getAlbum(matches.group(1)) - + + def getAlbum(album_id): """Returns deezer album infos - + :param int album_id: deezer album id """ global __albums_cache - + if str(album_id) in __albums_cache: return __albums_cache[str(album_id)] - + url = __API_INFO_URL + "album/" + str(album_id) data = request.request_json(url=url, headers=__HTTP_HEADERS, cookies=__cookies) - + if data and 'error' not in data: __albums_cache[str(album_id)] = data return data else: logger.debug("Deezloader: Can't load album infos") return None - + + def searchAlbums(search_term): """Search for deezer albums using search term - + :param str search_term: search term to search album for """ logger.info(u'Searching Deezer using term: "%s"' % search_term) @@ -108,7 +111,7 @@ def searchAlbums(search_term): data = request.request_json(url=url, headers=__HTTP_HEADERS, cookies=__cookies) albums = [] - + # Process content if data and 'total' in data and data['total'] > 0 and 'data' in data: for item in data['data']: @@ -121,13 +124,14 @@ def searchAlbums(search_term): return albums + def __matchAlbums(albums, artist_name, album_title, album_length): resultlist = [] - + for album in albums: total_size = 0 tracks_found = 0 - + for track in album['tracks']['data']: t = getTrack(track['id']) if t: @@ -139,29 +143,29 @@ def __matchAlbums(albums, artist_name, album_title, album_length): size = t["FILESIZE_MP3_128"] else: size = t["FILESIZE_MP3_64"] - + size = int(size) total_size += size tracks_found += 1 logger.debug(u'Found song "%s". Size: %s' % (t['SNG_TITLE'], helpers.bytes_to_mb(size))) - + matched = True mismatch_reason = 'matched!' - + if album_length > 0 and abs(int(album['duration']) - album_length) > 240: matched = False mismatch_reason = (u'duration mismatch: %i not in [%i, %i]' % (int(album['duration']), (album_length - 240), (album_length + 240))) - elif (helpers.latinToAscii(re.sub(r"\W", "", album_title, 0, re.UNICODE)).lower() != + elif (helpers.latinToAscii(re.sub(r"\W", "", album_title, 0, re.UNICODE)).lower() != helpers.latinToAscii(re.sub(r"\W", "", album['title'], 0, re.UNICODE)).lower()): matched = False mismatch_reason = (u'album name mismatch: %s != %s' % (album['title'], album_title)) - + elif (helpers.latinToAscii(re.sub(r"\W", "", artist_name, 0, re.UNICODE)).lower() != helpers.latinToAscii(re.sub(r"\W", "", album['artist']['name'], 0, re.UNICODE)).lower()): matched = False mismatch_reason = (u'artist name mismatch: %s != %s' % (album['artist']['name'], artist_name)) - + resultlist.append( (album['artist']['name'] + ' - ' + album['title'] + ' [' + album['release_date'][:4] + '] (' + str(tracks_found) + '/' + str(album['nb_tracks']) + ')', total_size, album['link'], PROVIDER_NAME, "ddl", matched) @@ -170,6 +174,7 @@ def __matchAlbums(albums, artist_name, album_title, album_length): return resultlist + def searchAlbum(artist_name, album_title, user_search_term=None, album_length=None): """Search for deezer specific album. This will iterate over deezer albums and try to find best matches @@ -182,7 +187,7 @@ def searchAlbum(artist_name, album_title, user_search_term=None, album_length=No # User search term by-pass normal search if user_search_term: return __matchAlbums(searchAlbums(user_search_term), artist_name, album_title, album_length) - + resultlist = __matchAlbums(searchAlbums((artist_name + ' ' + album_title).strip()), artist_name, album_title, album_length) if resultlist: return resultlist @@ -190,35 +195,36 @@ def searchAlbum(artist_name, album_title, user_search_term=None, album_length=No # Deezer API supports unicode, so just remove non alphanumeric characters clean_artist_name = re.sub(r"[^\w\s]", " ", artist_name, 0, re.UNICODE).strip() clean_album_name = re.sub(r"[^\w\s]", " ", album_title, 0, re.UNICODE).strip() - + resultlist = __matchAlbums(searchAlbums((clean_artist_name + ' ' + clean_album_name).strip()), artist_name, album_title, album_length) if resultlist: return resultlist - + resultlist = __matchAlbums(searchAlbums(clean_artist_name), artist_name, album_title, album_length) if resultlist: return resultlist - + return resultlist + def getTrack(sng_id, try_reload_api=True): """Returns deezer track infos - + :param int sng_id: deezer song id - :param bool try_reload_api: whether or not try reloading API if session expired + :param bool try_reload_api: whether or not try reloading API if session expired """ global __tracks_cache - + if str(sng_id) in __tracks_cache: return __tracks_cache[str(sng_id)] - + data = "[{\"method\":\"song.getListData\",\"params\":{\"sng_ids\":[" + str(sng_id) + "]}}]" json = request.request_json(url=__API_URL, headers=__HTTP_HEADERS, method='post', params=__api_queries, data=data, cookies=__cookies) - + results = [] error = None invalid_token = False - + if json: # Check for errors if 'error' in json: @@ -230,7 +236,7 @@ def getTrack(sng_id, try_reload_api=True): error = json[0]['error'] if 'VALID_TOKEN_REQUIRED' in json[0]['error'] and json[0]['error']['VALID_TOKEN_REQUIRED'] == u"Invalid CSRF token": invalid_token = True - + # Got invalid token error if error: if invalid_token and try_reload_api: @@ -245,26 +251,27 @@ def getTrack(sng_id, try_reload_api=True): if 'token' in item: logger.error(u"An unknown error occurred in the Deezer parser: Uploaded Files are currently not supported") return - + sng_id = item["SNG_ID"] md5Origin = item["MD5_ORIGIN"] sng_format = 3 - + if item["FILESIZE_MP3_320"] <= 0: if item["FILESIZE_MP3_256"] > 0: sng_format = 5 else: sng_format = 1 - + mediaVersion = int(item["MEDIA_VERSION"]) item['downloadUrl'] = __getDownloadUrl(md5Origin, sng_id, sng_format, mediaVersion) - + __tracks_cache[sng_id] = item return item - + except Exception as e: logger.error(u"An unknown error occurred in the Deezer parser: %s" % e) + def __getDownloadUrl(md5Origin, sng_id, sng_format, mediaVersion): urlPart = md5Origin.encode('utf-8') + b'\xa4' + str(sng_format) + b'\xa4' + str(sng_id) + b'\xa4' + str(mediaVersion) md5val = md5(urlPart).hexdigest() @@ -273,6 +280,7 @@ def __getDownloadUrl(md5Origin, sng_id, sng_format, mediaVersion): ciphertext = cipher.encrypt(__pad(urlPart, AES.block_size)) return "http://e-cdn-proxy-" + md5Origin[:1] + ".deezer.com/mobile/1/" + binascii.hexlify(ciphertext).lower() + def __pad(raw, block_size): if (len(raw) % block_size == 0): return raw @@ -281,10 +289,11 @@ def __pad(raw, block_size): data = raw + padding_required * padChar return data + def __tagTrack(path, track): try: album = getAlbum(track['ALB_ID']) - + f = MediaFile(path) f.artist = track['ART_NAME'] f.album = track['ALB_TITLE'] @@ -295,22 +304,23 @@ def __tagTrack(path, track): f.bpm = track['BPM'] f.date = datetime.strptime(album['release_date'], '%Y-%m-%d').date() f.albumartist = album['artist']['name'] - if u'genres' in album and u'data' in album['genres']: + if u'genres' in album and u'data' in album['genres']: f.genres = [genre['name'] for genre in album['genres']['data']] - + f.save() except Exception as e: logger.error(u'Unable to tag deezer track "%s": %s' % (path, e)) + def decryptTracks(paths): """Decrypt downloaded deezer tracks. - + :param paths: list of path to deezer tracks (*.dzr files). """ # Note: tracks can be from different albums decrypted_tracks = {} - + # First pass: load tracks data for path in paths: try: @@ -322,15 +332,15 @@ def decryptTracks(paths): if album_folder not in decrypted_tracks: decrypted_tracks[album_folder] = {} - + if disk_number not in decrypted_tracks[album_folder]: decrypted_tracks[album_folder][disk_number] = {} decrypted_tracks[album_folder][disk_number][track_number] = track - + except Exception as e: logger.error(u'Unable to load deezer track infos "%s": %s' % (path, e)) - + # Second pass: decrypt tracks for album_folder in decrypted_tracks: multi_disks = len(decrypted_tracks[album_folder]) > 1 @@ -339,12 +349,12 @@ def decryptTracks(paths): try: filename = helpers.replace_illegal_chars(track['SNG_TITLE']).strip() filename = ('{:02d}'.format(track_number) + '_' + filename + '.mp3') - + # Add a 'cd x' sub-folder if album has more than one disk disk_folder = os.path.join(album_folder, 'cd ' + str(disk_number)) if multi_disks else album_folder - + dest = os.path.join(disk_folder, filename).encode(headphones.SYS_ENCODING, 'replace') - + # Decrypt track if not already done if not os.path.exists(dest): try: @@ -358,12 +368,13 @@ def decryptTracks(paths): continue decrypted_tracks[album_folder][disk_number][track_number]['path'] = dest - + except Exception as e: logger.error(u'Unable to decrypt deezer track "%s": %s' % (path, e)) - + return decrypted_tracks + def __decryptDownload(source, sng_id, dest): interval_chunk = 3 chunk_size = 2048 @@ -383,7 +394,7 @@ def __decryptDownload(source, sng_id, dest): if(i % interval_chunk == 0): cipher = Blowfish.new(blowFishKey, Blowfish.MODE_CBC, iv) chunk = cipher.decrypt(__pad(chunk, Blowfish.block_size)) - + fout.write(chunk) i += 1 chunk = f.read(chunk_size) @@ -391,6 +402,7 @@ def __decryptDownload(source, sng_id, dest): f.close() fout.close() + def __getBlowFishKey(encryptionKey): if encryptionKey < 1: encryptionKey *= -1 @@ -402,14 +414,15 @@ def __getBlowFishKey(encryptionKey): return __xorHex(parts) + def __xorHex(parts): data = "" for i in range(0, 16): character = ord(parts[0][i]) - + for j in range(1, len(parts)): character ^= ord(parts[j][i]) - + data += chr(character) - + return data diff --git a/headphones/helpers_test.py b/headphones/helpers_test.py index 14b8540d..54c9e9bf 100644 --- a/headphones/helpers_test.py +++ b/headphones/helpers_test.py @@ -14,9 +14,9 @@ class HelpersTest(TestCase): u'Symphonęy Nº9': 'Symphoney No.9', u'ÆæßðÞIJij': u'AeaessdThIJıj', u'Obsessió (Cerebral Apoplexy remix)': 'obsessio cerebral ' - 'apoplexy remix', + 'apoplexy remix', u'Doktór Hałabała i siedmiu zbojów': 'doktor halabala i siedmiu ' - 'zbojow', + 'zbojow', u'Arbetets Söner och Döttrar': 'arbetets soner och dottrar', u'Björk Guðmundsdóttir': 'bjork gudmundsdottir', u'L\'Arc~en~Ciel': 'larc en ciel', diff --git a/headphones/postprocessor.py b/headphones/postprocessor.py index 336030af..c9621506 100755 --- a/headphones/postprocessor.py +++ b/headphones/postprocessor.py @@ -261,7 +261,7 @@ def verify(albumid, albumpath, Kind=None, forced=False, keep_original_folder=Fal if downloaded_deezer_list: logger.info('Decrypting deezer tracks') decrypted_deezer_list = deezloader.decryptTracks(downloaded_deezer_list) - + # Check if album is complete based on album duration only # (total track numbers is not determinant enough due to hidden tracks for eg) db_track_duration = 0 @@ -271,7 +271,7 @@ def verify(albumid, albumpath, Kind=None, forced=False, keep_original_folder=Fal db_track_duration += track['TrackDuration'] / 1000 except: downloaded_track_duration = False - + try: for disk_number in decrypted_deezer_list[albumpath]: for track in decrypted_deezer_list[albumpath][disk_number].values(): @@ -279,14 +279,14 @@ def verify(albumid, albumpath, Kind=None, forced=False, keep_original_folder=Fal downloaded_track_duration += int(track['DURATION']) except: downloaded_track_duration = False - + if not downloaded_track_duration or not db_track_duration or abs(downloaded_track_duration - db_track_duration) > 240: - logger.info("Looks like " + + logger.info("Looks like " + os.path.basename(albumpath).decode(headphones.SYS_ENCODING, 'replace') + " isn't complete yet (duration mismatch). Will try again on the next run") return - downloaded_track_list = list(set(downloaded_track_list)) # Remove duplicates + downloaded_track_list = list(set(downloaded_track_list)) # Remove duplicates # test #1: metadata - usually works logger.debug('Verifying metadata...') diff --git a/headphones/searcher.py b/headphones/searcher.py index 2516bc31..f6554cb1 100644 --- a/headphones/searcher.py +++ b/headphones/searcher.py @@ -55,6 +55,7 @@ redobj = None # Persistent Aria2 RPC object __aria2rpc_obj = None + def getAria2RPC(): global __aria2rpc_obj if not __aria2rpc_obj: @@ -67,10 +68,12 @@ def getAria2RPC(): ) return __aria2rpc_obj + def reconfigure(): global __aria2rpc_obj __aria2rpc_obj = None + def fix_url(s, charset="utf-8"): """ Fix the URL so it is proper formatted and encoded. @@ -896,7 +899,7 @@ def send_to_downloader(data, bestqual, album): except Exception as e: logger.error(u'Error sending torrent to Aria2. Are you sure it\'s running? (%s)' % e) return - + else: folder_name = '%s - %s [%s]' % ( helpers.latinToAscii(album['ArtistName']).encode('UTF-8').replace('/', '_'), @@ -1274,6 +1277,7 @@ def verifyresult(title, artistterm, term, lossless): return True + def searchDdl(album, new=False, losslessOnly=False, albumlength=None, choose_specific_download=False): reldate = album['ReleaseDate'] @@ -1313,7 +1317,7 @@ def searchDdl(album, new=False, losslessOnly=False, albumlength=None, logger.debug(u'Using search term: "%s"' % helpers.latinToAscii(term)) resultlist = [] - + # Deezer only provides lossy if headphones.CONFIG.DEEZLOADER and not losslessOnly: resultlist += deezloader.searchAlbum(album['ArtistName'], album['AlbumTitle'], album['SearchTerm'], int(albumlength / 1000)) diff --git a/headphones/webserve.py b/headphones/webserve.py index 19e2ffb3..7e97cefb 100644 --- a/headphones/webserve.py +++ b/headphones/webserve.py @@ -1587,7 +1587,7 @@ class WebInterface(object): # Reconfigure musicbrainz database connection with the new values mb.startmb() - + # Reconfigure Aria2 searcher.reconfigure()