Merge pull request #2925 from Kallys/master

Add deezloader provider and aria2 downloader (part 2)

data/interfaces/default/config.html

@@ -304,6 +304,45 @@
                                     <input type="text" name="usenet_retention" value="${config['usenet_retention']}" size="5">
                                 </div>
                             </fieldset>
+
+                            <fieldset title="Method for downloading direct download files.">
+                                <legend>Direct Download</legend>
+                                <input type="radio" name="ddl_downloader" id="ddl_downloader_aria" value="0" ${config['ddl_downloader_aria']}> Aria2
+                           </fieldset>
+                           <fieldset id="ddl_aria_options">
+                                <div class="row">
+                                    <label title="Aria2 RPC host, port and path.">
+                                        Aria2 Host
+                                    </label>
+                                    <input type="text" name="aria_host" value="${config['aria_host']}" size="30">
+                                    <small>usually http://localhost:6800/jsonrpc</small>
+                                </div>
+                                <div class="row">
+                                    <label title="Aria2 RPC username. Leave empty if not applicable.">
+                                        Aria2 Username
+                                    </label>
+                                    <input type="text" name="aria_username" value="${config['aria_username']}" size="20">
+                                </div>
+                                <div class="row">
+                                    <label title="Aria2 RPC password. Leave empty if not applicable.">
+                                        Aria2 Password
+                                    </label>
+                                    <input type="password" name="aria_password" value="${config['aria_password'] | h}" size="20">
+                                </div>
+                                <div class="row">
+                                    <label title="Aria2 RPC secret key. Leave empty if not applicable.">
+                                        Aria2 secret token
+                                    </label>
+                                    <input type="text" name="aria_token" value="${config['aria_token']}" size="36">
+                                </div>
+                                <div class="row">
+                                    <label title="Path to folder where Headphones can find the downloads.">
+                                        Music Download Directory:
+                                    </label>
+                                    <input type="text" name="download_ddl_dir" value="${config['download_ddl_dir']}" size="50">
+                                    <small>Full path where ddl client downloads your music, e.g. /Users/name/Downloads/music</small>
+                                </div>
+                            </fieldset>
                         </td>
                         <td>
                             <fieldset title="Method for downloading torrent files.">
@@ -461,6 +500,7 @@
                                     <label>Prefer</label>
                                     <input type="radio" name="prefer_torrents" id="prefer_torrents_0" value="0" ${config['prefer_torrents_0']}>NZBs
                                     <input type="radio" name="prefer_torrents" id="prefer_torrents_1" value="1" ${config['prefer_torrents_1']}>Torrents
+                                    <input type="radio" name="prefer_torrents" id="prefer_torrents_3" value="3" ${config['prefer_torrents_3']}>Direct Download
                                     <input type="radio" name="prefer_torrents" id="prefer_torrents_2" value="2" ${config['prefer_torrents_2']}>No Preference
                                 </div>
                             </fieldset>
@@ -573,6 +613,21 @@
                                     </div>
                                 </div>
                             </fieldset>
+
+                            <fieldset>
+                                <legend>Direct Download</legend>
+
+                                <fieldset>
+                                    <div class="row checkbox left">
+                                        <input id="use_deezloader" type="checkbox" class="bigcheck" name="use_deezloader" value="1" ${config['use_deezloader']} />
+                                       	<label for="use_deezloader">
+                                       		<span class="option">DeezLoader</span>
+                           					<small class="heading"><i class="fa fa-info-circle"></i> Note: this option requires <a href="https://pypi.python.org/pypi/pycrypto">pycrypto</a></small>
+                          				</label>
+                                    </div>
+                                </fieldset>
+
+                            </fieldset>
                         </td>
                         <td>
                             <fieldset>
@@ -2405,6 +2460,11 @@
                             $("#deluge_options").show();
                         }
 
+                if ($("#ddl_downloader_aria").is(":checked"))
+                {
+                    $("#ddl_aria_options").show();
+                }
+
                 $('input[type=radio]').change(function(){
                         if ($("#preferred_bitrate").is(":checked"))
                         {
@@ -2458,6 +2518,9 @@
                         {
                             $("#torrent_blackhole_options,#utorrent_options,#transmission_options,#qbittorrent_options").fadeOut("fast", function() { $("#deluge_options").fadeIn() });
                         }
+                        if ($("#ddl_downloader_aria").is(":checked"))
+                        {
+                        }
                 });
 
                 $("#mirror").change(handleNewServerSelection);
@@ -2560,6 +2623,7 @@
                 initConfigCheckbox("#enable_https");
                 initConfigCheckbox("#customauth");
                 initConfigCheckbox("#use_tquattrecentonze");
+                initConfigCheckbox("#use_deezloader");
 
 
                 $('#twitterStep1').click(function () {

headphones/aria2.py

@@ -0,0 +1,979 @@
+# -*- coding: utf8 -*-
+# Copyright (C) 2012-2016 Xyne
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# (version 2) as published by the Free Software Foundation.
+#
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+from __future__ import with_statement
+import base64
+import json
+import math
+import os
+import ssl
+import string
+import time
+import httplib
+import urllib2
+
+from headphones import logger
+
+# ################################ Constants ###################################
+
+DEFAULT_PORT = 6800
+SERVER_URI_FORMAT = '{}://{}:{:d}/jsonrpc'
+
+# Status values for unfinished downloads.
+TEMPORARY_STATUS = ('active', 'waiting', 'paused')
+# Status values for finished downloads.
+FINAL_STATUS = ('complete', 'error')
+
+ARIA2_CONTROL_FILE_EXT = '.aria2'
+
+# ########################## Convenience Functions #############################
+
+
+def to_json_list(objs):
+    '''
+    Wrap strings in lists. Other iterables are converted to lists directly.
+    '''
+    if isinstance(objs, str):
+        return [objs]
+    elif not isinstance(objs, list):
+        return list(objs)
+    else:
+        return objs
+
+
+def add_options_and_position(params, options=None, position=None):
+    '''
+    Convenience method for adding options and position to parameters.
+    '''
+    if options:
+        params.append(options)
+    if position:
+        if not isinstance(position, int):
+            try:
+                position = int(position)
+            except ValueError:
+                position = -1
+        if position >= 0:
+            params.append(position)
+    return params
+
+
+def get_status(response):
+    '''
+    Process a status response.
+    '''
+    if response:
+        try:
+            return response['status']
+        except KeyError:
+            logger.error('no status returned from Aria2 RPC server')
+            return 'error'
+    else:
+        logger.error('no response from server')
+        return 'error'
+
+
+def random_token(length, valid_chars=None):
+    '''
+    Get a random secret token for the Aria2 RPC server.
+
+    length:
+        The length of the token
+
+    valid_chars:
+        A list or other ordered and indexable iterable of valid characters. If not
+        given of None, asciinumberic characters with some punctuation characters
+        will be used.
+    '''
+    if not valid_chars:
+        valid_chars = string.ascii_letters + string.digits + '!@#$%^&*()-_=+'
+    number_of_chars = len(valid_chars)
+    bytes_to_read = math.ceil(math.log(number_of_chars) / math.log(0x100))
+    max_value = 0x100**bytes_to_read
+    max_index = number_of_chars - 1
+    token = ''
+    for _ in range(length):
+        value = int.from_bytes(os.urandom(bytes_to_read), byteorder='little')
+        index = round((value * max_index) / max_value)
+        token += valid_chars[index]
+    return token
+
+# ################ From python3-aur's ThreadedServers.common ###################
+
+
+def format_bytes(size):
+    '''Format bytes for inferior humans.'''
+    if size < 0x400:
+        return '{:d} B'.format(size)
+    else:
+        size = float(size) / 0x400
+    for prefix in ('KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB'):
+        if size < 0x400:
+            return '{:0.02f} {}'.format(size, prefix)
+        else:
+            size /= 0x400
+    return '{:0.02f} YiB'.format(size)
+
+
+def format_seconds(s):
+    '''Format seconds for inferior humans.'''
+    string = ''
+    for base, char in (
+        (60, 's'),
+        (60, 'm'),
+        (24, 'h')
+    ):
+        s, r = divmod(s, base)
+        if s == 0:
+            return '{:d}{}{}'.format(r, char, string)
+        elif r != 0:
+            string = '{:02d}{}{}'.format(r, char, string)
+    else:
+        return '{:d}d{}'.format(s, string)
+
+# ############################ Aria2JsonRpcError ###############################
+
+
+class Aria2JsonRpcError(Exception):
+    def __init__(self, msg, connection_error=False):
+        super(self.__class__, self).__init__(self, msg)
+        self.connection_error = connection_error
+
+# ############################ Aria2JsonRpc Class ##############################
+
+
+class Aria2JsonRpc(object):
+    '''
+    Interface class for interacting with an Aria2 RPC server.
+    '''
+    # TODO: certificate options, etc.
+    def __init__(
+        self, ID, uri,
+        mode='normal',
+        token=None,
+        http_user=None, http_passwd=None,
+        server_cert=None, client_cert=None, client_cert_password=None,
+        ssl_protocol=None,
+        setup_function=None
+    ):
+        '''
+        ID: the ID to send to the RPC interface
+
+        uri: the URI of the RPC interface
+
+        mode:
+            normal - process requests immediately
+            batch - queue requests (run with "process_queue")
+            format - return RPC request objects
+
+        token:
+            RPC method-level authorization token (set using `--rpc-secret`)
+
+        http_user, http_password:
+            HTTP Basic authentication credentials (deprecated)
+
+        server_cert:
+            server certificate for HTTPS connections
+
+        client_cert:
+            client certificate for HTTPS connections
+
+        client_cert_password:
+            prompt for client certificate password
+
+        ssl_protocol:
+            SSL protocol from the ssl module
+
+        setup_function:
+            A function to invoke prior to the first server call. This could be the
+            launch() method of an Aria2RpcServer instance, for example. This attribute
+            is set automatically in instances returned from Aria2RpcServer.get_a2jr()
+        '''
+        self.id = ID
+        self.uri = uri
+        self.mode = mode
+        self.queue = []
+        self.handlers = dict()
+        self.token = token
+        self.setup_function = setup_function
+
+        if None not in (http_user, http_passwd):
+            self.add_HTTPBasicAuthHandler(http_user, http_passwd)
+
+        if server_cert or client_cert:
+            self.add_HTTPSHandler(
+                server_cert=server_cert,
+                client_cert=client_cert,
+                client_cert_password=client_cert_password,
+                protocol=ssl_protocol
+            )
+
+        self.update_opener()
+
+    def iter_handlers(self):
+        '''
+        Iterate over handlers.
+        '''
+        for name in ('HTTPS', 'HTTPBasicAuth'):
+            try:
+                yield self.handlers[name]
+            except KeyError:
+                pass
+
+    def update_opener(self):
+        '''
+        Build an opener from the current handlers.
+        '''
+        self.opener = urllib2.build_opener(*self.iter_handlers())
+
+    def remove_handler(self, name):
+        '''
+        Remove a handler.
+        '''
+        try:
+            del self.handlers[name]
+        except KeyError:
+            pass
+
+    def add_HTTPBasicAuthHandler(self, user, passwd):
+        '''
+        Add a handler for HTTP Basic authentication.
+
+        If either user or passwd are None, the handler is removed.
+        '''
+        handler = urllib2.HTTPBasicAuthHandler()
+        handler.add_password(
+            realm='aria2',
+            uri=self.uri,
+            user=user,
+            passwd=passwd,
+        )
+        self.handlers['HTTPBasicAuth'] = handler
+
+    def remove_HTTPBasicAuthHandler(self):
+        self.remove_handler('HTTPBasicAuth')
+
+    def add_HTTPSHandler(
+        self,
+        server_cert=None,
+        client_cert=None,
+        client_cert_password=None,
+        protocol=None,
+    ):
+        '''
+        Add a handler for HTTPS connections with optional server and client
+        certificates.
+        '''
+        if not protocol:
+            protocol = ssl.PROTOCOL_TLSv1
+#       protocol = ssl.PROTOCOL_TLSv1_1 # openssl 1.0.1+
+#       protocol = ssl.PROTOCOL_TLSv1_2 # Python 3.4+
+        context = ssl.SSLContext(protocol)
+
+        if server_cert:
+            context.verify_mode = ssl.CERT_REQUIRED
+            context.load_verify_locations(cafile=server_cert)
+        else:
+            context.verify_mode = ssl.CERT_OPTIONAL
+
+        if client_cert:
+            context.load_cert_chain(client_cert, password=client_cert_password)
+
+        self.handlers['HTTPS'] = urllib2.HTTPSHandler(
+            context=context,
+            check_hostname=False
+        )
+
+    def remove_HTTPSHandler(self):
+        self.remove_handler('HTTPS')
+
+    def send_request(self, req_obj):
+        '''
+        Send the request and return the response.
+        '''
+        if self.setup_function:
+            self.setup_function()
+            self.setup_function = None
+        logger.debug("Aria req_obj: %s" % json.dumps(req_obj, indent=2, sort_keys=True))
+        req = json.dumps(req_obj).encode('UTF-8')
+        try:
+            f = self.opener.open(self.uri, req)
+            obj = json.loads(f.read())
+            try:
+                return obj['result']
+            except KeyError:
+                raise Aria2JsonRpcError('unexpected result: {}'.format(obj))
+        except (urllib2.URLError) as e:
+            # This should work but URLError does not set the errno attribute:
+            # e.errno == errno.ECONNREFUSED
+            raise Aria2JsonRpcError(
+                str(e),
+                connection_error=(
+                    '111' in str(e)
+                )
+            )
+        except httplib.BadStatusLine as e:
+            raise Aria2JsonRpcError('{}: BadStatusLine: {} (HTTPS error?)'.format(
+                self.__class__.__name__, e
+            ))
+
+    def jsonrpc(self, method, params=None, prefix='aria2.'):
+        '''
+        POST a request to the RPC interface.
+        '''
+        if not params:
+            params = []
+
+        if self.token is not None:
+            token_str = 'token:{}'.format(self.token)
+            if method == 'multicall':
+                for p in params[0]:
+                    try:
+                        p['params'].insert(0, token_str)
+                    except KeyError:
+                        p['params'] = [token_str]
+            else:
+                params.insert(0, token_str)
+
+        req_obj = {
+            'jsonrpc': '2.0',
+            'id': self.id,
+            'method': prefix + method,
+            'params': params,
+        }
+        if self.mode == 'batch':
+            self.queue.append(req_obj)
+            return None
+        elif self.mode == 'format':
+            return req_obj
+        else:
+            return self.send_request(req_obj)
+
+    def process_queue(self):
+        '''
+        Processed queued requests.
+        '''
+        req_obj = self.queue
+        self.queue = []
+        return self.send_request(req_obj)
+
+
+# ############################# Standard Methods ###############################
+    def addUri(self, uris, options=None, position=None):
+        '''
+        aria2.addUri method
+
+        uris: list of URIs
+
+        options: dictionary of additional options
+
+        position: position in queue
+
+        Returns a GID
+        '''
+        params = [uris]
+        params = add_options_and_position(params, options, position)
+        return self.jsonrpc('addUri', params)
+
+    def addTorrent(self, torrent, uris=None, options=None, position=None):
+        '''
+        aria2.addTorrent method
+
+        torrent: base64-encoded torrent file
+
+        uris: list of webseed URIs
+
+        options: dictionary of additional options
+
+        position: position in queue
+
+        Returns a GID.
+        '''
+        params = [torrent]
+        if uris:
+            params.append(uris)
+        params = add_options_and_position(params, options, position)
+        return self.jsonrpc('addTorrent', params)
+
+    def addMetalink(self, metalink, options=None, position=None):
+        '''
+        aria2.addMetalink method
+
+        metalink: base64-encoded metalink file
+
+        options: dictionary of additional options
+
+        position: position in queue
+
+        Returns an array of GIDs.
+        '''
+        params = [metalink]
+        params = add_options_and_position(params, options, position)
+        return self.jsonrpc('addTorrent', params)
+
+    def remove(self, gid):
+        '''
+        aria2.remove method
+
+        gid: GID to remove
+        '''
+        params = [gid]
+        return self.jsonrpc('remove', params)
+
+    def forceRemove(self, gid):
+        '''
+        aria2.forceRemove method
+
+        gid: GID to remove
+        '''
+        params = [gid]
+        return self.jsonrpc('forceRemove', params)
+
+    def pause(self, gid):
+        '''
+        aria2.pause method
+
+        gid: GID to pause
+        '''
+        params = [gid]
+        return self.jsonrpc('pause', params)
+
+    def pauseAll(self):
+        '''
+        aria2.pauseAll method
+        '''
+        return self.jsonrpc('pauseAll')
+
+    def forcePause(self, gid):
+        '''
+        aria2.forcePause method
+
+        gid: GID to pause
+        '''
+        params = [gid]
+        return self.jsonrpc('forcePause', params)
+
+    def forcePauseAll(self):
+        '''
+        aria2.forcePauseAll method
+        '''
+        return self.jsonrpc('forcePauseAll')
+
+    def unpause(self, gid):
+        '''
+        aria2.unpause method
+
+        gid: GID to unpause
+        '''
+        params = [gid]
+        return self.jsonrpc('unpause', params)
+
+    def unpauseAll(self):
+        '''
+        aria2.unpauseAll method
+        '''
+        return self.jsonrpc('unpauseAll')
+
+    def tellStatus(self, gid, keys=None):
+        '''
+        aria2.tellStatus method
+
+        gid: GID to query
+
+        keys: subset of status keys to return (all keys are returned otherwise)
+
+        Returns a dictionary.
+        '''
+        params = [gid]
+        if keys:
+            params.append(keys)
+        return self.jsonrpc('tellStatus', params)
+
+    def getUris(self, gid):
+        '''
+        aria2.getUris method
+
+        gid: GID to query
+
+        Returns a list of dictionaries.
+        '''
+        params = [gid]
+        return self.jsonrpc('getUris', params)
+
+    def getFiles(self, gid):
+        '''
+        aria2.getFiles method
+
+        gid: GID to query
+
+        Returns a list of dictionaries.
+        '''
+        params = [gid]
+        return self.jsonrpc('getFiles', params)
+
+    def getPeers(self, gid):
+        '''
+        aria2.getPeers method
+
+        gid: GID to query
+
+        Returns a list of dictionaries.
+        '''
+        params = [gid]
+        return self.jsonrpc('getPeers', params)
+
+    def getServers(self, gid):
+        '''
+        aria2.getServers method
+
+        gid: GID to query
+
+        Returns a list of dictionaries.
+        '''
+        params = [gid]
+        return self.jsonrpc('getServers', params)
+
+    def tellActive(self, keys=None):
+        '''
+        aria2.tellActive method
+
+        keys: same as tellStatus
+
+        Returns a list of dictionaries. The dictionaries are the same as those
+        returned by tellStatus.
+        '''
+        if keys:
+            params = [keys]
+        else:
+            params = None
+        return self.jsonrpc('tellActive', params)
+
+    def tellWaiting(self, offset, num, keys=None):
+        '''
+        aria2.tellWaiting method
+
+        offset: offset from start of waiting download queue
+                        (negative values are counted from the end of the queue)
+
+        num: number of downloads to return
+
+        keys: same as tellStatus
+
+        Returns a list of dictionaries. The dictionaries are the same as those
+        returned by tellStatus.
+        '''
+        params = [offset, num]
+        if keys:
+            params.append(keys)
+        return self.jsonrpc('tellWaiting', params)
+
+    def tellStopped(self, offset, num, keys=None):
+        '''
+        aria2.tellStopped method
+
+        offset: offset from oldest download (same semantics as tellWaiting)
+
+        num: same as tellWaiting
+
+        keys: same as tellStatus
+
+        Returns a list of dictionaries. The dictionaries are the same as those
+        returned by tellStatus.
+        '''
+        params = [offset, num]
+        if keys:
+            params.append(keys)
+        return self.jsonrpc('tellStopped', params)
+
+    def changePosition(self, gid, pos, how):
+        '''
+        aria2.changePosition method
+
+        gid: GID to change
+
+        pos: the position
+
+        how: "POS_SET", "POS_CUR" or "POS_END"
+        '''
+        params = [gid, pos, how]
+        return self.jsonrpc('changePosition', params)
+
+    def changeUri(self, gid, fileIndex, delUris, addUris, position=None):
+        '''
+        aria2.changePosition method
+
+        gid: GID to change
+
+        fileIndex: file to affect (1-based)
+
+        delUris: URIs to remove
+
+        addUris: URIs to add
+
+        position: where URIs are inserted, after URIs have been removed
+        '''
+        params = [gid, fileIndex, delUris, addUris]
+        if position:
+            params.append(position)
+        return self.jsonrpc('changePosition', params)
+
+    def getOption(self, gid):
+        '''
+        aria2.getOption method
+
+        gid: GID to query
+
+        Returns a dictionary of options.
+        '''
+        params = [gid]
+        return self.jsonrpc('getOption', params)
+
+    def changeOption(self, gid, options):
+        '''
+        aria2.changeOption method
+
+        gid: GID to change
+
+        options: dictionary of new options
+                         (not all options can be changed for active downloads)
+        '''
+        params = [gid, options]
+        return self.jsonrpc('changeOption', params)
+
+    def getGlobalOption(self):
+        '''
+        aria2.getGlobalOption method
+
+        Returns a dictionary.
+        '''
+        return self.jsonrpc('getGlobalOption')
+
+    def changeGlobalOption(self, options):
+        '''
+        aria2.changeGlobalOption method
+
+        options: dictionary of new options
+        '''
+        params = [options]
+        return self.jsonrpc('changeGlobalOption', params)
+
+    def getGlobalStat(self):
+        '''
+        aria2.getGlobalStat method
+
+        Returns a dictionary.
+        '''
+        return self.jsonrpc('getGlobalStat')
+
+    def purgeDownloadResult(self):
+        '''
+        aria2.purgeDownloadResult method
+        '''
+        self.jsonrpc('purgeDownloadResult')
+
+    def removeDownloadResult(self, gid):
+        '''
+        aria2.removeDownloadResult method
+
+        gid: GID to remove
+        '''
+        params = [gid]
+        return self.jsonrpc('removeDownloadResult', params)
+
+    def getVersion(self):
+        '''
+        aria2.getVersion method
+
+        Returns a dictionary.
+        '''
+        return self.jsonrpc('getVersion')
+
+    def getSessionInfo(self):
+        '''
+        aria2.getSessionInfo method
+
+        Returns a dictionary.
+        '''
+        return self.jsonrpc('getSessionInfo')
+
+    def shutdown(self):
+        '''
+        aria2.shutdown method
+        '''
+        return self.jsonrpc('shutdown')
+
+    def forceShutdown(self):
+        '''
+        aria2.forceShutdown method
+        '''
+        return self.jsonrpc('forceShutdown')
+
+    def multicall(self, methods):
+        '''
+        aria2.multicall method
+
+        methods: list of dictionaries (keys: methodName, params)
+
+        The method names must be those used by Aria2c, e.g. "aria2.tellStatus".
+        '''
+        return self.jsonrpc('multicall', [methods], prefix='system.')
+
+
+# ########################### Convenience Methods ##############################
+    def add_torrent(self, path, uris=None, options=None, position=None):
+        '''
+        A wrapper around addTorrent for loading files.
+        '''
+        with open(path, 'r') as f:
+            torrent = base64.encode(f.read())
+        return self.addTorrent(torrent, uris, options, position)
+
+    def add_metalink(self, path, uris=None, options=None, position=None):
+        '''
+        A wrapper around addMetalink for loading files.
+        '''
+        with open(path, 'r') as f:
+            metalink = base64.encode(f.read())
+        return self.addMetalink(metalink, uris, options, position)
+
+    def get_status(self, gid):
+        '''
+        Get the status of a single GID.
+        '''
+        response = self.tellStatus(gid, ['status'])
+        return get_status(response)
+
+    def wait_for_final_status(self, gid, interval=1):
+        '''
+        Wait for a GID to complete or fail and return its status.
+        '''
+        if not interval or interval < 0:
+            interval = 1
+        while True:
+            status = self.get_status(gid)
+            if status in TEMPORARY_STATUS:
+                time.sleep(interval)
+            else:
+                return status
+
+    def get_statuses(self, gids):
+        '''
+        Get the status of multiple GIDs. The status of each is yielded in order.
+        '''
+        methods = [
+            {
+                'methodName': 'aria2.tellStatus',
+                'params': [gid, ['gid', 'status']]
+            }
+            for gid in gids
+        ]
+        results = self.multicall(methods)
+        if results:
+            status = dict((r[0]['gid'], r[0]['status']) for r in results)
+            for gid in gids:
+                try:
+                    yield status[gid]
+                except KeyError:
+                    logger.error('Aria2 RPC server returned no status for GID {}'.format(gid))
+                    yield 'error'
+        else:
+            logger.error('no response from Aria2 RPC server')
+            for gid in gids:
+                yield 'error'
+
+    def wait_for_final_statuses(self, gids, interval=1):
+        '''
+        Wait for multiple GIDs to complete or fail and return their statuses in
+        order.
+
+        gids:
+            A flat list of GIDs.
+        '''
+        if not interval or interval < 0:
+            interval = 1
+        statusmap = dict((g, None) for g in gids)
+        remaining = list(
+            g for g, s in statusmap.items() if s is None
+        )
+        while remaining:
+            for g, s in zip(remaining, self.get_statuses(remaining)):
+                if s in TEMPORARY_STATUS:
+                    continue
+                else:
+                    statusmap[g] = s
+            remaining = list(
+                g for g, s in statusmap.items() if s is None
+            )
+            if remaining:
+                time.sleep(interval)
+        for g in gids:
+            yield statusmap[g]
+
+    def print_global_status(self):
+        '''
+        Print global status of the RPC server.
+        '''
+        status = self.getGlobalStat()
+        if status:
+            numWaiting = int(status['numWaiting'])
+            numStopped = int(status['numStopped'])
+            keys = ['totalLength', 'completedLength']
+            total = self.tellActive(keys)
+            waiting = self.tellWaiting(0, numWaiting, keys)
+            if waiting:
+                total += waiting
+            stopped = self.tellStopped(0, numStopped, keys)
+            if stopped:
+                total += stopped
+
+            downloadSpeed = int(status['downloadSpeed'])
+            uploadSpeed = int(status['uploadSpeed'])
+            totalLength = sum(int(x['totalLength']) for x in total)
+            completedLength = sum(int(x['completedLength']) for x in total)
+            remaining = totalLength - completedLength
+
+            status['downloadSpeed'] = format_bytes(downloadSpeed) + '/s'
+            status['uploadSpeed'] = format_bytes(uploadSpeed) + '/s'
+
+            preordered = ('downloadSpeed', 'uploadSpeed')
+
+            rows = list()
+            for k in sorted(status):
+                if k in preordered:
+                    continue
+                rows.append((k, status[k]))
+
+            rows.extend((x, status[x]) for x in preordered)
+
+            if totalLength > 0:
+                rows.append(('total', format(format_bytes(totalLength))))
+                rows.append(('completed', format(format_bytes(completedLength))))
+                rows.append(('remaining', format(format_bytes(remaining))))
+                if completedLength == totalLength:
+                    eta = 'finished'
+                else:
+                    try:
+                        eta = format_seconds(remaining // downloadSpeed)
+                    except ZeroDivisionError:
+                        eta = 'never'
+                rows.append(('ETA', eta))
+
+            l = max(len(r[0]) for r in rows)
+            r = max(len(r[1]) for r in rows)
+            r = max(r, len(self.uri) - (l + 2))
+            fmt = '{:<' + str(l) + 's}  {:>' + str(r) + 's}'
+
+            print(self.uri)
+            for k, v in rows:
+                print(fmt.format(k, v))
+
+    def queue_uris(self, uris, options, interval=None):
+        '''
+        Enqueue URIs and wait for download to finish while printing status at
+        regular intervals.
+        '''
+        gid = self.addUri(uris, options)
+        print('GID: {}'.format(gid))
+
+        if gid and interval is not None:
+            blanker = ''
+            while True:
+                response = self.tellStatus(gid, ['status'])
+                if response:
+                    try:
+                        status = response['status']
+                    except KeyError:
+                        print('error: no status returned from Aria2 RPC server')
+                        break
+                    print('{}\rstatus: {}'.format(blanker, status)),
+                    blanker = ' ' * len(status)
+                    if status in TEMPORARY_STATUS:
+                        time.sleep(interval)
+                    else:
+                        break
+                else:
+                    print('error: no response from server')
+                    break
+
+
+# ####################### Polymethod download handlers #########################
+    def polymethod_enqueue_many(self, downloads):
+        '''
+        Enqueue downloads.
+
+        downloads: Same as polymethod_download().
+        '''
+        methods = list(
+            {
+                'methodName': 'aria2.{}'.format(d[0]),
+                'params': list(d[1:])
+            } for d in downloads
+        )
+        return self.multicall(methods)
+
+    def polymethod_wait_many(self, gids, interval=1):
+        '''
+        Wait for the GIDs to complete or fail and return their statuses.
+
+        gids:
+            A list of lists of GIDs.
+        '''
+        # The flattened list of GIDs
+        gs = list(g for gs in gids for g in gs)
+        statusmap = dict(tuple(zip(
+            gs,
+            self.wait_for_final_statuses(gs, interval=interval)
+        )))
+        for gs in gids:
+            yield list(statusmap.get(g, 'error') for g in gs)
+
+    def polymethod_enqueue_one(self, download):
+        '''
+        Same as polymethod_enqueue_many but for one element.
+        '''
+        return getattr(self, download[0])(*download[1:])
+
+    def polymethod_download(self, downloads, interval=1):
+        '''
+        Enqueue a series of downloads and wait for them to finish. Iterate over the
+        status of each, in order.
+
+        downloads:
+            An iterable over (<type>, <args>, ...) where <type> indicates the "add"
+            method to use ('addUri', 'addTorrent', 'addMetalink') and everything that
+            follows are arguments to pass to that method.
+
+        interval:
+            The status check interval while waiting.
+
+        Iterates over the download status of finished downloads. "complete"
+        indicates success. Lists of statuses will be returned for downloads that
+        create multiple GIDs (e.g. metalinks).
+        '''
+        gids = self.polymethod_enqueue_many(downloads)
+        return self.polymethod_wait_many(gids, interval=interval)
+
+    def polymethod_download_bool(self, *args, **kwargs):
+        '''
+        A wrapper around polymethod_download() which returns a boolean for each
+        download to indicate success (True) or failure (False).
+        '''
+#     for status in self.polymethod_download(*args, **kwargs):
+#       yield all(s == 'complete' for s in status)
+        return list(
+            all(s == 'complete' for s in status)
+            for status in self.polymethod_download(*args, **kwargs)
+        )

headphones/classes.py

@@ -21,6 +21,7 @@
 import urllib
 
 from common import USER_AGENT
+from collections import OrderedDict
 
 
 class HeadphonesURLopener(urllib.FancyURLopener):
@@ -135,3 +136,33 @@ class Proper:
     def __str__(self):
         return str(self.date) + " " + self.name + " " + str(self.season) + "x" + str(
             self.episode) + " of " + str(self.tvdbid)
+
+
+class OptionalImport(object):
+    '''
+    Dummy class for optional import (imports needed for optional features).
+    '''
+    def __init__(self, name):
+        self.__name = name
+
+    def __getattr__(self, attr):
+        raise ImportError('The following package is required to use this feature: {0}'.format(self.__name))
+
+
+class CacheDict(OrderedDict):
+    '''
+    Ordered dictionary with fixed size, designed for caching.
+    '''
+    def __init__(self, *args, **kwds):
+        self.size_limit = kwds.pop("size_limit", None)
+        OrderedDict.__init__(self, *args, **kwds)
+        self._check_size_limit()
+
+    def __setitem__(self, key, value):
+        OrderedDict.__setitem__(self, key, value)
+        self._check_size_limit()
+
+    def _check_size_limit(self):
+        if self.size_limit is not None:
+            while len(self) > self.size_limit:
+                self.popitem(last=False)

headphones/config.py

@@ -46,6 +46,10 @@ _CONFIG_DEFINITIONS = {
     'APOLLO_RATIO': (str, 'Apollo.rip', ''),
     'APOLLO_USERNAME': (str, 'Apollo.rip', ''),
     'APOLLO_URL': (str, 'Apollo.rip', 'https://apollo.rip'),
+    'ARIA_HOST': (str, 'Aria2', ''),
+    'ARIA_PASSWORD': (str, 'Aria2', ''),
+    'ARIA_TOKEN': (str, 'Aria2', ''),
+    'ARIA_USERNAME': (str, 'Aria2', ''),
     'AUTOWANT_ALL': (int, 'General', 0),
     'AUTOWANT_MANUALLY_ADDED': (int, 'General', 1),
     'AUTOWANT_UPCOMING': (int, 'General', 1),
@@ -73,6 +77,8 @@ _CONFIG_DEFINITIONS = {
     'CUSTOMPORT': (int, 'General', 5000),
     'CUSTOMSLEEP': (int, 'General', 1),
     'CUSTOMUSER': (str, 'General', ''),
+    'DDL_DOWNLOADER': (int, 'General', 0),
+    'DEEZLOADER': (int, 'DeezLoader', 0),
     'DELETE_LOSSLESS_FILES': (int, 'General', 1),
     'DELUGE_HOST': (str, 'Deluge', ''),
     'DELUGE_CERT': (str, 'Deluge', ''),
@@ -86,6 +92,7 @@ _CONFIG_DEFINITIONS = {
     'DOWNLOAD_DIR': (path, 'General', ''),
     'DOWNLOAD_SCAN_INTERVAL': (int, 'General', 5),
     'DOWNLOAD_TORRENT_DIR': (path, 'General', ''),
+    'DOWNLOAD_DDL_DIR': (path, 'General', ''),
     'DO_NOT_OVERRIDE_GIT_BRANCH': (int, 'General', 0),
     'EMAIL_ENABLED': (int, 'Email', 0),
     'EMAIL_FROM': (str, 'Email', ''),

headphones/deezloader.py

@@ -0,0 +1,441 @@
+# -*- coding: utf-8 -*-
+# Deezloader (c) 2016 by ParadoxalManiak
+#
+# Deezloader is licensed under a
+# Creative Commons Attribution-NonCommercial-ShareAlike 3.0 Unported License.
+#
+# You should have received a copy of the license along with this
+# work. If not, see <http://creativecommons.org/licenses/by-nc-sa/3.0/>.
+#
+# Version 2.1.0
+# Maintained by ParadoxalManiak <https://www.reddit.com/user/ParadoxalManiak/>
+# Original work by ZzMTV <https://boerse.to/members/zzmtv.3378614/>
+#
+# Author's disclaimer:
+#  I am not responsible for the usage of this program by other people.
+#  I do not recommend you doing this illegally or against Deezer's terms of service.
+#  This project is licensed under CC BY-NC-SA 4.0
+
+import re
+import os
+from datetime import datetime
+from hashlib import md5
+import binascii
+
+from beets.mediafile import MediaFile
+from headphones import logger, request, helpers
+from headphones.classes import OptionalImport, CacheDict
+import headphones
+
+
+# Try to import optional Crypto.Cipher packages
+try:
+    from Crypto.Cipher import AES, Blowfish
+except ImportError:
+    AES = OptionalImport('Crypto.Cipher.AES')
+    Blowfish = OptionalImport('Crypto.Cipher.Blowfish')
+
+# Public constants
+PROVIDER_NAME = 'Deezer'
+
+# Internal constants
+__API_URL = "http://www.deezer.com/ajax/gw-light.php"
+__API_INFO_URL = "http://api.deezer.com/"
+__HTTP_HEADERS = {
+    "User-Agent": "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36",
+    "Content-Language": "en-US",
+    "Cache-Control": "max-age=0",
+    "Accept": "*/*",
+    "Accept-Charset": "utf-8,ISO-8859-1;q=0.7,*;q=0.3",
+    "Accept-Language": "de-DE,de;q=0.8,en-US;q=0.6,en;q=0.4"
+}
+
+# Internal variables
+__api_queries = {
+    'api_version': "1.0",
+    'api_token': "None",
+    'input': "3"
+}
+__cookies = None
+__tracks_cache = CacheDict(size_limit=512)
+__albums_cache = CacheDict(size_limit=64)
+
+
+def __getApiToken():
+    global __cookies
+    response = request.request_response(url="http://www.deezer.com/", headers=__HTTP_HEADERS)
+    __cookies = response.cookies
+    data = response.content
+    if data:
+        matches = re.search(r"checkForm\s*=\s*['|\"](.*)[\"|'];", data)
+        if matches:
+            token = matches.group(1)
+            __api_queries['api_token'] = token
+            logger.debug(u"Deezloader : api token loaded ('%s')" % token)
+
+    if not token:
+        logger.error(u"Deezloader: Unable to get api token")
+
+
+def getAlbumByLink(album_link):
+    """Returns deezer album infos using album link url
+
+    :param str album_link: deezer album link url (eg: 'http://www.deezer.com/album/1234567/')
+    """
+    matches = re.search(r"album\/([0-9]+)\/?$", album_link)
+    if matches:
+        return getAlbum(matches.group(1))
+
+
+def getAlbum(album_id):
+    """Returns deezer album infos
+
+    :param int album_id: deezer album id
+    """
+    global __albums_cache
+
+    if str(album_id) in __albums_cache:
+        return __albums_cache[str(album_id)]
+
+    url = __API_INFO_URL + "album/" + str(album_id)
+    data = request.request_json(url=url, headers=__HTTP_HEADERS, cookies=__cookies)
+
+    if data and 'error' not in data:
+        __albums_cache[str(album_id)] = data
+        return data
+    else:
+        logger.debug("Deezloader: Can't load album infos")
+        return None
+
+
+def searchAlbums(search_term):
+    """Search for deezer albums using search term
+
+    :param str search_term: search term to search album for
+    """
+    logger.info(u'Searching Deezer using term: "%s"' % search_term)
+
+    url = __API_INFO_URL + "search/album?q=" + search_term
+    data = request.request_json(url=url, headers=__HTTP_HEADERS, cookies=__cookies)
+
+    albums = []
+
+    # Process content
+    if data and 'total' in data and data['total'] > 0 and 'data' in data:
+        for item in data['data']:
+            try:
+                albums.append(getAlbum(item['id']))
+            except Exception as e:
+                logger.error(u"An unknown error occurred in the Deezer search album parser: %s" % e)
+    else:
+        logger.info(u'No results found from Deezer using term: "%s"' % search_term)
+
+    return albums
+
+
+def __matchAlbums(albums, artist_name, album_title, album_length):
+    resultlist = []
+
+    for album in albums:
+        total_size = 0
+        tracks_found = 0
+
+        for track in album['tracks']['data']:
+            t = getTrack(track['id'])
+            if t:
+                if t["FILESIZE_MP3_320"] > 0:
+                    size = t["FILESIZE_MP3_320"]
+                elif t["FILESIZE_MP3_256"] > 0:
+                    size = t["FILESIZE_MP3_256"]
+                elif t["FILESIZE_MP3_128"] > 0:
+                    size = t["FILESIZE_MP3_128"]
+                else:
+                    size = t["FILESIZE_MP3_64"]
+
+                size = int(size)
+                total_size += size
+                tracks_found += 1
+                logger.debug(u'Found song "%s". Size: %s' % (t['SNG_TITLE'], helpers.bytes_to_mb(size)))
+
+        if tracks_found == 0:
+            logger.info(u'Ignoring album "%s" (no tracks to download)' % album['title'])
+            continue
+
+        matched = True
+        mismatch_reason = 'matched!'
+
+        if album_length > 0 and abs(int(album['duration']) - album_length) > 240:
+            matched = False
+            mismatch_reason = (u'duration mismatch: %i not in [%i, %i]' % (int(album['duration']), (album_length - 240), (album_length + 240)))
+
+        elif (helpers.latinToAscii(re.sub(r"\W", "", album_title, 0, re.UNICODE)).lower() !=
+              helpers.latinToAscii(re.sub(r"\W", "", album['title'], 0, re.UNICODE)).lower()):
+            matched = False
+            mismatch_reason = (u'album name mismatch: %s != %s' % (album['title'], album_title))
+
+        elif (helpers.latinToAscii(re.sub(r"\W", "", artist_name, 0, re.UNICODE)).lower() !=
+              helpers.latinToAscii(re.sub(r"\W", "", album['artist']['name'], 0, re.UNICODE)).lower()):
+            matched = False
+            mismatch_reason = (u'artist name mismatch: %s != %s' % (album['artist']['name'], artist_name))
+
+        resultlist.append(
+            (album['artist']['name'] + ' - ' + album['title'] + ' [' + album['release_date'][:4] + '] (' + str(tracks_found) + '/' + str(album['nb_tracks']) + ')',
+            total_size, album['link'], PROVIDER_NAME, "ddl", matched)
+        )
+        logger.info(u'Found "%s". Tracks %i/%i. Size: %s (%s)' % (album['title'], tracks_found, album['nb_tracks'], helpers.bytes_to_mb(total_size), mismatch_reason))
+
+    return resultlist
+
+
+def searchAlbum(artist_name, album_title, user_search_term=None, album_length=None):
+    """Search for deezer specific album.
+    This will iterate over deezer albums and try to find best matches
+
+    :param str artist_name: album artist name
+    :param str album_title: album title
+    :param str user_search_term: search terms provided by user
+    :param int album_length: targeted album duration in seconds
+    """
+    # User search term by-pass normal search
+    if user_search_term:
+        return __matchAlbums(searchAlbums(user_search_term), artist_name, album_title, album_length)
+
+    resultlist = __matchAlbums(searchAlbums((artist_name + ' ' + album_title).strip()), artist_name, album_title, album_length)
+    if resultlist:
+        return resultlist
+
+    # Deezer API supports unicode, so just remove non alphanumeric characters
+    clean_artist_name = re.sub(r"[^\w\s]", " ", artist_name, 0, re.UNICODE).strip()
+    clean_album_name = re.sub(r"[^\w\s]", " ", album_title, 0, re.UNICODE).strip()
+
+    resultlist = __matchAlbums(searchAlbums((clean_artist_name + ' ' + clean_album_name).strip()), artist_name, album_title, album_length)
+    if resultlist:
+        return resultlist
+
+    resultlist = __matchAlbums(searchAlbums(clean_artist_name), artist_name, album_title, album_length)
+    if resultlist:
+        return resultlist
+
+    return resultlist
+
+
+def getTrack(sng_id, try_reload_api=True):
+    """Returns deezer track infos
+
+    :param int sng_id: deezer song id
+    :param bool try_reload_api: whether or not try reloading API if session expired
+    """
+    global __tracks_cache
+
+    if str(sng_id) in __tracks_cache:
+        return __tracks_cache[str(sng_id)]
+
+    data = "[{\"method\":\"song.getListData\",\"params\":{\"sng_ids\":[" + str(sng_id) + "]}}]"
+    json = request.request_json(url=__API_URL, headers=__HTTP_HEADERS, method='post', params=__api_queries, data=data, cookies=__cookies)
+
+    results = []
+    error = None
+    invalid_token = False
+
+    if json:
+        # Check for errors
+        if 'error' in json:
+            error = json['error']
+            if 'GATEWAY_ERROR' in json['error'] and json['error']['GATEWAY_ERROR'] == u"invalid api token":
+                invalid_token = True
+
+        elif 'error' in json[0] and json[0]['error']:
+            error = json[0]['error']
+            if 'VALID_TOKEN_REQUIRED' in json[0]['error'] and json[0]['error']['VALID_TOKEN_REQUIRED'] == u"Invalid CSRF token":
+                invalid_token = True
+
+        # Got invalid token error
+        if error:
+            if invalid_token and try_reload_api:
+                __getApiToken()
+                return getTrack(sng_id, False)
+            else:
+                logger.error(u"An unknown error occurred in the Deezer track parser: %s" % error)
+        else:
+            try:
+                results = json[0]['results']
+                item = results['data'][0]
+                if 'token' in item:
+                    logger.error(u"An unknown error occurred in the Deezer parser: Uploaded Files are currently not supported")
+                    return
+
+                sng_id = item["SNG_ID"]
+                md5Origin = item["MD5_ORIGIN"]
+                sng_format = 3
+
+                if item["FILESIZE_MP3_320"] <= 0:
+                    if item["FILESIZE_MP3_256"] > 0:
+                        sng_format = 5
+                    else:
+                        sng_format = 1
+
+                mediaVersion = int(item["MEDIA_VERSION"])
+                item['downloadUrl'] = __getDownloadUrl(md5Origin, sng_id, sng_format, mediaVersion)
+
+                __tracks_cache[sng_id] = item
+                return item
+
+            except Exception as e:
+                logger.error(u"An unknown error occurred in the Deezer track parser: %s" % e)
+
+
+def __getDownloadUrl(md5Origin, sng_id, sng_format, mediaVersion):
+    urlPart = md5Origin.encode('utf-8') + b'\xa4' + str(sng_format) + b'\xa4' + str(sng_id) + b'\xa4' + str(mediaVersion)
+    md5val = md5(urlPart).hexdigest()
+    urlPart = md5val + b'\xa4' + urlPart + b'\xa4'
+    cipher = AES.new('jo6aey6haid2Teih', AES.MODE_ECB)
+    ciphertext = cipher.encrypt(__pad(urlPart, AES.block_size))
+    return "http://e-cdn-proxy-" + md5Origin[:1] + ".deezer.com/mobile/1/" + binascii.hexlify(ciphertext).lower()
+
+
+def __pad(raw, block_size):
+    if (len(raw) % block_size == 0):
+        return raw
+    padding_required = block_size - (len(raw) % block_size)
+    padChar = b'\x00'
+    data = raw + padding_required * padChar
+    return data
+
+
+def __tagTrack(path, track):
+    try:
+        album = getAlbum(track['ALB_ID'])
+
+        f = MediaFile(path)
+        f.artist = track['ART_NAME']
+        f.album = track['ALB_TITLE']
+        f.title = track['SNG_TITLE']
+        f.track = track['TRACK_NUMBER']
+        f.tracktotal = album['nb_tracks']
+        f.disc = track['DISK_NUMBER']
+        f.bpm = track['BPM']
+        f.date = datetime.strptime(album['release_date'], '%Y-%m-%d').date()
+        f.albumartist = album['artist']['name']
+        if u'genres' in album and u'data' in album['genres']:
+            f.genres = [genre['name'] for genre in album['genres']['data']]
+
+        f.save()
+
+    except Exception as e:
+        logger.error(u'Unable to tag deezer track "%s": %s' % (path, e))
+
+
+def decryptTracks(paths):
+    """Decrypt downloaded deezer tracks.
+
+    :param paths: list of path to deezer tracks (*.dzr files).
+    """
+    # Note: tracks can be from different albums
+    decrypted_tracks = {}
+
+    # First pass: load tracks data
+    for path in paths:
+        try:
+            album_folder = os.path.dirname(path)
+            sng_id = os.path.splitext(os.path.basename(path))[0]
+            track = getTrack(sng_id)
+            if track:
+                track_number = int(track['TRACK_NUMBER'])
+                disk_number = int(track['DISK_NUMBER'])
+
+                if album_folder not in decrypted_tracks:
+                    decrypted_tracks[album_folder] = {}
+
+                if disk_number not in decrypted_tracks[album_folder]:
+                    decrypted_tracks[album_folder][disk_number] = {}
+
+                decrypted_tracks[album_folder][disk_number][track_number] = track
+
+        except Exception as e:
+            logger.error(u'Unable to load deezer track infos "%s": %s' % (path, e))
+
+    # Second pass: decrypt tracks
+    for album_folder in decrypted_tracks:
+        multi_disks = len(decrypted_tracks[album_folder]) > 1
+        for disk_number in decrypted_tracks[album_folder]:
+            for track_number, track in decrypted_tracks[album_folder][disk_number].items():
+                try:
+                    filename = helpers.replace_illegal_chars(track['SNG_TITLE']).strip()
+                    filename = ('{:02d}'.format(track_number) + '_' + filename + '.mp3')
+
+                    # Add a 'cd x' sub-folder if album has more than one disk
+                    disk_folder = os.path.join(album_folder, 'cd ' + str(disk_number)) if multi_disks else album_folder
+
+                    dest = os.path.join(disk_folder, filename).encode(headphones.SYS_ENCODING, 'replace')
+
+                    # Decrypt track if not already done
+                    if not os.path.exists(dest):
+                        try:
+                            __decryptDownload(path, sng_id, dest)
+                            __tagTrack(dest, track)
+                        except Exception as e:
+                            logger.error(u'Unable to decrypt deezer track "%s": %s' % (path, e))
+                            if os.path.exists(dest):
+                                os.remove(dest)
+                            decrypted_tracks[album_folder][disk_number].pop(track_number)
+                            continue
+
+                    decrypted_tracks[album_folder][disk_number][track_number]['path'] = dest
+
+                except Exception as e:
+                    logger.error(u'Unable to decrypt deezer track "%s": %s' % (path, e))
+
+    return decrypted_tracks
+
+
+def __decryptDownload(source, sng_id, dest):
+    interval_chunk = 3
+    chunk_size = 2048
+    blowFishKey = __getBlowFishKey(sng_id)
+    i = 0
+    iv = "\x00\x01\x02\x03\x04\x05\x06\x07"
+
+    dest_folder = os.path.dirname(dest)
+    if not os.path.exists(dest_folder):
+        os.makedirs(dest_folder)
+
+    f = open(source, "rb")
+    fout = open(dest, "wb")
+    try:
+        chunk = f.read(chunk_size)
+        while chunk:
+            if(i % interval_chunk == 0):
+                cipher = Blowfish.new(blowFishKey, Blowfish.MODE_CBC, iv)
+                chunk = cipher.decrypt(__pad(chunk, Blowfish.block_size))
+
+            fout.write(chunk)
+            i += 1
+            chunk = f.read(chunk_size)
+    finally:
+        f.close()
+        fout.close()
+
+
+def __getBlowFishKey(encryptionKey):
+    if encryptionKey < 1:
+        encryptionKey *= -1
+
+    hashcode = md5(str(encryptionKey)).hexdigest()
+    hPart = hashcode[0:16]
+    lPart = hashcode[16:32]
+    parts = ['g4el58wc0zvf9na1', hPart, lPart]
+
+    return __xorHex(parts)
+
+
+def __xorHex(parts):
+    data = ""
+    for i in range(0, 16):
+        character = ord(parts[0][i])
+
+        for j in range(1, len(parts)):
+            character ^= ord(parts[j][i])
+
+        data += chr(character)
+
+    return data

headphones/postprocessor.py

@@ -30,7 +30,7 @@ from beetsplug import lyrics as beetslyrics
 from headphones import notifiers, utorrent, transmission, deluge, qbittorrent
 from headphones import db, albumart, librarysync
 from headphones import logger, helpers, mb, music_encoder
-from headphones import metadata
+from headphones import metadata, deezloader
 
 postprocessor_lock = threading.Lock()
 
@@ -48,6 +48,8 @@ def checkFolder():
                 single = False
                 if album['Kind'] == 'nzb':
                     download_dir = headphones.CONFIG.DOWNLOAD_DIR
+                elif album['Kind'] == 'ddl':
+                    download_dir = headphones.CONFIG.DOWNLOAD_DDL_DIR
                 else:
                     if headphones.CONFIG.DELUGE_DONE_DIRECTORY and headphones.CONFIG.TORRENT_DOWNLOADER == 3:
                         download_dir = headphones.CONFIG.DELUGE_DONE_DIRECTORY
@@ -205,6 +207,7 @@ def verify(albumid, albumpath, Kind=None, forced=False, keep_original_folder=Fal
         tracks = myDB.select('SELECT * from tracks WHERE AlbumID=?', [albumid])
 
     downloaded_track_list = []
+    downloaded_deezer_list = []
     downloaded_cuecount = 0
 
     for r, d, f in os.walk(albumpath):
@@ -213,8 +216,10 @@ def verify(albumid, albumpath, Kind=None, forced=False, keep_original_folder=Fal
                 downloaded_track_list.append(os.path.join(r, files))
             elif files.lower().endswith('.cue'):
                 downloaded_cuecount += 1
+            elif files.lower().endswith('.dzr'):
+                downloaded_deezer_list.append(os.path.join(r, files))
             # if any of the files end in *.part, we know the torrent isn't done yet. Process if forced, though
-            elif files.lower().endswith(('.part', '.utpart')) and not forced:
+            elif files.lower().endswith(('.part', '.utpart', '.aria2')) and not forced:
                 logger.info(
                     "Looks like " + os.path.basename(albumpath).decode(headphones.SYS_ENCODING,
                                                                        'replace') + " isn't complete yet. Will try again on the next run")
@@ -253,6 +258,37 @@ def verify(albumid, albumpath, Kind=None, forced=False, keep_original_folder=Fal
             downloaded_track_list = helpers.get_downloaded_track_list(albumpath)
             keep_original_folder = False
 
+    # Decrypt deezer tracks
+    if downloaded_deezer_list:
+        logger.info('Decrypting deezer tracks')
+        decrypted_deezer_list = deezloader.decryptTracks(downloaded_deezer_list)
+
+        # Check if album is complete based on album duration only
+        # (total track numbers is not determinant enough due to hidden tracks for eg)
+        db_track_duration = 0
+        downloaded_track_duration = 0
+        try:
+            for track in tracks:
+                db_track_duration += track['TrackDuration'] / 1000
+        except:
+            downloaded_track_duration = False
+
+        try:
+            for disk_number in decrypted_deezer_list[albumpath]:
+                for track in decrypted_deezer_list[albumpath][disk_number].values():
+                    downloaded_track_list.append(track['path'])
+                    downloaded_track_duration += int(track['DURATION'])
+        except:
+            downloaded_track_duration = False
+
+        if not downloaded_track_duration or not db_track_duration or abs(downloaded_track_duration - db_track_duration) > 240:
+            logger.info("Looks like " +
+                        os.path.basename(albumpath).decode(headphones.SYS_ENCODING, 'replace') +
+                        " isn't complete yet (duration mismatch). Will try again on the next run")
+            return
+
+        downloaded_track_list = list(set(downloaded_track_list))  # Remove duplicates
+
     # test #1: metadata - usually works
     logger.debug('Verifying metadata...')
 

headphones/searcher.py

@@ -36,6 +36,7 @@ import headphones
 from headphones.common import USER_AGENT
 from headphones import logger, db, helpers, classes, sab, nzbget, request
 from headphones import utorrent, transmission, notifiers, rutracker, deluge, qbittorrent
+from headphones import deezloader, aria2
 from bencode import bencode, bdecode
 
 # Magnet to torrent services, for Black hole. Stolen from CouchPotato.
@@ -51,6 +52,27 @@ ruobj = None
 # Persistent RED API object
 redobj = None
 
+# Persistent Aria2 RPC object
+__aria2rpc_obj = None
+
+
+def getAria2RPC():
+    global __aria2rpc_obj
+    if not __aria2rpc_obj:
+        __aria2rpc_obj = aria2.Aria2JsonRpc(
+            ID='headphones',
+            uri=headphones.CONFIG.ARIA_HOST,
+            token=headphones.CONFIG.ARIA_TOKEN if headphones.CONFIG.ARIA_TOKEN else None,
+            http_user=headphones.CONFIG.ARIA_USERNAME if headphones.CONFIG.ARIA_USERNAME else None,
+            http_passwd=headphones.CONFIG.ARIA_PASSWORD if headphones.CONFIG.ARIA_PASSWORD else None
+        )
+    return __aria2rpc_obj
+
+
+def reconfigure():
+    global __aria2rpc_obj
+    __aria2rpc_obj = None
+
 
 def fix_url(s, charset="utf-8"):
     """
@@ -281,32 +303,53 @@ def do_sorted_search(album, new, losslessOnly, choose_specific_download=False):
                          headphones.CONFIG.STRIKE or
                          headphones.CONFIG.TQUATTRECENTONZE)
 
-    results = []
+    DDL_PROVIDERS = (headphones.CONFIG.DEEZLOADER)
+
     myDB = db.DBConnection()
     albumlength = myDB.select('SELECT sum(TrackDuration) from tracks WHERE AlbumID=?',
                               [album['AlbumID']])[0][0]
 
+    nzb_results = None
+    torrent_results = None
+    ddl_results = None
+
     if headphones.CONFIG.PREFER_TORRENTS == 0 and not choose_specific_download:
 
         if NZB_PROVIDERS and NZB_DOWNLOADERS:
-            results = searchNZB(album, new, losslessOnly, albumlength)
+            nzb_results = searchNZB(album, new, losslessOnly, albumlength)
 
-        if not results and TORRENT_PROVIDERS:
-            results = searchTorrent(album, new, losslessOnly, albumlength)
+        if not nzb_results:
+            if DDL_PROVIDERS:
+                ddl_results = searchDdl(album, new, losslessOnly, albumlength)
+
+            if TORRENT_PROVIDERS:
+                torrent_results = searchTorrent(album, new, losslessOnly, albumlength)
 
     elif headphones.CONFIG.PREFER_TORRENTS == 1 and not choose_specific_download:
 
         if TORRENT_PROVIDERS:
-            results = searchTorrent(album, new, losslessOnly, albumlength)
+            torrent_results = searchTorrent(album, new, losslessOnly, albumlength)
 
-        if not results and NZB_PROVIDERS and NZB_DOWNLOADERS:
-            results = searchNZB(album, new, losslessOnly, albumlength)
+        if not torrent_results:
+            if DDL_PROVIDERS:
+                ddl_results = searchDdl(album, new, losslessOnly, albumlength)
+
+            if NZB_PROVIDERS and NZB_DOWNLOADERS:
+                nzb_results = searchNZB(album, new, losslessOnly, albumlength)
+
+    elif headphones.CONFIG.PREFER_TORRENTS == 3 and not choose_specific_download:
+
+        if DDL_PROVIDERS:
+            ddl_results = searchDdl(album, new, losslessOnly, albumlength)
+
+        if not ddl_results:
+            if TORRENT_PROVIDERS:
+                torrent_results = searchTorrent(album, new, losslessOnly, albumlength)
+
+            if NZB_PROVIDERS and NZB_DOWNLOADERS:
+                nzb_results = searchNZB(album, new, losslessOnly, albumlength)
 
     else:
-
-        nzb_results = None
-        torrent_results = None
-
         if NZB_PROVIDERS and NZB_DOWNLOADERS:
             nzb_results = searchNZB(album, new, losslessOnly, albumlength, choose_specific_download)
 
@@ -314,13 +357,19 @@ def do_sorted_search(album, new, losslessOnly, choose_specific_download=False):
             torrent_results = searchTorrent(album, new, losslessOnly, albumlength,
                                             choose_specific_download)
 
-        if not nzb_results:
-            nzb_results = []
+        if DDL_PROVIDERS:
+            ddl_results = searchDdl(album, new, losslessOnly, albumlength, choose_specific_download)
 
-        if not torrent_results:
-            torrent_results = []
+    if not nzb_results:
+        nzb_results = []
 
-        results = nzb_results + torrent_results
+    if not torrent_results:
+        torrent_results = []
+
+    if not ddl_results:
+        ddl_results = []
+
+    results = nzb_results + torrent_results + ddl_results
 
     if choose_specific_download:
         return results
@@ -826,6 +875,31 @@ def send_to_downloader(data, bestqual, album):
             except Exception as e:
                 logger.error('Couldn\'t write NZB file: %s', e)
                 return
+
+    elif kind == 'ddl':
+        folder_name = '%s - %s [%s]' % (
+            helpers.latinToAscii(album['ArtistName']).encode('UTF-8').replace('/', '_'),
+            helpers.latinToAscii(album['AlbumTitle']).encode('UTF-8').replace('/', '_'),
+            get_year_from_release_date(album['ReleaseDate']))
+
+        # Aria2 downloader
+        if headphones.CONFIG.DDL_DOWNLOADER == 0:
+            logger.info("Sending download to Aria2")
+
+            try:
+                deezer_album = deezloader.getAlbumByLink(bestqual[2])
+
+                for album_track in deezer_album['tracks']['data']:
+                    track = deezloader.getTrack(album_track['id'])
+                    if track:
+                        filename = track['SNG_ID'] + '.dzr'
+                        logger.debug(u'Sending song "%s" to Aria' % track['SNG_TITLE'])
+                        getAria2RPC().addUri([track['downloadUrl']], {'out': filename, 'auto-file-renaming': 'false', 'continue': 'true', 'dir': folder_name})
+
+            except Exception as e:
+                logger.error(u'Error sending torrent to Aria2. Are you sure it\'s running? (%s)' % e)
+                return
+
     else:
         folder_name = '%s - %s [%s]' % (
             helpers.latinToAscii(album['ArtistName']).encode('UTF-8').replace('/', '_'),
@@ -1204,6 +1278,62 @@ def verifyresult(title, artistterm, term, lossless):
     return True
 
 
+def searchDdl(album, new=False, losslessOnly=False, albumlength=None,
+                  choose_specific_download=False):
+    reldate = album['ReleaseDate']
+    year = get_year_from_release_date(reldate)
+
+    # MERGE THIS WITH THE TERM CLEANUP FROM searchNZB
+    dic = {'...': '', ' & ': ' ', ' = ': ' ', '?': '', '$': 's', ' + ': ' ', '"': '', ',': ' ',
+           '*': ''}
+
+    semi_cleanalbum = helpers.replace_all(album['AlbumTitle'], dic)
+    cleanalbum = helpers.latinToAscii(semi_cleanalbum)
+    semi_cleanartist = helpers.replace_all(album['ArtistName'], dic)
+    cleanartist = helpers.latinToAscii(semi_cleanartist)
+
+    # Use provided term if available, otherwise build our own (this code needs to be cleaned up since a lot
+    # of these torrent providers are just using cleanartist/cleanalbum terms
+    if album['SearchTerm']:
+        term = album['SearchTerm']
+    elif album['Type'] == 'part of':
+        term = cleanalbum + " " + year
+    else:
+        # FLAC usually doesn't have a year for some reason so I'll leave it out
+        # Various Artist albums might be listed as VA, so I'll leave that out too
+        # Only use the year if the term could return a bunch of different albums, i.e. self-titled albums
+        if album['ArtistName'] in album['AlbumTitle'] or len(album['ArtistName']) < 4 or len(
+                album['AlbumTitle']) < 4:
+            term = cleanartist + ' ' + cleanalbum + ' ' + year
+        elif album['ArtistName'] == 'Various Artists':
+            term = cleanalbum + ' ' + year
+        else:
+            term = cleanartist + ' ' + cleanalbum
+
+    # Replace bad characters in the term and unicode it
+    term = re.sub('[\.\-\/]', ' ', term).encode('utf-8')
+    artistterm = re.sub('[\.\-\/]', ' ', cleanartist).encode('utf-8', 'replace')
+
+    logger.debug(u'Using search term: "%s"' % helpers.latinToAscii(term))
+
+    resultlist = []
+
+    # Deezer only provides lossy
+    if headphones.CONFIG.DEEZLOADER and not losslessOnly:
+        resultlist += deezloader.searchAlbum(album['ArtistName'], album['AlbumTitle'], album['SearchTerm'], int(albumlength / 1000))
+
+    # attempt to verify that this isn't a substring result
+    # when looking for "Foo - Foo" we don't want "Foobar"
+    # this should be less of an issue when it isn't a self-titled album so we'll only check vs artist
+    results = [result for result in resultlist if verifyresult(result[0], artistterm, term, losslessOnly)]
+
+    # Additional filtering for size etc
+    if results and not choose_specific_download:
+        results = more_filtering(results, album, albumlength, new)
+
+    return results
+
+
 def searchTorrent(album, new=False, losslessOnly=False, albumlength=None,
                   choose_specific_download=False):
     global apolloobj  # persistent apollo.rip api object to reduce number of login attempts
@@ -2036,7 +2166,10 @@ def searchTorrent(album, new=False, losslessOnly=False, albumlength=None,
 
 def preprocess(resultlist):
     for result in resultlist:
-        if result[4] == 'torrent':
+        if result[3] == deezloader.PROVIDER_NAME:
+            return True, result
+
+        elif result[4] == 'torrent':
 
             # rutracker always needs the torrent data
             if result[3] == 'rutracker.org':

headphones/webserve.py

@@ -1182,6 +1182,10 @@ class WebInterface(object):
             "utorrent_username": headphones.CONFIG.UTORRENT_USERNAME,
             "utorrent_password": headphones.CONFIG.UTORRENT_PASSWORD,
             "utorrent_label": headphones.CONFIG.UTORRENT_LABEL,
+            "aria_host": headphones.CONFIG.ARIA_HOST,
+            "aria_password": headphones.CONFIG.ARIA_PASSWORD,
+            "aria_token": headphones.CONFIG.ARIA_TOKEN,
+            "aria_username": headphones.CONFIG.ARIA_USERNAME,
             "nzb_downloader_sabnzbd": radio(headphones.CONFIG.NZB_DOWNLOADER, 0),
             "nzb_downloader_nzbget": radio(headphones.CONFIG.NZB_DOWNLOADER, 1),
             "nzb_downloader_blackhole": radio(headphones.CONFIG.NZB_DOWNLOADER, 2),
@@ -1190,6 +1194,7 @@ class WebInterface(object):
             "torrent_downloader_utorrent": radio(headphones.CONFIG.TORRENT_DOWNLOADER, 2),
             "torrent_downloader_deluge": radio(headphones.CONFIG.TORRENT_DOWNLOADER, 3),
             "torrent_downloader_qbittorrent": radio(headphones.CONFIG.TORRENT_DOWNLOADER, 4),
+            "ddl_downloader_aria": radio(headphones.CONFIG.DDL_DOWNLOADER, 0),
             "download_dir": headphones.CONFIG.DOWNLOAD_DIR,
             "use_blackhole": checked(headphones.CONFIG.BLACKHOLE),
             "blackhole_dir": headphones.CONFIG.BLACKHOLE_DIR,
@@ -1251,6 +1256,8 @@ class WebInterface(object):
             "use_tquattrecentonze": checked(headphones.CONFIG.TQUATTRECENTONZE),
             "tquattrecentonze_user": headphones.CONFIG.TQUATTRECENTONZE_USER,
             "tquattrecentonze_password": headphones.CONFIG.TQUATTRECENTONZE_PASSWORD,
+            "download_ddl_dir": headphones.CONFIG.DOWNLOAD_DDL_DIR,
+            "use_deezloader": checked(headphones.CONFIG.DEEZLOADER),
             "pref_qual_0": radio(headphones.CONFIG.PREFERRED_QUALITY, 0),
             "pref_qual_1": radio(headphones.CONFIG.PREFERRED_QUALITY, 1),
             "pref_qual_2": radio(headphones.CONFIG.PREFERRED_QUALITY, 2),
@@ -1296,6 +1303,7 @@ class WebInterface(object):
             "prefer_torrents_0": radio(headphones.CONFIG.PREFER_TORRENTS, 0),
             "prefer_torrents_1": radio(headphones.CONFIG.PREFER_TORRENTS, 1),
             "prefer_torrents_2": radio(headphones.CONFIG.PREFER_TORRENTS, 2),
+            "prefer_torrents_3": radio(headphones.CONFIG.PREFER_TORRENTS, 3),
             "magnet_links_0": radio(headphones.CONFIG.MAGNET_LINKS, 0),
             "magnet_links_1": radio(headphones.CONFIG.MAGNET_LINKS, 1),
             "magnet_links_2": radio(headphones.CONFIG.MAGNET_LINKS, 2),
@@ -1454,7 +1462,7 @@ class WebInterface(object):
             "launch_browser", "enable_https", "api_enabled", "use_blackhole", "headphones_indexer",
             "use_newznab", "newznab_enabled", "use_torznab", "torznab_enabled",
             "use_nzbsorg", "use_omgwtfnzbs", "use_kat", "use_piratebay", "use_oldpiratebay",
-            "use_mininova", "use_waffles", "use_rutracker",
+            "use_mininova", "use_waffles", "use_rutracker", "use_deezloader",
             "use_apollo", "use_redacted", "use_strike", "use_tquattrecentonze", "preferred_bitrate_allow_lossless",
             "detect_bitrate", "ignore_clean_releases", "freeze_db", "cue_split", "move_files",
             "rename_files", "correct_metadata", "cleanup_files", "keep_nfo", "add_album_art",
@@ -1588,6 +1596,9 @@ class WebInterface(object):
         # Reconfigure musicbrainz database connection with the new values
         mb.startmb()
 
+        # Reconfigure Aria2
+        searcher.reconfigure()
+
         raise cherrypy.HTTPRedirect("config")
 
     @cherrypy.expose