Add deezloader provider

Add aria2 downloader

data/interfaces/default/config.html

@@ -304,6 +304,45 @@
                                     <input type="text" name="usenet_retention" value="${config['usenet_retention']}" size="5">
                                 </div>
                             </fieldset>
+
+                            <fieldset title="Method for downloading direct download files.">
+                                <legend>Direct Download</legend>
+                                <input type="radio" name="ddl_downloader" id="ddl_downloader_aria" value="0" ${config['ddl_downloader_aria']}> Aria2
+                           </fieldset>
+                           <fieldset id="ddl_aria_options">
+                                <div class="row">
+                                    <label title="Aria2 RPC host, port and path.">
+                                        Aria2 Host
+                                    </label>
+                                    <input type="text" name="aria_host" value="${config['aria_host']}" size="30">
+                                    <small>usually http://localhost:6800/jsonrpc</small>
+                                </div>
+                                <div class="row">
+                                    <label title="Aria2 RPC username. Leave empty if not applicable.">
+                                        Aria2 Username
+                                    </label>
+                                    <input type="text" name="aria_username" value="${config['aria_username']}" size="20">
+                                </div>
+                                <div class="row">
+                                    <label title="Aria2 RPC password. Leave empty if not applicable.">
+                                        Aria2 Password
+                                    </label>
+                                    <input type="password" name="aria_password" value="${config['aria_password'] | h}" size="20">
+                                </div>
+                                <div class="row">
+                                    <label title="Aria2 RPC secret key. Leave empty if not applicable.">
+                                        Aria2 secret token
+                                    </label>
+                                    <input type="text" name="aria_token" value="${config['aria_token']}" size="36">
+                                </div>
+                                <div class="row">
+                                    <label title="Path to folder where Headphones can find the downloads.">
+                                        Music Download Directory:
+                                    </label>
+                                    <input type="text" name="download_ddl_dir" value="${config['download_ddl_dir']}" size="50">
+                                    <small>Full path where ddl client downloads your music, e.g. /Users/name/Downloads/music</small>
+                                </div>
+                            </fieldset>
                         </td>
                         <td>
                             <fieldset title="Method for downloading torrent files.">
@@ -461,6 +500,7 @@
                                     <label>Prefer</label>
                                     <input type="radio" name="prefer_torrents" id="prefer_torrents_0" value="0" ${config['prefer_torrents_0']}>NZBs
                                     <input type="radio" name="prefer_torrents" id="prefer_torrents_1" value="1" ${config['prefer_torrents_1']}>Torrents
+                                    <input type="radio" name="prefer_torrents" id="prefer_torrents_3" value="3" ${config['prefer_torrents_3']}>Direct Download
                                     <input type="radio" name="prefer_torrents" id="prefer_torrents_2" value="2" ${config['prefer_torrents_2']}>No Preference
                                 </div>
                             </fieldset>
@@ -573,6 +613,17 @@
                                     </div>
                                 </div>
                             </fieldset>
+
+                            <fieldset>
+                                <legend>Direct Download</legend>
+
+                                <fieldset>
+                                    <div class="row checkbox left">
+                                        <input id="use_deezloader" type="checkbox" class="bigcheck" name="use_deezloader" value="1" ${config['use_deezloader']} /><label for="use_deezloader"><span class="option">DeezLoader</span></label>
+                                    </div>
+                                </fieldset>
+
+                            </fieldset>
                         </td>
                         <td>
                             <fieldset>
@@ -804,7 +855,6 @@
                                         </div>
                                     </div>
                                 </fieldset>
-
                             </fieldset>
                         </td>
                     </tr>
@@ -2405,6 +2455,11 @@
                             $("#deluge_options").show();
                         }
 
+                if ($("#ddl_downloader_aria").is(":checked"))
+                {
+                    $("#ddl_aria_options").show();
+                }
+
                 $('input[type=radio]').change(function(){
                         if ($("#preferred_bitrate").is(":checked"))
                         {
@@ -2458,6 +2513,9 @@
                         {
                             $("#torrent_blackhole_options,#utorrent_options,#transmission_options,#qbittorrent_options").fadeOut("fast", function() { $("#deluge_options").fadeIn() });
                         }
+                        if ($("#ddl_downloader_aria").is(":checked"))
+                        {
+                        }
                 });
 
                 $("#mirror").change(handleNewServerSelection);
@@ -2560,6 +2618,7 @@
                 initConfigCheckbox("#enable_https");
                 initConfigCheckbox("#customauth");
                 initConfigCheckbox("#use_tquattrecentonze");
+                initConfigCheckbox("#use_deezloader");
 
 
                 $('#twitterStep1').click(function () {

headphones/config.py

@@ -46,6 +46,10 @@ _CONFIG_DEFINITIONS = {
     'APOLLO_RATIO': (str, 'Apollo.rip', ''),
     'APOLLO_USERNAME': (str, 'Apollo.rip', ''),
     'APOLLO_URL': (str, 'Apollo.rip', 'https://apollo.rip'),
+    'ARIA_HOST': (str, 'Aria2', ''),
+    'ARIA_PASSWORD': (str, 'Aria2', ''),
+    'ARIA_TOKEN': (str, 'Aria2', ''),
+    'ARIA_USERNAME': (str, 'Aria2', ''),
     'AUTOWANT_ALL': (int, 'General', 0),
     'AUTOWANT_MANUALLY_ADDED': (int, 'General', 1),
     'AUTOWANT_UPCOMING': (int, 'General', 1),
@@ -73,6 +77,8 @@ _CONFIG_DEFINITIONS = {
     'CUSTOMPORT': (int, 'General', 5000),
     'CUSTOMSLEEP': (int, 'General', 1),
     'CUSTOMUSER': (str, 'General', ''),
+    'DDL_DOWNLOADER': (int, 'General', 0),
+    'DEEZLOADER': (int, 'DeezLoader', 0),
     'DELETE_LOSSLESS_FILES': (int, 'General', 1),
     'DELUGE_HOST': (str, 'Deluge', ''),
     'DELUGE_CERT': (str, 'Deluge', ''),
@@ -86,6 +92,7 @@ _CONFIG_DEFINITIONS = {
     'DOWNLOAD_DIR': (path, 'General', ''),
     'DOWNLOAD_SCAN_INTERVAL': (int, 'General', 5),
     'DOWNLOAD_TORRENT_DIR': (path, 'General', ''),
+    'DOWNLOAD_DDL_DIR': (path, 'General', ''),
     'DO_NOT_OVERRIDE_GIT_BRANCH': (int, 'General', 0),
     'EMAIL_ENABLED': (int, 'Email', 0),
     'EMAIL_FROM': (str, 'Email', ''),

headphones/deezloader.py

@@ -0,0 +1,415 @@
+# -*- coding: utf-8 -*-
+# Deezloader (c) 2016 by ParadoxalManiak
+#
+# Deezloader is licensed under a
+# Creative Commons Attribution-NonCommercial-ShareAlike 3.0 Unported License.
+#
+# You should have received a copy of the license along with this
+# work. If not, see <http://creativecommons.org/licenses/by-nc-sa/3.0/>.
+#
+# Version 2.1.0
+# Maintained by ParadoxalManiak <https://www.reddit.com/user/ParadoxalManiak/>
+# Original work by ZzMTV <https://boerse.to/members/zzmtv.3378614/>
+#
+# Author's disclaimer:
+#  I am not responsible for the usage of this program by other people.
+#  I do not recommend you doing this illegally or against Deezer's terms of service.
+#  This project is licensed under CC BY-NC-SA 4.0
+
+import re
+import os
+from datetime import datetime
+from Crypto.Cipher import AES, Blowfish
+from hashlib import md5
+import binascii
+
+from beets.mediafile import MediaFile
+from headphones import logger, request, helpers
+import headphones
+from twisted.conch.insults import helper
+
+# Public constants
+PROVIDER_NAME = 'Deezer'
+
+# Internal constants
+__API_URL = "http://www.deezer.com/ajax/gw-light.php"
+__API_INFO_URL = "http://api.deezer.com/"
+__HTTP_HEADERS = {
+    "User-Agent": "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36",
+    "Content-Language": "en-US",
+    "Cache-Control": "max-age=0",
+    "Accept": "*/*",
+    "Accept-Charset": "utf-8,ISO-8859-1;q=0.7,*;q=0.3",
+    "Accept-Language": "de-DE,de;q=0.8,en-US;q=0.6,en;q=0.4"
+}
+
+# Internal variables
+__api_queries = {
+    'api_version': "1.0",
+    'api_token': "None",
+    'input': "3"
+}
+__cookies = None
+__tracks_cache = {}
+__albums_cache = {}
+
+def __getApiToken():
+    global __cookies
+    response = request.request_response(url="http://www.deezer.com/", headers=__HTTP_HEADERS)
+    __cookies = response.cookies
+    data = response.content
+    if data:
+        matches = re.search(r"checkForm\s*=\s*['|\"](.*)[\"|'];", data)
+        if matches:
+            token = matches.group(1)
+            __api_queries['api_token'] = token
+            logger.debug(u"Deezloader : api token loeaded ('%s')" % token)
+
+    if not token:
+        logger.error(u"Deezloader: Unable to get api token")
+    
+def getAlbumByLink(album_link):
+    """Returns deezer album infos using album link url
+    
+    :param str album_link: deezer album link url (eg: 'http://www.deezer.com/album/1234567/')
+    """
+    matches = re.search(r"album\/([0-9]+)\/?$", album_link)
+    if matches:
+        return getAlbum(matches.group(1))
+        
+def getAlbum(album_id):
+    """Returns deezer album infos
+    
+    :param int album_id: deezer album id
+    """
+    global __albums_cache
+    
+    if str(album_id) in __albums_cache:
+        return __albums_cache[str(album_id)]
+    
+    url = __API_INFO_URL + "album/" + str(album_id)
+    data = request.request_json(url=url, headers=__HTTP_HEADERS, cookies=__cookies)
+    
+    if data and 'error' not in data:
+        __albums_cache[str(album_id)] = data
+        return data
+    else:
+        logger.debug("Deezloader: Can't load album infos")
+        return None
+    
+def searchAlbums(search_term):
+    """Search for deezer albums using search term
+    
+    :param str search_term: search term to search album for
+    """
+    logger.info(u'Searching Deezer using term: "%s"' % search_term)
+
+    url = __API_INFO_URL + "search/album?q=" + search_term
+    data = request.request_json(url=url, headers=__HTTP_HEADERS, cookies=__cookies)
+
+    albums = []
+    
+    # Process content
+    if data and 'total' in data and data['total'] > 0 and 'data' in data:
+        for item in data['data']:
+            try:
+                albums.append(getAlbum(item['id']))
+            except Exception as e:
+                logger.error(u"An unknown error occurred in the Deezer parser: %s" % e)
+    else:
+        logger.info(u'No results found from Deezer using term: "%s"' % search_term)
+
+    return albums
+
+def __matchAlbums(albums, artist_name, album_title, album_length):
+    resultlist = []
+    
+    for album in albums:
+        total_size = 0
+        tracks_found = 0
+        
+        for track in album['tracks']['data']:
+            t = getTrack(track['id'])
+            if t:
+                if t["FILESIZE_MP3_320"] > 0:
+                    size = t["FILESIZE_MP3_320"]
+                elif t["FILESIZE_MP3_256"] > 0:
+                    size = t["FILESIZE_MP3_256"]
+                elif t["FILESIZE_MP3_128"] > 0:
+                    size = t["FILESIZE_MP3_128"]
+                else:
+                    size = t["FILESIZE_MP3_64"]
+                
+                size = int(size)
+                total_size += size
+                tracks_found += 1
+                logger.debug(u'Found song "%s". Size: %s' % (t['SNG_TITLE'], helpers.bytes_to_mb(size)))
+                
+        matched = True
+        mismatch_reason = 'matched!'
+        
+        if album_length > 0 and abs(int(album['duration']) - album_length) > 240:
+            matched = False
+            mismatch_reason = (u'duration mismatch: %i not in [%i, %i]' % (int(album['duration']), (album_length - 240), (album_length + 240)))
+
+        elif (helpers.latinToAscii(re.sub(r"\W", "", album_title, 0, re.UNICODE)).lower() != 
+              helpers.latinToAscii(re.sub(r"\W", "", album['title'], 0, re.UNICODE)).lower()):
+            matched = False
+            mismatch_reason = (u'album name mismatch: %s != %s' % (album['title'], album_title))
+            
+        elif (helpers.latinToAscii(re.sub(r"\W", "", artist_name, 0, re.UNICODE)).lower() !=
+              helpers.latinToAscii(re.sub(r"\W", "", album['artist']['name'], 0, re.UNICODE)).lower()):
+            matched = False
+            mismatch_reason = (u'artist name mismatch: %s != %s' % (album['artist']['name'], artist_name))
+        
+        resultlist.append(
+            (album['artist']['name'] + ' - ' + album['title'] + ' [' + album['release_date'][:4] + '] (' + str(tracks_found) + '/' + str(album['nb_tracks']) + ')',
+            total_size, album['link'], PROVIDER_NAME, "ddl", matched)
+        )
+        logger.info(u'Found "%s". Tracks %i/%i. Size: %s (%s)' % (album['title'], tracks_found, album['nb_tracks'], helpers.bytes_to_mb(total_size), mismatch_reason))
+
+    return resultlist
+
+def searchAlbum(artist_name, album_title, user_search_term=None, album_length=None):
+    """Search for deezer specific album.
+    This will iterate over deezer albums and try to find best matches
+
+    :param str artist_name: album artist name
+    :param str album_title: album title
+    :param str user_search_term: search terms provided by user
+    :param int album_length: targeted album duration in seconds
+    """
+    # User search term by-pass normal search
+    if user_search_term:
+        return __matchAlbums(searchAlbums(user_search_term), artist_name, album_title, album_length)
+    
+    resultlist = __matchAlbums(searchAlbums((artist_name + ' ' + album_title).strip()), artist_name, album_title, album_length)
+    if resultlist:
+        return resultlist
+
+    # Deezer API supports unicode, so just remove non alphanumeric characters
+    clean_artist_name = re.sub(r"[^\w\s]", " ", artist_name, 0, re.UNICODE).strip()
+    clean_album_name = re.sub(r"[^\w\s]", " ", album_title, 0, re.UNICODE).strip()
+    
+    resultlist = __matchAlbums(searchAlbums((clean_artist_name + ' ' + clean_album_name).strip()), artist_name, album_title, album_length)
+    if resultlist:
+        return resultlist
+        
+    resultlist = __matchAlbums(searchAlbums(clean_artist_name), artist_name, album_title, album_length)
+    if resultlist:
+        return resultlist
+        
+    return resultlist
+
+def getTrack(sng_id, try_reload_api=True):
+    """Returns deezer track infos
+    
+    :param int sng_id: deezer song id
+    :param bool try_reload_api: whether or not try reloading API if session expired 
+    """
+    global __tracks_cache
+    
+    if str(sng_id) in __tracks_cache:
+        return __tracks_cache[str(sng_id)]
+    
+    data = "[{\"method\":\"song.getListData\",\"params\":{\"sng_ids\":[" + str(sng_id) + "]}}]"
+    json = request.request_json(url=__API_URL, headers=__HTTP_HEADERS, method='post', params=__api_queries, data=data, cookies=__cookies)
+    
+    results = []
+    error = None
+    invalid_token = False
+    
+    if json:
+        # Check for errors
+        if 'error' in json:
+            error = json['error']
+            if 'GATEWAY_ERROR' in json['error'] and json['error']['GATEWAY_ERROR'] == u"invalid api token":
+                invalid_token = True
+
+        elif 'error' in json[0] and json[0]['error']:
+            error = json[0]['error']
+            if 'VALID_TOKEN_REQUIRED' in json[0]['error'] and json[0]['error']['VALID_TOKEN_REQUIRED'] == u"Invalid CSRF token":
+                invalid_token = True
+        
+        # Got invalid token error
+        if error:
+            if invalid_token and try_reload_api:
+                __getApiToken()
+                return getTrack(sng_id, False)
+            else:
+                logger.error(u"An unknown error occurred in the Deezer parser: %s" % error)
+        else:
+            try:
+                results = json[0]['results']
+                item = results['data'][0]
+                if 'token' in item:
+                    logger.error(u"An unknown error occurred in the Deezer parser: Uploaded Files are currently not supported")
+                    return
+                
+                sng_id = item["SNG_ID"]
+                md5Origin = item["MD5_ORIGIN"]
+                sng_format = 3
+                
+                if item["FILESIZE_MP3_320"] <= 0:
+                    if item["FILESIZE_MP3_256"] > 0:
+                        sng_format = 5
+                    else:
+                        sng_format = 1
+                
+                mediaVersion = int(item["MEDIA_VERSION"])
+                item['downloadUrl'] = __getDownloadUrl(md5Origin, sng_id, sng_format, mediaVersion)
+                
+                __tracks_cache[sng_id] = item
+                return item
+                
+            except Exception as e:
+                logger.error(u"An unknown error occurred in the Deezer parser: %s" % e)
+
+def __getDownloadUrl(md5Origin, sng_id, sng_format, mediaVersion):
+    urlPart = md5Origin.encode('utf-8') + b'\xa4' + str(sng_format) + b'\xa4' + str(sng_id) + b'\xa4' + str(mediaVersion)
+    md5val = md5(urlPart).hexdigest()
+    urlPart = md5val + b'\xa4' + urlPart + b'\xa4'
+    cipher = AES.new('jo6aey6haid2Teih', AES.MODE_ECB)
+    ciphertext = cipher.encrypt(__pad(urlPart, AES.block_size))
+    return "http://e-cdn-proxy-" + md5Origin[:1] + ".deezer.com/mobile/1/" + binascii.hexlify(ciphertext).lower()
+
+def __pad(raw, block_size):
+    if (len(raw) % block_size == 0):
+        return raw
+    padding_required = block_size - (len(raw) % block_size)
+    padChar = b'\x00'
+    data = raw + padding_required * padChar
+    return data
+
+def __tagTrack(path, track):
+    try:
+        album = getAlbum(track['ALB_ID'])
+    
+        f = MediaFile(path)
+        f.artist = track['ART_NAME']
+        f.album = track['ALB_TITLE']
+        f.title = track['SNG_TITLE']
+        f.track = track['TRACK_NUMBER']
+        f.tracktotal = album['nb_tracks']
+        f.disc = track['DISK_NUMBER']
+        f.bpm = track['BPM']
+        f.date = datetime.strptime(album['release_date'], '%Y-%m-%d').date()
+        f.albumartist = album['artist']['name']
+        if u'genres' in album and u'data' in album['genres']: 
+            f.genres = [genre['name'] for genre in album['genres']['data']]
+    
+        f.save()
+
+    except Exception as e:
+        logger.error(u'Unable to tag deezer track "%s": %s' % (path, e))
+
+def decryptTracks(paths):
+    """Decrypt downloaded deezer tracks.
+    
+    :param paths: list of path to deezer tracks (*.dzr files).
+    """
+    # Note: tracks can be from different albums
+    decrypted_tracks = {}
+    
+    # First pass: load tracks data
+    for path in paths:
+        try:
+            album_folder = os.path.dirname(path)
+            sng_id = os.path.splitext(os.path.basename(path))[0]
+            track = getTrack(sng_id)
+            track_number = int(track['TRACK_NUMBER'])
+            disk_number = int(track['DISK_NUMBER'])
+
+            if album_folder not in decrypted_tracks:
+                decrypted_tracks[album_folder] = {}
+                
+            if disk_number not in decrypted_tracks[album_folder]:
+                decrypted_tracks[album_folder][disk_number] = {}
+
+            decrypted_tracks[album_folder][disk_number][track_number] = track
+        
+        except Exception as e:
+            logger.error(u'Unable to load deezer track infos "%s": %s' % (path, e))
+    
+    # Second pass: decrypt tracks
+    for album_folder in decrypted_tracks:
+        multi_disks = len(decrypted_tracks[album_folder]) > 1
+        for disk_number in decrypted_tracks[album_folder]:
+            for track_number, track in decrypted_tracks[album_folder][disk_number].items():
+                try:
+                    filename = helpers.replace_illegal_chars(track['SNG_TITLE']).strip()
+                    filename = ('{:02d}'.format(track_number) + '_' + filename + '.mp3')
+                    
+                    # Add a 'cd x' sub-folder if album has more than one disk
+                    disk_folder = os.path.join(album_folder, 'cd ' + str(disk_number)) if multi_disks else album_folder
+                    
+                    dest = os.path.join(disk_folder, filename).encode(headphones.SYS_ENCODING, 'replace')
+        
+                    # Decrypt track if not already done
+                    if not os.path.exists(dest):
+                        try:
+                            __decryptDownload(path, sng_id, dest)
+                            __tagTrack(dest, track)
+                        except Exception as e:
+                            logger.error(u'Unable to decrypt deezer track "%s": %s' % (path, e))
+                            if os.path.exists(dest):
+                                os.remove(dest)
+                            decrypted_tracks[album_folder][disk_number].pop(track_number)
+                            continue
+
+                    decrypted_tracks[album_folder][disk_number][track_number]['path'] = dest
+                
+                except Exception as e:
+                    logger.error(u'Unable to decrypt deezer track "%s": %s' % (path, e))
+                
+    return decrypted_tracks
+
+def __decryptDownload(source, sng_id, dest):
+    interval_chunk = 3
+    chunk_size = 2048
+    blowFishKey = __getBlowFishKey(sng_id)
+    i = 0
+    iv = "\x00\x01\x02\x03\x04\x05\x06\x07"
+
+    dest_folder = os.path.dirname(dest)
+    if not os.path.exists(dest_folder):
+        os.makedirs(dest_folder)
+
+    f = open(source, "rb")
+    fout = open(dest, "wb")
+    try:
+        chunk = f.read(chunk_size)
+        while chunk:
+            if(i % interval_chunk == 0):
+                cipher = Blowfish.new(blowFishKey, Blowfish.MODE_CBC, iv)
+                chunk = cipher.decrypt(__pad(chunk, Blowfish.block_size))
+            
+            fout.write(chunk)
+            i += 1
+            chunk = f.read(chunk_size)
+    finally:
+        f.close()
+        fout.close()
+
+def __getBlowFishKey(encryptionKey):
+    if encryptionKey < 1:
+        encryptionKey *= -1
+
+    hashcode = md5(str(encryptionKey)).hexdigest()
+    hPart = hashcode[0:16]
+    lPart = hashcode[16:32]
+    parts = ['g4el58wc0zvf9na1', hPart, lPart]
+
+    return __xorHex(parts)
+
+def __xorHex(parts):
+    data = ""
+    for i in range(0, 16):
+        character = ord(parts[0][i])
+        
+        for j in range(1, len(parts)):
+            character ^= ord(parts[j][i])
+            
+        data += chr(character)
+        
+    return data

headphones/postprocessor.py

@@ -29,7 +29,7 @@ from beetsplug import lyrics as beetslyrics
 from headphones import notifiers, utorrent, transmission, deluge, qbittorrent
 from headphones import db, albumart, librarysync
 from headphones import logger, helpers, mb, music_encoder
-from headphones import metadata
+from headphones import metadata, deezloader
 
 postprocessor_lock = threading.Lock()
 
@@ -47,6 +47,8 @@ def checkFolder():
                 single = False
                 if album['Kind'] == 'nzb':
                     download_dir = headphones.CONFIG.DOWNLOAD_DIR
+                elif album['Kind'] == 'ddl':
+                    download_dir = headphones.CONFIG.DOWNLOAD_DDL_DIR
                 else:
                     if headphones.CONFIG.DELUGE_DONE_DIRECTORY and headphones.CONFIG.TORRENT_DOWNLOADER == 3:
                         download_dir = headphones.CONFIG.DELUGE_DONE_DIRECTORY
@@ -204,6 +206,7 @@ def verify(albumid, albumpath, Kind=None, forced=False, keep_original_folder=Fal
         tracks = myDB.select('SELECT * from tracks WHERE AlbumID=?', [albumid])
 
     downloaded_track_list = []
+    downloaded_deezer_list = []
     downloaded_cuecount = 0
 
     for r, d, f in os.walk(albumpath):
@@ -212,8 +215,10 @@ def verify(albumid, albumpath, Kind=None, forced=False, keep_original_folder=Fal
                 downloaded_track_list.append(os.path.join(r, files))
             elif files.lower().endswith('.cue'):
                 downloaded_cuecount += 1
+            elif files.lower().endswith('.dzr'):
+                downloaded_deezer_list.append(os.path.join(r, files))
             # if any of the files end in *.part, we know the torrent isn't done yet. Process if forced, though
-            elif files.lower().endswith(('.part', '.utpart')) and not forced:
+            elif files.lower().endswith(('.part', '.utpart', '.aria2')) and not forced:
                 logger.info(
                     "Looks like " + os.path.basename(albumpath).decode(headphones.SYS_ENCODING,
                                                                        'replace') + " isn't complete yet. Will try again on the next run")
@@ -252,6 +257,37 @@ def verify(albumid, albumpath, Kind=None, forced=False, keep_original_folder=Fal
             downloaded_track_list = helpers.get_downloaded_track_list(albumpath)
             keep_original_folder = False
 
+    # Decrypt deezer tracks
+    if downloaded_deezer_list:
+        logger.info('Decrypting deezer tracks')
+        decrypted_deezer_list = deezloader.decryptTracks(downloaded_deezer_list)
+        
+        # Check if album is complete based on album duration only
+        # (total track numbers is not determinant enough due to hidden tracks for eg)
+        db_track_duration = 0
+        downloaded_track_duration = 0
+        try:
+            for track in tracks:
+                db_track_duration += track['TrackDuration'] / 1000
+        except:
+            downloaded_track_duration = False
+        
+        try:
+            for disk_number in decrypted_deezer_list[albumpath]:
+                for track in decrypted_deezer_list[albumpath][disk_number].values():
+                    downloaded_track_list.append(track['path'])
+                    downloaded_track_duration += int(track['DURATION'])
+        except:
+            downloaded_track_duration = False
+        
+        if not downloaded_track_duration or not db_track_duration or abs(downloaded_track_duration - db_track_duration) > 240:
+            logger.info("Looks like " + 
+                        os.path.basename(albumpath).decode(headphones.SYS_ENCODING, 'replace') +
+                        " isn't complete yet (duration mismatch). Will try again on the next run")
+            return
+
+        downloaded_track_list = list(set(downloaded_track_list)) # Remove duplicates
+
     # test #1: metadata - usually works
     logger.debug('Verifying metadata...')
 

headphones/searcher.py

@@ -36,6 +36,7 @@ import headphones
 from headphones.common import USER_AGENT
 from headphones import logger, db, helpers, classes, sab, nzbget, request
 from headphones import utorrent, transmission, notifiers, rutracker, deluge, qbittorrent
+from headphones import deezloader, aria2
 from bencode import bencode, bdecode
 
 # Magnet to torrent services, for Black hole. Stolen from CouchPotato.
@@ -51,6 +52,24 @@ ruobj = None
 # Persistent RED API object
 redobj = None
 
+# Persistent Aria2 RPC object
+__aria2rpc_obj = None
+
+def getAria2RPC():
+    global __aria2rpc_obj
+    if not __aria2rpc_obj:
+        __aria2rpc_obj = aria2.Aria2JsonRpc(
+            ID='headphones',
+            uri=headphones.CONFIG.ARIA_HOST,
+            token=headphones.CONFIG.ARIA_TOKEN if headphones.CONFIG.ARIA_TOKEN else None,
+            http_user=headphones.CONFIG.ARIA_USERNAME if headphones.CONFIG.ARIA_USERNAME else None,
+            http_passwd=headphones.CONFIG.ARIA_PASSWORD if headphones.CONFIG.ARIA_PASSWORD else None
+        )
+    return __aria2rpc_obj
+
+def reconfigure():
+    global __aria2rpc_obj
+    __aria2rpc_obj = None
 
 def fix_url(s, charset="utf-8"):
     """
@@ -281,32 +300,53 @@ def do_sorted_search(album, new, losslessOnly, choose_specific_download=False):
                          headphones.CONFIG.STRIKE or
                          headphones.CONFIG.TQUATTRECENTONZE)
 
-    results = []
+    DDL_PROVIDERS = (headphones.CONFIG.DEEZLOADER)
+
     myDB = db.DBConnection()
     albumlength = myDB.select('SELECT sum(TrackDuration) from tracks WHERE AlbumID=?',
                               [album['AlbumID']])[0][0]
 
+    nzb_results = None
+    torrent_results = None
+    ddl_results = None
+
     if headphones.CONFIG.PREFER_TORRENTS == 0 and not choose_specific_download:
 
         if NZB_PROVIDERS and NZB_DOWNLOADERS:
-            results = searchNZB(album, new, losslessOnly, albumlength)
+            nzb_results = searchNZB(album, new, losslessOnly, albumlength)
 
-        if not results and TORRENT_PROVIDERS:
-            results = searchTorrent(album, new, losslessOnly, albumlength)
+        if not nzb_results:
+            if DDL_PROVIDERS:
+                ddl_results = searchDdl(album, new, losslessOnly, albumlength)
+
+            if TORRENT_PROVIDERS:
+                torrent_results = searchTorrent(album, new, losslessOnly, albumlength)
 
     elif headphones.CONFIG.PREFER_TORRENTS == 1 and not choose_specific_download:
 
         if TORRENT_PROVIDERS:
-            results = searchTorrent(album, new, losslessOnly, albumlength)
+            torrent_results = searchTorrent(album, new, losslessOnly, albumlength)
 
-        if not results and NZB_PROVIDERS and NZB_DOWNLOADERS:
-            results = searchNZB(album, new, losslessOnly, albumlength)
+        if not torrent_results:
+            if DDL_PROVIDERS:
+                ddl_results = searchDdl(album, new, losslessOnly, albumlength)
+
+            if NZB_PROVIDERS and NZB_DOWNLOADERS:
+                nzb_results = searchNZB(album, new, losslessOnly, albumlength)
+
+    elif headphones.CONFIG.PREFER_TORRENTS == 3 and not choose_specific_download:
+
+        if DDL_PROVIDERS:
+            ddl_results = searchDdl(album, new, losslessOnly, albumlength)
+
+        if not ddl_results:
+            if TORRENT_PROVIDERS:
+                torrent_results = searchTorrent(album, new, losslessOnly, albumlength)
+
+            if NZB_PROVIDERS and NZB_DOWNLOADERS:
+                nzb_results = searchNZB(album, new, losslessOnly, albumlength)
 
     else:
-
-        nzb_results = None
-        torrent_results = None
-
         if NZB_PROVIDERS and NZB_DOWNLOADERS:
             nzb_results = searchNZB(album, new, losslessOnly, albumlength, choose_specific_download)
 
@@ -314,13 +354,19 @@ def do_sorted_search(album, new, losslessOnly, choose_specific_download=False):
             torrent_results = searchTorrent(album, new, losslessOnly, albumlength,
                                             choose_specific_download)
 
-        if not nzb_results:
-            nzb_results = []
+        if DDL_PROVIDERS:
+            ddl_results = searchDdl(album, new, losslessOnly, albumlength, choose_specific_download)
 
-        if not torrent_results:
-            torrent_results = []
+    if not nzb_results:
+        nzb_results = []
 
-        results = nzb_results + torrent_results
+    if not torrent_results:
+        torrent_results = []
+
+    if not ddl_results:
+        ddl_results = []
+
+    results = nzb_results + torrent_results + ddl_results
 
     if choose_specific_download:
         return results
@@ -826,6 +872,31 @@ def send_to_downloader(data, bestqual, album):
             except Exception as e:
                 logger.error('Couldn\'t write NZB file: %s', e)
                 return
+
+    elif kind == 'ddl':
+        folder_name = '%s - %s [%s]' % (
+            helpers.latinToAscii(album['ArtistName']).encode('UTF-8').replace('/', '_'),
+            helpers.latinToAscii(album['AlbumTitle']).encode('UTF-8').replace('/', '_'),
+            get_year_from_release_date(album['ReleaseDate']))
+
+        # Aria2 downloader
+        if headphones.CONFIG.DDL_DOWNLOADER == 0:
+            logger.info("Sending download to Aria2")
+
+            try:
+                deezer_album = deezloader.getAlbumByLink(bestqual[2])
+
+                for album_track in deezer_album['tracks']['data']:
+                    track = deezloader.getTrack(album_track['id'])
+                    if track:
+                        filename = track['SNG_ID'] + '.dzr'
+                        logger.debug(u'Sending song "%s" to Aria' % track['SNG_TITLE'])
+                        getAria2RPC().addUri([track['downloadUrl']], {'out': filename, 'auto-file-renaming': 'false', 'continue': 'true', 'dir': folder_name})
+
+            except Exception as e:
+                logger.error(u'Error sending torrent to Aria2. Are you sure it\'s running? (%s)' % e)
+                return
+                
     else:
         folder_name = '%s - %s [%s]' % (
             helpers.latinToAscii(album['ArtistName']).encode('UTF-8').replace('/', '_'),
@@ -1203,6 +1274,61 @@ def verifyresult(title, artistterm, term, lossless):
 
     return True
 
+def searchDdl(album, new=False, losslessOnly=False, albumlength=None,
+                  choose_specific_download=False):
+    reldate = album['ReleaseDate']
+    year = get_year_from_release_date(reldate)
+
+    # MERGE THIS WITH THE TERM CLEANUP FROM searchNZB
+    dic = {'...': '', ' & ': ' ', ' = ': ' ', '?': '', '$': 's', ' + ': ' ', '"': '', ',': ' ',
+           '*': ''}
+
+    semi_cleanalbum = helpers.replace_all(album['AlbumTitle'], dic)
+    cleanalbum = helpers.latinToAscii(semi_cleanalbum)
+    semi_cleanartist = helpers.replace_all(album['ArtistName'], dic)
+    cleanartist = helpers.latinToAscii(semi_cleanartist)
+
+    # Use provided term if available, otherwise build our own (this code needs to be cleaned up since a lot
+    # of these torrent providers are just using cleanartist/cleanalbum terms
+    if album['SearchTerm']:
+        term = album['SearchTerm']
+    elif album['Type'] == 'part of':
+        term = cleanalbum + " " + year
+    else:
+        # FLAC usually doesn't have a year for some reason so I'll leave it out
+        # Various Artist albums might be listed as VA, so I'll leave that out too
+        # Only use the year if the term could return a bunch of different albums, i.e. self-titled albums
+        if album['ArtistName'] in album['AlbumTitle'] or len(album['ArtistName']) < 4 or len(
+                album['AlbumTitle']) < 4:
+            term = cleanartist + ' ' + cleanalbum + ' ' + year
+        elif album['ArtistName'] == 'Various Artists':
+            term = cleanalbum + ' ' + year
+        else:
+            term = cleanartist + ' ' + cleanalbum
+
+    # Replace bad characters in the term and unicode it
+    term = re.sub('[\.\-\/]', ' ', term).encode('utf-8')
+    artistterm = re.sub('[\.\-\/]', ' ', cleanartist).encode('utf-8', 'replace')
+
+    logger.debug(u'Using search term: "%s"' % helpers.latinToAscii(term))
+
+    resultlist = []
+    
+    # Deezer only provides lossy
+    if headphones.CONFIG.DEEZLOADER and not losslessOnly:
+        resultlist += deezloader.searchAlbum(album['ArtistName'], album['AlbumTitle'], album['SearchTerm'], int(albumlength / 1000))
+
+    # attempt to verify that this isn't a substring result
+    # when looking for "Foo - Foo" we don't want "Foobar"
+    # this should be less of an issue when it isn't a self-titled album so we'll only check vs artist
+    results = [result for result in resultlist if verifyresult(result[0], artistterm, term, losslessOnly)]
+
+    # Additional filtering for size etc
+    if results and not choose_specific_download:
+        results = more_filtering(results, album, albumlength, new)
+
+    return results
+
 
 def searchTorrent(album, new=False, losslessOnly=False, albumlength=None,
                   choose_specific_download=False):
@@ -2036,7 +2162,10 @@ def searchTorrent(album, new=False, losslessOnly=False, albumlength=None,
 
 def preprocess(resultlist):
     for result in resultlist:
-        if result[4] == 'torrent':
+        if result[3] == deezloader.PROVIDER_NAME:
+            return True, result
+
+        elif result[4] == 'torrent':
 
             # rutracker always needs the torrent data
             if result[3] == 'rutracker.org':

headphones/webserve.py

@@ -1174,6 +1174,10 @@ class WebInterface(object):
             "utorrent_username": headphones.CONFIG.UTORRENT_USERNAME,
             "utorrent_password": headphones.CONFIG.UTORRENT_PASSWORD,
             "utorrent_label": headphones.CONFIG.UTORRENT_LABEL,
+            "aria_host": headphones.CONFIG.ARIA_HOST,
+            "aria_password": headphones.CONFIG.ARIA_PASSWORD,
+            "aria_token": headphones.CONFIG.ARIA_TOKEN,
+            "aria_username": headphones.CONFIG.ARIA_USERNAME,
             "nzb_downloader_sabnzbd": radio(headphones.CONFIG.NZB_DOWNLOADER, 0),
             "nzb_downloader_nzbget": radio(headphones.CONFIG.NZB_DOWNLOADER, 1),
             "nzb_downloader_blackhole": radio(headphones.CONFIG.NZB_DOWNLOADER, 2),
@@ -1182,6 +1186,7 @@ class WebInterface(object):
             "torrent_downloader_utorrent": radio(headphones.CONFIG.TORRENT_DOWNLOADER, 2),
             "torrent_downloader_deluge": radio(headphones.CONFIG.TORRENT_DOWNLOADER, 3),
             "torrent_downloader_qbittorrent": radio(headphones.CONFIG.TORRENT_DOWNLOADER, 4),
+            "ddl_downloader_aria": radio(headphones.CONFIG.DDL_DOWNLOADER, 0),
             "download_dir": headphones.CONFIG.DOWNLOAD_DIR,
             "use_blackhole": checked(headphones.CONFIG.BLACKHOLE),
             "blackhole_dir": headphones.CONFIG.BLACKHOLE_DIR,
@@ -1243,6 +1248,8 @@ class WebInterface(object):
             "use_tquattrecentonze": checked(headphones.CONFIG.TQUATTRECENTONZE),
             "tquattrecentonze_user": headphones.CONFIG.TQUATTRECENTONZE_USER,
             "tquattrecentonze_password": headphones.CONFIG.TQUATTRECENTONZE_PASSWORD,
+            "download_ddl_dir": headphones.CONFIG.DOWNLOAD_DDL_DIR,
+            "use_deezloader": checked(headphones.CONFIG.DEEZLOADER),
             "pref_qual_0": radio(headphones.CONFIG.PREFERRED_QUALITY, 0),
             "pref_qual_1": radio(headphones.CONFIG.PREFERRED_QUALITY, 1),
             "pref_qual_2": radio(headphones.CONFIG.PREFERRED_QUALITY, 2),
@@ -1288,6 +1295,7 @@ class WebInterface(object):
             "prefer_torrents_0": radio(headphones.CONFIG.PREFER_TORRENTS, 0),
             "prefer_torrents_1": radio(headphones.CONFIG.PREFER_TORRENTS, 1),
             "prefer_torrents_2": radio(headphones.CONFIG.PREFER_TORRENTS, 2),
+            "prefer_torrents_3": radio(headphones.CONFIG.PREFER_TORRENTS, 3),
             "magnet_links_0": radio(headphones.CONFIG.MAGNET_LINKS, 0),
             "magnet_links_1": radio(headphones.CONFIG.MAGNET_LINKS, 1),
             "magnet_links_2": radio(headphones.CONFIG.MAGNET_LINKS, 2),
@@ -1446,7 +1454,7 @@ class WebInterface(object):
             "launch_browser", "enable_https", "api_enabled", "use_blackhole", "headphones_indexer",
             "use_newznab", "newznab_enabled", "use_torznab", "torznab_enabled",
             "use_nzbsorg", "use_omgwtfnzbs", "use_kat", "use_piratebay", "use_oldpiratebay",
-            "use_mininova", "use_waffles", "use_rutracker",
+            "use_mininova", "use_waffles", "use_rutracker", "use_deezloader",
             "use_apollo", "use_redacted", "use_strike", "use_tquattrecentonze", "preferred_bitrate_allow_lossless",
             "detect_bitrate", "ignore_clean_releases", "freeze_db", "cue_split", "move_files",
             "rename_files", "correct_metadata", "cleanup_files", "keep_nfo", "add_album_art",
@@ -1579,6 +1587,9 @@ class WebInterface(object):
 
         # Reconfigure musicbrainz database connection with the new values
         mb.startmb()
+        
+        # Reconfigure Aria2
+        searcher.reconfigure()
 
         raise cherrypy.HTTPRedirect("config")