gronod/headphones
1
0
Fork 1
mirror of https://github.com/rembo10/headphones.git synced 2026-04-13 08:29:26 +01:00
Code Issues Packages Projects Releases Wiki Activity

Fix travis errors

Browse Source
This commit is contained in:
Kallys
2017-04-19 19:55:41 +02:00
parent f0fe23899a
commit 34b8a11145
6 changed files with 920 additions and 1019 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1798
headphones/aria2.py
View File

File diff suppressed because it is too large Load Diff

117
headphones/deezloader.py
View File

@@ -26,7 +26,6 @@ import binascii
from beets.mediafile import MediaFile
from headphones import logger, request, helpers
import headphones
from twisted.conch.insults import helper
# Public constants
PROVIDER_NAME = 'Deezer'
@@ -53,6 +52,7 @@ __cookies = None
__tracks_cache = {}
__albums_cache = {}
def __getApiToken():
global __cookies
response = request.request_response(url="http://www.deezer.com/", headers=__HTTP_HEADERS)
@@ -67,39 +67,42 @@ def __getApiToken():
if not token:
logger.error(u"Deezloader: Unable to get api token")
def getAlbumByLink(album_link):
"""Returns deezer album infos using album link url
:param str album_link: deezer album link url (eg: 'http://www.deezer.com/album/1234567/')
"""
matches = re.search(r"album\/([0-9]+)\/?$", album_link)
if matches:
return getAlbum(matches.group(1))
def getAlbum(album_id):
"""Returns deezer album infos
:param int album_id: deezer album id
"""
global __albums_cache
if str(album_id) in __albums_cache:
return __albums_cache[str(album_id)]
url = __API_INFO_URL + "album/" + str(album_id)
data = request.request_json(url=url, headers=__HTTP_HEADERS, cookies=__cookies)
if data and 'error' not in data:
__albums_cache[str(album_id)] = data
return data
else:
logger.debug("Deezloader: Can't load album infos")
return None
def searchAlbums(search_term):
"""Search for deezer albums using search term
:param str search_term: search term to search album for
"""
logger.info(u'Searching Deezer using term: "%s"' % search_term)
@@ -108,7 +111,7 @@ def searchAlbums(search_term):
data = request.request_json(url=url, headers=__HTTP_HEADERS, cookies=__cookies)
albums = []
# Process content
if data and 'total' in data and data['total'] > 0 and 'data' in data:
for item in data['data']:
@@ -121,13 +124,14 @@ def searchAlbums(search_term):
return albums
def __matchAlbums(albums, artist_name, album_title, album_length):
resultlist = []
for album in albums:
total_size = 0
tracks_found = 0
for track in album['tracks']['data']:
t = getTrack(track['id'])
if t:
@@ -139,29 +143,29 @@ def __matchAlbums(albums, artist_name, album_title, album_length):
size = t["FILESIZE_MP3_128"]
else:
size = t["FILESIZE_MP3_64"]
size = int(size)
total_size += size
tracks_found += 1
logger.debug(u'Found song "%s". Size: %s' % (t['SNG_TITLE'], helpers.bytes_to_mb(size)))
matched = True
mismatch_reason = 'matched!'
if album_length > 0 and abs(int(album['duration']) - album_length) > 240:
matched = False
mismatch_reason = (u'duration mismatch: %i not in [%i, %i]' % (int(album['duration']), (album_length - 240), (album_length + 240)))
elif (helpers.latinToAscii(re.sub(r"\W", "", album_title, 0, re.UNICODE)).lower() !=
elif (helpers.latinToAscii(re.sub(r"\W", "", album_title, 0, re.UNICODE)).lower() !=
helpers.latinToAscii(re.sub(r"\W", "", album['title'], 0, re.UNICODE)).lower()):
matched = False
mismatch_reason = (u'album name mismatch: %s != %s' % (album['title'], album_title))
elif (helpers.latinToAscii(re.sub(r"\W", "", artist_name, 0, re.UNICODE)).lower() !=
helpers.latinToAscii(re.sub(r"\W", "", album['artist']['name'], 0, re.UNICODE)).lower()):
matched = False
mismatch_reason = (u'artist name mismatch: %s != %s' % (album['artist']['name'], artist_name))
resultlist.append(
(album['artist']['name'] + ' - ' + album['title'] + ' [' + album['release_date'][:4] + '] (' + str(tracks_found) + '/' + str(album['nb_tracks']) + ')',
total_size, album['link'], PROVIDER_NAME, "ddl", matched)
@@ -170,6 +174,7 @@ def __matchAlbums(albums, artist_name, album_title, album_length):
return resultlist
def searchAlbum(artist_name, album_title, user_search_term=None, album_length=None):
"""Search for deezer specific album.
This will iterate over deezer albums and try to find best matches
@@ -182,7 +187,7 @@ def searchAlbum(artist_name, album_title, user_search_term=None, album_length=No
# User search term by-pass normal search
if user_search_term:
return __matchAlbums(searchAlbums(user_search_term), artist_name, album_title, album_length)
resultlist = __matchAlbums(searchAlbums((artist_name + ' ' + album_title).strip()), artist_name, album_title, album_length)
if resultlist:
return resultlist
@@ -190,35 +195,36 @@ def searchAlbum(artist_name, album_title, user_search_term=None, album_length=No
# Deezer API supports unicode, so just remove non alphanumeric characters
clean_artist_name = re.sub(r"[^\w\s]", " ", artist_name, 0, re.UNICODE).strip()
clean_album_name = re.sub(r"[^\w\s]", " ", album_title, 0, re.UNICODE).strip()
resultlist = __matchAlbums(searchAlbums((clean_artist_name + ' ' + clean_album_name).strip()), artist_name, album_title, album_length)
if resultlist:
return resultlist
resultlist = __matchAlbums(searchAlbums(clean_artist_name), artist_name, album_title, album_length)
if resultlist:
return resultlist
return resultlist
def getTrack(sng_id, try_reload_api=True):
"""Returns deezer track infos
:param int sng_id: deezer song id
:param bool try_reload_api: whether or not try reloading API if session expired
:param bool try_reload_api: whether or not try reloading API if session expired
"""
global __tracks_cache
if str(sng_id) in __tracks_cache:
return __tracks_cache[str(sng_id)]
data = "[{\"method\":\"song.getListData\",\"params\":{\"sng_ids\":[" + str(sng_id) + "]}}]"
json = request.request_json(url=__API_URL, headers=__HTTP_HEADERS, method='post', params=__api_queries, data=data, cookies=__cookies)
results = []
error = None
invalid_token = False
if json:
# Check for errors
if 'error' in json:
@@ -230,7 +236,7 @@ def getTrack(sng_id, try_reload_api=True):
error = json[0]['error']
if 'VALID_TOKEN_REQUIRED' in json[0]['error'] and json[0]['error']['VALID_TOKEN_REQUIRED'] == u"Invalid CSRF token":
invalid_token = True
# Got invalid token error
if error:
if invalid_token and try_reload_api:
@@ -245,26 +251,27 @@ def getTrack(sng_id, try_reload_api=True):
if 'token' in item:
logger.error(u"An unknown error occurred in the Deezer parser: Uploaded Files are currently not supported")
return
sng_id = item["SNG_ID"]
md5Origin = item["MD5_ORIGIN"]
sng_format = 3
if item["FILESIZE_MP3_320"] <= 0:
if item["FILESIZE_MP3_256"] > 0:
sng_format = 5
else:
sng_format = 1
mediaVersion = int(item["MEDIA_VERSION"])
item['downloadUrl'] = __getDownloadUrl(md5Origin, sng_id, sng_format, mediaVersion)
__tracks_cache[sng_id] = item
return item
except Exception as e:
logger.error(u"An unknown error occurred in the Deezer parser: %s" % e)
def __getDownloadUrl(md5Origin, sng_id, sng_format, mediaVersion):
urlPart = md5Origin.encode('utf-8') + b'\xa4' + str(sng_format) + b'\xa4' + str(sng_id) + b'\xa4' + str(mediaVersion)
md5val = md5(urlPart).hexdigest()
@@ -273,6 +280,7 @@ def __getDownloadUrl(md5Origin, sng_id, sng_format, mediaVersion):
ciphertext = cipher.encrypt(__pad(urlPart, AES.block_size))
return "http://e-cdn-proxy-" + md5Origin[:1] + ".deezer.com/mobile/1/" + binascii.hexlify(ciphertext).lower()
def __pad(raw, block_size):
if (len(raw) % block_size == 0):
return raw
@@ -281,10 +289,11 @@ def __pad(raw, block_size):
data = raw + padding_required * padChar
return data
def __tagTrack(path, track):
try:
album = getAlbum(track['ALB_ID'])
f = MediaFile(path)
f.artist = track['ART_NAME']
f.album = track['ALB_TITLE']
@@ -295,22 +304,23 @@ def __tagTrack(path, track):
f.bpm = track['BPM']
f.date = datetime.strptime(album['release_date'], '%Y-%m-%d').date()
f.albumartist = album['artist']['name']
if u'genres' in album and u'data' in album['genres']:
if u'genres' in album and u'data' in album['genres']:
f.genres = [genre['name'] for genre in album['genres']['data']]
f.save()
except Exception as e:
logger.error(u'Unable to tag deezer track "%s": %s' % (path, e))
def decryptTracks(paths):
"""Decrypt downloaded deezer tracks.
:param paths: list of path to deezer tracks (*.dzr files).
"""
# Note: tracks can be from different albums
decrypted_tracks = {}
# First pass: load tracks data
for path in paths:
try:
@@ -322,15 +332,15 @@ def decryptTracks(paths):
if album_folder not in decrypted_tracks:
decrypted_tracks[album_folder] = {}
if disk_number not in decrypted_tracks[album_folder]:
decrypted_tracks[album_folder][disk_number] = {}
decrypted_tracks[album_folder][disk_number][track_number] = track
except Exception as e:
logger.error(u'Unable to load deezer track infos "%s": %s' % (path, e))
# Second pass: decrypt tracks
for album_folder in decrypted_tracks:
multi_disks = len(decrypted_tracks[album_folder]) > 1
@@ -339,12 +349,12 @@ def decryptTracks(paths):
try:
filename = helpers.replace_illegal_chars(track['SNG_TITLE']).strip()
filename = ('{:02d}'.format(track_number) + '_' + filename + '.mp3')
# Add a 'cd x' sub-folder if album has more than one disk
disk_folder = os.path.join(album_folder, 'cd ' + str(disk_number)) if multi_disks else album_folder
dest = os.path.join(disk_folder, filename).encode(headphones.SYS_ENCODING, 'replace')
# Decrypt track if not already done
if not os.path.exists(dest):
try:
@@ -358,12 +368,13 @@ def decryptTracks(paths):
continue
decrypted_tracks[album_folder][disk_number][track_number]['path'] = dest
except Exception as e:
logger.error(u'Unable to decrypt deezer track "%s": %s' % (path, e))
return decrypted_tracks
def __decryptDownload(source, sng_id, dest):
interval_chunk = 3
chunk_size = 2048
@@ -383,7 +394,7 @@ def __decryptDownload(source, sng_id, dest):
if(i % interval_chunk == 0):
cipher = Blowfish.new(blowFishKey, Blowfish.MODE_CBC, iv)
chunk = cipher.decrypt(__pad(chunk, Blowfish.block_size))
fout.write(chunk)
i += 1
chunk = f.read(chunk_size)
@@ -391,6 +402,7 @@ def __decryptDownload(source, sng_id, dest):
f.close()
fout.close()
def __getBlowFishKey(encryptionKey):
if encryptionKey < 1:
encryptionKey *= -1
@@ -402,14 +414,15 @@ def __getBlowFishKey(encryptionKey):
return __xorHex(parts)
def __xorHex(parts):
data = ""
for i in range(0, 16):
character = ord(parts[0][i])
for j in range(1, len(parts)):
character ^= ord(parts[j][i])
data += chr(character)
return data

4
headphones/helpers_test.py
View File

@@ -14,9 +14,9 @@ class HelpersTest(TestCase):
u'Symphonęy Nº9': 'Symphoney No.9',
u'ÆæßðÞIJij': u'AeaessdThIJıj',
u'Obsessió (Cerebral Apoplexy remix)': 'obsessio cerebral '
'apoplexy remix',
'apoplexy remix',
u'Doktór Hałabała i siedmiu zbojów': 'doktor halabala i siedmiu '
'zbojow',
'zbojow',
u'Arbetets Söner och Döttrar': 'arbetets soner och dottrar',
u'Björk Guðmundsdóttir': 'bjork gudmundsdottir',
u'L\'Arc~en~Ciel': 'larc en ciel',

10
headphones/postprocessor.py
View File

@@ -261,7 +261,7 @@ def verify(albumid, albumpath, Kind=None, forced=False, keep_original_folder=Fal
if downloaded_deezer_list:
logger.info('Decrypting deezer tracks')
decrypted_deezer_list = deezloader.decryptTracks(downloaded_deezer_list)
# Check if album is complete based on album duration only
# (total track numbers is not determinant enough due to hidden tracks for eg)
db_track_duration = 0
@@ -271,7 +271,7 @@ def verify(albumid, albumpath, Kind=None, forced=False, keep_original_folder=Fal
db_track_duration += track['TrackDuration'] / 1000
except:
downloaded_track_duration = False
try:
for disk_number in decrypted_deezer_list[albumpath]:
for track in decrypted_deezer_list[albumpath][disk_number].values():
@@ -279,14 +279,14 @@ def verify(albumid, albumpath, Kind=None, forced=False, keep_original_folder=Fal
downloaded_track_duration += int(track['DURATION'])
except:
downloaded_track_duration = False
if not downloaded_track_duration or not db_track_duration or abs(downloaded_track_duration - db_track_duration) > 240:
logger.info("Looks like " +
logger.info("Looks like " +
os.path.basename(albumpath).decode(headphones.SYS_ENCODING, 'replace') +
" isn't complete yet (duration mismatch). Will try again on the next run")
return
downloaded_track_list = list(set(downloaded_track_list)) # Remove duplicates
downloaded_track_list = list(set(downloaded_track_list)) # Remove duplicates
# test #1: metadata - usually works
logger.debug('Verifying metadata...')

8
headphones/searcher.py
View File

@@ -55,6 +55,7 @@ redobj = None
# Persistent Aria2 RPC object
__aria2rpc_obj = None
def getAria2RPC():
global __aria2rpc_obj
if not __aria2rpc_obj:
@@ -67,10 +68,12 @@ def getAria2RPC():
)
return __aria2rpc_obj
def reconfigure():
global __aria2rpc_obj
__aria2rpc_obj = None
def fix_url(s, charset="utf-8"):
"""
Fix the URL so it is proper formatted and encoded.
@@ -896,7 +899,7 @@ def send_to_downloader(data, bestqual, album):
except Exception as e:
logger.error(u'Error sending torrent to Aria2. Are you sure it\'s running? (%s)' % e)
return
else:
folder_name = '%s - %s [%s]' % (
helpers.latinToAscii(album['ArtistName']).encode('UTF-8').replace('/', '_'),
@@ -1274,6 +1277,7 @@ def verifyresult(title, artistterm, term, lossless):
return True
def searchDdl(album, new=False, losslessOnly=False, albumlength=None,
choose_specific_download=False):
reldate = album['ReleaseDate']
@@ -1313,7 +1317,7 @@ def searchDdl(album, new=False, losslessOnly=False, albumlength=None,
logger.debug(u'Using search term: "%s"' % helpers.latinToAscii(term))
resultlist = []
# Deezer only provides lossy
if headphones.CONFIG.DEEZLOADER and not losslessOnly:
resultlist += deezloader.searchAlbum(album['ArtistName'], album['AlbumTitle'], album['SearchTerm'], int(albumlength / 1000))

2
headphones/webserve.py
View File

@@ -1587,7 +1587,7 @@ class WebInterface(object):
# Reconfigure musicbrainz database connection with the new values
mb.startmb()
# Reconfigure Aria2
searcher.reconfigure()