From d037147c23e676a6792fd96a907cf304f51e1eb1 Mon Sep 17 00:00:00 2001 From: Ade Date: Fri, 15 Aug 2014 20:16:26 +1200 Subject: [PATCH 01/16] Manage Unmatched fix Potential fix for https://github.com/rembo10/headphones/issues/1808 --- headphones/webserve.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/headphones/webserve.py b/headphones/webserve.py index 7a8314c7..f40206a6 100644 --- a/headphones/webserve.py +++ b/headphones/webserve.py @@ -460,8 +460,9 @@ class WebInterface(object): have_album_dictionary.append(have_dict) headphones_albums = myDB.select('SELECT ArtistName, AlbumTitle from albums ORDER BY ArtistName') for albums in headphones_albums: - headphones_dict = { 'ArtistName' : albums['ArtistName'], 'AlbumTitle' : albums['AlbumTitle'] } - headphones_album_dictionary.append(headphones_dict) + if albums['ArtistName'] and albums['AlbumTitle']: + headphones_dict = { 'ArtistName' : albums['ArtistName'], 'AlbumTitle' : albums['AlbumTitle'] } + headphones_album_dictionary.append(headphones_dict) #unmatchedalbums = [f for f in have_album_dictionary if f not in [x for x in headphones_album_dictionary]] check = set([(cleanName(d['ArtistName']).lower(), cleanName(d['AlbumTitle']).lower()) for d in headphones_album_dictionary]) From 59b4cefd53bab74cc95d708e04616aafb3dc0f97 Mon Sep 17 00:00:00 2001 From: David Date: Fri, 15 Aug 2014 17:02:34 +0200 Subject: [PATCH 02/16] Creating two functions from duplicated code --- headphones/postprocessor.py | 41 +++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/headphones/postprocessor.py b/headphones/postprocessor.py index c80fc267..0c053495 100644 --- a/headphones/postprocessor.py +++ b/headphones/postprocessor.py @@ -30,6 +30,47 @@ from headphones import logger, helpers, request, mb, music_encoder postprocessor_lock = threading.Lock() + +def find_in_path(albumpath, extra_formats=None, use_MF=True): + """ Takes a path and optionally extra formats. + Finds files matching the MEDIA_FORMATS and returns them as a list. + If use_MF is disabled MEDIA_FORMATS will be ignored. + """ + found_tracks = [] + if extra_formats: + if type(extra_formats) is not list: + extra_formats = [extra_formats] + if not use_MF: + mf = extra_formats + else: + mf = headphones.MEDIA_FORMATS + extra_formats + else: + if not use_MF: + # if not using MF then extra_formats must be set + return False + mf = headphones.MEDIA_FORMATS + for r, d, f in os.walk(albumpath): + for file in f: + if any(file.lower().endswith('.' + x.lower()) for x in mf): + found_tracks.append(os.path.join(r, file)) + return found_tracks + + +def count_matches(track_list, match, inverted=False): + """ Takes a list of tracks and a pattern to match. + Returns the number of matched items and a list of the matches. + If inverted is set to True the returned list will be inverted and only + contain the NON-matches. + """ + if not track_list or not match: + return False, False + new_list = [x for x in track_list if x.lower().endswith(match)] + nr_of_matches = len(new_list) + if inverted: + new_list = [x for x in track_list if not x.lower().endswith(match)] + return nr_of_matches, new_list + + def checkFolder(): with postprocessor_lock: From 405ebe5ee2b6ca71053659b1e6cdf2759fccd26c Mon Sep 17 00:00:00 2001 From: David Date: Fri, 15 Aug 2014 17:46:01 +0200 Subject: [PATCH 03/16] Removing duplications, replacing with functions --- headphones/postprocessor.py | 71 ++++++++++++++++--------------------- 1 file changed, 31 insertions(+), 40 deletions(-) diff --git a/headphones/postprocessor.py b/headphones/postprocessor.py index 0c053495..38c2c2fb 100644 --- a/headphones/postprocessor.py +++ b/headphones/postprocessor.py @@ -193,20 +193,19 @@ def verify(albumid, albumpath, Kind=None, forced=False): release = myDB.action('SELECT * from albums WHERE AlbumID=?', [albumid]).fetchone() tracks = myDB.select('SELECT * from tracks WHERE AlbumID=?', [albumid]) - downloaded_track_list = [] - downloaded_cuecount = 0 - - for r,d,f in os.walk(albumpath): - for files in f: - if any(files.lower().endswith('.' + x.lower()) for x in headphones.MEDIA_FORMATS): - downloaded_track_list.append(os.path.join(r, files)) - elif files.lower().endswith('.cue'): - downloaded_cuecount += 1 - # if any of the files end in *.part, we know the torrent isn't done yet. Process if forced, though - elif files.lower().endswith(('.part', '.utpart')) and not forced: - logger.info("Looks like " + os.path.basename(albumpath).decode(headphones.SYS_ENCODING, 'replace') + " isn't complete yet. Will try again on the next run") - return + all_matched = find_in_path(albumpath, ["cue", "part", "upart"]) + incomplete_matches, remaining_matches = count_matches(all_matched, + (".part", ".upart"), + inverted = True) + if incomplete_matches and not forced: + _a = os.path.basename(albumpath).decode(headphones.SYS_ENCODING, + 'replace') + logger.info("Looks like " + _a + " isn't complete yet. Will try again on the next run") + return + downloaded_cuecount, downloaded_track_list = count_matches(remaining_matches, + ".cue", + inverted = True) # use xld to split cue @@ -248,11 +247,7 @@ def verify(albumid, albumpath, Kind=None, forced=False): # count files, should now be more than original if xld successfully split - new_downloaded_track_list_count = 0 - for r,d,f in os.walk(albumpath): - for file in f: - if any(file.lower().endswith('.' + x.lower()) for x in headphones.MEDIA_FORMATS): - new_downloaded_track_list_count += 1 + new_downloaded_track_list_count = len(find_in_path(albumpath)) if new_downloaded_track_list_count > len(downloaded_track_list): @@ -261,12 +256,8 @@ def verify(albumid, albumpath, Kind=None, forced=False): os.rename(downloaded_track, downloaded_track + '.original') #reload + downloaded_track_list = find_in_path(albumpath) - downloaded_track_list = [] - for r,d,f in os.walk(albumpath): - for file in f: - if any(file.lower().endswith('.' + x.lower()) for x in headphones.MEDIA_FORMATS): - downloaded_track_list.append(os.path.join(r, file)) # test #1: metadata - usually works logger.debug('Verifying metadata...') @@ -371,15 +362,12 @@ def doPostProcessing(albumid, albumpath, release, tracks, downloaded_track_list, # Need to update the downloaded track list with the new location. # Could probably just throw in the "headphones-modified" folder, # but this is good to make sure we're not counting files that may have failed to move - downloaded_track_list = [] - downloaded_cuecount = 0 - for r,d,f in os.walk(albumpath): - for files in f: - if any(files.lower().endswith('.' + x.lower()) for x in headphones.MEDIA_FORMATS): - downloaded_track_list.append(os.path.join(r, files)) - elif files.lower().endswith('.cue'): - downloaded_cuecount += 1 + all_matches = find_in_path(albumpath, "cue") + downloaded_cuecount, downloaded_track_list = count_matches(all_matches, + ".cue", + True) + # Check if files are valid media files and are writeable, before the steps # below are executed. This simplifies errors and prevents unfinished steps. @@ -618,15 +606,18 @@ def addAlbumArt(artwork, albumpath, release): def cleanupFiles(albumpath): logger.info('Cleaning up files') - for r,d,f in os.walk(albumpath): - for files in f: - if not any(files.lower().endswith('.' + x.lower()) for x in headphones.MEDIA_FORMATS): - if not (headphones.KEEP_NFO and files.lower().endswith('.nfo')): - logger.debug('Removing: %s' % files) - try: - os.remove(os.path.join(r, files)) - except Exception, e: - logger.error(u'Could not remove file: %s. Error: %s' % (files.decode(headphones.SYS_ENCODING, 'replace'), e)) + if headphones.KEEP_NFO: + files = find_in_path(albumpath, extra_formats="nfo", inverted=True) + else: + files = find_in_path(albumpath, inverted=True) + for _f in files: + logger.debug('Removing: %s' % _f) + try: + os.remove(_f) + except Exception, e: + _f_decoded = _f.decode(headphones.SYS_ENCODING, 'replace') + logger.error(u'Could not remove file: %s. Error: %s' % (_f_decoded, + e)) def renameNFO(albumpath): for r,d,f in os.walk(albumpath): From 08f483d77d9b6838b074300afd3c1beba46e665b Mon Sep 17 00:00:00 2001 From: David Date: Fri, 15 Aug 2014 18:57:44 +0200 Subject: [PATCH 04/16] Removing duplications, replacing with functions --- headphones/postprocessor.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/headphones/postprocessor.py b/headphones/postprocessor.py index 38c2c2fb..f0debe29 100644 --- a/headphones/postprocessor.py +++ b/headphones/postprocessor.py @@ -620,15 +620,16 @@ def cleanupFiles(albumpath): e)) def renameNFO(albumpath): - for r,d,f in os.walk(albumpath): - for file in f: - if file.lower().endswith('.nfo'): - logger.debug('Renaming: "%s" to "%s"' % (file.decode(headphones.SYS_ENCODING, 'replace'), file.decode(headphones.SYS_ENCODING, 'replace') + '-orig')) - try: - new_file_name = os.path.join(r, file)[:-3] + 'orig.nfo' - os.rename(os.path.join(r, file), new_file_name) - except Exception, e: - logger.error(u'Could not rename file: %s. Error: %s' % (os.path.join(r, file).decode(headphones.SYS_ENCODING, 'replace'), e)) + files = find_in_path(albumpath, extra_formats="nfo", use_MF=False) + for _f in files: + _f_decoded = _f.decode(headphones.SYS_ENCODING, 'replace') + logger.debug('Renaming: "%s" to "%s"' % (_f_decoded, _f_decoded + '-orig')) + try: + new_file_name = _f[:-3] + 'orig.nfo' + os.rename(_f, new_file_name) + except Exception, e: + logger.error(u'Could not rename file: %s. Error: %s' % (_f_decoded, + e)) def moveFiles(albumpath, release, tracks): From ef3cd00ef1724ae234add7d35dc063db379e43ad Mon Sep 17 00:00:00 2001 From: David Date: Fri, 15 Aug 2014 19:12:02 +0200 Subject: [PATCH 05/16] Removing duplications, replacing with functions --- headphones/postprocessor.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/headphones/postprocessor.py b/headphones/postprocessor.py index f0debe29..a4f65674 100644 --- a/headphones/postprocessor.py +++ b/headphones/postprocessor.py @@ -695,13 +695,14 @@ def moveFiles(albumpath, release, tracks): lossy_media = False lossless_media = False - for r,d,f in os.walk(albumpath): - for files in f: - files_to_move.append(os.path.join(r, files)) - if any(files.lower().endswith('.' + x.lower()) for x in headphones.LOSSY_MEDIA_FORMATS): - lossy_media = True - if any(files.lower().endswith('.' + x.lower()) for x in headphones.LOSSLESS_MEDIA_FORMATS): - lossless_media = True + if find_in_path(albumpath, + extra_formats=headphones.LOSSY_MEDIA_FORMATS, + use_MF=False): + lossy_media = True + if find_in_path(albumpath, + extra_formats=headphones.LOSSLESS_MEDIA_FORMATS, + use_MF=False): + lossless_media = True # Do some sanity checking to see what directories we need to create: make_lossy_folder = False From fc1c26f58f23714cd974c44c2975c375601a6d5e Mon Sep 17 00:00:00 2001 From: Ade Date: Sat, 16 Aug 2014 09:55:37 +1200 Subject: [PATCH 06/16] Sanitize name too Fixes https://github.com/rembo10/headphones/issues/1706 --- headphones/sab.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/headphones/sab.py b/headphones/sab.py index 5249e914..bf746d31 100644 --- a/headphones/sab.py +++ b/headphones/sab.py @@ -62,7 +62,7 @@ def sendNZB(nzb): # Sanitize the file a bit, since we can only use ascii chars with MultiPartPostHandler nzbdata = helpers.latinToAscii(nzb.extraInfo[0]) params['mode'] = 'addfile' - multiPartParams = {"nzbfile": (nzb.name+".nzb", nzbdata)} + multiPartParams = {"nzbfile": (helpers.latinToAscii(nzb.name)+".nzb", nzbdata)} if not headphones.SAB_HOST.startswith('http'): headphones.SAB_HOST = 'http://' + headphones.SAB_HOST From 06822e25146c8e41df0b64de508556c41cb9832e Mon Sep 17 00:00:00 2001 From: Ade Date: Sat, 16 Aug 2014 10:49:42 +1200 Subject: [PATCH 07/16] Ignore Artist fix Fixes https://github.com/rembo10/headphones/issues/1660 --- headphones/webserve.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/headphones/webserve.py b/headphones/webserve.py index f40206a6..db0f52c6 100644 --- a/headphones/webserve.py +++ b/headphones/webserve.py @@ -299,8 +299,8 @@ class WebInterface(object): if action == "ignore": myDB = db.DBConnection() for artist in args: - myDB.action('DELETE FROM newartists WHERE ArtistName=?', [artist]) - myDB.action('UPDATE have SET Matched="Ignored" WHERE ArtistName=?', [artist]) + myDB.action('DELETE FROM newartists WHERE ArtistName=?', [artist.decode(headphones.SYS_ENCODING, 'replace')]) + myDB.action('UPDATE have SET Matched="Ignored" WHERE ArtistName=?', [artist.decode(headphones.SYS_ENCODING, 'replace')]) logger.info("Artist %s removed from new artist list and set to ignored" % artist) raise cherrypy.HTTPRedirect("home") addArtists.exposed = True From 17ef29887e48c4e9e35b63896429214f78cf4795 Mon Sep 17 00:00:00 2001 From: David Date: Sat, 16 Aug 2014 01:57:05 +0200 Subject: [PATCH 08/16] Use _file instead of file since file is a built-in function --- headphones/postprocessor.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/headphones/postprocessor.py b/headphones/postprocessor.py index a4f65674..e102a4d2 100644 --- a/headphones/postprocessor.py +++ b/headphones/postprocessor.py @@ -50,9 +50,9 @@ def find_in_path(albumpath, extra_formats=None, use_MF=True): return False mf = headphones.MEDIA_FORMATS for r, d, f in os.walk(albumpath): - for file in f: - if any(file.lower().endswith('.' + x.lower()) for x in mf): - found_tracks.append(os.path.join(r, file)) + for _file in f: + if any(_file.lower().endswith('.' + x.lower()) for x in mf): + found_tracks.append(os.path.join(r, _file)) return found_tracks @@ -420,7 +420,7 @@ def doPostProcessing(albumid, albumpath, release, tracks, downloaded_track_list, if headphones.KEEP_NFO: renameNFO(albumpath) - + if headphones.ADD_ALBUM_ART and artwork: addAlbumArt(artwork, albumpath, release) @@ -655,12 +655,12 @@ def moveFiles(albumpath, release, tracks): firstchar = '0-9' else: firstchar = sortname[0] - + for r,d,f in os.walk(albumpath): try: origfolder = os.path.basename(os.path.normpath(r)) except: - origfolder = '' + origfolder = '' values = { '$Artist': artist, '$SortArtist': sortname, From a7d1f128ad70c3e44d5d3a76320e221f2639fa87 Mon Sep 17 00:00:00 2001 From: Ade Date: Sun, 17 Aug 2014 18:14:18 +1200 Subject: [PATCH 09/16] Create alltracks when adding album Potential fix for https://github.com/rembo10/headphones/issues/1586 --- headphones/importer.py | 2 ++ headphones/webserve.py | 8 ++++++++ 2 files changed, 10 insertions(+) diff --git a/headphones/importer.py b/headphones/importer.py index 4bb3f9a7..815f1390 100644 --- a/headphones/importer.py +++ b/headphones/importer.py @@ -488,6 +488,7 @@ def addArtisttoDB(artistid, extrasonly=False, forcefull=False): if skip_log == 0: logger.info(u"[%s] No new releases, so no changes made to %s" % (artist['artist_name'], rg['title'])) + time.sleep(3) finalize_update(artistid, artist['artist_name'], errors) logger.info(u"Seeing if we need album art for: %s" % artist['artist_name']) @@ -666,6 +667,7 @@ def addReleaseById(rid, rgid=None): #myDB.action('DELETE from have WHERE Location=?', [match['Location']]) myDB.upsert("tracks", newValueDict, controlValueDict) + myDB.upsert("alltracks", newValueDict, controlValueDict) # Reset status if status == 'Loading': diff --git a/headphones/webserve.py b/headphones/webserve.py index db0f52c6..5d197369 100644 --- a/headphones/webserve.py +++ b/headphones/webserve.py @@ -378,6 +378,14 @@ class WebInterface(object): def deleteAlbum(self, AlbumID, ArtistID=None): logger.info(u"Deleting all traces of album: " + AlbumID) myDB = db.DBConnection() + + namecheck = myDB.select('SELECT ArtistName, AlbumTitle from albums where AlbumID=?', [AlbumID]) + for name in namecheck: + artist = name['ArtistName'] + album = name['AlbumTitle'] + + myDB.action('UPDATE have SET Matched=NULL WHERE ArtistName=? AND AlbumTitle=?', (artist, album)) + myDB.action('DELETE from albums WHERE AlbumID=?', [AlbumID]) myDB.action('DELETE from tracks WHERE AlbumID=?', [AlbumID]) myDB.action('DELETE from allalbums WHERE AlbumID=?', [AlbumID]) From 1b65df6c4574b094d9d6115eede6b4547bd80eac Mon Sep 17 00:00:00 2001 From: David Date: Sun, 17 Aug 2014 19:48:47 +0200 Subject: [PATCH 10/16] Return false on non existing dirs --- headphones/postprocessor.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/headphones/postprocessor.py b/headphones/postprocessor.py index e102a4d2..b4381934 100644 --- a/headphones/postprocessor.py +++ b/headphones/postprocessor.py @@ -36,6 +36,8 @@ def find_in_path(albumpath, extra_formats=None, use_MF=True): Finds files matching the MEDIA_FORMATS and returns them as a list. If use_MF is disabled MEDIA_FORMATS will be ignored. """ + if not os.path.isdir(albumpath): + return False found_tracks = [] if extra_formats: if type(extra_formats) is not list: From 72d2201dd6545891605c87620fdb10105da06c45 Mon Sep 17 00:00:00 2001 From: David Date: Mon, 18 Aug 2014 21:17:28 +0200 Subject: [PATCH 11/16] Remove unused import --- headphones/torrentfinished.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/headphones/torrentfinished.py b/headphones/torrentfinished.py index a86614e2..0c468631 100644 --- a/headphones/torrentfinished.py +++ b/headphones/torrentfinished.py @@ -15,7 +15,7 @@ import threading import headphones -from headphones import db, utorrent, transmission, logger +from headphones import db, utorrent, transmission postprocessor_lock = threading.Lock() From bb884eeab778f879b5340fac426b024416b94a72 Mon Sep 17 00:00:00 2001 From: Ade Date: Tue, 19 Aug 2014 21:49:03 +1200 Subject: [PATCH 12/16] Duplicate track issue Possible fix for https://github.com/rembo10/headphones/issues/1504 --- headphones/importer.py | 1 - headphones/librarysync.py | 16 +++++++++++----- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/headphones/importer.py b/headphones/importer.py index 815f1390..47363b4a 100644 --- a/headphones/importer.py +++ b/headphones/importer.py @@ -667,7 +667,6 @@ def addReleaseById(rid, rgid=None): #myDB.action('DELETE from have WHERE Location=?', [match['Location']]) myDB.upsert("tracks", newValueDict, controlValueDict) - myDB.upsert("alltracks", newValueDict, controlValueDict) # Reset status if status == 'Loading': diff --git a/headphones/librarysync.py b/headphones/librarysync.py index 416023fc..0fdd98c8 100644 --- a/headphones/librarysync.py +++ b/headphones/librarysync.py @@ -50,7 +50,7 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, cron=Fal if not append: # Clean up bad filepaths - tracks = myDB.select('SELECT Location, TrackID from alltracks WHERE Location IS NOT NULL') + tracks = myDB.select('SELECT Location from alltracks WHERE Location IS NOT NULL UNION SELECT Location from tracks WHERE Location IS NOT NULL') for track in tracks: encoded_track_string = track['Location'].encode(headphones.SYS_ENCODING) @@ -216,6 +216,7 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, cron=Fal if song['ArtistName'] and song['AlbumTitle'] and song['TrackTitle']: track = myDB.action('SELECT ArtistName, AlbumTitle, TrackTitle, AlbumID from tracks WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [song['ArtistName'], song['AlbumTitle'], song['TrackTitle']]).fetchone() + have_updated = False if track: controlValueDict = { 'ArtistName' : track['ArtistName'], 'AlbumTitle' : track['AlbumTitle'], @@ -228,6 +229,7 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, cron=Fal controlValueDict2 = { 'Location' : song['Location']} newValueDict2 = { 'Matched' : track['AlbumID']} myDB.upsert("have", newValueDict2, controlValueDict2) + have_updated = True else: track = myDB.action('SELECT CleanName, AlbumID from tracks WHERE CleanName LIKE ?', [song['CleanName']]).fetchone() if track: @@ -240,11 +242,12 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, cron=Fal controlValueDict2 = { 'Location' : song['Location']} newValueDict2 = { 'Matched' : track['AlbumID']} myDB.upsert("have", newValueDict2, controlValueDict2) + have_updated = True else: controlValueDict2 = { 'Location' : song['Location']} newValueDict2 = { 'Matched' : "Failed"} myDB.upsert("have", newValueDict2, controlValueDict2) - + have_updated = True alltrack = myDB.action('SELECT ArtistName, AlbumTitle, TrackTitle, AlbumID from alltracks WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [song['ArtistName'], song['AlbumTitle'], song['TrackTitle']]).fetchone() if alltrack: @@ -272,9 +275,12 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, cron=Fal newValueDict2 = { 'Matched' : alltrack['AlbumID']} myDB.upsert("have", newValueDict2, controlValueDict2) else: - controlValueDict2 = { 'Location' : song['Location']} - newValueDict2 = { 'Matched' : "Failed"} - myDB.upsert("have", newValueDict2, controlValueDict2) + # alltracks may not exist if adding album manually, have should only be set to failed if not already updated in tracks + if not have_updated: + controlValueDict2 = { 'Location' : song['Location']} + newValueDict2 = { 'Matched' : "Failed"} + myDB.upsert("have", newValueDict2, controlValueDict2) + else: controlValueDict2 = { 'Location' : song['Location']} newValueDict2 = { 'Matched' : "Failed"} From aa6206a235d5efae51ff107492d77707d319499b Mon Sep 17 00:00:00 2001 From: Ade Date: Sat, 23 Aug 2014 09:08:37 +1200 Subject: [PATCH 13/16] Remove from have when deleting Album --- headphones/webserve.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/headphones/webserve.py b/headphones/webserve.py index 5d197369..92debc30 100644 --- a/headphones/webserve.py +++ b/headphones/webserve.py @@ -379,12 +379,10 @@ class WebInterface(object): logger.info(u"Deleting all traces of album: " + AlbumID) myDB = db.DBConnection() - namecheck = myDB.select('SELECT ArtistName, AlbumTitle from albums where AlbumID=?', [AlbumID]) - for name in namecheck: - artist = name['ArtistName'] - album = name['AlbumTitle'] - - myDB.action('UPDATE have SET Matched=NULL WHERE ArtistName=? AND AlbumTitle=?', (artist, album)) + myDB.action('DELETE from have WHERE Matched=?', [AlbumID]) + album = myDB.action('SELECT ArtistName, AlbumTitle from albums where AlbumID=?', [AlbumID]).fetchone() + if album: + myDB.action('DELETE from have WHERE ArtistName=? AND AlbumTitle=?', [album['ArtistName'], album['AlbumTitle']]) myDB.action('DELETE from albums WHERE AlbumID=?', [AlbumID]) myDB.action('DELETE from tracks WHERE AlbumID=?', [AlbumID]) From b950bcbd0b651b7d9929050d5c1fba334a0913e2 Mon Sep 17 00:00:00 2001 From: Ade Date: Mon, 25 Aug 2014 18:46:09 +1200 Subject: [PATCH 14/16] Extras changes - Append secondary type to type description if different, e.g. Album + Mixtape/Street - Keep Extras in order - Fix bug where albums not deleted from db if not in mb results when refreshing Artist --- headphones/importer.py | 2 +- headphones/mb.py | 34 ++++++++++++++++++++++++++++++---- headphones/webserve.py | 25 ++++++++++++++----------- 3 files changed, 45 insertions(+), 16 deletions(-) diff --git a/headphones/importer.py b/headphones/importer.py index 47363b4a..5c34cddf 100644 --- a/headphones/importer.py +++ b/headphones/importer.py @@ -200,7 +200,7 @@ def addArtisttoDB(artistid, extrasonly=False, forcefull=False): if len(artist['releasegroups']) != 0 and not extrasonly: for groups in artist['releasegroups']: group_list.append(groups['id']) - remove_missing_groups_from_albums = myDB.action("SELECT AlbumID FROM albums WHERE ArtistID=?", [artistid]) + remove_missing_groups_from_albums = myDB.select("SELECT AlbumID FROM albums WHERE ArtistID=?", [artistid]) for items in remove_missing_groups_from_albums: if items['AlbumID'] not in group_list: # Remove all from albums/tracks that aren't in release groups diff --git a/headphones/mb.py b/headphones/mb.py index 9ce2442a..b5fe1102 100644 --- a/headphones/mb.py +++ b/headphones/mb.py @@ -174,6 +174,14 @@ def findRelease(name, limit=1, artist=None): formats += str(count) + 'x' formats += format + rg_type = '' + if 'type' in result['release-group']: + rg_type = result['release-group']['type'] + if 'secondary-type-list' in result['release-group']: + secondary_type = result['release-group']['secondary-type-list'][0] + if secondary_type != rg_type: + rg_type += ' + ' + secondary_type + releaselist.append({ 'uniquename': unicode(result['artist-credit'][0]['artist']['name']), 'title': unicode(title), @@ -187,7 +195,7 @@ def findRelease(name, limit=1, artist=None): 'formats': unicode(formats), 'tracks': unicode(tracks), 'rgid': unicode(result['release-group']['id']), - 'rgtype': unicode(result['release-group']['type']) if 'type' in result['release-group'] else '' + 'rgtype': unicode(rg_type) }) return releaselist @@ -263,9 +271,9 @@ def getArtist(artistid, extrasonly=False): if includeExtras: - # Need to convert extras string from something like '2,5.6' to ['ep','live','remix'] + # Need to convert extras string from something like '2,5.6' to ['ep','live','remix'] (append new extras to end) extras = db_artist['Extras'] - extras_list = ["single", "ep", "compilation", "soundtrack", "live", "remix", "dj-mix", "mixtape/street", "spokenword", "audiobook", "broadcast", "interview", "other"] + extras_list = ["single", "ep", "compilation", "soundtrack", "live", "remix", "spokenword", "audiobook", "other", "dj-mix", "mixtape/street", "broadcast", "interview"] includes = [] i = 1 @@ -289,11 +297,18 @@ def getArtist(artistid, extrasonly=False): time.sleep(5) for rg in mb_extras_list: + + rg_type = rg['type'] + if 'secondary-type-list' in rg: + secondary_type = rg['secondary-type-list'][0] + if secondary_type != rg_type: + rg_type += ' + ' + secondary_type + releasegroups.append({ 'title': unicode(rg['title']), 'id': unicode(rg['id']), 'url': u"http://musicbrainz.org/release-group/" + rg['id'], - 'type': unicode(rg['type']) + 'type': unicode(rg_type) }) artist_dict['releasegroups'] = releasegroups @@ -364,8 +379,15 @@ def getRelease(releaseid, include_artist_info=True): release['rg_title'] = unicode(results['release-group']['title']) try: release['rg_type'] = unicode(results['release-group']['type']) + + if 'secondary-type-list' in results['release-group']: + secondary_type = unicode(results['release-group']['secondary-type-list'][0]) + if secondary_type != release['rg_type']: + release['rg_type'] += ' + ' + secondary_type + except KeyError: release['rg_type'] = u'Unknown' + else: logger.warn("Release " + releaseid + "had no ReleaseGroup associated") @@ -445,6 +467,10 @@ def get_new_releases(rgid,includeExtras=False,forcefull=False): raise Exception('No release group associated with release id ' + releasedata['id'] + ' album id' + rgid) release['Type'] = unicode(releasedata['release-group']['type']) + if 'secondary-type-list' in releasedata['release-group']: + secondary_type = unicode(releasedata['release-group']['secondary-type-list'][0]) + if secondary_type != release['Type']: + release['Type'] += ' + ' + secondary_type #making the assumption that the most important artist will be first in the list if 'artist-credit' in releasedata: diff --git a/headphones/webserve.py b/headphones/webserve.py index 92debc30..4d001e61 100644 --- a/headphones/webserve.py +++ b/headphones/webserve.py @@ -34,6 +34,7 @@ from headphones import logger, searcher, db, importer, mb, lastfm, librarysync, from headphones.helpers import checked, radio,today, cleanName import lib.simplejson as simplejson +from lib.simplejson import OrderedDict import sys @@ -84,9 +85,10 @@ class WebInterface(object): if not artist: raise cherrypy.HTTPRedirect("home") - # Serve the extras up as a dict to make things easier for new templates - extras_list = ["single", "ep", "compilation", "soundtrack", "live", "remix", "djmix", "mixtape_street", "spokenword", "audiobook", "broadcast", "interview", "other"] - extras_dict = {} + # Serve the extras up as a dict to make things easier for new templates (append new extras to the end) + extras_list = ["single", "ep", "compilation", "soundtrack", "live", "remix", "spokenword", "audiobook", "other", "djmix", "mixtape_street", "broadcast", "interview"] + + extras_dict = OrderedDict() if not artist['Extras']: artist_extras = "" @@ -161,7 +163,7 @@ class WebInterface(object): temp_extras_list = [] # TODO: Put these extras as a global variable i = 1 - for extra in ["single", "ep", "compilation", "soundtrack", "live", "remix", "djmix", "mixtape_street", "spokenword", "audiobook", "broadcast", "interview", "other"]: + for extra in ["single", "ep", "compilation", "soundtrack", "live", "remix", "spokenword", "audiobook", "other", "djmix", "mixtape_street", "broadcast", "interview"]: if extra in kwargs: temp_extras_list.append(i) i += 1 @@ -1150,9 +1152,10 @@ class WebInterface(object): "mpc_enabled": checked(headphones.MPC_ENABLED) } - # Need to convert EXTRAS to a dictionary we can pass to the config: it'll come in as a string like 2,5,6,8 - extras_list = ["single", "ep", "compilation", "soundtrack", "live", "remix", "dj-mix", "mixtape/street", "spokenword", "audiobook", "broadcast", "interview", "other"] - extras_dict = {} + # Need to convert EXTRAS to a dictionary we can pass to the config: it'll come in as a string like 2,5,6,8 (append new extras to the end) + extras_list = ["single", "ep", "compilation", "soundtrack", "live", "remix", "spokenword", "audiobook", "other", "djmix", "mixtape_street", "broadcast", "interview"] + + extras_dict = OrderedDict() i = 1 for extra in extras_list: @@ -1175,8 +1178,8 @@ class WebInterface(object): preferred_words=None, required_words=None, ignored_words=None, preferred_quality=0, preferred_bitrate=None, detect_bitrate=0, move_files=0, torrentblackhole_dir=None, download_torrent_dir=None, numberofseeders=None, use_piratebay=0, piratebay_proxy_url=None, piratebay_ratio=None, use_kat=0, kat_proxy_url=None, kat_ratio=None, use_mininova=0, mininova_ratio=None, waffles=0, waffles_uid=None, waffles_passkey=None, waffles_ratio=None, whatcd=0, whatcd_username=None, whatcd_password=None, whatcd_ratio=None, rutracker=0, rutracker_user=None, rutracker_password=None, rutracker_ratio=None, rename_files=0, correct_metadata=0, cleanup_files=0, keep_nfo=0, add_album_art=0, album_art_format=None, embed_album_art=0, embed_lyrics=0, replace_existing_folders=False, - destination_dir=None, lossless_destination_dir=None, folder_format=None, file_format=None, file_underscores=0, include_extras=0, single=0, ep=0, compilation=0, soundtrack=0, live=0, - remix=0, djmix=0, mixtape_street=0, broadcast=0, interview=0, spokenword=0, audiobook=0, other=0, autowant_upcoming=False, autowant_all=False, keep_torrent_files=False, prefer_torrents=0, open_magnet_links=0, interface=None, log_dir=None, cache_dir=None, music_encoder=0, encoder=None, xldprofile=None, + destination_dir=None, lossless_destination_dir=None, folder_format=None, file_format=None, file_underscores=0, include_extras=0, single=0, ep=0, compilation=0, soundtrack=0, live=0, remix=0, spokenword=0, audiobook=0, other=0, djmix=0, mixtape_street=0, broadcast=0, interview=0, + autowant_upcoming=False, autowant_all=False, keep_torrent_files=False, prefer_torrents=0, open_magnet_links=0, interface=None, log_dir=None, cache_dir=None, music_encoder=0, encoder=None, xldprofile=None, bitrate=None, samplingfrequency=None, encoderfolder=None, advancedencoder=None, encoderoutputformat=None, encodervbrcbr=None, encoderquality=None, encoderlossless=0, delete_lossless_files=0, growl_enabled=0, growl_onsnatch=0, growl_host=None, growl_password=None, prowl_enabled=0, prowl_onsnatch=0, prowl_keys=None, prowl_priority=0, xbmc_enabled=0, xbmc_host=None, xbmc_username=None, xbmc_password=None, xbmc_update=0, xbmc_notify=0, nma_enabled=False, nma_apikey=None, nma_priority=0, nma_onsnatch=0, pushalot_enabled=False, pushalot_apikey=None, pushalot_onsnatch=0, synoindex_enabled=False, lms_enabled=0, lms_host=None, @@ -1389,9 +1392,9 @@ class WebInterface(object): headphones.EXTRA_NEWZNABS.append((newznab_host, newznab_api, newznab_enabled)) - # Convert the extras to list then string. Coming in as 0 or 1 + # Convert the extras to list then string. Coming in as 0 or 1 (append new extras to the end) temp_extras_list = [] - extras_list = [single, ep, compilation, soundtrack, live, remix, djmix, mixtape_street, spokenword, audiobook, broadcast, interview, other] + extras_list = [single, ep, compilation, soundtrack, live, remix, spokenword, audiobook, other, djmix, mixtape_street, broadcast, interview] i = 1 for extra in extras_list: From b3746dc58ef4c7fc6dc39b33989ca5435aa1e6f7 Mon Sep 17 00:00:00 2001 From: David Date: Tue, 26 Aug 2014 20:20:21 +0200 Subject: [PATCH 15/16] Long list of calls -> for loop --- headphones/__init__.py | 38 ++++++++------------------------------ 1 file changed, 8 insertions(+), 30 deletions(-) diff --git a/headphones/__init__.py b/headphones/__init__.py index 8248799e..8692af65 100644 --- a/headphones/__init__.py +++ b/headphones/__init__.py @@ -376,36 +376,14 @@ def initialize(): return False # Make sure all the config sections exist - CheckSection('General') - CheckSection('SABnzbd') - CheckSection('NZBget') - CheckSection('Transmission') - CheckSection('uTorrent') - CheckSection('Headphones') - CheckSection('Newznab') - CheckSection('NZBsorg') - CheckSection('omgwtfnzbs') - CheckSection('Piratebay') - CheckSection('Kat') - CheckSection('Mininova') - CheckSection('Waffles') - CheckSection('Rutracker') - CheckSection('What.cd') - CheckSection('Growl') - CheckSection('Prowl') - CheckSection('Pushover') - CheckSection('PushBullet') - CheckSection('XBMC') - CheckSection('LMS') - CheckSection('Plex') - CheckSection('NMA') - CheckSection('Pushalot') - CheckSection('Synoindex') - CheckSection('Twitter') - CheckSection('OSX_Notify') - CheckSection('Boxcar') - CheckSection('Songkick') - CheckSection('Advanced') + for section in ('General', 'SABnzbd', 'NZBget', 'Transmission', + 'uTorrent', 'Headphones', 'Newznab', 'NZBsorg', + 'omgwtfnzbs', 'Piratebay', 'Kat', 'Mininova', 'Waffles', + 'Rutracker', 'What.cd', 'Growl', 'Prowl', 'Pushover', + 'PushBullet', 'XBMC', 'LMS', 'Plex', 'NMA', 'Pushalot', + 'Synoindex', 'Twitter', 'OSX_Notify', 'Boxcar', + 'Songkick', 'Advanced'): + CheckSection(section) # Set global variables based on config file or use defaults CONFIG_VERSION = check_setting_str(CFG, 'General', 'config_version', '0') From 22bff59789e65dd5641ff94c3d982e86d192fc9e Mon Sep 17 00:00:00 2001 From: Ade Date: Wed, 27 Aug 2014 18:59:31 +1200 Subject: [PATCH 16/16] More Extras MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Artist Refresh, if removing extras from Modify/Fetch Extras then also remove from db - mb, use secondary type if type = Album and different, e.g DJ-mix - utorrent - only check “Put new downloads in” directory --- headphones/importer.py | 20 +++++++++++--------- headphones/mb.py | 24 ++++++++++++++---------- headphones/utorrent.py | 6 +++--- headphones/webserve.py | 17 ++++++++++------- 4 files changed, 38 insertions(+), 29 deletions(-) diff --git a/headphones/importer.py b/headphones/importer.py index 5c34cddf..f082e4c0 100644 --- a/headphones/importer.py +++ b/headphones/importer.py @@ -193,14 +193,17 @@ def addArtisttoDB(artistid, extrasonly=False, forcefull=False): except IndexError: includeExtras = False - #Clean all references to release group in dB that are no longer referenced in musicbrainz + #Clean all references to release group in dB that are no longer referenced from the musicbrainz refresh group_list = [] force_repackage = 0 #Don't nuke the database if there's a MusicBrainz error - if len(artist['releasegroups']) != 0 and not extrasonly: + if len(artist['releasegroups']) != 0: for groups in artist['releasegroups']: group_list.append(groups['id']) - remove_missing_groups_from_albums = myDB.select("SELECT AlbumID FROM albums WHERE ArtistID=?", [artistid]) + if not extrasonly: + remove_missing_groups_from_albums = myDB.select("SELECT AlbumID FROM albums WHERE ArtistID=?", [artistid]) + else: + remove_missing_groups_from_albums = myDB.select('SELECT AlbumID FROM albums WHERE ArtistID=? AND Status="Skipped" AND Type!="Album"', [artistid]) for items in remove_missing_groups_from_albums: if items['AlbumID'] not in group_list: # Remove all from albums/tracks that aren't in release groups @@ -209,13 +212,12 @@ def addArtisttoDB(artistid, extrasonly=False, forcefull=False): myDB.action("DELETE FROM tracks WHERE AlbumID=?", [items['AlbumID']]) myDB.action("DELETE FROM alltracks WHERE AlbumID=?", [items['AlbumID']]) myDB.action('DELETE from releases WHERE ReleaseGroupID=?', [items['AlbumID']]) - logger.info("[%s] Removing all references to release group %s to reflect MusicBrainz" % (artist['artist_name'], items['AlbumID'])) - force_repackage = 1 - elif extrasonly: - # Not really sure what we're doing here but don't want to log the message below if we're fetching extras only - pass + logger.info("[%s] Removing all references to release group %s to reflect MusicBrainz refresh" % (artist['artist_name'], items['AlbumID'])) + if not extrasonly: + force_repackage = 1 else: - logger.info("[%s] There was either an error pulling data from MusicBrainz or there might not be any releases for this category" % artist['artist_name']) + if not extrasonly: + logger.info("[%s] There was either an error pulling data from MusicBrainz or there might not be any releases for this category" % artist['artist_name']) # Then search for releases within releasegroups, if releases don't exist, then remove from allalbums/alltracks album_searches = [] diff --git a/headphones/mb.py b/headphones/mb.py index b5fe1102..c8a4dd06 100644 --- a/headphones/mb.py +++ b/headphones/mb.py @@ -177,10 +177,10 @@ def findRelease(name, limit=1, artist=None): rg_type = '' if 'type' in result['release-group']: rg_type = result['release-group']['type'] - if 'secondary-type-list' in result['release-group']: + if rg_type == 'Album' and 'secondary-type-list' in result['release-group']: secondary_type = result['release-group']['secondary-type-list'][0] if secondary_type != rg_type: - rg_type += ' + ' + secondary_type + rg_type = secondary_type releaselist.append({ 'uniquename': unicode(result['artist-credit'][0]['artist']['name']), @@ -272,13 +272,17 @@ def getArtist(artistid, extrasonly=False): if includeExtras: # Need to convert extras string from something like '2,5.6' to ['ep','live','remix'] (append new extras to end) - extras = db_artist['Extras'] + if db_artist['Extras']: + extras = map(int, db_artist['Extras'].split(',')) + else: + extras = [] extras_list = ["single", "ep", "compilation", "soundtrack", "live", "remix", "spokenword", "audiobook", "other", "dj-mix", "mixtape/street", "broadcast", "interview"] + includes = [] i = 1 for extra in extras_list: - if str(i) in extras: + if i in extras: includes.append(extra) i += 1 @@ -299,10 +303,10 @@ def getArtist(artistid, extrasonly=False): for rg in mb_extras_list: rg_type = rg['type'] - if 'secondary-type-list' in rg: + if rg_type == 'Album' and 'secondary-type-list' in rg: secondary_type = rg['secondary-type-list'][0] if secondary_type != rg_type: - rg_type += ' + ' + secondary_type + rg_type = secondary_type releasegroups.append({ 'title': unicode(rg['title']), @@ -380,10 +384,10 @@ def getRelease(releaseid, include_artist_info=True): try: release['rg_type'] = unicode(results['release-group']['type']) - if 'secondary-type-list' in results['release-group']: + if release['rg_type'] == 'Album' and 'secondary-type-list' in results['release-group']: secondary_type = unicode(results['release-group']['secondary-type-list'][0]) if secondary_type != release['rg_type']: - release['rg_type'] += ' + ' + secondary_type + release['rg_type'] = secondary_type except KeyError: release['rg_type'] = u'Unknown' @@ -467,10 +471,10 @@ def get_new_releases(rgid,includeExtras=False,forcefull=False): raise Exception('No release group associated with release id ' + releasedata['id'] + ' album id' + rgid) release['Type'] = unicode(releasedata['release-group']['type']) - if 'secondary-type-list' in releasedata['release-group']: + if release['Type'] == 'Album' and 'secondary-type-list' in releasedata['release-group']: secondary_type = unicode(releasedata['release-group']['secondary-type-list'][0]) if secondary_type != release['Type']: - release['Type'] += ' + ' + secondary_type + release['Type'] = secondary_type #making the assumption that the most important artist will be first in the list if 'artist-credit' in releasedata: diff --git a/headphones/utorrent.py b/headphones/utorrent.py index 0856df33..c34558f8 100644 --- a/headphones/utorrent.py +++ b/headphones/utorrent.py @@ -180,7 +180,7 @@ def setSeedRatio(hash, ratio): uTorrentClient.setprops(hash,'seed_ratio', ratio * 10) else: # TODO passing -1 should be unlimited - uTorrentClient.setprops(hash,'seed_ratio', -1.00) + uTorrentClient.setprops(hash,'seed_ratio', -10) def dirTorrent(hash, cacheid=None, return_name=None): @@ -214,8 +214,8 @@ def addTorrent(link, hash): # Get Active Directory from settings active_dir, completed_dir = getSettingsDirectories() - if not active_dir or not completed_dir: - logger.error('Could not get "Put new downloads in:" or "Move completed downloads to:" directories from uTorrent settings, please ensure they are set') + if not active_dir: + logger.error('Could not get "Put new downloads in:" directory from uTorrent settings, please ensure it is set') return None uTorrentClient.add_url(link) diff --git a/headphones/webserve.py b/headphones/webserve.py index 4d001e61..41a4cc07 100644 --- a/headphones/webserve.py +++ b/headphones/webserve.py @@ -87,17 +87,16 @@ class WebInterface(object): # Serve the extras up as a dict to make things easier for new templates (append new extras to the end) extras_list = ["single", "ep", "compilation", "soundtrack", "live", "remix", "spokenword", "audiobook", "other", "djmix", "mixtape_street", "broadcast", "interview"] + if artist['Extras']: + artist_extras = map(int, artist['Extras'].split(',')) + else: + artist_extras = [] extras_dict = OrderedDict() - if not artist['Extras']: - artist_extras = "" - else: - artist_extras = artist['Extras'] - i = 1 for extra in extras_list: - if str(i) in artist_extras: + if i in artist_extras: extras_dict[extra] = "checked" else: extras_dict[extra] = "" @@ -1154,12 +1153,16 @@ class WebInterface(object): # Need to convert EXTRAS to a dictionary we can pass to the config: it'll come in as a string like 2,5,6,8 (append new extras to the end) extras_list = ["single", "ep", "compilation", "soundtrack", "live", "remix", "spokenword", "audiobook", "other", "djmix", "mixtape_street", "broadcast", "interview"] + if headphones.EXTRAS: + extras = map(int, headphones.EXTRAS.split(',')) + else: + extras = [] extras_dict = OrderedDict() i = 1 for extra in extras_list: - if str(i) in headphones.EXTRAS: + if i in extras: extras_dict[extra] = "checked" else: extras_dict[extra] = ""