From 81082be1be35f83bc0ba9f75829dabaed8c0c381 Mon Sep 17 00:00:00 2001 From: Patrick Speiser Date: Mon, 3 Sep 2012 18:09:36 +0200 Subject: [PATCH 01/44] Cheap fix for Issue #796, not actually tested on mac but filters the directories on Linux and Windows --- headphones/librarysync.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/headphones/librarysync.py b/headphones/librarysync.py index 7001c14c..eeff6e8c 100644 --- a/headphones/librarysync.py +++ b/headphones/librarysync.py @@ -56,6 +56,11 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None): song_list = [] for r,d,f in os.walk(dir): + try: + d.remove('.AppleDouble') + d.remove('.DS_Store') + except ValueError: + pass for files in f: # MEDIA_FORMATS = music file extensions, e.g. mp3, flac, etc if any(files.lower().endswith('.' + x.lower()) for x in headphones.MEDIA_FORMATS): From 1efdd64978784eeffe24213a7e0aa621aea8b02e Mon Sep 17 00:00:00 2001 From: Patrick Speiser Date: Mon, 3 Sep 2012 18:21:53 +0200 Subject: [PATCH 02/44] Better fix for Issue #796, filters all hidden Unix directories --- headphones/librarysync.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/headphones/librarysync.py b/headphones/librarysync.py index eeff6e8c..ea95efa6 100644 --- a/headphones/librarysync.py +++ b/headphones/librarysync.py @@ -56,11 +56,11 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None): song_list = [] for r,d,f in os.walk(dir): - try: - d.remove('.AppleDouble') - d.remove('.DS_Store') - except ValueError: - pass + #need to abuse slicing to get a copy of the list, doing it directly will skip the element after a deleted one + #using a list comprehension will not work correctly for nested subdirectories (os.walk keeps its original list) + for directory in d[:]: + if directory.startswith("."): + d.remove(directory) for files in f: # MEDIA_FORMATS = music file extensions, e.g. mp3, flac, etc if any(files.lower().endswith('.' + x.lower()) for x in headphones.MEDIA_FORMATS): From fa171deca1c7e075b16588a2cc9297ee97343ebf Mon Sep 17 00:00:00 2001 From: Patrick Speiser Date: Mon, 3 Sep 2012 23:23:57 +0200 Subject: [PATCH 03/44] Fixes Issue 811 by detecting and failing early if getHybridRelease gets an empty fullreleaselist passed into it. --- headphones/importer.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/headphones/importer.py b/headphones/importer.py index d8ceb281..c3c513ac 100644 --- a/headphones/importer.py +++ b/headphones/importer.py @@ -251,7 +251,13 @@ def addArtisttoDB(artistid, extrasonly=False): myDB.upsert("alltracks", newValueDict, controlValueDict) # Basically just do the same thing again for the hybrid release - hybridrelease = getHybridRelease(fullreleaselist) + # This may end up being called with an empty fullreleaselist + try: + hybridrelease = getHybridRelease(fullreleaselist) + except Exception, e: + errors = True + logger.warn('Unable to get hybrid release information for %s: %s' % (rg['title'],e)) + continue # Use the ReleaseGroupID as the ReleaseID for the hybrid release to differentiate it # We can then use the condition WHERE ReleaseID == ReleaseGroupID to select it @@ -584,6 +590,8 @@ def getHybridRelease(fullreleaselist): """ Returns a dictionary of best group of tracks from the list of releases & earliest release date """ + if len(fullreleaselist) == 0: + raise Exception("getHybridRelease was called with an empty fullreleaselist") sortable_release_list = [] for release in fullreleaselist: From 84b121ecfe6ef92024aa39d8a7682ebccbc0ce91 Mon Sep 17 00:00:00 2001 From: Patrick Speiser Date: Mon, 3 Sep 2012 23:23:57 +0200 Subject: [PATCH 04/44] Fixes Issue 811 by detecting and failing early if getHybridRelease gets an empty fullreleaselist passed into it. --- headphones/importer.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/headphones/importer.py b/headphones/importer.py index d8ceb281..c3c513ac 100644 --- a/headphones/importer.py +++ b/headphones/importer.py @@ -251,7 +251,13 @@ def addArtisttoDB(artistid, extrasonly=False): myDB.upsert("alltracks", newValueDict, controlValueDict) # Basically just do the same thing again for the hybrid release - hybridrelease = getHybridRelease(fullreleaselist) + # This may end up being called with an empty fullreleaselist + try: + hybridrelease = getHybridRelease(fullreleaselist) + except Exception, e: + errors = True + logger.warn('Unable to get hybrid release information for %s: %s' % (rg['title'],e)) + continue # Use the ReleaseGroupID as the ReleaseID for the hybrid release to differentiate it # We can then use the condition WHERE ReleaseID == ReleaseGroupID to select it @@ -584,6 +590,8 @@ def getHybridRelease(fullreleaselist): """ Returns a dictionary of best group of tracks from the list of releases & earliest release date """ + if len(fullreleaselist) == 0: + raise Exception("getHybridRelease was called with an empty fullreleaselist") sortable_release_list = [] for release in fullreleaselist: From 4aa233e9e4dd838fe95bdd8b2e44dc0a5cfe61ac Mon Sep 17 00:00:00 2001 From: Patrick Speiser Date: Mon, 3 Sep 2012 23:32:29 +0200 Subject: [PATCH 05/44] Better fix for Issue #796, filters all hidden Unix directories --- headphones/librarysync.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/headphones/librarysync.py b/headphones/librarysync.py index 7001c14c..ea95efa6 100644 --- a/headphones/librarysync.py +++ b/headphones/librarysync.py @@ -56,6 +56,11 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None): song_list = [] for r,d,f in os.walk(dir): + #need to abuse slicing to get a copy of the list, doing it directly will skip the element after a deleted one + #using a list comprehension will not work correctly for nested subdirectories (os.walk keeps its original list) + for directory in d[:]: + if directory.startswith("."): + d.remove(directory) for files in f: # MEDIA_FORMATS = music file extensions, e.g. mp3, flac, etc if any(files.lower().endswith('.' + x.lower()) for x in headphones.MEDIA_FORMATS): From 196ea11cedc3719b1835c27638e92e44c3a57770 Mon Sep 17 00:00:00 2001 From: Patrick Speiser Date: Sat, 8 Sep 2012 09:13:18 +0200 Subject: [PATCH 06/44] Possible Fix for Issue #533 --- headphones/webstart.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/headphones/webstart.py b/headphones/webstart.py index b16d6765..c8f53e9a 100644 --- a/headphones/webstart.py +++ b/headphones/webstart.py @@ -31,6 +31,9 @@ def initialize(options={}): 'server.socket_port': options['http_port'], 'server.socket_host': options['http_host'], 'engine.autoreload_on': False, + 'tools.encode.on' : True, + 'tools.encode.encoding' : 'utf-8', + 'tools.decode.on' : True, }) conf = { From be330afc54468511b46ce1120ad00bb61fcb6c1f Mon Sep 17 00:00:00 2001 From: Patrick Speiser Date: Sat, 8 Sep 2012 13:51:39 +0200 Subject: [PATCH 07/44] Added new valid include to browse_recordings --- lib/musicbrainzngs/musicbrainz.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/musicbrainzngs/musicbrainz.py b/lib/musicbrainzngs/musicbrainz.py index b0e94fed..e16c0816 100644 --- a/lib/musicbrainzngs/musicbrainz.py +++ b/lib/musicbrainzngs/musicbrainz.py @@ -552,6 +552,7 @@ def _do_mb_query(entity, id, includes=[], params={}): # Build the endpoint components. path = '%s/%s' % (entity, id) + logging.debug(str(path)) return _mb_request(path, 'GET', auth_required, args=args) def _do_mb_search(entity, query='', fields={}, @@ -776,7 +777,7 @@ def browse_recordings(artist=None, release=None, includes=[], limit=None, offset def browse_releases(artist=None, label=None, recording=None, release_group=None, release_status=[], release_type=[], includes=[], limit=None, offset=None): # track_artist param doesn't work yet - valid_includes = ["artist-credits", "labels", "recordings", "release-groups"] + valid_includes = ["artist-credits", "labels", "recordings", "release-groups","media"] params = {"artist": artist, "label": label, "recording": recording, From 0445859fb44280901277ddacbc42417c6a066ee4 Mon Sep 17 00:00:00 2001 From: Patrick Speiser Date: Sat, 8 Sep 2012 13:52:34 +0200 Subject: [PATCH 08/44] Initial Prototype for get_all_releases --- headphones/mb.py | 57 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/headphones/mb.py b/headphones/mb.py index 36eeeae8..b7307b49 100644 --- a/headphones/mb.py +++ b/headphones/mb.py @@ -367,6 +367,63 @@ def getRelease(releaseid, include_artist_info=True): release['tracks'] = tracks return release +def get_all_releases(releasegroup,include_artist_info=True): + results = None + q, sleepytime = startmb() + try: + if include_artist_info: + results = musicbrainzngs.browse_releases(release_group=releasegroup,includes=['artist-credits','labels','recordings','release-groups','media']) + else: + results = musicbrainzngs.browse_releases(release_group=releasegroup,includes=['labels','recordings','release-groups','media']) + except WebServiceError, e: + logger.warn('Attempt to retrieve information from MusicBrainz for releasegroup "%s" failed (%s)' % (releasegroup, str(e))) + time.sleep(5) + + if not results or 'release-list' not in results: + return False + + results = results['release-list'] + releases = [] + for releasedata in results: + release = {} + release['AlbumASIN'] = unicode(releasedata['asin']) + release['AlbumID'] = unicode(releasedata['release-group']['id']) + release['Type'] = unicode(releasedata['release-group']['type']) + release['AlbumTitle'] = unicode(releasedata['title']) + #making the assumption that the most important artist will be first in the list + if include_artist_info: + release['ArtistID'] = unicode(releasedata['artist-credit'][0]['artist']['id']) + release['ArtistName'] = unicode(releasedata['artist-credit-phrase']) + release['ReleaseCountry'] = unicode(releasedata['country']) + release['ReleaseDate'] = unicode(releasedata['date']) + #assuming that the list will contain media at all and that the format will be consistent + try: + release['ReleaseFormat'] = unicode(releasedata['medium-list'][0]['format']) + except: + release['ReleaseFormat'] = u'Unknown' + release['ReleaseID'] = releasedata['id'] + + #pasted in from getRelease + totalTracks = 1 + tracks = [] + for medium in releasedata['medium-list']: + for track in medium['track-list']: + tracks.append({ + 'number': totalTracks, + 'title': unicode(track['recording']['title']), + 'id': unicode(track['recording']['id']), + 'url': u"http://musicbrainz.org/track/" + track['recording']['id'], + 'duration': int(track['length']) if 'length' in track else 0 + }) + totalTracks += 1 + release['Tracks'] = tracks + releases.append(release) + + + + + + return releases # Used when there is a disambiguation def findArtistbyAlbum(name): From b71bb6938df5e836067970a4999486610dc6e6a9 Mon Sep 17 00:00:00 2001 From: Patrick Speiser Date: Sat, 8 Sep 2012 16:13:33 +0200 Subject: [PATCH 09/44] get_all_releases now produces identical output to the final contents of the fullreleaselist in addArtisttoDB --- headphones/mb.py | 39 ++++++++++++++++++++++++--------------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/headphones/mb.py b/headphones/mb.py index b7307b49..febd423e 100644 --- a/headphones/mb.py +++ b/headphones/mb.py @@ -367,41 +367,50 @@ def getRelease(releaseid, include_artist_info=True): release['tracks'] = tracks return release -def get_all_releases(releasegroup,include_artist_info=True): +def get_all_releases(rgid): results = None q, sleepytime = startmb() try: - if include_artist_info: - results = musicbrainzngs.browse_releases(release_group=releasegroup,includes=['artist-credits','labels','recordings','release-groups','media']) - else: - results = musicbrainzngs.browse_releases(release_group=releasegroup,includes=['labels','recordings','release-groups','media']) + results = musicbrainzngs.browse_releases(release_group=rgid,includes=['artist-credits','labels','recordings','release-groups','media']) except WebServiceError, e: - logger.warn('Attempt to retrieve information from MusicBrainz for releasegroup "%s" failed (%s)' % (releasegroup, str(e))) - time.sleep(5) + logger.warn('Attempt to retrieve information from MusicBrainz for release group "%s" failed (%s)' % (rgid, str(e))) + time.sleep(5) + return False if not results or 'release-list' not in results: return False results = results['release-list'] + releases = [] for releasedata in results: release = {} - release['AlbumASIN'] = unicode(releasedata['asin']) - release['AlbumID'] = unicode(releasedata['release-group']['id']) - release['Type'] = unicode(releasedata['release-group']['type']) release['AlbumTitle'] = unicode(releasedata['title']) + release['AlbumID'] = unicode(rgid) + release['AlbumASIN'] = unicode(releasedata['asin']) + release['ReleaseDate'] = unicode(releasedata['date']) + release['ReleaseID'] = releasedata['id'] + if 'release-group' not in releasedata: + raise Exception('No release group associated with release id ' + releasedata['id'] + ' album id' + rgid) + release['Type'] = unicode(releasedata['release-group']['type']) + + #making the assumption that the most important artist will be first in the list - if include_artist_info: + if 'artist-credit' in releasedata: release['ArtistID'] = unicode(releasedata['artist-credit'][0]['artist']['id']) release['ArtistName'] = unicode(releasedata['artist-credit-phrase']) - release['ReleaseCountry'] = unicode(releasedata['country']) - release['ReleaseDate'] = unicode(releasedata['date']) - #assuming that the list will contain media at all and that the format will be consistent + else: + raise Exception('Release ' + releasedata['id'] + ' has no Artists associated.') + + + release['ReleaseCountry'] = unicode(releasedata['country']) if 'country' in releasedata else u'Unknown' + #assuming that the list will contain media and that the format will be consistent try: release['ReleaseFormat'] = unicode(releasedata['medium-list'][0]['format']) except: release['ReleaseFormat'] = u'Unknown' - release['ReleaseID'] = releasedata['id'] + + #pasted in from getRelease totalTracks = 1 From fe8220a000d679869e4f1d80a6dfb996d4f97512 Mon Sep 17 00:00:00 2001 From: Patrick Speiser Date: Sat, 8 Sep 2012 18:54:11 +0200 Subject: [PATCH 10/44] Improved addArtisttoDB, it now uses get_all_releases(releasegroupid) to get the data. This has reduced the number of calls to the musicbrainz database by 90% in a test where i added U2, i expect similar effects for all artists. --- headphones/importer.py | 70 ++++++++++++++++-------------------------- headphones/mb.py | 7 +++-- 2 files changed, 31 insertions(+), 46 deletions(-) diff --git a/headphones/importer.py b/headphones/importer.py index d8ceb281..b25eb6f5 100644 --- a/headphones/importer.py +++ b/headphones/importer.py @@ -167,69 +167,51 @@ def addArtisttoDB(artistid, extrasonly=False): # check if the album already exists rg_exists = myDB.action("SELECT * from albums WHERE AlbumID=?", [rg['id']]).fetchone() - try: - releaselist = mb.getReleaseGroup(rgid) - except Exception, e: - logger.info('Unable to get release information for %s - there may not be any official releases in this release group' % rg['title']) - continue - - if not releaselist: + releases = mb.get_all_releases(rgid) + if not releases: errors = True + logger.info('Unable to get release information for %s - there may not be any official releases in this release group' % rg['title']) continue # This will be used later to build a hybrid release fullreleaselist = [] - - for release in releaselist: + + for release in releases: # What we're doing here now is first updating the allalbums & alltracks table to the most # current info, then moving the appropriate release into the album table and its associated # tracks into the tracks table - - releaseid = release['id'] - - try: - releasedict = mb.getRelease(releaseid, include_artist_info=False) - except Exception, e: - errors = True - logger.info('Unable to get release information for %s: %s' % (release['id'], e)) - continue - - if not releasedict: - errors = True - continue + controlValueDict = {"ReleaseID" : release['ReleaseID']} - controlValueDict = {"ReleaseID": release['id']} - - newValueDict = {"ArtistID": artistid, - "ArtistName": artist['artist_name'], - "AlbumTitle": rg['title'], - "AlbumID": rg['id'], - "AlbumASIN": releasedict['asin'], - "ReleaseDate": releasedict['date'], - "Type": rg['type'], - "ReleaseCountry": releasedict['country'], - "ReleaseFormat": releasedict['format'] + newValueDict = {"ArtistID": release['ArtistID'], + "ArtistName": release['ArtistName'], + "AlbumTitle": release['AlbumTitle'], + "AlbumID": release['AlbumID'], + "AlbumASIN": release['AlbumASIN'], + "ReleaseDate": release['ReleaseDate'], + "Type": release['Type'], + "ReleaseCountry": release['ReleaseCountry'], + "ReleaseFormat": release['ReleaseFormat'] } - + myDB.upsert("allalbums", newValueDict, controlValueDict) # Build the dictionary for the fullreleaselist - newValueDict['ReleaseID'] = release['id'] - newValueDict['Tracks'] = releasedict['tracks'] + newValueDict['ReleaseID'] = release['ReleaseID'] + newValueDict['Tracks'] = release['Tracks'] fullreleaselist.append(newValueDict) - for track in releasedict['tracks']: + for track in release['Tracks']: cleanname = helpers.cleanName(artist['artist_name'] + ' ' + rg['title'] + ' ' + track['title']) controlValueDict = {"TrackID": track['id'], - "ReleaseID": release['id']} + "ReleaseID": release['ReleaseID']} - newValueDict = {"ArtistID": artistid, - "ArtistName": artist['artist_name'], - "AlbumTitle": rg['title'], - "AlbumASIN": releasedict['asin'], - "AlbumID": rg['id'], + newValueDict = {"ArtistID": release['ArtistID'], + "ArtistName": release['ArtistName'], + "AlbumTitle": release['AlbumTitle'], + "AlbumID": release['AlbumID'], + "AlbumASIN": release['AlbumASIN'], "TrackTitle": track['title'], "TrackDuration": track['duration'], "TrackNumber": track['number'], @@ -584,6 +566,8 @@ def getHybridRelease(fullreleaselist): """ Returns a dictionary of best group of tracks from the list of releases & earliest release date """ + if len(fullreleaselist) == 0: + raise Exception("getHybridRelease was called with an empty fullreleaselist") sortable_release_list = [] for release in fullreleaselist: diff --git a/headphones/mb.py b/headphones/mb.py index febd423e..b9a629ac 100644 --- a/headphones/mb.py +++ b/headphones/mb.py @@ -387,8 +387,8 @@ def get_all_releases(rgid): release = {} release['AlbumTitle'] = unicode(releasedata['title']) release['AlbumID'] = unicode(rgid) - release['AlbumASIN'] = unicode(releasedata['asin']) - release['ReleaseDate'] = unicode(releasedata['date']) + release['AlbumASIN'] = unicode(releasedata['asin']) if 'asin' in releasedata else None + release['ReleaseDate'] = unicode(releasedata['date']) if 'date' in releasedata else None release['ReleaseID'] = releasedata['id'] if 'release-group' not in releasedata: raise Exception('No release group associated with release id ' + releasedata['id'] + ' album id' + rgid) @@ -400,7 +400,8 @@ def get_all_releases(rgid): release['ArtistID'] = unicode(releasedata['artist-credit'][0]['artist']['id']) release['ArtistName'] = unicode(releasedata['artist-credit-phrase']) else: - raise Exception('Release ' + releasedata['id'] + ' has no Artists associated.') + logger.warn('Release ' + releasedata['id'] + ' has no Artists associated.') + return False release['ReleaseCountry'] = unicode(releasedata['country']) if 'country' in releasedata else u'Unknown' From ea9f896310eacb24cfdd539c904fb263ef15df2c Mon Sep 17 00:00:00 2001 From: Patrick Speiser Date: Sun, 9 Sep 2012 07:02:59 +0200 Subject: [PATCH 11/44] Added paging to get_all_releases, this will ensure that any future (however unlikely) release groups containing over 100 releases will be read correctly --- headphones/mb.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/headphones/mb.py b/headphones/mb.py index b9a629ac..dd161633 100644 --- a/headphones/mb.py +++ b/headphones/mb.py @@ -368,19 +368,26 @@ def getRelease(releaseid, include_artist_info=True): return release def get_all_releases(rgid): - results = None + results = [] q, sleepytime = startmb() try: - results = musicbrainzngs.browse_releases(release_group=rgid,includes=['artist-credits','labels','recordings','release-groups','media']) + limit = 100 + newResults = None + while newResults == None or len(newResults) >= limit: + newResults = musicbrainzngs.browse_releases(release_group=rgid,includes=['artist-credits','labels','recordings','release-groups','media'],limit=limit,offset=len(results)) + if 'release-list' not in newResults: + break #may want to raise an exception here instead ? + newResults = newResults['release-list'] + results += newResults + except WebServiceError, e: logger.warn('Attempt to retrieve information from MusicBrainz for release group "%s" failed (%s)' % (rgid, str(e))) time.sleep(5) return False - if not results or 'release-list' not in results: + if not results or len(results) == 0: return False - results = results['release-list'] releases = [] for releasedata in results: From 11946b80da2d07d1c730e44c484b70eda1088894 Mon Sep 17 00:00:00 2001 From: Patrick Speiser Date: Sun, 9 Sep 2012 07:14:13 +0200 Subject: [PATCH 12/44] Revert "Possible Fix for Issue #533" This reverts commit 196ea11cedc3719b1835c27638e92e44c3a57770. --- headphones/webstart.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/headphones/webstart.py b/headphones/webstart.py index c8f53e9a..b16d6765 100644 --- a/headphones/webstart.py +++ b/headphones/webstart.py @@ -31,9 +31,6 @@ def initialize(options={}): 'server.socket_port': options['http_port'], 'server.socket_host': options['http_host'], 'engine.autoreload_on': False, - 'tools.encode.on' : True, - 'tools.encode.encoding' : 'utf-8', - 'tools.decode.on' : True, }) conf = { From db9df9b563f989d7e651dbb909e432cb79f877f0 Mon Sep 17 00:00:00 2001 From: Patrick Speiser Date: Tue, 11 Sep 2012 22:35:23 +0200 Subject: [PATCH 13/44] Allow deletion of empty artists (artists where you don't have any songs OR that haven't released any albums), should also work to fix Issue #569 and will in general be helpful --- data/interfaces/default/manage.html | 1 + headphones/webserve.py | 14 ++++++++++++++ 2 files changed, 15 insertions(+) diff --git a/data/interfaces/default/manage.html b/data/interfaces/default/manage.html index 19cc9ac4..88e26d8a 100644 --- a/data/interfaces/default/manage.html +++ b/data/interfaces/default/manage.html @@ -97,6 +97,7 @@ Force Update Active Artists Force Post-Process Albums in Download Folder Check for Headphones Updates + Delete empty Artists diff --git a/headphones/webserve.py b/headphones/webserve.py index 799d84f3..cbb82e4e 100644 --- a/headphones/webserve.py +++ b/headphones/webserve.py @@ -190,6 +190,20 @@ class WebInterface(object): raise cherrypy.HTTPRedirect("home") deleteArtist.exposed = True + + def deleteEmptyArtists(self): + logger.info(u"Deleting all empty artists") + myDB = db.DBConnection() + emptyArtistIDs = [row['ArtistID'] for row in myDB.select("SELECT ArtistID FROM artists WHERE HaveTracks == 0 OR LatestAlbum IS NULL")] + for ArtistID in emptyArtistIDs: + logger.info(u"Deleting all traces of artist: " + ArtistID) + myDB.action('DELETE from artists WHERE ArtistID=?', [ArtistID]) + myDB.action('DELETE from albums WHERE ArtistID=?', [ArtistID]) + myDB.action('DELETE from tracks WHERE ArtistID=?', [ArtistID]) + myDB.action('INSERT OR REPLACE into blacklist VALUES (?)', [ArtistID]) + deleteEmptyArtists.exposed = True + + def refreshArtist(self, ArtistID): threading.Thread(target=importer.addArtisttoDB, args=[ArtistID]).start() raise cherrypy.HTTPRedirect("artistPage?ArtistID=%s" % ArtistID) From 47240bd5cd9a181361f4769acc5dc9e0b1421985 Mon Sep 17 00:00:00 2001 From: Patrick Speiser Date: Wed, 12 Sep 2012 12:16:08 +0200 Subject: [PATCH 14/44] Removed useless return values from startmb --- headphones/mb.py | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/headphones/mb.py b/headphones/mb.py index dd161633..daca0659 100644 --- a/headphones/mb.py +++ b/headphones/mb.py @@ -71,13 +71,10 @@ def startmb(forcemb=False): logger.warn("No username or password set for VIP server") else: musicbrainzngs.hpauth(mbuser,mbpass) - - # Don't really need to return q anymore since ngs, but maybe we can return an 'initialized=True' instead? - q = musicbrainzngs logger.debug('Using the following server values:\nMBHost: %s ; MBPort: %i ; Sleep Interval: %i ' % (mbhost, mbport, sleepytime)) - return (q, sleepytime) + return True def findArtist(name, limit=1): @@ -89,7 +86,7 @@ def findArtist(name, limit=1): if any((c in chars) for c in name): name = '"'+name+'"' - q, sleepytime = startmb(forcemb=True) + startmb(forcemb=True) try: artistResults = musicbrainzngs.search_artists(query=name,limit=limit)['artist-list'] @@ -141,7 +138,7 @@ def findRelease(name, limit=1): if any((c in chars) for c in name): name = '"'+name+'"' - q, sleepytime = startmb(forcemb=True) + startmb(forcemb=True) try: releaseResultsngs = musicbrainzngs.search_releases(query=name,limit=limit)['release-list'] @@ -172,7 +169,7 @@ def getArtist(artistid, extrasonly=False): artist = None - q, sleepytime = startmb() + startmb() try: limit = 100 @@ -286,7 +283,7 @@ def getReleaseGroup(rgid): releaseGroup = None - q, sleepytime = startmb() + startmb() try: releaseGroup = musicbrainzngs.get_release_group_by_id(rgid,["artists","releases","media","discids",])['release-group'] @@ -308,7 +305,7 @@ def getRelease(releaseid, include_artist_info=True): release = {} results = None - q, sleepytime = startmb() + startmb() try: if include_artist_info: @@ -369,7 +366,7 @@ def getRelease(releaseid, include_artist_info=True): return release def get_all_releases(rgid): results = [] - q, sleepytime = startmb() + startmb() try: limit = 100 newResults = None @@ -460,7 +457,7 @@ def findArtistbyAlbum(name): results = None - q, sleepytime = startmb(forcemb=True) + startmb(forcemb=True) try: results = musicbrainzngs.search_release_groups(term).get('release-group-list') @@ -495,7 +492,7 @@ def findAlbumID(artist=None, album=None): results_ngs = None - q, sleepytime = startmb(forcemb=True) + startmb(forcemb=True) try: term = '"'+album+'" AND artist:"'+artist+'"' From 9a760f8f0c11898725919acc560efd7aba3851ec Mon Sep 17 00:00:00 2001 From: Patrick Speiser Date: Wed, 12 Sep 2012 12:17:36 +0200 Subject: [PATCH 15/44] Removed dead musicbrainz mirror --- headphones/mb.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/headphones/mb.py b/headphones/mb.py index daca0659..bf6bf752 100644 --- a/headphones/mb.py +++ b/headphones/mb.py @@ -54,9 +54,7 @@ def startmb(forcemb=False): mbpass = headphones.HPPASS sleepytime = 0 else: - mbhost = "tbueter.com" - mbport = 5000 - sleepytime = 0 + return False musicbrainzngs.set_useragent("headphones","0.0","https://github.com/rembo10/headphones") musicbrainzngs.set_hostname(mbhost + ":" + str(mbport)) From 296ef01f637d802342ba87f7fa743f0394bcc2d4 Mon Sep 17 00:00:00 2001 From: Patrick Speiser Date: Wed, 12 Sep 2012 14:20:09 +0200 Subject: [PATCH 16/44] startmb will now be called once at startup and every time a new configuration is submitted, removed option to forcemb because the value was ignored --- Headphones.py | 3 +++ headphones/mb.py | 40 +++++----------------------------------- headphones/webserve.py | 3 +++ 3 files changed, 11 insertions(+), 35 deletions(-) diff --git a/Headphones.py b/Headphones.py index 999c8d1f..18bb7970 100644 --- a/Headphones.py +++ b/Headphones.py @@ -109,6 +109,9 @@ def main(): if headphones.DAEMON: headphones.daemonize() + + #configure the connection to the musicbrainz database + headphones.mb.startmb() # Force the http port if neccessary if args.port: diff --git a/headphones/mb.py b/headphones/mb.py index bf6bf752..1542d782 100644 --- a/headphones/mb.py +++ b/headphones/mb.py @@ -30,16 +30,12 @@ mb_lock = threading.Lock() # Quick fix to add mirror switching on the fly. Need to probably return the mbhost & mbport that's # being used, so we can send those values to the log -def startmb(forcemb=False): +def startmb(): mbuser = None mbpass = None - # Can use headphones mirror for queries - if headphones.MIRROR == "headphones" or "custom": - forcemb=False - - if forcemb or headphones.MIRROR == "musicbrainz.org": + if headphones.MIRROR == "musicbrainz.org": mbhost = "musicbrainz.org" mbport = 80 sleepytime = 1 @@ -84,16 +80,12 @@ def findArtist(name, limit=1): if any((c in chars) for c in name): name = '"'+name+'"' - startmb(forcemb=True) - try: artistResults = musicbrainzngs.search_artists(query=name,limit=limit)['artist-list'] except WebServiceError, e: logger.warn('Attempt to query MusicBrainz for %s failed (%s)' % (name, str(e))) time.sleep(5) - time.sleep(sleepytime) - if not artistResults: return False for result in artistResults: @@ -136,16 +128,12 @@ def findRelease(name, limit=1): if any((c in chars) for c in name): name = '"'+name+'"' - startmb(forcemb=True) - try: releaseResultsngs = musicbrainzngs.search_releases(query=name,limit=limit)['release-list'] except WebServiceError, e: #need to update exceptions logger.warn('Attempt to query MusicBrainz for "%s" failed: %s' % (name, str(e))) time.sleep(5) - - time.sleep(sleepytime) - + if not releaseResultsngs: return False for result in releaseResultsngs: @@ -167,8 +155,6 @@ def getArtist(artistid, extrasonly=False): artist = None - startmb() - try: limit = 100 artist = musicbrainzngs.get_artist_by_id(artistid)['artist'] @@ -186,8 +172,6 @@ def getArtist(artistid, extrasonly=False): if not artist: return False - time.sleep(sleepytime) - #if 'disambiguation' in artist: # uniquename = unicode(artist['sort-name'] + " (" + artist['disambiguation'] + ")") #else: @@ -280,9 +264,7 @@ def getReleaseGroup(rgid): releaselist = [] releaseGroup = None - - startmb() - + try: releaseGroup = musicbrainzngs.get_release_group_by_id(rgid,["artists","releases","media","discids",])['release-group'] except WebServiceError, e: @@ -303,8 +285,6 @@ def getRelease(releaseid, include_artist_info=True): release = {} results = None - startmb() - try: if include_artist_info: results = musicbrainzngs.get_release_by_id(releaseid,["artists","release-groups","media","recordings"]).get('release') @@ -316,8 +296,6 @@ def getRelease(releaseid, include_artist_info=True): if not results: return False - - time.sleep(sleepytime) release['title'] = unicode(results['title']) release['id'] = unicode(results['id']) @@ -364,7 +342,6 @@ def getRelease(releaseid, include_artist_info=True): return release def get_all_releases(rgid): results = [] - startmb() try: limit = 100 newResults = None @@ -455,15 +432,12 @@ def findArtistbyAlbum(name): results = None - startmb(forcemb=True) - try: results = musicbrainzngs.search_release_groups(term).get('release-group-list') except WebServiceError, e: logger.warn('Attempt to query MusicBrainz for %s failed (%s)' % (name, str(e))) time.sleep(5) - time.sleep(sleepytime) if not results: return False @@ -490,17 +464,13 @@ def findAlbumID(artist=None, album=None): results_ngs = None - startmb(forcemb=True) - try: term = '"'+album+'" AND artist:"'+artist+'"' results_ngs = musicbrainzngs.search_release_groups(term,1).get('release-group-list') except WebServiceError, e: logger.warn('Attempt to query MusicBrainz for %s - %s failed (%s)' % (artist, album, str(e))) time.sleep(5) - - time.sleep(sleepytime) - + if not results_ngs: return False diff --git a/headphones/webserve.py b/headphones/webserve.py index cbb82e4e..1d7d9e52 100644 --- a/headphones/webserve.py +++ b/headphones/webserve.py @@ -694,6 +694,9 @@ class WebInterface(object): # Write the config headphones.config_write() + #reconfigure musicbrainz database connection with the new values + mb.startmb() + raise cherrypy.HTTPRedirect("config") configUpdate.exposed = True From e337bb4d624158a572cd351d7115615240e28377 Mon Sep 17 00:00:00 2001 From: Patrick Speiser Date: Thu, 13 Sep 2012 12:19:02 +0200 Subject: [PATCH 17/44] Fixed rate limiting, the provided sleep interval was pretty much ignored because any value > 0 was considered to be a limit of 1 request/second --- headphones/mb.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/headphones/mb.py b/headphones/mb.py index 1542d782..1096b813 100644 --- a/headphones/mb.py +++ b/headphones/mb.py @@ -57,7 +57,8 @@ def startmb(): if sleepytime == 0: musicbrainzngs.set_rate_limit(False) else: - musicbrainzngs.set_rate_limit(True) + #calling it with an it ends up blocking all requests after the first + musicbrainzngs.set_rate_limit(limit_or_interval=float(sleepytime)) # Add headphones credentials if headphones.MIRROR == "headphones": From 8f3a20841c6c0c2867a433b28aadbcae307db3ae Mon Sep 17 00:00:00 2001 From: Patrick Speiser Date: Thu, 13 Sep 2012 12:33:50 +0200 Subject: [PATCH 18/44] Removed unnecessary and ugly "ngs" linendings from variables. --- headphones/mb.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/headphones/mb.py b/headphones/mb.py index 1096b813..25111637 100644 --- a/headphones/mb.py +++ b/headphones/mb.py @@ -122,23 +122,23 @@ def findArtist(name, limit=1): def findRelease(name, limit=1): with mb_lock: - releaselistngs = [] - releaseResultsngs = None + releaselist = [] + releaseResults = None chars = set('!?') if any((c in chars) for c in name): name = '"'+name+'"' try: - releaseResultsngs = musicbrainzngs.search_releases(query=name,limit=limit)['release-list'] + releaseResults = musicbrainzngs.search_releases(query=name,limit=limit)['release-list'] except WebServiceError, e: #need to update exceptions logger.warn('Attempt to query MusicBrainz for "%s" failed: %s' % (name, str(e))) time.sleep(5) - if not releaseResultsngs: + if not releaseResults: return False - for result in releaseResultsngs: - releaselistngs.append({ + for result in releaseResults: + releaselist.append({ 'uniquename': unicode(result['artist-credit'][0]['artist']['name']), 'title': unicode(result['title']), 'id': unicode(result['artist-credit'][0]['artist']['id']), @@ -147,7 +147,7 @@ def findRelease(name, limit=1): 'albumurl': unicode("http://musicbrainz.org/release/" + result['id']),#probably needs to be changed 'score': int(result['ext:score']) }) - return releaselistngs + return releaselist def getArtist(artistid, extrasonly=False): @@ -463,19 +463,19 @@ def findArtistbyAlbum(name): def findAlbumID(artist=None, album=None): - results_ngs = None + results = None try: term = '"'+album+'" AND artist:"'+artist+'"' - results_ngs = musicbrainzngs.search_release_groups(term,1).get('release-group-list') + results = musicbrainzngs.search_release_groups(term,1).get('release-group-list') except WebServiceError, e: logger.warn('Attempt to query MusicBrainz for %s - %s failed (%s)' % (artist, album, str(e))) time.sleep(5) - if not results_ngs: + if not results: return False - if len(results_ngs) < 1: + if len(results) < 1: return False - rgid_ngs = unicode(results_ngs[0]['id']) - return rgid_ngs + rgid = unicode(results[0]['id']) + return rgid From b7064ab6a718823d99ed96d42d2a3d4c7c1b8e26 Mon Sep 17 00:00:00 2001 From: Patrick Speiser Date: Thu, 13 Sep 2012 12:47:48 +0200 Subject: [PATCH 19/44] Moved the release data track extraction to a new function. --- headphones/mb.py | 54 +++++++++++++++++------------------------------- 1 file changed, 19 insertions(+), 35 deletions(-) diff --git a/headphones/mb.py b/headphones/mb.py index 25111637..4dc15956 100644 --- a/headphones/mb.py +++ b/headphones/mb.py @@ -324,23 +324,11 @@ def getRelease(releaseid, include_artist_info=True): release['artist_name'] = unicode(results['artist-credit'][0]['artist']['name']) release['artist_id'] = unicode(results['artist-credit'][0]['artist']['id']) - - totalTracks = 1 - tracks = [] - for medium in results['medium-list']: - for track in medium['track-list']: - tracks.append({ - 'number': totalTracks, - 'title': unicode(track['recording']['title']), - 'id': unicode(track['recording']['id']), - 'url': u"http://musicbrainz.org/track/" + track['recording']['id'], - 'duration': int(track['length']) if 'length' in track else 0 - }) - totalTracks += 1 - release['tracks'] = tracks + release['tracks'] = getTracksFromRelease(results) return release + def get_all_releases(rgid): results = [] try: @@ -390,31 +378,27 @@ def get_all_releases(rgid): release['ReleaseFormat'] = unicode(releasedata['medium-list'][0]['format']) except: release['ReleaseFormat'] = u'Unknown' - - - - #pasted in from getRelease - totalTracks = 1 - tracks = [] - for medium in releasedata['medium-list']: - for track in medium['track-list']: - tracks.append({ - 'number': totalTracks, - 'title': unicode(track['recording']['title']), - 'id': unicode(track['recording']['id']), - 'url': u"http://musicbrainz.org/track/" + track['recording']['id'], - 'duration': int(track['length']) if 'length' in track else 0 - }) - totalTracks += 1 - release['Tracks'] = tracks + + release['Tracks'] = getTracksFromRelease(releasedata) releases.append(release) - - - - return releases +def getTracksFromRelease(release): + totalTracks = 1 + tracks = [] + for medium in release['medium-list']: + for track in medium['track-list']: + tracks.append({ + 'number': totalTracks, + 'title': unicode(track['recording']['title']), + 'id': unicode(track['recording']['id']), + 'url': u"http://musicbrainz.org/track/" + track['recording']['id'], + 'duration': int(track['length']) if 'length' in track else 0 + }) + totalTracks += 1 + return tracks + # Used when there is a disambiguation def findArtistbyAlbum(name): From 490b4bc90952436656d3adba24b403407a755d63 Mon Sep 17 00:00:00 2001 From: Patrick Speiser Date: Fri, 14 Sep 2012 11:40:02 +0200 Subject: [PATCH 20/44] Ignore bootlegs and similar that have their release group type set to "Album". Fixes Issue #536 --- headphones/importer.py | 15 ++++++++++++++- headphones/mb.py | 7 ++++++- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/headphones/importer.py b/headphones/importer.py index 98e3c46e..54d88399 100644 --- a/headphones/importer.py +++ b/headphones/importer.py @@ -158,6 +158,16 @@ def addArtisttoDB(artistid, extrasonly=False): myDB.upsert("artists", newValueDict, controlValueDict) + # See if we need to grab extras. Artist specific extras take precedence over global option + # Global options are set when adding a new artist + myDB = db.DBConnection() + + try: + db_artist = myDB.action('SELECT IncludeExtras, Extras from artists WHERE ArtistID=?', [artistid]).fetchone() + includeExtras = db_artist['IncludeExtras'] + except IndexError: + includeExtras = False + for rg in artist['releasegroups']: logger.info("Now adding/updating: " + rg['title']) @@ -167,7 +177,10 @@ def addArtisttoDB(artistid, extrasonly=False): # check if the album already exists rg_exists = myDB.action("SELECT * from albums WHERE AlbumID=?", [rg['id']]).fetchone() - releases = mb.get_all_releases(rgid) + releases = mb.get_all_releases(rgid,includeExtras) + if releases == []: + logger.info('No official releases in release group %s' % rg['title']) + continue if not releases: errors = True logger.info('Unable to get release information for %s - there may not be any official releases in this release group' % rg['title']) diff --git a/headphones/mb.py b/headphones/mb.py index 4dc15956..8dbcc5e8 100644 --- a/headphones/mb.py +++ b/headphones/mb.py @@ -329,7 +329,7 @@ def getRelease(releaseid, include_artist_info=True): return release -def get_all_releases(rgid): +def get_all_releases(rgid,includeExtras=False): results = [] try: limit = 100 @@ -352,6 +352,11 @@ def get_all_releases(rgid): releases = [] for releasedata in results: + #releasedata.get will return None if it doesn't have a status + #all official releases should have the Official status included + if not includeExtras and releasedata.get('status') != 'Official': + continue + release = {} release['AlbumTitle'] = unicode(releasedata['title']) release['AlbumID'] = unicode(rgid) From 6304d27266d11ae151152edbb56f7dce2bc7c8ee Mon Sep 17 00:00:00 2001 From: Patrick Speiser Date: Sat, 15 Sep 2012 23:58:09 +0200 Subject: [PATCH 21/44] Changed some sqlite settings to improve performance --- headphones/db.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/headphones/db.py b/headphones/db.py index 601b6286..a1f1d96d 100644 --- a/headphones/db.py +++ b/headphones/db.py @@ -40,6 +40,12 @@ class DBConnection: self.filename = filename self.connection = sqlite3.connect(dbFilename(filename), timeout=20) + #don't wait for the disk to finish writing + self.connection.execute("PRAGMA synchronous = OFF") + #journal disabled since we never do rollbacks + self.connection.execute("PRAGMA journal_mode = OFF") + #64mb of cache memory,probably need to make it user configurable + self.connection.execute("PRAGMA cache_size=-65536") self.connection.row_factory = sqlite3.Row def action(self, query, args=None): From e9c66b8de210013b8e9fb8781f346d4952ab9fcf Mon Sep 17 00:00:00 2001 From: Patrick Speiser Date: Sun, 16 Sep 2012 11:27:48 +0200 Subject: [PATCH 22/44] SQL Cache size is now user configurable (best performance when its at least as large as the database itself) --- data/interfaces/default/config.html | 4 ++++ headphones/__init__.py | 7 ++++++- headphones/db.py | 2 +- headphones/webserve.py | 6 ++++-- 4 files changed, 15 insertions(+), 4 deletions(-) diff --git a/data/interfaces/default/config.html b/data/interfaces/default/config.html index 643af20e..bbe45b0b 100644 --- a/data/interfaces/default/config.html +++ b/data/interfaces/default/config.html @@ -538,6 +538,10 @@ m<%inherit file="base.html"/> +
+ + +
Interface diff --git a/headphones/__init__.py b/headphones/__init__.py index 0e06ba4c..90218bcd 100644 --- a/headphones/__init__.py +++ b/headphones/__init__.py @@ -193,6 +193,8 @@ CUSTOMSLEEP = None HPUSER = None HPPASS = None +CACHE_SIZEMB = 32 + def CheckSection(sec): """ Check if INI section exists, if not create it """ try: @@ -256,7 +258,7 @@ def initialize(): ENCODERVBRCBR, ENCODERLOSSLESS, DELETE_LOSSLESS_FILES, PROWL_ENABLED, PROWL_PRIORITY, PROWL_KEYS, PROWL_ONSNATCH, MIRRORLIST, \ MIRROR, CUSTOMHOST, CUSTOMPORT, CUSTOMSLEEP, HPUSER, HPPASS, XBMC_ENABLED, XBMC_HOST, XBMC_USERNAME, XBMC_PASSWORD, XBMC_UPDATE, \ XBMC_NOTIFY, NMA_ENABLED, NMA_APIKEY, NMA_PRIORITY, NMA_ONSNATCH, SYNOINDEX_ENABLED, ALBUM_COMPLETION_PCT, PREFERRED_BITRATE_HIGH_BUFFER, \ - PREFERRED_BITRATE_LOW_BUFFER + PREFERRED_BITRATE_LOW_BUFFER,CACHE_SIZEMB if __INITIALIZED__: return False @@ -412,6 +414,8 @@ def initialize(): CUSTOMSLEEP = check_setting_int(CFG, 'General', 'customsleep', 1) HPUSER = check_setting_str(CFG, 'General', 'hpuser', '') HPPASS = check_setting_str(CFG, 'General', 'hppass', '') + + CACHE_SIZEMB = check_setting_int(CFG,'Advanced','cache_sizemb',32) ALBUM_COMPLETION_PCT = check_setting_int(CFG, 'Advanced', 'album_completion_pct', 80) @@ -708,6 +712,7 @@ def config_write(): new_config['Advanced'] = {} new_config['Advanced']['album_completion_pct'] = ALBUM_COMPLETION_PCT + new_config['Advanced']['cache_sizemb'] = CACHE_SIZEMB new_config.write() diff --git a/headphones/db.py b/headphones/db.py index a1f1d96d..985f6b72 100644 --- a/headphones/db.py +++ b/headphones/db.py @@ -45,7 +45,7 @@ class DBConnection: #journal disabled since we never do rollbacks self.connection.execute("PRAGMA journal_mode = OFF") #64mb of cache memory,probably need to make it user configurable - self.connection.execute("PRAGMA cache_size=-65536") + self.connection.execute("PRAGMA cache_size=-%s" % (headphones.CACHE_SIZEMB*1024)) self.connection.row_factory = sqlite3.Row def action(self, query, args=None): diff --git a/headphones/webserve.py b/headphones/webserve.py index 1d7d9e52..1b652053 100644 --- a/headphones/webserve.py +++ b/headphones/webserve.py @@ -533,7 +533,8 @@ class WebInterface(object): "customport": headphones.CUSTOMPORT, "customsleep": headphones.CUSTOMSLEEP, "hpuser": headphones.HPUSER, - "hppass": headphones.HPPASS + "hppass": headphones.HPPASS, + "cache_sizemb":headphones.CACHE_SIZEMB, } # Need to convert EXTRAS to a dictionary we can pass to the config: it'll come in as a string like 2,5,6,8 @@ -566,7 +567,7 @@ class WebInterface(object): encoderoutputformat=None, encodervbrcbr=None, encoderquality=None, encoderlossless=0, delete_lossless_files=0, prowl_enabled=0, prowl_onsnatch=0, prowl_keys=None, prowl_priority=0, xbmc_enabled=0, xbmc_host=None, xbmc_username=None, xbmc_password=None, xbmc_update=0, xbmc_notify=0, nma_enabled=False, nma_apikey=None, nma_priority=0, nma_onsnatch=0, synoindex_enabled=False, mirror=None, customhost=None, customport=None, customsleep=None, hpuser=None, hppass=None, - preferred_bitrate_high_buffer=None, preferred_bitrate_low_buffer=None, **kwargs): + preferred_bitrate_high_buffer=None, preferred_bitrate_low_buffer=None,cache_sizemb=32, **kwargs): headphones.HTTP_HOST = http_host headphones.HTTP_PORT = http_port @@ -662,6 +663,7 @@ class WebInterface(object): headphones.CUSTOMSLEEP = customsleep headphones.HPUSER = hpuser headphones.HPPASS = hppass + headphones.CACHE_SIZEMB = cache_sizemb # Handle the variable config options. Note - keys with False values aren't getting passed From 3cbc6d3ba6bf9305466e4c5619dfa08b14b07037 Mon Sep 17 00:00:00 2001 From: Patrick Speiser Date: Fri, 28 Sep 2012 10:20:20 +0200 Subject: [PATCH 23/44] Block special purpose artists http://musicbrainz.org/doc/Style/Unknown_and_untitled/Special_purpose_artist --- headphones/importer.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/headphones/importer.py b/headphones/importer.py index d8ceb281..f57543d3 100644 --- a/headphones/importer.py +++ b/headphones/importer.py @@ -21,7 +21,12 @@ from lib.beets.mediafile import MediaFile import headphones from headphones import logger, helpers, db, mb, albumart, lastfm -various_artists_mbid = '89ad4ac3-39f7-470e-963a-56509c546377' +#[anonymous],[data],[no artist],[traditional],[unknown],Various Artists +blacklisted_special_artists = ['f731ccc4-e22a-43af-a747-64213329e088','33cf029c-63b0-41a0-9855-be2a3665fb3b',\ + '314e1c25-dde7-4e4d-b2f4-0a7b9f7c56dc','eec63d3c-3b81-4ad4-b1e4-7c147d4d2b61',\ + '9be7f096-97ec-4615-8957-8d40b5dcbc41','125ec42a-7229-4250-afc5-e057484327fe',\ + '89ad4ac3-39f7-470e-963a-56509c546377'] + def is_exists(artistid): @@ -63,7 +68,7 @@ def artistlist_to_mbids(artistlist, forced=False): if not forced: bl_artist = myDB.action('SELECT * FROM blacklist WHERE ArtistID=?', [artistid]).fetchone() - if bl_artist or artistid == various_artists_mbid: + if bl_artist or artistid in blacklisted_special_artists: logger.info("Artist ID for '%s' is either blacklisted or Various Artists. To add artist, you must do it manually (Artist ID: %s)" % (artist, artistid)) continue @@ -99,8 +104,8 @@ def addArtisttoDB(artistid, extrasonly=False): from headphones import cache # Can't add various artists - throws an error from MB - if artistid == various_artists_mbid: - logger.warn('Cannot import Various Artists.') + if artistid in blacklisted_special_artists: + logger.warn('Cannot import blocked special purpose artist with id' + artistid) return # We'll use this to see if we should update the 'LastUpdated' time stamp From 1b4669bb2f573cebf9cadfadd47766d0d2046b1a Mon Sep 17 00:00:00 2001 From: Patrick Speiser Date: Fri, 28 Sep 2012 16:34:38 +0200 Subject: [PATCH 24/44] Stop cherrypy from complaining about a relative favicon path --- headphones/webstart.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/headphones/webstart.py b/headphones/webstart.py index b16d6765..0472a033 100644 --- a/headphones/webstart.py +++ b/headphones/webstart.py @@ -56,7 +56,7 @@ def initialize(options={}): }, '/favicon.ico':{ 'tools.staticfile.on': True, - 'tools.staticfile.filename': "images/favicon.ico" + 'tools.staticfile.filename': os.path.join(os.path.abspath(os.curdir),"images" + os.sep + "favicon.ico") }, '/cache':{ 'tools.staticdir.on': True, From a6c8a756f70e97337d8a77e29c0b0de6322a1301 Mon Sep 17 00:00:00 2001 From: Patrick Speiser Date: Fri, 28 Sep 2012 18:00:36 +0200 Subject: [PATCH 25/44] Changed Logs page to use ajax for retrieving log messages as needed instead of getting all at the start and freezing the browser --- data/interfaces/default/logs.html | 56 ++++++++++++++++--------------- headphones/webserve.py | 32 ++++++++++++++++++ 2 files changed, 61 insertions(+), 27 deletions(-) diff --git a/data/interfaces/default/logs.html b/data/interfaces/default/logs.html index 022be43e..84bdc1f9 100644 --- a/data/interfaces/default/logs.html +++ b/data/interfaces/default/logs.html @@ -16,21 +16,6 @@ lossless<%inherit file="base.html"/> - %for line in lineList: - <% - timestamp, message, level, threadname = line - - if level == 'WARNING' or level == 'ERROR': - grade = 'X' - else: - grade = 'Z' - %> - - ${timestamp} - ${level} - ${message} - - %endfor @@ -42,22 +27,39 @@ lossless<%inherit file="base.html"/> <%def name="javascriptIncludes()"> \ No newline at end of file diff --git a/headphones/webserve.py b/headphones/webserve.py index 799d84f3..59718b9e 100644 --- a/headphones/webserve.py +++ b/headphones/webserve.py @@ -394,6 +394,38 @@ class WebInterface(object): return serve_template(templatename="logs.html", title="Log", lineList=headphones.LOG_LIST) logs.exposed = True + + def getLog(self,iDisplayStart=0,iDisplayLength=100,iSortCol_0=0,iSortCol_1=0,iSortCol_2=0, + sSortDir_0="desc",sSortDir_1="desc",sSortDir_2="desc", + sSearch="",**kwargs): + + iDisplayStart = int(iDisplayStart) + iDisplayLength = int(iDisplayLength) + + filtered = [] + if sSearch == "": + filtered = headphones.LOG_LIST[::] + else: + filtered = [row for row in headphones.LOG_LIST for column in row if sSearch in column] + + sortcolumn = 0 + if iSortCol_0 == '1': + sortcolumn = 2 + elif iSortCol_0 == '2': + sortcolumn = 1 + filtered.sort(key=lambda x:x[sortcolumn],reverse=sSortDir_0 == "desc") + + rows = filtered[iDisplayStart:(iDisplayStart+iDisplayLength)] + rows = [[row[0],row[2],row[1]] for row in rows] + + dict = {'iTotalDisplayRecords':len(filtered), + 'iTotalRecords':len(headphones.LOG_LIST), + 'aaData':rows, + } + s = simplejson.dumps(dict) + return s + getLog.exposed = True + def clearhistory(self, type=None): myDB = db.DBConnection() if type == 'all': From 18d75db4233df7f38618403b58896fefb302a62e Mon Sep 17 00:00:00 2001 From: Patrick Speiser Date: Fri, 28 Sep 2012 18:12:01 +0200 Subject: [PATCH 26/44] Removed unnecessary getLog parameters --- headphones/webserve.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/headphones/webserve.py b/headphones/webserve.py index 59718b9e..f193cd00 100644 --- a/headphones/webserve.py +++ b/headphones/webserve.py @@ -395,9 +395,7 @@ class WebInterface(object): logs.exposed = True - def getLog(self,iDisplayStart=0,iDisplayLength=100,iSortCol_0=0,iSortCol_1=0,iSortCol_2=0, - sSortDir_0="desc",sSortDir_1="desc",sSortDir_2="desc", - sSearch="",**kwargs): + def getLog(self,iDisplayStart=0,iDisplayLength=100,iSortCol_0=0,sSortDir_0="desc",sSearch="",**kwargs): iDisplayStart = int(iDisplayStart) iDisplayLength = int(iDisplayLength) From e3145974f6eee87018d49b53656c903a870548b1 Mon Sep 17 00:00:00 2001 From: Patrick Speiser Date: Fri, 28 Sep 2012 20:05:24 +0200 Subject: [PATCH 27/44] Added GetArtists.json as the backend for the ajax enabled main page --- headphones/webserve.py | 46 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/headphones/webserve.py b/headphones/webserve.py index 87a59336..803bbe5f 100644 --- a/headphones/webserve.py +++ b/headphones/webserve.py @@ -29,6 +29,7 @@ from headphones import logger, searcher, db, importer, mb, lastfm, librarysync from headphones.helpers import checked, radio import lib.simplejson as simplejson +import json @@ -438,6 +439,51 @@ class WebInterface(object): return s getLog.exposed = True + def getArtists_json(self,iDisplayStart=0,iDisplayLength=100,sSearch="",iSortCol_0='0',sSortDir_0='asc'): + iDisplayStart = int(iDisplayStart) + iDisplayLength = int(iDisplayLength) + + filtered = [] + totalcount = 0 + myDB = db.DBConnection() + + if sSearch == "": + filtered = myDB.select('SELECT * from artists order by ArtistSortName COLLATE NOCASE') + totalcount = len(filtered) + else: + filtered = myDB.select('SELECT * from artists order by ArtistSortName WHERE ArtistSortName LIKE %s OR LatestAlbum LIKE %s COLLATE NOCASE' % (sSearch,sSearch)) + totalcount = myDB.select('SELECT COUNT(*) from artists') + + sortcolumn = 0 + if iSortCol_0 == '1': + sortcolumn = 2 + elif iSortCol_0 == '2': + sortcolumn = 1 + filtered.sort(key=lambda x:x[sortcolumn],reverse=sSortDir_0 == "desc") + artists = filtered[iDisplayStart:(iDisplayStart+iDisplayLength)] + rows = [] + for artist in artists: + row = {"ArtistID":artist['ArtistID'], + "ArtistSortName":artist["ArtistSortName"], + "Status":artist["Status"], + "TotalTracks":artist["TotalTracks"], + "HaveTracks":artist["HaveTracks"] if 'HaveTracks' in artist else 0, + "LatestAlbum":"", + "ReleaseDate":"", + } + + rows.append(row) + + + dict = {'iTotalDisplayRecords':len(filtered), + 'iTotalRecords':totalcount, + 'aaData':rows, + } + s = json.dumps(dict) + cherrypy.response.headers['Content-type'] = 'application/json' + return s + getArtists_json.exposed=True + def clearhistory(self, type=None): myDB = db.DBConnection() if type == 'all': From 203e7ca3a127270873931b623b0ba2bac955d1f0 Mon Sep 17 00:00:00 2001 From: Patrick Speiser Date: Fri, 28 Sep 2012 21:28:35 +0200 Subject: [PATCH 28/44] More groundwork for home page ajax --- .gitignore | 32 + data/interfaces/default/index.html | 96 ++- data/js/libs/jquery.dataTables.min.js | 801 +++++--------------------- data/js/libs/jquery.js | 2 + headphones/webserve.py | 11 +- 5 files changed, 233 insertions(+), 709 deletions(-) create mode 100644 data/js/libs/jquery.js diff --git a/.gitignore b/.gitignore index 74e29c78..b557314a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,10 @@ + # Compiled source # ################### *.pyc *.py~ +*.pyproj +*.sln # Logs and databases # ###################### @@ -19,3 +22,32 @@ cache/* ehthumbs.db Icon? Thumbs.db + +#ignore thumbnails created by windows +Thumbs.db +#Ignore files build by Visual Studio +*.obj +*.exe +*.pdb +*.user +*.aps +*.pch +*.vspscc +*_i.c +*_p.c +*.ncb +*.suo +*.tlb +*.tlh +*.bak +*.cache +*.ilk +*.log +[Bb]in +[Dd]ebug*/ +*.lib +*.sbr +obj/ +[Rr]elease*/ +_ReSharper*/ +[Tt]est[Rr]esult* \ No newline at end of file diff --git a/data/interfaces/default/index.html b/data/interfaces/default/index.html index bcd8cf2b..f33920e0 100644 --- a/data/interfaces/default/index.html +++ b/data/interfaces/default/index.html @@ -8,58 +8,13 @@ - Artist Name - Status - Latest Album - Have + Artist Name + Status + Latest Album + Have - %for artist in artists: - <% - totaltracks = artist['TotalTracks'] - havetracks = artist['HaveTracks'] - if not havetracks: - havetracks = 0 - try: - percent = (havetracks*100.0)/totaltracks - if percent > 100: - percent = 100 - except (ZeroDivisionError, TypeError): - percent = 0 - totaltracks = '?' - - if artist['ReleaseDate'] and artist['LatestAlbum']: - releasedate = artist['ReleaseDate'] - albumdisplay = '%s (%s)' % (artist['LatestAlbum'], artist['ReleaseDate']) - if releasedate > helpers.today(): - grade = 'A' - else: - grade = 'Z' - elif artist['LatestAlbum']: - releasedate = '' - grade = 'Z' - albumdisplay = '%s' % artist['LatestAlbum'] - else: - releasedate = '' - grade = 'Z' - albumdisplay = 'None' - - if artist['Status'] == 'Paused': - grade = 'X' - - if artist['Status'] == 'Loading': - grade = 'L' - - %> - -
- ${artist['ArtistName']} - ${artist['Status']} - ${albumdisplay} -
${havetracks}/${totaltracks}
- - %endfor @@ -72,7 +27,9 @@