From 625b0efd13024ee373abfc68a9a8b4e7bbdf5f1e Mon Sep 17 00:00:00 2001 From: rembo10 Date: Thu, 16 Aug 2012 22:54:33 +0530 Subject: [PATCH] A couple of changes to library sync --- data/interfaces/default/album.html | 4 +- headphones/importer.py | 4 +- headphones/librarysync.py | 244 +++++++++++++++++++++++++---- 3 files changed, 218 insertions(+), 34 deletions(-) diff --git a/data/interfaces/default/album.html b/data/interfaces/default/album.html index 20a8f659..fce97271 100644 --- a/data/interfaces/default/album.html +++ b/data/interfaces/default/album.html @@ -176,8 +176,6 @@ $('#refresh_artist').click(function() { $('#dialog').dialog("close"); }); - getAlbumInfo(); - getAlbumArt(); initActions(); setTimeout(function(){ initFancybox(); @@ -193,6 +191,8 @@ }; $(document).ready(function() { + getAlbumInfo(); + getAlbumArt(); initThisPage(); }); diff --git a/headphones/importer.py b/headphones/importer.py index cf58fa63..c92a0d49 100644 --- a/headphones/importer.py +++ b/headphones/importer.py @@ -236,7 +236,7 @@ def addArtisttoDB(artistid, extrasonly=False): newValueDict['Location'] = match['Location'] newValueDict['BitRate'] = match['BitRate'] newValueDict['Format'] = match['Format'] - myDB.action('UPDATE tracks SET Matched="True" WHERE Location=?', match['Location']) + myDB.action('UPDATE have SET Matched="True" WHERE Location=?', [match['Location']]) myDB.upsert("alltracks", newValueDict, controlValueDict) @@ -287,7 +287,7 @@ def addArtisttoDB(artistid, extrasonly=False): newValueDict['Location'] = match['Location'] newValueDict['BitRate'] = match['BitRate'] newValueDict['Format'] = match['Format'] - myDB.action('UPDATE tracks SET Matched="True" WHERE Location=?', match['Location']) + myDB.action('UPDATE have SET Matched="True" WHERE Location=?', [match['Location']]) myDB.upsert("alltracks", newValueDict, controlValueDict) diff --git a/headphones/librarysync.py b/headphones/librarysync.py index 8dfd4dd5..b7484b1c 100644 --- a/headphones/librarysync.py +++ b/headphones/librarysync.py @@ -45,6 +45,8 @@ def libraryScan(dir=None): new_artists = [] bitrates = [] + + song_list = [] myDB.action('DELETE from have') @@ -69,44 +71,224 @@ def libraryScan(dir=None): # Grab the bitrates for the auto detect bit rate option if f.bitrate: bitrates.append(f.bitrate) - - # Try to find a match based on artist/album/tracktitle + + # Use the album artist over the artist if available if f.albumartist: f_artist = f.albumartist elif f.artist: f_artist = f.artist else: - continue - - if f_artist and f.album and f.title: - - track = myDB.action('SELECT TrackID from tracks WHERE CleanName LIKE ?', [helpers.cleanName(f_artist +' '+f.album+' '+f.title)]).fetchone() - - if not track: - track = myDB.action('SELECT TrackID from tracks WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [f_artist, f.album, f.title]).fetchone() + f_artist = None - if track: - myDB.action('UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE TrackID=?', [unicode_song_path, f.bitrate, f.format, track['TrackID']]) - continue - - # Try to match on mbid if available and we couldn't find a match based on metadata - if f.mb_trackid: + # Add the song to our song list - + # TODO: skip adding songs without the minimum requisite information (just a matter of putting together the right if statements) - # Wondering if theres a better way to do this -> do one thing if the row exists, - # do something else if it doesn't - track = myDB.action('SELECT TrackID from tracks WHERE TrackID=?', [f.mb_trackid]).fetchone() + song_dict = { 'TrackID' : f.mb_trackid, + 'ReleaseID' : f.mb_albumid, + 'ArtistName' : f_artist, + 'AlbumTitle' : f.album, + 'TrackNumber': f.track, + 'TrackLength': f.length, + 'Genre' : f.genre, + 'Date' : f.date, + 'TrackTitle' : f.title, + 'BitRate' : f.bitrate, + 'Format' : f.format, + 'Location' : unicode_song_path } + + song_list.append(song_dict) + + # Now we start track matching + total_number_of_songs = len(song_list) + logger.info("Found " + str(total_number_of_songs) + " tracks in: '" + dir + "'. Matching tracks to the appropriate releases....") + + # Sort the song_list by most vague (e.g. no trackid or releaseid) to most specific (both trackid & releaseid) + # When we insert into the database, the tracks with the most specific information will overwrite the more general matches + + song_list = helpers.multikeysort(song_list, ['ReleaseID', 'TrackID']) + + # We'll use this to give a % completion, just because the track matching might take a while + song_count = 0 + + for song in song_list: - if track: - myDB.action('UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE TrackID=?', [unicode_song_path, f.bitrate, f.format, track['TrackID']]) - continue - - # if we can't find a match in the database on a track level, it might be a new artist or it might be on a non-mb release - new_artists.append(f_artist) - - # The have table will become the new database for unmatched tracks (i.e. tracks with no associated links in the database - myDB.action('INSERT INTO have (ArtistName, AlbumTitle, TrackNumber, TrackTitle, TrackLength, BitRate, Genre, Date, TrackID, Location, CleanName, Format) VALUES( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', [f_artist, f.album, f.track, f.title, f.length, f.bitrate, f.genre, f.date, f.mb_trackid, unicode_song_path, helpers.cleanName(f_artist+' '+f.album+' '+f.title), f.format]) + song_count += 1 + completion_percentage = float(song_count)/total_number_of_songs * 100 + + if completion_percentage%10 == 0: + logger.info("Track matching is " + str(completion_percentage) + "% complete") + + # If the track has a trackid & releaseid (beets: albumid) that the most surefire way + # of identifying a track to a specific release so we'll use that first + if song['TrackID'] and song['ReleaseID']: - logger.info('Completed scanning directory: %s' % dir) + # Check both the tracks table & alltracks table in case they haven't populated the alltracks table yet + track = myDB.action('SELECT TrackID, ReleaseID, AlbumID from alltracks WHERE TrackID=? AND ReleaseID=?', [song['TrackID'], song['ReleaseID']]).fetchone() + + # It might be the case that the alltracks table isn't populated yet, so maybe we can only find a match in the tracks table + if not track: + track = myDB.action('SELECT TrackID, ReleaseID, AlbumID from tracks WHERE TrackID=? AND ReleaseID=?', [song['TrackID'], song['ReleaseID']]).fetchone() + + if track: + # Use TrackID & ReleaseID here since there can only be one possible match with a TrackID & ReleaseID query combo + controlValueDict = { 'TrackID' : track['TrackID'], + 'ReleaseID' : track['ReleaseID'] } + + # Insert it into the Headphones hybrid release (ReleaseID == AlbumID) + hybridControlValueDict = { 'TrackID' : track['TrackID'], + 'ReleaseID' : track['AlbumID'] } + + newValueDict = { 'Location' : song['Location'], + 'BitRate' : song['BitRate'], + 'Format' : song['Format'] } + + # Update both the tracks table and the alltracks table using the controlValueDict and hybridControlValueDict + myDB.upsert("alltracks", newValueDict, controlValueDict) + myDB.upsert("tracks", newValueDict, controlValueDict) + + myDB.upsert("alltracks", newValueDict, hybridControlValueDict) + myDB.upsert("tracks", newValueDict, hybridControlValueDict) + + # Matched. Move on to the next one: + continue + + # If we can't find it with TrackID & ReleaseID, next most specific will be + # releaseid + tracktitle, although perhaps less reliable due to a higher + # likelihood of variations in the song title (e.g. feat. artists) + if song['ReleaseID'] and song['TrackTitle']: + + track = myDB.action('SELECT TrackID, ReleaseID, AlbumID from alltracks WHERE ReleaseID=? AND TrackTitle=?', [song['ReleaseID'], song['TrackTitle']]).fetchone() + + if not track: + track = myDB.action('SELECT TrackID, ReleaseID, AlbumID from tracks WHERE ReleaseID=? AND TrackTitle=?', [song['ReleaseID'], song['TrackTitle']]).fetchone() + + if track: + # There can also only be one match for this query as well (although it might be on both the tracks and alltracks table) + # So use both TrackID & ReleaseID as the control values + controlValueDict = { 'TrackID' : track['TrackID'], + 'ReleaseID' : track['ReleaseID'] } + + hybridControlValueDict = { 'TrackID' : track['TrackID'], + 'ReleaseID' : track['AlbumID'] } + + newValueDict = { 'Location' : song['Location'], + 'BitRate' : song['BitRate'], + 'Format' : song['Format'] } + + # Update both tables here as well + myDB.upsert("alltracks", newValueDict, controlValueDict) + myDB.upsert("tracks", newValueDict, controlValueDict) + + myDB.upsert("alltracks", newValueDict, hybridControlValueDict) + myDB.upsert("tracks", newValueDict, hybridControlValueDict) + + # Done + continue + + # Next most specific will be the opposite: a TrackID and an AlbumTitle + # TrackIDs span multiple releases so if something is on an official album + # and a compilation, for example, this will match it to the right one + # However - there may be multiple matches here + if song['TrackID'] and song['AlbumTitle']: + + # Even though there might be multiple matches, we just need to grab one to confirm a match + track = myDB.action('SELECT TrackID, AlbumTitle from alltracks WHERE TrackID=? AND AlbumTitle LIKE ?', [song['TrackID'], song['AlbumTitle']]).fetchone() + + if not track: + track = myDB.action('SELECT TrackID, AlbumTitle from tracks WHERE TrackID=? AND AlbumTitle LIKE ?', [song['TrackID'], song['AlbumTitle']]).fetchone() + + if track: + # Don't need the hybridControlValueDict here since ReleaseID is not unique + controlValueDict = { 'TrackID' : track['TrackID'], + 'AlbumTitle' : track['AlbumTitle'] } + + newValueDict = { 'Location' : song['Location'], + 'BitRate' : song['BitRate'], + 'Format' : song['Format'] } + + myDB.upsert("alltracks", newValueDict, controlValueDict) + myDB.upsert("tracks", newValueDict, controlValueDict) + + continue + + # Next most specific is the ArtistName + AlbumTitle + TrackTitle combo (but probably + # even more unreliable than the previous queries, and might span multiple releases) + if song['ArtistName'] and song['AlbumTitle'] and song['TrackTitle']: + + track = myDB.action('SELECT ArtistName, AlbumTitle, TrackTitle from alltracks WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [song['ArtistName'], song['AlbumTitle'], song['TrackTitle']]).fetchone() + + if not track: + track = myDB.action('SELECT ArtistName, AlbumTitle, TrackTitle from tracks WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [song['ArtistName'], song['AlbumTitle'], song['TrackTitle']]).fetchone() + + if track: + controlValueDict = { 'ArtistName' : track['ArtistName'], + 'AlbumTitle' : track['AlbumTitle'], + 'TrackTitle' : track['TrackTitle'] } + + newValueDict = { 'Location' : song['Location'], + 'BitRate' : song['BitRate'], + 'Format' : song['Format'] } + + myDB.upsert("alltracks", newValueDict, controlValueDict) + myDB.upsert("tracks", newValueDict, controlValueDict) + + continue + + # Use the "CleanName" (ArtistName + AlbumTitle + TrackTitle stripped of punctuation, capitalization, etc) + # This is more reliable than the former but requires some string manipulation so we'll do it only + # if we can't find a match with the original data + if song['ArtistName'] and song['AlbumTitle'] and song['TrackTitle']: + + CleanName = helpers.cleanName(song['ArtistName'] +' '+ song['AlbumTitle'] +' '+song['TrackTitle']) + + track = myDB.action('SELECT CleanName from alltracks WHERE CleanName LIKE ?', [CleanName]).fetchone() + + if not track: + track = myDB.action('SELECT CleanName from tracks WHERE CleanName LIKE ?', [CleanName]).fetchone() + + if track: + controlValueDict = { 'CleanName' : track['CleanName'] } + + newValueDict = { 'Location' : song['Location'], + 'BitRate' : song['BitRate'], + 'Format' : song['Format'] } + + myDB.upsert("alltracks", newValueDict, controlValueDict) + myDB.upsert("tracks", newValueDict, controlValueDict) + + continue + + # Match on TrackID alone if we can't find it using any of the above methods. This method is reliable + # but spans multiple releases - but that's why we're putting at the beginning as a last resort. If a track + # with more specific information exists in the library, it'll overwrite these values + if song['TrackID']: + + track = myDB.action('SELECT TrackID from alltracks WHERE TrackID=?', [song['TrackID']]).fetchone() + + if not track: + track = myDB.action('SELECT TrackID from tracks WHERE TrackID=?', [song['TrackID']]).fetchone() + + if track: + controlValueDict = { 'TrackID' : track['TrackID'] } + + newValueDict = { 'Location' : song['Location'], + 'BitRate' : song['BitRate'], + 'Format' : song['Format'] } + + myDB.upsert("alltracks", newValueDict, controlValueDict) + myDB.upsert("tracks", newValueDict, controlValueDict) + + continue + + # if we can't find a match in the database on a track level, it might be a new artist or it might be on a non-mb release + new_artists.append(song['ArtistName']) + + # The have table will become the new database for unmatched tracks (i.e. tracks with no associated links in the database + CleanName = helpers.cleanName(song['ArtistName'] +' '+ song['AlbumTitle'] +' '+song['TrackTitle']) + + myDB.action('INSERT INTO have (ArtistName, AlbumTitle, TrackNumber, TrackTitle, TrackLength, BitRate, Genre, Date, TrackID, Location, CleanName, Format) VALUES( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', [song['ArtistName'], song['AlbumTitle'], song['TrackNumber'], song['TrackTitle'], song['TrackLength'], song['BitRate'], song['Genre'], song['Date'], song['TrackID'], song['Location'], CleanName, song['Format']]) + + logger.info('Completed matching tracks from directory: %s' % dir) # Clean up the new artist list unique_artists = {}.fromkeys(new_artists).keys() @@ -115,9 +297,11 @@ def libraryScan(dir=None): artist_list = [f for f in unique_artists if f.lower() not in [x[0].lower() for x in current_artists]] # Update track counts - logger.info('Updating track counts') + logger.info('Updating current artist track counts') for artist in current_artists: + # Have tracks are selected from tracks table and not all tracks because of duplicates + # We update the track count upon an album switch to compliment this havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistID like ? AND Location IS NOT NULL', [artist['ArtistID']])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ?', [artist['ArtistName']])) myDB.action('UPDATE artists SET HaveTracks=? WHERE ArtistID=?', [havetracks, artist['ArtistID']])