A couple of changes to library sync

data/interfaces/default/album.html

@@ -176,8 +176,6 @@
 		$('#refresh_artist').click(function() {
 			$('#dialog').dialog("close");
 		});
-		getAlbumInfo();
-		getAlbumArt();
 		initActions();
 		setTimeout(function(){
 			initFancybox();
@@ -193,6 +191,8 @@
 	};
 	
 	$(document).ready(function()	{
+		getAlbumInfo();
+		getAlbumArt();
 		initThisPage();
 	});
 

headphones/importer.py

@@ -236,7 +236,7 @@ def addArtisttoDB(artistid, extrasonly=False):
                     newValueDict['Location'] = match['Location']
                     newValueDict['BitRate'] = match['BitRate']
                     newValueDict['Format'] = match['Format']
-                    myDB.action('UPDATE tracks SET Matched="True" WHERE Location=?', match['Location'])
+                    myDB.action('UPDATE have SET Matched="True" WHERE Location=?', [match['Location']])
                                 
                 myDB.upsert("alltracks", newValueDict, controlValueDict)
 
@@ -287,7 +287,7 @@ def addArtisttoDB(artistid, extrasonly=False):
                 newValueDict['Location'] = match['Location']
                 newValueDict['BitRate'] = match['BitRate']
                 newValueDict['Format'] = match['Format']
-                myDB.action('UPDATE tracks SET Matched="True" WHERE Location=?', match['Location'])
+                myDB.action('UPDATE have SET Matched="True" WHERE Location=?', [match['Location']])
                             
             myDB.upsert("alltracks", newValueDict, controlValueDict)
         

headphones/librarysync.py

@@ -46,6 +46,8 @@ def libraryScan(dir=None):
     new_artists = []
     bitrates = []
     
+    song_list = []
+
     myDB.action('DELETE from have')
     
     for r,d,f in os.walk(dir):
@@ -70,43 +72,223 @@ def libraryScan(dir=None):
                 if f.bitrate:
                     bitrates.append(f.bitrate)
                     
-                # Try to find a match based on artist/album/tracktitle
+                # Use the album artist over the artist if available
                 if f.albumartist:
                     f_artist = f.albumartist
                 elif f.artist:
                     f_artist = f.artist
                 else:
-                    continue
+                    f_artist = None
                     
-                if f_artist and f.album and f.title:
+                # Add the song to our song list - 
+                # TODO: skip adding songs without the minimum requisite information (just a matter of putting together the right if statements)
 
-                    track = myDB.action('SELECT TrackID from tracks WHERE CleanName LIKE ?', [helpers.cleanName(f_artist +' '+f.album+' '+f.title)]).fetchone()
+                song_dict = { 'TrackID' : f.mb_trackid,
+                              'ReleaseID' : f.mb_albumid,
+                              'ArtistName' : f_artist,
+                              'AlbumTitle' : f.album,
+                              'TrackNumber': f.track,
+                              'TrackLength': f.length,
+                              'Genre'      : f.genre,
+                              'Date'       : f.date,
+                              'TrackTitle' : f.title,
+                              'BitRate'    : f.bitrate,
+                              'Format'     : f.format,
+                              'Location'   : unicode_song_path }
                               
-                    if not track:
+                song_list.append(song_dict)
-                        track = myDB.action('SELECT TrackID from tracks WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [f_artist, f.album, f.title]).fetchone()
 
-                    if track:
+    # Now we start track matching
-                        myDB.action('UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE TrackID=?', [unicode_song_path, f.bitrate, f.format, track['TrackID']])
+    total_number_of_songs = len(song_list)
-                        continue        
+    logger.info("Found " + str(total_number_of_songs) + " tracks in: '" + dir + "'. Matching tracks to the appropriate releases....")
     
-                # Try to match on mbid if available and we couldn't find a match based on metadata
+    # Sort the song_list by most vague (e.g. no trackid or releaseid) to most specific (both trackid & releaseid)
-                if f.mb_trackid:
+    # When we insert into the database, the tracks with the most specific information will overwrite the more general matches
     
-                    # Wondering if theres a better way to do this -> do one thing if the row exists,
+    song_list = helpers.multikeysort(song_list, ['ReleaseID', 'TrackID'])
-                    # do something else if it doesn't
-                    track = myDB.action('SELECT TrackID from tracks WHERE TrackID=?', [f.mb_trackid]).fetchone()
     
-                    if track:
+    # We'll use this to give a % completion, just because the track matching might take a while
-                        myDB.action('UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE TrackID=?', [unicode_song_path, f.bitrate, f.format, track['TrackID']])
+    song_count = 0
-                        continue                
     
-                # if we can't find a match in the database on a track level, it might be a new artist or it might be on a non-mb release
+    for song in song_list:
-                new_artists.append(f_artist)
         
-                # The have table will become the new database for unmatched tracks (i.e. tracks with no associated links in the database                
+        song_count += 1
-                myDB.action('INSERT INTO have (ArtistName, AlbumTitle, TrackNumber, TrackTitle, TrackLength, BitRate, Genre, Date, TrackID, Location, CleanName, Format) VALUES( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', [f_artist, f.album, f.track, f.title, f.length, f.bitrate, f.genre, f.date, f.mb_trackid, unicode_song_path, helpers.cleanName(f_artist+' '+f.album+' '+f.title), f.format])
+        completion_percentage = float(song_count)/total_number_of_songs * 100
         
-    logger.info('Completed scanning directory: %s' % dir)
+        if completion_percentage%10 == 0:
+            logger.info("Track matching is " + str(completion_percentage) + "% complete")
+        
+        # If the track has a trackid & releaseid (beets: albumid) that the most surefire way
+        # of identifying a track to a specific release so we'll use that first
+        if song['TrackID'] and song['ReleaseID']:
+
+            # Check both the tracks table & alltracks table in case they haven't populated the alltracks table yet
+            track = myDB.action('SELECT TrackID, ReleaseID, AlbumID from alltracks WHERE TrackID=? AND ReleaseID=?', [song['TrackID'], song['ReleaseID']]).fetchone()
+            
+            # It might be the case that the alltracks table isn't populated yet, so maybe we can only find a match in the tracks table
+            if not track:
+                track = myDB.action('SELECT TrackID, ReleaseID, AlbumID from tracks WHERE TrackID=? AND ReleaseID=?', [song['TrackID'], song['ReleaseID']]).fetchone()
+    
+            if track:
+                # Use TrackID & ReleaseID here since there can only be one possible match with a TrackID & ReleaseID query combo
+                controlValueDict = { 'TrackID'   : track['TrackID'],
+                                     'ReleaseID' : track['ReleaseID'] }
+                
+                # Insert it into the Headphones hybrid release (ReleaseID == AlbumID)                   
+                hybridControlValueDict = { 'TrackID'   : track['TrackID'],
+                                           'ReleaseID' : track['AlbumID'] }
+                                     
+                newValueDict = { 'Location' : song['Location'],
+                                 'BitRate'  : song['BitRate'],
+                                 'Format'   : song['Format'] }
+                                 
+                # Update both the tracks table and the alltracks table using the controlValueDict and hybridControlValueDict
+                myDB.upsert("alltracks", newValueDict, controlValueDict)
+                myDB.upsert("tracks", newValueDict, controlValueDict)
+                
+                myDB.upsert("alltracks", newValueDict, hybridControlValueDict)
+                myDB.upsert("tracks", newValueDict, hybridControlValueDict)
+                
+                # Matched. Move on to the next one:
+                continue
+    
+        # If we can't find it with TrackID & ReleaseID, next most specific will be 
+        # releaseid + tracktitle, although perhaps less reliable due to a higher 
+        # likelihood of variations in the song title (e.g. feat. artists)
+        if song['ReleaseID'] and song['TrackTitle']:
+    
+            track = myDB.action('SELECT TrackID, ReleaseID, AlbumID from alltracks WHERE ReleaseID=? AND TrackTitle=?', [song['ReleaseID'], song['TrackTitle']]).fetchone()
+    
+            if not track:
+                track = myDB.action('SELECT TrackID, ReleaseID, AlbumID from tracks WHERE ReleaseID=? AND TrackTitle=?', [song['ReleaseID'], song['TrackTitle']]).fetchone()
+                
+            if track:
+                # There can also only be one match for this query as well (although it might be on both the tracks and alltracks table)
+                # So use both TrackID & ReleaseID as the control values
+                controlValueDict = { 'TrackID'   : track['TrackID'],
+                                     'ReleaseID' : track['ReleaseID'] }
+                                     
+                hybridControlValueDict = { 'TrackID'   : track['TrackID'],
+                                           'ReleaseID' : track['AlbumID'] }
+                                     
+                newValueDict = { 'Location' : song['Location'],
+                                 'BitRate'  : song['BitRate'],
+                                 'Format'   : song['Format'] }
+                                 
+                # Update both tables here as well
+                myDB.upsert("alltracks", newValueDict, controlValueDict)
+                myDB.upsert("tracks", newValueDict, controlValueDict)
+                
+                myDB.upsert("alltracks", newValueDict, hybridControlValueDict)
+                myDB.upsert("tracks", newValueDict, hybridControlValueDict)
+                
+                # Done
+                continue
+                
+        # Next most specific will be the opposite: a TrackID and an AlbumTitle
+        # TrackIDs span multiple releases so if something is on an official album
+        # and a compilation, for example, this will match it to the right one
+        # However - there may be multiple matches here
+        if song['TrackID'] and song['AlbumTitle']:
+    
+            # Even though there might be multiple matches, we just need to grab one to confirm a match
+            track = myDB.action('SELECT TrackID, AlbumTitle from alltracks WHERE TrackID=? AND AlbumTitle LIKE ?', [song['TrackID'], song['AlbumTitle']]).fetchone()
+    
+            if not track:
+                track = myDB.action('SELECT TrackID, AlbumTitle from tracks WHERE TrackID=? AND AlbumTitle LIKE ?', [song['TrackID'], song['AlbumTitle']]).fetchone()
+                
+            if track:
+                # Don't need the hybridControlValueDict here since ReleaseID is not unique
+                controlValueDict = { 'TrackID'   : track['TrackID'],
+                                     'AlbumTitle' : track['AlbumTitle'] }
+                                     
+                newValueDict = { 'Location' : song['Location'],
+                                 'BitRate'  : song['BitRate'],
+                                 'Format'   : song['Format'] }
+
+                myDB.upsert("alltracks", newValueDict, controlValueDict)
+                myDB.upsert("tracks", newValueDict, controlValueDict)
+
+                continue   
+        
+        # Next most specific is the ArtistName + AlbumTitle + TrackTitle combo (but probably 
+        # even more unreliable than the previous queries, and might span multiple releases)
+        if song['ArtistName'] and song['AlbumTitle'] and song['TrackTitle']:
+            
+            track = myDB.action('SELECT ArtistName, AlbumTitle, TrackTitle from alltracks WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [song['ArtistName'], song['AlbumTitle'], song['TrackTitle']]).fetchone()
+    
+            if not track:
+                track = myDB.action('SELECT ArtistName, AlbumTitle, TrackTitle from tracks WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [song['ArtistName'], song['AlbumTitle'], song['TrackTitle']]).fetchone()
+                
+            if track:
+                controlValueDict = { 'ArtistName' : track['ArtistName'],
+                                     'AlbumTitle' : track['AlbumTitle'],
+                                     'TrackTitle' : track['TrackTitle'] }
+                                     
+                newValueDict = { 'Location' : song['Location'],
+                                 'BitRate'  : song['BitRate'],
+                                 'Format'   : song['Format'] }
+
+                myDB.upsert("alltracks", newValueDict, controlValueDict)
+                myDB.upsert("tracks", newValueDict, controlValueDict)
+
+                continue
+        
+        # Use the "CleanName" (ArtistName + AlbumTitle + TrackTitle stripped of punctuation, capitalization, etc)
+        # This is more reliable than the former but requires some string manipulation so we'll do it only
+        # if we can't find a match with the original data
+        if song['ArtistName'] and song['AlbumTitle'] and song['TrackTitle']:
+            
+            CleanName = helpers.cleanName(song['ArtistName'] +' '+ song['AlbumTitle'] +' '+song['TrackTitle'])
+            
+            track = myDB.action('SELECT CleanName from alltracks WHERE CleanName LIKE ?', [CleanName]).fetchone()
+            
+            if not track:
+                track = myDB.action('SELECT CleanName from tracks WHERE CleanName LIKE ?', [CleanName]).fetchone()
+    
+            if track:
+                controlValueDict = { 'CleanName' : track['CleanName'] }
+                                     
+                newValueDict = { 'Location' : song['Location'],
+                                 'BitRate'  : song['BitRate'],
+                                 'Format'   : song['Format'] }
+
+                myDB.upsert("alltracks", newValueDict, controlValueDict)
+                myDB.upsert("tracks", newValueDict, controlValueDict)
+
+                continue     
+        
+        # Match on TrackID alone if we can't find it using any of the above methods. This method is reliable
+        # but spans multiple releases - but that's why we're putting at the beginning as a last resort. If a track
+        # with more specific information exists in the library, it'll overwrite these values
+        if song['TrackID']:
+    
+            track = myDB.action('SELECT TrackID from alltracks WHERE TrackID=?', [song['TrackID']]).fetchone()
+            
+            if not track:
+                track = myDB.action('SELECT TrackID from tracks WHERE TrackID=?', [song['TrackID']]).fetchone()
+    
+            if track:
+                controlValueDict = { 'TrackID' : track['TrackID'] }
+                                     
+                newValueDict = { 'Location' : song['Location'],
+                                 'BitRate'  : song['BitRate'],
+                                 'Format'   : song['Format'] }
+
+                myDB.upsert("alltracks", newValueDict, controlValueDict)
+                myDB.upsert("tracks", newValueDict, controlValueDict)
+
+                continue          
+        
+        # if we can't find a match in the database on a track level, it might be a new artist or it might be on a non-mb release
+        new_artists.append(song['ArtistName'])
+        
+        # The have table will become the new database for unmatched tracks (i.e. tracks with no associated links in the database                
+        CleanName = helpers.cleanName(song['ArtistName'] +' '+ song['AlbumTitle'] +' '+song['TrackTitle'])
+        
+        myDB.action('INSERT INTO have (ArtistName, AlbumTitle, TrackNumber, TrackTitle, TrackLength, BitRate, Genre, Date, TrackID, Location, CleanName, Format) VALUES( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', [song['ArtistName'], song['AlbumTitle'], song['TrackNumber'], song['TrackTitle'], song['TrackLength'], song['BitRate'], song['Genre'], song['Date'], song['TrackID'], song['Location'], CleanName, song['Format']])
+
+    logger.info('Completed matching tracks from directory: %s' % dir)
     
     # Clean up the new artist list
     unique_artists = {}.fromkeys(new_artists).keys()
@@ -115,9 +297,11 @@ def libraryScan(dir=None):
     artist_list = [f for f in unique_artists if f.lower() not in [x[0].lower() for x in current_artists]]
     
     # Update track counts
-    logger.info('Updating track counts')
+    logger.info('Updating current artist track counts')
 
     for artist in current_artists:
+        # Have tracks are selected from tracks table and not all tracks because of duplicates
+        # We update the track count upon an album switch to compliment this
         havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistID like ? AND Location IS NOT NULL', [artist['ArtistID']])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ?', [artist['ArtistName']]))
         myDB.action('UPDATE artists SET HaveTracks=? WHERE ArtistID=?', [havetracks, artist['ArtistID']])