From 693e5b1624a7b24f091768c1995bf3545ca24a1c Mon Sep 17 00:00:00 2001 From: Ade Date: Wed, 15 Aug 2018 14:51:27 +1200 Subject: [PATCH] Improve database locking issues - Added new indexes - Retry if locked - Fix upsert race condition - Various changes to track matching --- headphones/__init__.py | 16 +- headphones/db.py | 122 +++++++++--- headphones/importer.py | 31 ++- headphones/librarysync.py | 374 +++++++++++++++++++++--------------- headphones/postprocessor.py | 6 +- headphones/webserve.py | 69 +++---- 6 files changed, 391 insertions(+), 227 deletions(-) diff --git a/headphones/__init__.py b/headphones/__init__.py index df4fdb07..e470b635 100644 --- a/headphones/__init__.py +++ b/headphones/__init__.py @@ -421,7 +421,6 @@ def dbcheck(): 'CREATE INDEX IF NOT EXISTS tracks_artistid ON tracks(ArtistID ASC)') # Speed up album page - c.execute('CREATE INDEX IF NOT EXISTS have_matched ON have(Matched ASC)') c.execute('CREATE INDEX IF NOT EXISTS allalbums_albumid ON allalbums(AlbumID ASC)') c.execute('CREATE INDEX IF NOT EXISTS alltracks_albumid ON alltracks(AlbumID ASC)') c.execute('CREATE INDEX IF NOT EXISTS releases_albumid ON releases(ReleaseGroupID ASC)') @@ -432,6 +431,21 @@ def dbcheck(): c.execute('CREATE INDEX IF NOT EXISTS alltracks_artistid ON alltracks(ArtistID ASC)') c.execute('CREATE INDEX IF NOT EXISTS descriptions_artistid ON descriptions(ArtistID ASC)') + # Speed up Artist refresh hybrid release + c.execute('CREATE INDEX IF NOT EXISTS albums_releaseid ON albums(ReleaseID ASC)') + c.execute('CREATE INDEX IF NOT EXISTS tracks_releaseid ON tracks(ReleaseID ASC)') + + # Speed up scanning and track matching + c.execute('CREATE INDEX IF NOT EXISTS artist_artistname ON artists(ArtistName COLLATE NOCASE ASC)') + + # General speed up + c.execute('CREATE INDEX IF NOT EXISTS artist_artistsortname ON artists(ArtistSortName COLLATE NOCASE ASC)') + + exists = c.execute('SELECT * FROM pragma_index_info("have_matched_artist_album")').fetchone() + if not exists: + c.execute('CREATE INDEX have_matched_artist_album ON have(Matched ASC, ArtistName COLLATE NOCASE ASC, AlbumTitle COLLATE NOCASE ASC)') + c.execute('DROP INDEX IF EXISTS have_matched') + try: c.execute('SELECT IncludeExtras from artists') except sqlite3.OperationalError: diff --git a/headphones/db.py b/headphones/db.py index 8d864aba..35b447d6 100644 --- a/headphones/db.py +++ b/headphones/db.py @@ -19,6 +19,8 @@ from __future__ import with_statement +import time + import sqlite3 import os @@ -45,36 +47,97 @@ class DBConnection: self.connection = sqlite3.connect(dbFilename(filename), timeout=20) # don't wait for the disk to finish writing self.connection.execute("PRAGMA synchronous = OFF") - # journal disabled since we never do rollbacks + # default set to Write-Ahead Logging WAL self.connection.execute("PRAGMA journal_mode = %s" % headphones.CONFIG.JOURNAL_MODE) # 64mb of cache memory,probably need to make it user configurable self.connection.execute("PRAGMA cache_size=-%s" % (getCacheSize() * 1024)) self.connection.row_factory = sqlite3.Row - def action(self, query, args=None): + def action(self, query, args=None, upsert_insert_qry=None): if query is None: return sqlResult = None + attempts = 0 + dberror = None - try: - with self.connection as c: - if args is None: - sqlResult = c.execute(query) + while attempts < 10: + try: + with self.connection as c: + + # log that previous attempt was locked and we're trying again + if dberror: + if args is None: + logger.debug('SQL: Database was previously locked, trying again. Attempt number %i. Query: %s', attempts + 1, query) + else: + logger.debug('SQL: Database was previously locked, trying again. Attempt number %i. Query: %s. Args: %s', attempts + 1, query, args) + + # debugging + # try: + # explain_query = 'EXPLAIN QUERY PLAN ' + query + # if not args: + # sql_results = c.execute(explain_query) + # else: + # sql_results = c.execute(explain_query, args) + # if not args: + # print(explain_query) + # else: + # print(explain_query + ' ' + str(args)) + # explain_results = sql_results + # for row in explain_results: + # print row + # except Exception as e: + # print(e) + + # Execute query + + # time0 = time.time() + + if args is None: + sqlResult = c.execute(query) + # logger.debug('SQL: ' + query) + else: + sqlResult = c.execute(query, args) + # logger.debug('SQL: %s. Args: %s', query, args) + + # INSERT part of upsert query + if upsert_insert_qry: + sqlResult = c.execute(upsert_insert_qry, args) + # logger.debug('SQL: %s. Args: %s', upsert_insert_qry, args) + + # debugging: loose test to log queries taking longer than 5 seconds + # seconds = time.time() - time0 + # if seconds > 5: + # if args is None: + # logger.debug("SQL: Query ran for %s seconds: %s", seconds, query) + # else: + # logger.debug("SQL: Query ran for %s seconds: %s with args %s", seconds, query, args) + + break + + except sqlite3.OperationalError, e: + if "unable to open database file" in e.message or "database is locked" in e.message: + dberror = e + if args is None: + logger.debug('Database error: %s. Query: %s', e, query) + else: + logger.debug('Database error: %s. Query: %s. Args: %s', e, query, args) + attempts += 1 + time.sleep(1) else: - sqlResult = c.execute(query, args) - - except sqlite3.OperationalError, e: - if "unable to open database file" in e.message or "database is locked" in e.message: - logger.warn('Database Error: %s', e) - else: - logger.error('Database error: %s', e) + logger.error('Database error: %s', e) + raise + except sqlite3.DatabaseError, e: + logger.error('Fatal Error executing %s :: %s', query, e) raise - except sqlite3.DatabaseError, e: - logger.error('Fatal Error executing %s :: %s', query, e) - raise + # log if no results returned due to lock + if not sqlResult and attempts: + if args is None: + logger.warn('SQL: Query failed due to database error: %s. Query: %s', dberror, query) + else: + logger.warn('SQL: Query failed due to database error: %s. Query: %s. Args: %s', dberror, query, args) return sqlResult @@ -88,24 +151,19 @@ class DBConnection: return sqlResults def upsert(self, tableName, valueDict, keyDict): - + """ + Transactions an Update or Insert to a table based on key. + If the table is not updated then the 'WHERE changes' will be 0 and the table inserted + """ def genParams(myDict): return [x + " = ?" for x in myDict.keys()] - changesBefore = self.connection.total_changes + update_query = "UPDATE " + tableName + " SET " + ", ".join(genParams(valueDict)) + " WHERE " + " AND ".join(genParams(keyDict)) - update_query = "UPDATE " + tableName + " SET " + ", ".join( - genParams(valueDict)) + " WHERE " + " AND ".join(genParams(keyDict)) + insert_query = ("INSERT INTO " + tableName + " (" + ", ".join(valueDict.keys() + keyDict.keys()) + ")" + " SELECT " + ", ".join( + ["?"] * len(valueDict.keys() + keyDict.keys())) + " WHERE changes()=0") - self.action(update_query, valueDict.values() + keyDict.values()) - - if self.connection.total_changes == changesBefore: - insert_query = ( - "INSERT INTO " + tableName + " (" + ", ".join( - valueDict.keys() + keyDict.keys()) + ")" + - " VALUES (" + ", ".join(["?"] * len(valueDict.keys() + keyDict.keys())) + ")" - ) - try: - self.action(insert_query, valueDict.values() + keyDict.values()) - except sqlite3.IntegrityError: - logger.info('Queries failed: %s and %s', update_query, insert_query) + try: + self.action(update_query, valueDict.values() + keyDict.values(), upsert_insert_qry=insert_query) + except sqlite3.IntegrityError: + logger.info('Queries failed: %s and %s', update_query, insert_query) diff --git a/headphones/importer.py b/headphones/importer.py index 0426004d..5595d37e 100644 --- a/headphones/importer.py +++ b/headphones/importer.py @@ -89,11 +89,13 @@ def artistlist_to_mbids(artistlist, forced=False): addArtisttoDB(artistid) # Just update the tracks if it does - else: - havetracks = len( - myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=?', [artistid])) + len( - myDB.select('SELECT TrackTitle from have WHERE ArtistName like ?', [artist])) - myDB.action('UPDATE artists SET HaveTracks=? WHERE ArtistID=?', [havetracks, artistid]) + + # not sure this is correct and we're updating during scanning in librarysync + # else: + # havetracks = len( + # myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=?', [artistid])) + len( + # myDB.select('SELECT TrackTitle from have WHERE ArtistName like ?', [artist])) + # myDB.action('UPDATE artists SET HaveTracks=? WHERE ArtistID=?', [havetracks, artistid]) # Delete it from the New Artists if the request came from there if forced: @@ -105,7 +107,7 @@ def artistlist_to_mbids(artistlist, forced=False): try: lastfm.getSimilar() except Exception as e: - logger.warn('Failed to update arist information from Last.fm: %s' % e) + logger.warn('Failed to update artist information from Last.fm: %s' % e) def addArtistIDListToDB(artistidlist): @@ -309,7 +311,7 @@ def addArtisttoDB(artistid, extrasonly=False, forcefull=False, type="artist"): # This will be used later to build a hybrid release fullreleaselist = [] # Search for releases within a release group - find_hybrid_releases = myDB.action("SELECT * from allalbums WHERE AlbumID=?", + find_hybrid_releases = myDB.select("SELECT * from allalbums WHERE AlbumID=?", [rg['id']]) # Build the dictionary for the fullreleaselist @@ -516,7 +518,10 @@ def addArtisttoDB(artistid, extrasonly=False, forcefull=False, type="artist"): logger.info( u"[%s] Seeing if we need album art for %s" % (artist['artist_name'], rg['title'])) - cache.getThumb(AlbumID=rg['id']) + try: + cache.getThumb(AlbumID=rg['id']) + except Exception as e: + logger.error("Error getting album art: %s", e) # Start a search for the album if it's new, hasn't been marked as # downloaded and autowant_all is selected. This search is deferred, @@ -532,10 +537,16 @@ def addArtisttoDB(artistid, extrasonly=False, forcefull=False, type="artist"): finalize_update(artistid, artist['artist_name'], errors) logger.info(u"Seeing if we need album art for: %s" % artist['artist_name']) - cache.getThumb(ArtistID=artistid) + try: + cache.getThumb(ArtistID=artistid) + except Exception as e: + logger.error("Error getting album art: %s", e) logger.info(u"Fetching Metacritic reviews for: %s" % artist['artist_name']) - metacritic.update(artistid, artist['artist_name'], artist['releasegroups']) + try: + metacritic.update(artistid, artist['artist_name'], artist['releasegroups']) + except Exception as e: + logger.error("Error getting Metacritic reviews: %s", e) if errors: logger.info( diff --git a/headphones/librarysync.py b/headphones/librarysync.py index ba246ff0..2791ea5d 100644 --- a/headphones/librarysync.py +++ b/headphones/librarysync.py @@ -50,27 +50,43 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, logger.info('Scanning music directory: %s' % dir.decode(headphones.SYS_ENCODING, 'replace')) if not append: - # Clean up bad filepaths - tracks = myDB.select( - 'SELECT Location from alltracks WHERE Location IS NOT NULL UNION SELECT Location from tracks WHERE Location IS NOT NULL') + # Clean up bad filepaths. Queries can take some time, ensure all results are loaded before processing + if ArtistID: + tracks = myDB.action( + 'SELECT Location FROM alltracks WHERE ArtistID = ? AND Location IS NOT NULL UNION SELECT Location FROM tracks WHERE ArtistID = ? AND Location ' + 'IS NOT NULL', + [ArtistID, ArtistID]) + else: + tracks = myDB.action( + 'SELECT Location FROM alltracks WHERE Location IS NOT NULL UNION SELECT Location FROM tracks WHERE Location IS NOT NULL') + + locations = [] for track in tracks: - encoded_track_string = track['Location'].encode(headphones.SYS_ENCODING, 'replace') + locations.append(track['Location']) + for location in locations: + encoded_track_string = location.encode(headphones.SYS_ENCODING, 'replace') if not os.path.isfile(encoded_track_string): myDB.action('UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE Location=?', - [None, None, None, track['Location']]) + [None, None, None, location]) myDB.action('UPDATE alltracks SET Location=?, BitRate=?, Format=? WHERE Location=?', - [None, None, None, track['Location']]) + [None, None, None, location]) - del_have_tracks = myDB.select('SELECT Location, Matched, ArtistName from have') + if ArtistName: + del_have_tracks = myDB.select('SELECT Location, Matched, ArtistName FROM have WHERE ArtistName = ? COLLATE NOCASE', [ArtistName]) + else: + del_have_tracks = myDB.select('SELECT Location, Matched, ArtistName FROM have') + locations = [] for track in del_have_tracks: - encoded_track_string = track['Location'].encode(headphones.SYS_ENCODING, 'replace') + locations.append([track['Location'], track['ArtistName']]) + for location in locations: + encoded_track_string = location[0].encode(headphones.SYS_ENCODING, 'replace') if not os.path.isfile(encoded_track_string): - if track['ArtistName']: + if location[1]: # Make sure deleted files get accounted for when updating artist track counts - new_artists.append(track['ArtistName']) - myDB.action('DELETE FROM have WHERE Location=?', [track['Location']]) + new_artists.append(location[1]) + myDB.action('DELETE FROM have WHERE Location=?', [location[0]]) logger.info( 'File %s removed from Headphones, as it is no longer on disk' % encoded_track_string.decode( headphones.SYS_ENCODING, 'replace')) @@ -216,6 +232,8 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, song_count = 0 latest_artist = [] last_completion_percentage = 0 + prev_artist_name = None + artistid = None for song in song_list: @@ -237,87 +255,68 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, # ARE NEVER FOUND, AND THE OTHER CLAUSES WERE NEVER HIT. FURTHERMORE, OTHER MATCHING FUNCTIONS IN THIS PROGRAM # (IMPORTER.PY, MB.PY) SIMPLY DO A [ARTIST/ALBUM/TRACK] OR CLEANNAME MATCH, SO IT'S ALL CONSISTENT. - if song['ArtistName'] and song['AlbumTitle'] and song['TrackTitle']: + albumid = None - track = myDB.action( - 'SELECT ArtistName, AlbumTitle, TrackTitle, AlbumID from tracks WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', - [song['ArtistName'], song['AlbumTitle'], song['TrackTitle']]).fetchone() - have_updated = False - if track: - controlValueDict = {'ArtistName': track['ArtistName'], - 'AlbumTitle': track['AlbumTitle'], - 'TrackTitle': track['TrackTitle']} - newValueDict = {'Location': song['Location'], - 'BitRate': song['BitRate'], - 'Format': song['Format']} - myDB.upsert("tracks", newValueDict, controlValueDict) + if song['ArtistName'] and song['CleanName']: + artist_name = song['ArtistName'] + clean_name = song['CleanName'] - controlValueDict2 = {'Location': song['Location']} - newValueDict2 = {'Matched': track['AlbumID']} - myDB.upsert("have", newValueDict2, controlValueDict2) - have_updated = True - else: - track = myDB.action('SELECT CleanName, AlbumID from tracks WHERE CleanName LIKE ?', - [song['CleanName']]).fetchone() + # Only update if artist is in the db + if artist_name != prev_artist_name: + prev_artist_name = artist_name + artistid = None + artist_lookup = '"' + artist_name + '"' + + dbartist = myDB.select('SELECT DISTINCT ArtistID, ArtistName FROM artists WHERE ArtistName LIKE ' + artist_lookup + '') + if not dbartist: + dbartist = myDB.select('SELECT DISTINCT ArtistID, ArtistName FROM tracks WHERE CleanName = ?', [clean_name]) + if not dbartist: + dbartist = myDB.select('SELECT DISTINCT ArtistID, ArtistName FROM alltracks WHERE CleanName = ?', [clean_name]) + if not dbartist: + clean_artist = helpers.clean_name(artist_name) + if clean_artist: + dbartist = myDB.select('SELECT DISTINCT ArtistID, ArtistName FROM tracks WHERE CleanName >= ? and CleanName < ?', + [clean_artist, clean_artist + '{']) + if not dbartist: + dbartist = myDB.select('SELECT DISTINCT ArtistID, ArtistName FROM alltracks WHERE CleanName >= ? and CleanName < ?', + [clean_artist, clean_artist + '{']) + + if dbartist: + artistid = dbartist[0][0] + + if artistid: + + # This was previously using Artist, Album, Title with a SELECT LIKE ? and was not using an index + # (Possible issue: https://stackoverflow.com/questions/37845854/python-sqlite3-not-using-index-with-like) + # Now selects/updates using CleanName index (may have to revert if not working) + + # matching on CleanName should be enough, ensure it's the same artist just in case + + # Update tracks + track = myDB.action('SELECT AlbumID, ArtistName FROM tracks WHERE CleanName = ? AND ArtistID = ?', [clean_name, artistid]).fetchone() if track: - controlValueDict = {'CleanName': track['CleanName']} - newValueDict = {'Location': song['Location'], - 'BitRate': song['BitRate'], - 'Format': song['Format']} - myDB.upsert("tracks", newValueDict, controlValueDict) + albumid = track['AlbumID'] + myDB.action( + 'UPDATE tracks SET Location = ?, BitRate = ?, Format = ? WHERE CleanName = ? AND ArtistID = ?', + [song['Location'], song['BitRate'], song['Format'], clean_name, artistid]) - controlValueDict2 = {'Location': song['Location']} - newValueDict2 = {'Matched': track['AlbumID']} - myDB.upsert("have", newValueDict2, controlValueDict2) - have_updated = True - else: - controlValueDict2 = {'Location': song['Location']} - newValueDict2 = {'Matched': "Failed"} - myDB.upsert("have", newValueDict2, controlValueDict2) - have_updated = True - - alltrack = myDB.action( - 'SELECT ArtistName, AlbumTitle, TrackTitle, AlbumID from alltracks WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', - [song['ArtistName'], song['AlbumTitle'], song['TrackTitle']]).fetchone() - if alltrack: - controlValueDict = {'ArtistName': alltrack['ArtistName'], - 'AlbumTitle': alltrack['AlbumTitle'], - 'TrackTitle': alltrack['TrackTitle']} - newValueDict = {'Location': song['Location'], - 'BitRate': song['BitRate'], - 'Format': song['Format']} - myDB.upsert("alltracks", newValueDict, controlValueDict) - - controlValueDict2 = {'Location': song['Location']} - newValueDict2 = {'Matched': alltrack['AlbumID']} - myDB.upsert("have", newValueDict2, controlValueDict2) - else: - alltrack = myDB.action( - 'SELECT CleanName, AlbumID from alltracks WHERE CleanName LIKE ?', - [song['CleanName']]).fetchone() + # Update alltracks + alltrack = myDB.action('SELECT AlbumID, ArtistName FROM alltracks WHERE CleanName = ? AND ArtistID = ?', [clean_name, artistid]).fetchone() if alltrack: - controlValueDict = {'CleanName': alltrack['CleanName']} - newValueDict = {'Location': song['Location'], - 'BitRate': song['BitRate'], - 'Format': song['Format']} - myDB.upsert("alltracks", newValueDict, controlValueDict) - - controlValueDict2 = {'Location': song['Location']} - newValueDict2 = {'Matched': alltrack['AlbumID']} - myDB.upsert("have", newValueDict2, controlValueDict2) - else: - # alltracks may not exist if adding album manually, have should only be set to failed if not already updated in tracks - if not have_updated: - controlValueDict2 = {'Location': song['Location']} - newValueDict2 = {'Matched': "Failed"} - myDB.upsert("have", newValueDict2, controlValueDict2) + albumid = alltrack['AlbumID'] + myDB.action( + 'UPDATE alltracks SET Location = ?, BitRate = ?, Format = ? WHERE CleanName = ? AND ArtistID = ?', + [song['Location'], song['BitRate'], song['Format'], clean_name, artistid]) + # Update have + controlValueDict2 = {'Location': song['Location']} + if albumid: + newValueDict2 = {'Matched': albumid} else: - controlValueDict2 = {'Location': song['Location']} newValueDict2 = {'Matched': "Failed"} - myDB.upsert("have", newValueDict2, controlValueDict2) + myDB.upsert("have", newValueDict2, controlValueDict2) - # myDB.action('INSERT INTO have (ArtistName, AlbumTitle, TrackNumber, TrackTitle, TrackLength, BitRate, Genre, Date, TrackID, Location, CleanName, Format) VALUES( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', [song['ArtistName'], song['AlbumTitle'], song['TrackNumber'], song['TrackTitle'], song['TrackLength'], song['BitRate'], song['Genre'], song['Date'], song['TrackID'], song['Location'], CleanName, song['Format']]) + # myDB.action('INSERT INTO have (ArtistName, AlbumTitle, TrackNumber, TrackTitle, TrackLength, BitRate, Genre, Date, TrackID, Location, CleanName, Format) VALUES( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', [song['ArtistName'], song['AlbumTitle'], song['TrackNumber'], song['TrackTitle'], song['TrackLength'], song['BitRate'], song['Genre'], song['Date'], song['TrackID'], song['Location'], CleanName, song['Format']]) logger.info('Completed matching tracks from directory: %s' % dir.decode(headphones.SYS_ENCODING, 'replace')) @@ -327,49 +326,98 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, # Clean up the new artist list unique_artists = {}.fromkeys(new_artists).keys() - current_artists = myDB.select('SELECT ArtistName, ArtistID from artists') - # There was a bug where artists with special characters (-,') would show up in new artists. - artist_list = [ - x for x in unique_artists - if helpers.clean_name(x).lower() not in [ - helpers.clean_name(y[0]).lower() - for y in current_artists - ] - ] - artists_checked = [ - x for x in unique_artists - if helpers.clean_name(x).lower() in [ - helpers.clean_name(y[0]).lower() - for y in current_artists - ] - ] + # # Don't think we need to do this, check the db instead below + # + # # artist scan + # if ArtistName: + # current_artists = [[ArtistName]] + # # directory scan + # else: + # current_artists = myDB.select('SELECT ArtistName, ArtistID FROM artists WHERE ArtistName IS NOT NULL') + # + # # There was a bug where artists with special characters (-,') would show up in new artists. + # + # # artist_list = scanned artists not in the db + # artist_list = [ + # x for x in unique_artists + # if helpers.clean_name(x).lower() not in [ + # helpers.clean_name(y[0]).lower() + # for y in current_artists + # ] + # ] + # + # # artists_checked = scanned artists that exist in the db + # artists_checked = [ + # x for x in unique_artists + # if helpers.clean_name(x).lower() in [ + # helpers.clean_name(y[0]).lower() + # for y in current_artists + # ] + # ] - # Update track counts - for artist in artists_checked: - # Have tracks are selected from tracks table and not all tracks because of duplicates - # We update the track count upon an album switch to compliment this - havetracks = ( - len(myDB.select( - 'SELECT TrackTitle from tracks WHERE ArtistName like ? AND Location IS NOT NULL', - [artist])) + len(myDB.select( - 'SELECT TrackTitle from have WHERE ArtistName like ? AND Matched = "Failed"', - [artist])) - ) - # Note: some people complain about having "artist have tracks" > # of tracks total in artist official releases - # (can fix by getting rid of second len statement) - myDB.action('UPDATE artists SET HaveTracks=? WHERE ArtistName=?', [havetracks, artist]) + new_artist_list = [] - logger.info('Found %i new artists' % len(artist_list)) + for artist in unique_artists: - if artist_list: + # check if artist is already in the db + artist_lookup = '"' + artist + '"' + dbartist = myDB.select('SELECT DISTINCT ArtistID, ArtistName FROM artists WHERE ArtistName LIKE ' + artist_lookup + '') + if not dbartist: + clean_artist = helpers.clean_name(artist) + if clean_artist: + dbartist = myDB.select('SELECT DISTINCT ArtistID, ArtistName FROM tracks WHERE CleanName >= ? and CleanName < ?', + [clean_artist, clean_artist + '{']) + if not dbartist: + dbartist = myDB.select('SELECT DISTINCT ArtistID, ArtistName FROM alltracks WHERE CleanName >= ? and CleanName < ?', + [clean_artist, clean_artist + '{']) + + # new artist not in db, add to list + if not dbartist: + new_artist_list.append(artist) + else: + + # artist in db, update have track counts + artistid = dbartist[0][0] + + # Have tracks are selected from tracks table and not all tracks because of duplicates + # We update the track count upon an album switch to compliment this + + # havetracks = ( + # len(myDB.select( + # 'SELECT TrackTitle from tracks WHERE ArtistName like ? AND Location IS NOT NULL', + # [artist])) + len(myDB.select( + # 'SELECT TrackTitle from have WHERE ArtistName like ? AND Matched = "Failed"', + # [artist])) + # ) + + havetracks = ( + len(myDB.select( + 'SELECT ArtistID From tracks WHERE ArtistID = ? AND Location IS NOT NULL', + [artistid])) + len(myDB.select( + 'SELECT ArtistName FROM have WHERE ArtistName LIKE ' + artist_lookup + ' AND Matched = "Failed"')) + ) + + # Note: some people complain about having "artist have tracks" > # of tracks total in artist official releases + # (can fix by getting rid of second len statement) + + myDB.action('UPDATE artists SET HaveTracks = ? WHERE ArtistID = ?', [havetracks, artistid]) + + # Update albums to downloaded + if havetracks: + update_album_status(ArtistID=artistid) + + logger.info('Found %i new artists' % len(new_artist_list)) + + # Add scanned artists not in the db + if new_artist_list: if headphones.CONFIG.AUTO_ADD_ARTISTS: - logger.info('Importing %i new artists' % len(artist_list)) - importer.artistlist_to_mbids(artist_list) + logger.info('Importing %i new artists' % len(new_artist_list)) + importer.artistlist_to_mbids(new_artist_list) else: logger.info('To add these artists, go to Manage->Manage New Artists') # myDB.action('DELETE from newartists') - for artist in artist_list: + for artist in new_artist_list: myDB.action('INSERT OR IGNORE INTO newartists VALUES (?)', [artist]) if headphones.CONFIG.DETECT_BITRATE and bitrates: @@ -379,50 +427,73 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, # If we're appending a new album to the database, update the artists total track counts logger.info('Updating artist track counts') + artist_lookup = '"' + ArtistName + '"' havetracks = len( - myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=? AND Location IS NOT NULL', + myDB.select('SELECT ArtistID FROM tracks WHERE ArtistID = ? AND Location IS NOT NULL', [ArtistID])) + len(myDB.select( - 'SELECT TrackTitle from have WHERE ArtistName like ? AND Matched = "Failed"', - [ArtistName])) + 'SELECT ArtistName FROM have WHERE ArtistName LIKE ' + artist_lookup + ' AND Matched = "Failed"')) myDB.action('UPDATE artists SET HaveTracks=? WHERE ArtistID=?', [havetracks, ArtistID]) - if not append: - update_album_status() + # Moved above to call for each artist + # if not append: + # update_album_status() if not append and not artistScan: lastfm.getSimilar() - logger.info('Library scan complete') + if ArtistName: + logger.info('Scanning complete for artist: %s', ArtistName) + else: + logger.info('Library scan complete') # ADDED THIS SECTION TO MARK ALBUMS AS DOWNLOADED IF ARTISTS ARE ADDED EN MASSE BEFORE LIBRARY IS SCANNED +# Think the above comment relates to calling from Manage Unmatched -def update_album_status(AlbumID=None): +# This used to select and update all albums and would clobber the db, changed to run by ArtistID. + +def update_album_status(AlbumID=None, ArtistID=None): myDB = db.DBConnection() - logger.info('Counting matched tracks to mark albums as skipped/downloaded') + # logger.info('Counting matched tracks to mark albums as skipped/downloaded') + if AlbumID: album_status_updater = myDB.action( - 'SELECT AlbumID, AlbumTitle, Status from albums WHERE AlbumID=?', [AlbumID]) + 'SELECT' + ' a.AlbumID, a.ArtistName, a.AlbumTitle, a.Status, AVG(t.Location IS NOT NULL) * 100 AS album_completion ' + 'FROM' + ' albums AS a ' + 'JOIN tracks AS t ON t.AlbumID = a.AlbumID ' + 'WHERE' + ' a.AlbumID = ? AND a.Status != "Downloaded" ' + 'GROUP BY' + ' a.AlbumID ' + 'HAVING' + ' AVG(t.Location IS NOT NULL) * 100 >= ?', + [AlbumID, headphones.CONFIG.ALBUM_COMPLETION_PCT] + ) else: - album_status_updater = myDB.action('SELECT AlbumID, AlbumTitle, Status from albums') - for album in album_status_updater: - track_counter = myDB.action('SELECT Location from tracks where AlbumID=?', - [album['AlbumID']]) - total_tracks = 0 - have_tracks = 0 - for track in track_counter: - total_tracks += 1 - if track['Location']: - have_tracks += 1 - if total_tracks != 0: - album_completion = float(float(have_tracks) / float(total_tracks)) * 100 - else: - album_completion = 0 - logger.info('Album %s does not have any tracks in database' % album['AlbumTitle']) + album_status_updater = myDB.action( + 'SELECT' + ' a.AlbumID, a.ArtistID, a.ArtistName, a.AlbumTitle, a.Status, AVG(t.Location IS NOT NULL) * 100 AS album_completion ' + 'FROM' + ' albums AS a ' + 'JOIN tracks AS t ON t.AlbumID = a.AlbumID ' + 'WHERE' + ' a.ArtistID = ? AND a.Status != "Downloaded" ' + 'GROUP BY' + ' a.AlbumID ' + 'HAVING' + ' AVG(t.Location IS NOT NULL) * 100 >= ?', + [ArtistID, headphones.CONFIG.ALBUM_COMPLETION_PCT] + ) - if album_completion >= headphones.CONFIG.ALBUM_COMPLETION_PCT: - new_album_status = "Downloaded" + new_album_status = "Downloaded" + + albums = [] + for album in album_status_updater: + albums.append([album['AlbumID'], album['ArtistName'], album['AlbumTitle']]) + for album in albums: # I don't think we want to change Downloaded->Skipped..... # I think we can only automatically change Skipped->Downloaded when updating @@ -433,10 +504,13 @@ def update_album_status(AlbumID=None): # new_album_status = "Skipped" # else: # new_album_status = album['Status'] - else: - new_album_status = album['Status'] + # else: + # new_album_status = album['Status'] + # + # myDB.upsert("albums", {'Status': new_album_status}, {'AlbumID': album['AlbumID']}) + # if new_album_status != album['Status']: + # logger.info('Album %s changed to %s' % (album['AlbumTitle'], new_album_status)) + # logger.info('Album status update complete') - myDB.upsert("albums", {'Status': new_album_status}, {'AlbumID': album['AlbumID']}) - if new_album_status != album['Status']: - logger.info('Album %s changed to %s' % (album['AlbumTitle'], new_album_status)) - logger.info('Album status update complete') + myDB.action('UPDATE albums SET Status = ? WHERE AlbumID = ?', [new_album_status, album[0]]) + logger.info('Album: %s - %s. Status updated to %s' % (album[1], album[2], new_album_status)) diff --git a/headphones/postprocessor.py b/headphones/postprocessor.py index ed3c136b..ab801a45 100755 --- a/headphones/postprocessor.py +++ b/headphones/postprocessor.py @@ -183,13 +183,17 @@ def verify(albumid, albumpath, Kind=None, forced=False, keep_original_folder=Fal controlValueDict = {"TrackID": track['id'], "AlbumID": albumid} + clean_name = helpers.clean_name( + release_dict['artist_name'] + ' ' + release_dict['title'] + ' ' + track['title']) + newValueDict = {"ArtistID": release_dict['artist_id'], "ArtistName": release_dict['artist_name'], "AlbumTitle": release_dict['title'], "AlbumASIN": release_dict['asin'], "TrackTitle": track['title'], "TrackDuration": track['duration'], - "TrackNumber": track['number'] + "TrackNumber": track['number'], + "CleanName": clean_name } myDB.upsert("tracks", newValueDict, controlValueDict) diff --git a/headphones/webserve.py b/headphones/webserve.py index 62548e6c..1880e566 100644 --- a/headphones/webserve.py +++ b/headphones/webserve.py @@ -640,6 +640,7 @@ class WebInterface(object): 'SELECT Matched, CleanName, Location, BitRate, Format FROM have WHERE ArtistName=?', [existing_artist]) update_count = 0 + artist_id = None for entry in have_tracks: old_clean_filename = entry['CleanName'] if old_clean_filename.startswith(existing_artist_clean): @@ -648,31 +649,30 @@ class WebInterface(object): myDB.action( 'UPDATE have SET CleanName=? WHERE ArtistName=? AND CleanName=?', [new_clean_filename, existing_artist, old_clean_filename]) - controlValueDict = {"CleanName": new_clean_filename} - newValueDict = {"Location": entry['Location'], - "BitRate": entry['BitRate'], - "Format": entry['Format'] - } + # Attempt to match tracks with new CleanName match_alltracks = myDB.action( - 'SELECT CleanName from alltracks WHERE CleanName=?', + 'SELECT CleanName FROM alltracks WHERE CleanName = ?', [new_clean_filename]).fetchone() if match_alltracks: - myDB.upsert("alltracks", newValueDict, controlValueDict) + myDB.action( + 'UPDATE alltracks SET Location = ?, BitRate = ?, Format = ? WHERE CleanName = ?', + [entry['Location'], entry['BitRate'], entry['Format'], new_clean_filename]) + match_tracks = myDB.action( - 'SELECT CleanName, AlbumID from tracks WHERE CleanName=?', + 'SELECT ArtistID, CleanName, AlbumID FROM tracks WHERE CleanName = ?', [new_clean_filename]).fetchone() if match_tracks: - myDB.upsert("tracks", newValueDict, controlValueDict) + myDB.action( + 'UPDATE tracks SET Location = ?, BitRate = ?, Format = ? WHERE CleanName = ?', + [entry['Location'], entry['BitRate'], entry['Format'], new_clean_filename]) myDB.action('UPDATE have SET Matched="Manual" WHERE CleanName=?', [new_clean_filename]) update_count += 1 - # This was throwing errors and I don't know why, but it seems to be working fine. - # else: - # logger.info("There was an error modifying Artist %s. This should not have happened" % existing_artist) + artist_id = match_tracks['Artist_ID'] logger.info("Manual matching yielded %s new matches for Artist: %s" % (update_count, new_artist)) - if update_count > 0: - librarysync.update_album_status() + if artist_id: + librarysync.update_album_status(ArtistID=artist_id) else: logger.info( "Artist %s already named appropriately; nothing to modify" % existing_artist) @@ -698,29 +698,28 @@ class WebInterface(object): 'UPDATE have SET CleanName=? WHERE ArtistName=? AND AlbumTitle=? AND CleanName=?', [new_clean_filename, existing_artist, existing_album, old_clean_filename]) - controlValueDict = {"CleanName": new_clean_filename} - newValueDict = {"Location": entry['Location'], - "BitRate": entry['BitRate'], - "Format": entry['Format'] - } + # Attempt to match tracks with new CleanName match_alltracks = myDB.action( - 'SELECT CleanName from alltracks WHERE CleanName=?', + 'SELECT CleanName FROM alltracks WHERE CleanName = ?', [new_clean_filename]).fetchone() if match_alltracks: - myDB.upsert("alltracks", newValueDict, controlValueDict) + myDB.action( + 'UPDATE alltracks SET Location = ?, BitRate = ?, Format = ? WHERE CleanName = ?', + [entry['Location'], entry['BitRate'], entry['Format'], new_clean_filename]) + match_tracks = myDB.action( - 'SELECT CleanName, AlbumID from tracks WHERE CleanName=?', + 'SELECT CleanName, AlbumID FROM tracks WHERE CleanName = ?', [new_clean_filename]).fetchone() if match_tracks: - myDB.upsert("tracks", newValueDict, controlValueDict) + myDB.action( + 'UPDATE tracks SET Location = ?, BitRate = ?, Format = ? WHERE CleanName = ?', + [entry['Location'], entry['BitRate'], entry['Format'], new_clean_filename]) myDB.action('UPDATE have SET Matched="Manual" WHERE CleanName=?', [new_clean_filename]) album_id = match_tracks['AlbumID'] update_count += 1 - # This was throwing errors and I don't know why, but it seems to be working fine. - # else: - # logger.info("There was an error modifying Artist %s / Album %s with clean name %s" % (existing_artist, existing_album, existing_clean_string)) + logger.info("Manual matching yielded %s new matches for Artist: %s / Album: %s" % ( update_count, new_artist, new_album)) if update_count > 0: @@ -785,25 +784,29 @@ class WebInterface(object): album = tracks['AlbumTitle'] track_title = tracks['TrackTitle'] if tracks['CleanName'] != original_clean: + artist_id_check = myDB.action('SELECT ArtistID FROM tracks WHERE CleanName = ?', + [tracks['CleanName']]).fetchone() + if artist_id_check: + artist_id = artist_id_check[0] myDB.action( - 'UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE CleanName=?', + 'UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE CleanName = ?', [None, None, None, tracks['CleanName']]) myDB.action( - 'UPDATE alltracks SET Location=?, BitRate=?, Format=? WHERE CleanName=?', + 'UPDATE alltracks SET Location=?, BitRate=?, Format=? WHERE CleanName = ?', [None, None, None, tracks['CleanName']]) myDB.action( 'UPDATE have SET CleanName=?, Matched="Failed" WHERE ArtistName=? AND AlbumTitle=? AND TrackTitle=?', (original_clean, artist, album, track_title)) update_count += 1 if update_count > 0: - librarysync.update_album_status() + librarysync.update_album_status(ArtistID=artist_id) logger.info("Artist: %s successfully restored to unmatched list" % artist) elif action == "unmatchAlbum": artist = existing_artist album = existing_album update_clean = myDB.select( - 'SELECT ArtistName, AlbumTitle, TrackTitle, CleanName, Matched from have WHERE ArtistName=? AND AlbumTitle=?', + 'SELECT ArtistName, AlbumTitle, TrackTitle, CleanName, Matched FROM have WHERE ArtistName=? AND AlbumTitle=?', (artist, album)) update_count = 0 for tracks in update_clean: @@ -812,15 +815,15 @@ class WebInterface(object): 'TrackTitle']).lower() track_title = tracks['TrackTitle'] if tracks['CleanName'] != original_clean: - album_id_check = myDB.action('SELECT AlbumID from tracks WHERE CleanName=?', + album_id_check = myDB.action('SELECT AlbumID FROM tracks WHERE CleanName = ?', [tracks['CleanName']]).fetchone() if album_id_check: album_id = album_id_check[0] myDB.action( - 'UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE CleanName=?', + 'UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE CleanName = ?', [None, None, None, tracks['CleanName']]) myDB.action( - 'UPDATE alltracks SET Location=?, BitRate=?, Format=? WHERE CleanName=?', + 'UPDATE alltracks SET Location=?, BitRate=?, Format=? WHERE CleanName = ?', [None, None, None, tracks['CleanName']]) myDB.action( 'UPDATE have SET CleanName=?, Matched="Failed" WHERE ArtistName=? AND AlbumTitle=? AND TrackTitle=?',