Improve database locking issues

- Added new indexes
- Retry if locked
- Fix upsert race condition
- Various changes to track matching
This commit is contained in:
Ade
2018-08-15 14:51:27 +12:00
parent 287278c049
commit 693e5b1624
6 changed files with 391 additions and 227 deletions

View File

@@ -421,7 +421,6 @@ def dbcheck():
'CREATE INDEX IF NOT EXISTS tracks_artistid ON tracks(ArtistID ASC)')
# Speed up album page
c.execute('CREATE INDEX IF NOT EXISTS have_matched ON have(Matched ASC)')
c.execute('CREATE INDEX IF NOT EXISTS allalbums_albumid ON allalbums(AlbumID ASC)')
c.execute('CREATE INDEX IF NOT EXISTS alltracks_albumid ON alltracks(AlbumID ASC)')
c.execute('CREATE INDEX IF NOT EXISTS releases_albumid ON releases(ReleaseGroupID ASC)')
@@ -432,6 +431,21 @@ def dbcheck():
c.execute('CREATE INDEX IF NOT EXISTS alltracks_artistid ON alltracks(ArtistID ASC)')
c.execute('CREATE INDEX IF NOT EXISTS descriptions_artistid ON descriptions(ArtistID ASC)')
# Speed up Artist refresh hybrid release
c.execute('CREATE INDEX IF NOT EXISTS albums_releaseid ON albums(ReleaseID ASC)')
c.execute('CREATE INDEX IF NOT EXISTS tracks_releaseid ON tracks(ReleaseID ASC)')
# Speed up scanning and track matching
c.execute('CREATE INDEX IF NOT EXISTS artist_artistname ON artists(ArtistName COLLATE NOCASE ASC)')
# General speed up
c.execute('CREATE INDEX IF NOT EXISTS artist_artistsortname ON artists(ArtistSortName COLLATE NOCASE ASC)')
exists = c.execute('SELECT * FROM pragma_index_info("have_matched_artist_album")').fetchone()
if not exists:
c.execute('CREATE INDEX have_matched_artist_album ON have(Matched ASC, ArtistName COLLATE NOCASE ASC, AlbumTitle COLLATE NOCASE ASC)')
c.execute('DROP INDEX IF EXISTS have_matched')
try:
c.execute('SELECT IncludeExtras from artists')
except sqlite3.OperationalError:

View File

@@ -19,6 +19,8 @@
from __future__ import with_statement
import time
import sqlite3
import os
@@ -45,36 +47,97 @@ class DBConnection:
self.connection = sqlite3.connect(dbFilename(filename), timeout=20)
# don't wait for the disk to finish writing
self.connection.execute("PRAGMA synchronous = OFF")
# journal disabled since we never do rollbacks
# default set to Write-Ahead Logging WAL
self.connection.execute("PRAGMA journal_mode = %s" % headphones.CONFIG.JOURNAL_MODE)
# 64mb of cache memory,probably need to make it user configurable
self.connection.execute("PRAGMA cache_size=-%s" % (getCacheSize() * 1024))
self.connection.row_factory = sqlite3.Row
def action(self, query, args=None):
def action(self, query, args=None, upsert_insert_qry=None):
if query is None:
return
sqlResult = None
attempts = 0
dberror = None
try:
with self.connection as c:
if args is None:
sqlResult = c.execute(query)
while attempts < 10:
try:
with self.connection as c:
# log that previous attempt was locked and we're trying again
if dberror:
if args is None:
logger.debug('SQL: Database was previously locked, trying again. Attempt number %i. Query: %s', attempts + 1, query)
else:
logger.debug('SQL: Database was previously locked, trying again. Attempt number %i. Query: %s. Args: %s', attempts + 1, query, args)
# debugging
# try:
# explain_query = 'EXPLAIN QUERY PLAN ' + query
# if not args:
# sql_results = c.execute(explain_query)
# else:
# sql_results = c.execute(explain_query, args)
# if not args:
# print(explain_query)
# else:
# print(explain_query + ' ' + str(args))
# explain_results = sql_results
# for row in explain_results:
# print row
# except Exception as e:
# print(e)
# Execute query
# time0 = time.time()
if args is None:
sqlResult = c.execute(query)
# logger.debug('SQL: ' + query)
else:
sqlResult = c.execute(query, args)
# logger.debug('SQL: %s. Args: %s', query, args)
# INSERT part of upsert query
if upsert_insert_qry:
sqlResult = c.execute(upsert_insert_qry, args)
# logger.debug('SQL: %s. Args: %s', upsert_insert_qry, args)
# debugging: loose test to log queries taking longer than 5 seconds
# seconds = time.time() - time0
# if seconds > 5:
# if args is None:
# logger.debug("SQL: Query ran for %s seconds: %s", seconds, query)
# else:
# logger.debug("SQL: Query ran for %s seconds: %s with args %s", seconds, query, args)
break
except sqlite3.OperationalError, e:
if "unable to open database file" in e.message or "database is locked" in e.message:
dberror = e
if args is None:
logger.debug('Database error: %s. Query: %s', e, query)
else:
logger.debug('Database error: %s. Query: %s. Args: %s', e, query, args)
attempts += 1
time.sleep(1)
else:
sqlResult = c.execute(query, args)
except sqlite3.OperationalError, e:
if "unable to open database file" in e.message or "database is locked" in e.message:
logger.warn('Database Error: %s', e)
else:
logger.error('Database error: %s', e)
logger.error('Database error: %s', e)
raise
except sqlite3.DatabaseError, e:
logger.error('Fatal Error executing %s :: %s', query, e)
raise
except sqlite3.DatabaseError, e:
logger.error('Fatal Error executing %s :: %s', query, e)
raise
# log if no results returned due to lock
if not sqlResult and attempts:
if args is None:
logger.warn('SQL: Query failed due to database error: %s. Query: %s', dberror, query)
else:
logger.warn('SQL: Query failed due to database error: %s. Query: %s. Args: %s', dberror, query, args)
return sqlResult
@@ -88,24 +151,19 @@ class DBConnection:
return sqlResults
def upsert(self, tableName, valueDict, keyDict):
"""
Transactions an Update or Insert to a table based on key.
If the table is not updated then the 'WHERE changes' will be 0 and the table inserted
"""
def genParams(myDict):
return [x + " = ?" for x in myDict.keys()]
changesBefore = self.connection.total_changes
update_query = "UPDATE " + tableName + " SET " + ", ".join(genParams(valueDict)) + " WHERE " + " AND ".join(genParams(keyDict))
update_query = "UPDATE " + tableName + " SET " + ", ".join(
genParams(valueDict)) + " WHERE " + " AND ".join(genParams(keyDict))
insert_query = ("INSERT INTO " + tableName + " (" + ", ".join(valueDict.keys() + keyDict.keys()) + ")" + " SELECT " + ", ".join(
["?"] * len(valueDict.keys() + keyDict.keys())) + " WHERE changes()=0")
self.action(update_query, valueDict.values() + keyDict.values())
if self.connection.total_changes == changesBefore:
insert_query = (
"INSERT INTO " + tableName + " (" + ", ".join(
valueDict.keys() + keyDict.keys()) + ")" +
" VALUES (" + ", ".join(["?"] * len(valueDict.keys() + keyDict.keys())) + ")"
)
try:
self.action(insert_query, valueDict.values() + keyDict.values())
except sqlite3.IntegrityError:
logger.info('Queries failed: %s and %s', update_query, insert_query)
try:
self.action(update_query, valueDict.values() + keyDict.values(), upsert_insert_qry=insert_query)
except sqlite3.IntegrityError:
logger.info('Queries failed: %s and %s', update_query, insert_query)

View File

@@ -89,11 +89,13 @@ def artistlist_to_mbids(artistlist, forced=False):
addArtisttoDB(artistid)
# Just update the tracks if it does
else:
havetracks = len(
myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=?', [artistid])) + len(
myDB.select('SELECT TrackTitle from have WHERE ArtistName like ?', [artist]))
myDB.action('UPDATE artists SET HaveTracks=? WHERE ArtistID=?', [havetracks, artistid])
# not sure this is correct and we're updating during scanning in librarysync
# else:
# havetracks = len(
# myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=?', [artistid])) + len(
# myDB.select('SELECT TrackTitle from have WHERE ArtistName like ?', [artist]))
# myDB.action('UPDATE artists SET HaveTracks=? WHERE ArtistID=?', [havetracks, artistid])
# Delete it from the New Artists if the request came from there
if forced:
@@ -105,7 +107,7 @@ def artistlist_to_mbids(artistlist, forced=False):
try:
lastfm.getSimilar()
except Exception as e:
logger.warn('Failed to update arist information from Last.fm: %s' % e)
logger.warn('Failed to update artist information from Last.fm: %s' % e)
def addArtistIDListToDB(artistidlist):
@@ -309,7 +311,7 @@ def addArtisttoDB(artistid, extrasonly=False, forcefull=False, type="artist"):
# This will be used later to build a hybrid release
fullreleaselist = []
# Search for releases within a release group
find_hybrid_releases = myDB.action("SELECT * from allalbums WHERE AlbumID=?",
find_hybrid_releases = myDB.select("SELECT * from allalbums WHERE AlbumID=?",
[rg['id']])
# Build the dictionary for the fullreleaselist
@@ -516,7 +518,10 @@ def addArtisttoDB(artistid, extrasonly=False, forcefull=False, type="artist"):
logger.info(
u"[%s] Seeing if we need album art for %s" % (artist['artist_name'], rg['title']))
cache.getThumb(AlbumID=rg['id'])
try:
cache.getThumb(AlbumID=rg['id'])
except Exception as e:
logger.error("Error getting album art: %s", e)
# Start a search for the album if it's new, hasn't been marked as
# downloaded and autowant_all is selected. This search is deferred,
@@ -532,10 +537,16 @@ def addArtisttoDB(artistid, extrasonly=False, forcefull=False, type="artist"):
finalize_update(artistid, artist['artist_name'], errors)
logger.info(u"Seeing if we need album art for: %s" % artist['artist_name'])
cache.getThumb(ArtistID=artistid)
try:
cache.getThumb(ArtistID=artistid)
except Exception as e:
logger.error("Error getting album art: %s", e)
logger.info(u"Fetching Metacritic reviews for: %s" % artist['artist_name'])
metacritic.update(artistid, artist['artist_name'], artist['releasegroups'])
try:
metacritic.update(artistid, artist['artist_name'], artist['releasegroups'])
except Exception as e:
logger.error("Error getting Metacritic reviews: %s", e)
if errors:
logger.info(

View File

@@ -50,27 +50,43 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None,
logger.info('Scanning music directory: %s' % dir.decode(headphones.SYS_ENCODING, 'replace'))
if not append:
# Clean up bad filepaths
tracks = myDB.select(
'SELECT Location from alltracks WHERE Location IS NOT NULL UNION SELECT Location from tracks WHERE Location IS NOT NULL')
# Clean up bad filepaths. Queries can take some time, ensure all results are loaded before processing
if ArtistID:
tracks = myDB.action(
'SELECT Location FROM alltracks WHERE ArtistID = ? AND Location IS NOT NULL UNION SELECT Location FROM tracks WHERE ArtistID = ? AND Location '
'IS NOT NULL',
[ArtistID, ArtistID])
else:
tracks = myDB.action(
'SELECT Location FROM alltracks WHERE Location IS NOT NULL UNION SELECT Location FROM tracks WHERE Location IS NOT NULL')
locations = []
for track in tracks:
encoded_track_string = track['Location'].encode(headphones.SYS_ENCODING, 'replace')
locations.append(track['Location'])
for location in locations:
encoded_track_string = location.encode(headphones.SYS_ENCODING, 'replace')
if not os.path.isfile(encoded_track_string):
myDB.action('UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE Location=?',
[None, None, None, track['Location']])
[None, None, None, location])
myDB.action('UPDATE alltracks SET Location=?, BitRate=?, Format=? WHERE Location=?',
[None, None, None, track['Location']])
[None, None, None, location])
del_have_tracks = myDB.select('SELECT Location, Matched, ArtistName from have')
if ArtistName:
del_have_tracks = myDB.select('SELECT Location, Matched, ArtistName FROM have WHERE ArtistName = ? COLLATE NOCASE', [ArtistName])
else:
del_have_tracks = myDB.select('SELECT Location, Matched, ArtistName FROM have')
locations = []
for track in del_have_tracks:
encoded_track_string = track['Location'].encode(headphones.SYS_ENCODING, 'replace')
locations.append([track['Location'], track['ArtistName']])
for location in locations:
encoded_track_string = location[0].encode(headphones.SYS_ENCODING, 'replace')
if not os.path.isfile(encoded_track_string):
if track['ArtistName']:
if location[1]:
# Make sure deleted files get accounted for when updating artist track counts
new_artists.append(track['ArtistName'])
myDB.action('DELETE FROM have WHERE Location=?', [track['Location']])
new_artists.append(location[1])
myDB.action('DELETE FROM have WHERE Location=?', [location[0]])
logger.info(
'File %s removed from Headphones, as it is no longer on disk' % encoded_track_string.decode(
headphones.SYS_ENCODING, 'replace'))
@@ -216,6 +232,8 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None,
song_count = 0
latest_artist = []
last_completion_percentage = 0
prev_artist_name = None
artistid = None
for song in song_list:
@@ -237,87 +255,68 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None,
# ARE NEVER FOUND, AND THE OTHER CLAUSES WERE NEVER HIT. FURTHERMORE, OTHER MATCHING FUNCTIONS IN THIS PROGRAM
# (IMPORTER.PY, MB.PY) SIMPLY DO A [ARTIST/ALBUM/TRACK] OR CLEANNAME MATCH, SO IT'S ALL CONSISTENT.
if song['ArtistName'] and song['AlbumTitle'] and song['TrackTitle']:
albumid = None
track = myDB.action(
'SELECT ArtistName, AlbumTitle, TrackTitle, AlbumID from tracks WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?',
[song['ArtistName'], song['AlbumTitle'], song['TrackTitle']]).fetchone()
have_updated = False
if track:
controlValueDict = {'ArtistName': track['ArtistName'],
'AlbumTitle': track['AlbumTitle'],
'TrackTitle': track['TrackTitle']}
newValueDict = {'Location': song['Location'],
'BitRate': song['BitRate'],
'Format': song['Format']}
myDB.upsert("tracks", newValueDict, controlValueDict)
if song['ArtistName'] and song['CleanName']:
artist_name = song['ArtistName']
clean_name = song['CleanName']
controlValueDict2 = {'Location': song['Location']}
newValueDict2 = {'Matched': track['AlbumID']}
myDB.upsert("have", newValueDict2, controlValueDict2)
have_updated = True
else:
track = myDB.action('SELECT CleanName, AlbumID from tracks WHERE CleanName LIKE ?',
[song['CleanName']]).fetchone()
# Only update if artist is in the db
if artist_name != prev_artist_name:
prev_artist_name = artist_name
artistid = None
artist_lookup = '"' + artist_name + '"'
dbartist = myDB.select('SELECT DISTINCT ArtistID, ArtistName FROM artists WHERE ArtistName LIKE ' + artist_lookup + '')
if not dbartist:
dbartist = myDB.select('SELECT DISTINCT ArtistID, ArtistName FROM tracks WHERE CleanName = ?', [clean_name])
if not dbartist:
dbartist = myDB.select('SELECT DISTINCT ArtistID, ArtistName FROM alltracks WHERE CleanName = ?', [clean_name])
if not dbartist:
clean_artist = helpers.clean_name(artist_name)
if clean_artist:
dbartist = myDB.select('SELECT DISTINCT ArtistID, ArtistName FROM tracks WHERE CleanName >= ? and CleanName < ?',
[clean_artist, clean_artist + '{'])
if not dbartist:
dbartist = myDB.select('SELECT DISTINCT ArtistID, ArtistName FROM alltracks WHERE CleanName >= ? and CleanName < ?',
[clean_artist, clean_artist + '{'])
if dbartist:
artistid = dbartist[0][0]
if artistid:
# This was previously using Artist, Album, Title with a SELECT LIKE ? and was not using an index
# (Possible issue: https://stackoverflow.com/questions/37845854/python-sqlite3-not-using-index-with-like)
# Now selects/updates using CleanName index (may have to revert if not working)
# matching on CleanName should be enough, ensure it's the same artist just in case
# Update tracks
track = myDB.action('SELECT AlbumID, ArtistName FROM tracks WHERE CleanName = ? AND ArtistID = ?', [clean_name, artistid]).fetchone()
if track:
controlValueDict = {'CleanName': track['CleanName']}
newValueDict = {'Location': song['Location'],
'BitRate': song['BitRate'],
'Format': song['Format']}
myDB.upsert("tracks", newValueDict, controlValueDict)
albumid = track['AlbumID']
myDB.action(
'UPDATE tracks SET Location = ?, BitRate = ?, Format = ? WHERE CleanName = ? AND ArtistID = ?',
[song['Location'], song['BitRate'], song['Format'], clean_name, artistid])
controlValueDict2 = {'Location': song['Location']}
newValueDict2 = {'Matched': track['AlbumID']}
myDB.upsert("have", newValueDict2, controlValueDict2)
have_updated = True
else:
controlValueDict2 = {'Location': song['Location']}
newValueDict2 = {'Matched': "Failed"}
myDB.upsert("have", newValueDict2, controlValueDict2)
have_updated = True
alltrack = myDB.action(
'SELECT ArtistName, AlbumTitle, TrackTitle, AlbumID from alltracks WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?',
[song['ArtistName'], song['AlbumTitle'], song['TrackTitle']]).fetchone()
if alltrack:
controlValueDict = {'ArtistName': alltrack['ArtistName'],
'AlbumTitle': alltrack['AlbumTitle'],
'TrackTitle': alltrack['TrackTitle']}
newValueDict = {'Location': song['Location'],
'BitRate': song['BitRate'],
'Format': song['Format']}
myDB.upsert("alltracks", newValueDict, controlValueDict)
controlValueDict2 = {'Location': song['Location']}
newValueDict2 = {'Matched': alltrack['AlbumID']}
myDB.upsert("have", newValueDict2, controlValueDict2)
else:
alltrack = myDB.action(
'SELECT CleanName, AlbumID from alltracks WHERE CleanName LIKE ?',
[song['CleanName']]).fetchone()
# Update alltracks
alltrack = myDB.action('SELECT AlbumID, ArtistName FROM alltracks WHERE CleanName = ? AND ArtistID = ?', [clean_name, artistid]).fetchone()
if alltrack:
controlValueDict = {'CleanName': alltrack['CleanName']}
newValueDict = {'Location': song['Location'],
'BitRate': song['BitRate'],
'Format': song['Format']}
myDB.upsert("alltracks", newValueDict, controlValueDict)
controlValueDict2 = {'Location': song['Location']}
newValueDict2 = {'Matched': alltrack['AlbumID']}
myDB.upsert("have", newValueDict2, controlValueDict2)
else:
# alltracks may not exist if adding album manually, have should only be set to failed if not already updated in tracks
if not have_updated:
controlValueDict2 = {'Location': song['Location']}
newValueDict2 = {'Matched': "Failed"}
myDB.upsert("have", newValueDict2, controlValueDict2)
albumid = alltrack['AlbumID']
myDB.action(
'UPDATE alltracks SET Location = ?, BitRate = ?, Format = ? WHERE CleanName = ? AND ArtistID = ?',
[song['Location'], song['BitRate'], song['Format'], clean_name, artistid])
# Update have
controlValueDict2 = {'Location': song['Location']}
if albumid:
newValueDict2 = {'Matched': albumid}
else:
controlValueDict2 = {'Location': song['Location']}
newValueDict2 = {'Matched': "Failed"}
myDB.upsert("have", newValueDict2, controlValueDict2)
myDB.upsert("have", newValueDict2, controlValueDict2)
# myDB.action('INSERT INTO have (ArtistName, AlbumTitle, TrackNumber, TrackTitle, TrackLength, BitRate, Genre, Date, TrackID, Location, CleanName, Format) VALUES( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', [song['ArtistName'], song['AlbumTitle'], song['TrackNumber'], song['TrackTitle'], song['TrackLength'], song['BitRate'], song['Genre'], song['Date'], song['TrackID'], song['Location'], CleanName, song['Format']])
# myDB.action('INSERT INTO have (ArtistName, AlbumTitle, TrackNumber, TrackTitle, TrackLength, BitRate, Genre, Date, TrackID, Location, CleanName, Format) VALUES( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', [song['ArtistName'], song['AlbumTitle'], song['TrackNumber'], song['TrackTitle'], song['TrackLength'], song['BitRate'], song['Genre'], song['Date'], song['TrackID'], song['Location'], CleanName, song['Format']])
logger.info('Completed matching tracks from directory: %s' % dir.decode(headphones.SYS_ENCODING,
'replace'))
@@ -327,49 +326,98 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None,
# Clean up the new artist list
unique_artists = {}.fromkeys(new_artists).keys()
current_artists = myDB.select('SELECT ArtistName, ArtistID from artists')
# There was a bug where artists with special characters (-,') would show up in new artists.
artist_list = [
x for x in unique_artists
if helpers.clean_name(x).lower() not in [
helpers.clean_name(y[0]).lower()
for y in current_artists
]
]
artists_checked = [
x for x in unique_artists
if helpers.clean_name(x).lower() in [
helpers.clean_name(y[0]).lower()
for y in current_artists
]
]
# # Don't think we need to do this, check the db instead below
#
# # artist scan
# if ArtistName:
# current_artists = [[ArtistName]]
# # directory scan
# else:
# current_artists = myDB.select('SELECT ArtistName, ArtistID FROM artists WHERE ArtistName IS NOT NULL')
#
# # There was a bug where artists with special characters (-,') would show up in new artists.
#
# # artist_list = scanned artists not in the db
# artist_list = [
# x for x in unique_artists
# if helpers.clean_name(x).lower() not in [
# helpers.clean_name(y[0]).lower()
# for y in current_artists
# ]
# ]
#
# # artists_checked = scanned artists that exist in the db
# artists_checked = [
# x for x in unique_artists
# if helpers.clean_name(x).lower() in [
# helpers.clean_name(y[0]).lower()
# for y in current_artists
# ]
# ]
# Update track counts
for artist in artists_checked:
# Have tracks are selected from tracks table and not all tracks because of duplicates
# We update the track count upon an album switch to compliment this
havetracks = (
len(myDB.select(
'SELECT TrackTitle from tracks WHERE ArtistName like ? AND Location IS NOT NULL',
[artist])) + len(myDB.select(
'SELECT TrackTitle from have WHERE ArtistName like ? AND Matched = "Failed"',
[artist]))
)
# Note: some people complain about having "artist have tracks" > # of tracks total in artist official releases
# (can fix by getting rid of second len statement)
myDB.action('UPDATE artists SET HaveTracks=? WHERE ArtistName=?', [havetracks, artist])
new_artist_list = []
logger.info('Found %i new artists' % len(artist_list))
for artist in unique_artists:
if artist_list:
# check if artist is already in the db
artist_lookup = '"' + artist + '"'
dbartist = myDB.select('SELECT DISTINCT ArtistID, ArtistName FROM artists WHERE ArtistName LIKE ' + artist_lookup + '')
if not dbartist:
clean_artist = helpers.clean_name(artist)
if clean_artist:
dbartist = myDB.select('SELECT DISTINCT ArtistID, ArtistName FROM tracks WHERE CleanName >= ? and CleanName < ?',
[clean_artist, clean_artist + '{'])
if not dbartist:
dbartist = myDB.select('SELECT DISTINCT ArtistID, ArtistName FROM alltracks WHERE CleanName >= ? and CleanName < ?',
[clean_artist, clean_artist + '{'])
# new artist not in db, add to list
if not dbartist:
new_artist_list.append(artist)
else:
# artist in db, update have track counts
artistid = dbartist[0][0]
# Have tracks are selected from tracks table and not all tracks because of duplicates
# We update the track count upon an album switch to compliment this
# havetracks = (
# len(myDB.select(
# 'SELECT TrackTitle from tracks WHERE ArtistName like ? AND Location IS NOT NULL',
# [artist])) + len(myDB.select(
# 'SELECT TrackTitle from have WHERE ArtistName like ? AND Matched = "Failed"',
# [artist]))
# )
havetracks = (
len(myDB.select(
'SELECT ArtistID From tracks WHERE ArtistID = ? AND Location IS NOT NULL',
[artistid])) + len(myDB.select(
'SELECT ArtistName FROM have WHERE ArtistName LIKE ' + artist_lookup + ' AND Matched = "Failed"'))
)
# Note: some people complain about having "artist have tracks" > # of tracks total in artist official releases
# (can fix by getting rid of second len statement)
myDB.action('UPDATE artists SET HaveTracks = ? WHERE ArtistID = ?', [havetracks, artistid])
# Update albums to downloaded
if havetracks:
update_album_status(ArtistID=artistid)
logger.info('Found %i new artists' % len(new_artist_list))
# Add scanned artists not in the db
if new_artist_list:
if headphones.CONFIG.AUTO_ADD_ARTISTS:
logger.info('Importing %i new artists' % len(artist_list))
importer.artistlist_to_mbids(artist_list)
logger.info('Importing %i new artists' % len(new_artist_list))
importer.artistlist_to_mbids(new_artist_list)
else:
logger.info('To add these artists, go to Manage->Manage New Artists')
# myDB.action('DELETE from newartists')
for artist in artist_list:
for artist in new_artist_list:
myDB.action('INSERT OR IGNORE INTO newartists VALUES (?)', [artist])
if headphones.CONFIG.DETECT_BITRATE and bitrates:
@@ -379,50 +427,73 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None,
# If we're appending a new album to the database, update the artists total track counts
logger.info('Updating artist track counts')
artist_lookup = '"' + ArtistName + '"'
havetracks = len(
myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=? AND Location IS NOT NULL',
myDB.select('SELECT ArtistID FROM tracks WHERE ArtistID = ? AND Location IS NOT NULL',
[ArtistID])) + len(myDB.select(
'SELECT TrackTitle from have WHERE ArtistName like ? AND Matched = "Failed"',
[ArtistName]))
'SELECT ArtistName FROM have WHERE ArtistName LIKE ' + artist_lookup + ' AND Matched = "Failed"'))
myDB.action('UPDATE artists SET HaveTracks=? WHERE ArtistID=?', [havetracks, ArtistID])
if not append:
update_album_status()
# Moved above to call for each artist
# if not append:
# update_album_status()
if not append and not artistScan:
lastfm.getSimilar()
logger.info('Library scan complete')
if ArtistName:
logger.info('Scanning complete for artist: %s', ArtistName)
else:
logger.info('Library scan complete')
# ADDED THIS SECTION TO MARK ALBUMS AS DOWNLOADED IF ARTISTS ARE ADDED EN MASSE BEFORE LIBRARY IS SCANNED
# Think the above comment relates to calling from Manage Unmatched
def update_album_status(AlbumID=None):
# This used to select and update all albums and would clobber the db, changed to run by ArtistID.
def update_album_status(AlbumID=None, ArtistID=None):
myDB = db.DBConnection()
logger.info('Counting matched tracks to mark albums as skipped/downloaded')
# logger.info('Counting matched tracks to mark albums as skipped/downloaded')
if AlbumID:
album_status_updater = myDB.action(
'SELECT AlbumID, AlbumTitle, Status from albums WHERE AlbumID=?', [AlbumID])
'SELECT'
' a.AlbumID, a.ArtistName, a.AlbumTitle, a.Status, AVG(t.Location IS NOT NULL) * 100 AS album_completion '
'FROM'
' albums AS a '
'JOIN tracks AS t ON t.AlbumID = a.AlbumID '
'WHERE'
' a.AlbumID = ? AND a.Status != "Downloaded" '
'GROUP BY'
' a.AlbumID '
'HAVING'
' AVG(t.Location IS NOT NULL) * 100 >= ?',
[AlbumID, headphones.CONFIG.ALBUM_COMPLETION_PCT]
)
else:
album_status_updater = myDB.action('SELECT AlbumID, AlbumTitle, Status from albums')
for album in album_status_updater:
track_counter = myDB.action('SELECT Location from tracks where AlbumID=?',
[album['AlbumID']])
total_tracks = 0
have_tracks = 0
for track in track_counter:
total_tracks += 1
if track['Location']:
have_tracks += 1
if total_tracks != 0:
album_completion = float(float(have_tracks) / float(total_tracks)) * 100
else:
album_completion = 0
logger.info('Album %s does not have any tracks in database' % album['AlbumTitle'])
album_status_updater = myDB.action(
'SELECT'
' a.AlbumID, a.ArtistID, a.ArtistName, a.AlbumTitle, a.Status, AVG(t.Location IS NOT NULL) * 100 AS album_completion '
'FROM'
' albums AS a '
'JOIN tracks AS t ON t.AlbumID = a.AlbumID '
'WHERE'
' a.ArtistID = ? AND a.Status != "Downloaded" '
'GROUP BY'
' a.AlbumID '
'HAVING'
' AVG(t.Location IS NOT NULL) * 100 >= ?',
[ArtistID, headphones.CONFIG.ALBUM_COMPLETION_PCT]
)
if album_completion >= headphones.CONFIG.ALBUM_COMPLETION_PCT:
new_album_status = "Downloaded"
new_album_status = "Downloaded"
albums = []
for album in album_status_updater:
albums.append([album['AlbumID'], album['ArtistName'], album['AlbumTitle']])
for album in albums:
# I don't think we want to change Downloaded->Skipped.....
# I think we can only automatically change Skipped->Downloaded when updating
@@ -433,10 +504,13 @@ def update_album_status(AlbumID=None):
# new_album_status = "Skipped"
# else:
# new_album_status = album['Status']
else:
new_album_status = album['Status']
# else:
# new_album_status = album['Status']
#
# myDB.upsert("albums", {'Status': new_album_status}, {'AlbumID': album['AlbumID']})
# if new_album_status != album['Status']:
# logger.info('Album %s changed to %s' % (album['AlbumTitle'], new_album_status))
# logger.info('Album status update complete')
myDB.upsert("albums", {'Status': new_album_status}, {'AlbumID': album['AlbumID']})
if new_album_status != album['Status']:
logger.info('Album %s changed to %s' % (album['AlbumTitle'], new_album_status))
logger.info('Album status update complete')
myDB.action('UPDATE albums SET Status = ? WHERE AlbumID = ?', [new_album_status, album[0]])
logger.info('Album: %s - %s. Status updated to %s' % (album[1], album[2], new_album_status))

View File

@@ -183,13 +183,17 @@ def verify(albumid, albumpath, Kind=None, forced=False, keep_original_folder=Fal
controlValueDict = {"TrackID": track['id'],
"AlbumID": albumid}
clean_name = helpers.clean_name(
release_dict['artist_name'] + ' ' + release_dict['title'] + ' ' + track['title'])
newValueDict = {"ArtistID": release_dict['artist_id'],
"ArtistName": release_dict['artist_name'],
"AlbumTitle": release_dict['title'],
"AlbumASIN": release_dict['asin'],
"TrackTitle": track['title'],
"TrackDuration": track['duration'],
"TrackNumber": track['number']
"TrackNumber": track['number'],
"CleanName": clean_name
}
myDB.upsert("tracks", newValueDict, controlValueDict)

View File

@@ -640,6 +640,7 @@ class WebInterface(object):
'SELECT Matched, CleanName, Location, BitRate, Format FROM have WHERE ArtistName=?',
[existing_artist])
update_count = 0
artist_id = None
for entry in have_tracks:
old_clean_filename = entry['CleanName']
if old_clean_filename.startswith(existing_artist_clean):
@@ -648,31 +649,30 @@ class WebInterface(object):
myDB.action(
'UPDATE have SET CleanName=? WHERE ArtistName=? AND CleanName=?',
[new_clean_filename, existing_artist, old_clean_filename])
controlValueDict = {"CleanName": new_clean_filename}
newValueDict = {"Location": entry['Location'],
"BitRate": entry['BitRate'],
"Format": entry['Format']
}
# Attempt to match tracks with new CleanName
match_alltracks = myDB.action(
'SELECT CleanName from alltracks WHERE CleanName=?',
'SELECT CleanName FROM alltracks WHERE CleanName = ?',
[new_clean_filename]).fetchone()
if match_alltracks:
myDB.upsert("alltracks", newValueDict, controlValueDict)
myDB.action(
'UPDATE alltracks SET Location = ?, BitRate = ?, Format = ? WHERE CleanName = ?',
[entry['Location'], entry['BitRate'], entry['Format'], new_clean_filename])
match_tracks = myDB.action(
'SELECT CleanName, AlbumID from tracks WHERE CleanName=?',
'SELECT ArtistID, CleanName, AlbumID FROM tracks WHERE CleanName = ?',
[new_clean_filename]).fetchone()
if match_tracks:
myDB.upsert("tracks", newValueDict, controlValueDict)
myDB.action(
'UPDATE tracks SET Location = ?, BitRate = ?, Format = ? WHERE CleanName = ?',
[entry['Location'], entry['BitRate'], entry['Format'], new_clean_filename])
myDB.action('UPDATE have SET Matched="Manual" WHERE CleanName=?',
[new_clean_filename])
update_count += 1
# This was throwing errors and I don't know why, but it seems to be working fine.
# else:
# logger.info("There was an error modifying Artist %s. This should not have happened" % existing_artist)
artist_id = match_tracks['Artist_ID']
logger.info("Manual matching yielded %s new matches for Artist: %s" % (update_count, new_artist))
if update_count > 0:
librarysync.update_album_status()
if artist_id:
librarysync.update_album_status(ArtistID=artist_id)
else:
logger.info(
"Artist %s already named appropriately; nothing to modify" % existing_artist)
@@ -698,29 +698,28 @@ class WebInterface(object):
'UPDATE have SET CleanName=? WHERE ArtistName=? AND AlbumTitle=? AND CleanName=?',
[new_clean_filename, existing_artist, existing_album,
old_clean_filename])
controlValueDict = {"CleanName": new_clean_filename}
newValueDict = {"Location": entry['Location'],
"BitRate": entry['BitRate'],
"Format": entry['Format']
}
# Attempt to match tracks with new CleanName
match_alltracks = myDB.action(
'SELECT CleanName from alltracks WHERE CleanName=?',
'SELECT CleanName FROM alltracks WHERE CleanName = ?',
[new_clean_filename]).fetchone()
if match_alltracks:
myDB.upsert("alltracks", newValueDict, controlValueDict)
myDB.action(
'UPDATE alltracks SET Location = ?, BitRate = ?, Format = ? WHERE CleanName = ?',
[entry['Location'], entry['BitRate'], entry['Format'], new_clean_filename])
match_tracks = myDB.action(
'SELECT CleanName, AlbumID from tracks WHERE CleanName=?',
'SELECT CleanName, AlbumID FROM tracks WHERE CleanName = ?',
[new_clean_filename]).fetchone()
if match_tracks:
myDB.upsert("tracks", newValueDict, controlValueDict)
myDB.action(
'UPDATE tracks SET Location = ?, BitRate = ?, Format = ? WHERE CleanName = ?',
[entry['Location'], entry['BitRate'], entry['Format'], new_clean_filename])
myDB.action('UPDATE have SET Matched="Manual" WHERE CleanName=?',
[new_clean_filename])
album_id = match_tracks['AlbumID']
update_count += 1
# This was throwing errors and I don't know why, but it seems to be working fine.
# else:
# logger.info("There was an error modifying Artist %s / Album %s with clean name %s" % (existing_artist, existing_album, existing_clean_string))
logger.info("Manual matching yielded %s new matches for Artist: %s / Album: %s" % (
update_count, new_artist, new_album))
if update_count > 0:
@@ -785,25 +784,29 @@ class WebInterface(object):
album = tracks['AlbumTitle']
track_title = tracks['TrackTitle']
if tracks['CleanName'] != original_clean:
artist_id_check = myDB.action('SELECT ArtistID FROM tracks WHERE CleanName = ?',
[tracks['CleanName']]).fetchone()
if artist_id_check:
artist_id = artist_id_check[0]
myDB.action(
'UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE CleanName=?',
'UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE CleanName = ?',
[None, None, None, tracks['CleanName']])
myDB.action(
'UPDATE alltracks SET Location=?, BitRate=?, Format=? WHERE CleanName=?',
'UPDATE alltracks SET Location=?, BitRate=?, Format=? WHERE CleanName = ?',
[None, None, None, tracks['CleanName']])
myDB.action(
'UPDATE have SET CleanName=?, Matched="Failed" WHERE ArtistName=? AND AlbumTitle=? AND TrackTitle=?',
(original_clean, artist, album, track_title))
update_count += 1
if update_count > 0:
librarysync.update_album_status()
librarysync.update_album_status(ArtistID=artist_id)
logger.info("Artist: %s successfully restored to unmatched list" % artist)
elif action == "unmatchAlbum":
artist = existing_artist
album = existing_album
update_clean = myDB.select(
'SELECT ArtistName, AlbumTitle, TrackTitle, CleanName, Matched from have WHERE ArtistName=? AND AlbumTitle=?',
'SELECT ArtistName, AlbumTitle, TrackTitle, CleanName, Matched FROM have WHERE ArtistName=? AND AlbumTitle=?',
(artist, album))
update_count = 0
for tracks in update_clean:
@@ -812,15 +815,15 @@ class WebInterface(object):
'TrackTitle']).lower()
track_title = tracks['TrackTitle']
if tracks['CleanName'] != original_clean:
album_id_check = myDB.action('SELECT AlbumID from tracks WHERE CleanName=?',
album_id_check = myDB.action('SELECT AlbumID FROM tracks WHERE CleanName = ?',
[tracks['CleanName']]).fetchone()
if album_id_check:
album_id = album_id_check[0]
myDB.action(
'UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE CleanName=?',
'UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE CleanName = ?',
[None, None, None, tracks['CleanName']])
myDB.action(
'UPDATE alltracks SET Location=?, BitRate=?, Format=? WHERE CleanName=?',
'UPDATE alltracks SET Location=?, BitRate=?, Format=? WHERE CleanName = ?',
[None, None, None, tracks['CleanName']])
myDB.action(
'UPDATE have SET CleanName=?, Matched="Failed" WHERE ArtistName=? AND AlbumTitle=? AND TrackTitle=?',