diff --git a/data/interfaces/default/album.html b/data/interfaces/default/album.html index bec42cfb..2496bd4c 100644 --- a/data/interfaces/default/album.html +++ b/data/interfaces/default/album.html @@ -138,7 +138,7 @@ %endfor <% - unmatched = myDB.select('SELECT * from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ?', [album['ArtistName'], album['AlbumTitle']]) + unmatched_temp = myDB.select('SELECT * from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND Matched is null ORDER BY CAST(TrackNumber AS INTEGER)', [album['ArtistName'], album['AlbumTitle']]) %> %if unmatched: %for track in unmatched: diff --git a/data/interfaces/default/artist.html b/data/interfaces/default/artist.html index e891b1fa..845f6719 100644 --- a/data/interfaces/default/artist.html +++ b/data/interfaces/default/artist.html @@ -93,7 +93,7 @@ myDB = db.DBConnection() totaltracks = len(myDB.select('SELECT TrackTitle from tracks WHERE AlbumID=?', [album['AlbumID']])) - havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE AlbumID=? AND Location IS NOT NULL', [album['AlbumID']])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ? AND AlbumTitle LIKE ?', [album['ArtistName'], album['AlbumTitle']])) + havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE AlbumID=? AND Location IS NOT NULL', [album['AlbumID']])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ? AND AlbumTitle LIKE ? AND Matched IS NULL', [album['ArtistName'], album['AlbumTitle']])) try: percent = (havetracks*100.0)/totaltracks diff --git a/data/interfaces/default/index.html b/data/interfaces/default/index.html index b67aaf7d..e0201252 100644 --- a/data/interfaces/default/index.html +++ b/data/interfaces/default/index.html @@ -69,7 +69,7 @@ } if (artist['ReleaseInFuture'] === 'True') { - grade = 'gradeA'; + grade = 'gradeA';6666666666666666 } else { diff --git a/data/interfaces/default/manage.html b/data/interfaces/default/manage.html index 4c0de24c..b4ebc824 100644 --- a/data/interfaces/default/manage.html +++ b/data/interfaces/default/manage.html @@ -22,6 +22,7 @@ %if not headphones.ADD_ARTISTS: Manage New Artists %endif + Manage Unmatched @@ -35,6 +36,7 @@
  • Scan Music Library
  • Imports
  • Force Actions
    • +
  • Force Legacy
    • @@ -120,7 +122,6 @@
    Force Check for Wanted Albums Force Update Active Artists [Fast] - Force Update Active Artists [Comprehensive] Force Post-Process Albums in Download Folder Check for Headphones Updates Delete empty Artists @@ -140,6 +141,23 @@
    + + +
    +
    + Force Legacy +

    Comprehensive Updating

    +

    Please note that these functions will take a significant amount of time to complete.

    +
    + Force Update Active Artists [Comprehensive] + Force Re-scan Library [Comprehensive] +
    +
    + +
    + + + <%def name="javascriptIncludes()"> diff --git a/data/interfaces/default/managenew.html b/data/interfaces/default/managenew.html index 17d567f7..41f9263d 100644 --- a/data/interfaces/default/managenew.html +++ b/data/interfaces/default/managenew.html @@ -20,7 +20,10 @@
    - Add selected artists +
    diff --git a/data/interfaces/default/manageunmatched.html b/data/interfaces/default/manageunmatched.html new file mode 100644 index 00000000..2054382c --- /dev/null +++ b/data/interfaces/default/manageunmatched.html @@ -0,0 +1,157 @@ +<%inherit file="base.html" /> +<%! + import headphones + import json + from headphones import db, helpers + myDB = db.DBConnection() + artist_json = {} + counter = 0 + artist_list = myDB.action("SELECT ArtistName from artists ORDER BY ArtistName COLLATE NOCASE") + for artist in artist_list: + artist_json[counter] = artist['ArtistName'] + counter+=1 + json_artists = json.dumps(artist_json) +%> + +<%def name="headerIncludes()"> +
    +
    +
    +
    + « Back to manage overview + + + +<%def name="body()"> +
    +
    +

    manageManage Unmatched Albums

    +
    + +
    + + + + + + + + + + <% count_albums=0 %> + %for album in unmatchedalbums: + + + + + + + <% count_albums+=1 %> + %endfor + +
    Local ArtistLocal AlbumMatch ArtistMatch Album
    ${album['ArtistName']} + + + + + + ${album['AlbumTitle']} +
    + + + + +
    +     		+ +
    + + + + +
    +     		+ +
    + + + + + + +
    +
    + + + +<%def name="headIncludes()"> + + + +<%def name="javascriptIncludes()"> + + + diff --git a/headphones/__init__.py b/headphones/__init__.py index 8c9af893..758b28dc 100644 --- a/headphones/__init__.py +++ b/headphones/__init__.py @@ -954,169 +954,7 @@ def dbcheck(): c.execute('CREATE INDEX IF NOT EXISTS tracks_albumid ON tracks(AlbumID ASC)') c.execute('CREATE INDEX IF NOT EXISTS album_artistid_reldate ON albums(ArtistID ASC, ReleaseDate DESC)') - try: - c.execute('SELECT IncludeExtras from artists') - except sqlite3.OperationalError: - c.execute('ALTER TABLE artists ADD COLUMN IncludeExtras INTEGER DEFAULT 0') - try: - c.execute('SELECT LatestAlbum from artists') - except sqlite3.OperationalError: - c.execute('ALTER TABLE artists ADD COLUMN LatestAlbum TEXT') - - try: - c.execute('SELECT ReleaseDate from artists') - except sqlite3.OperationalError: - c.execute('ALTER TABLE artists ADD COLUMN ReleaseDate TEXT') - - try: - c.execute('SELECT AlbumID from artists') - except sqlite3.OperationalError: - c.execute('ALTER TABLE artists ADD COLUMN AlbumID TEXT') - - try: - c.execute('SELECT HaveTracks from artists') - except sqlite3.OperationalError: - c.execute('ALTER TABLE artists ADD COLUMN HaveTracks INTEGER DEFAULT 0') - - try: - c.execute('SELECT TotalTracks from artists') - except sqlite3.OperationalError: - c.execute('ALTER TABLE artists ADD COLUMN TotalTracks INTEGER DEFAULT 0') - - try: - c.execute('SELECT Type from albums') - except sqlite3.OperationalError: - c.execute('ALTER TABLE albums ADD COLUMN Type TEXT DEFAULT "Album"') - - try: - c.execute('SELECT TrackNumber from tracks') - except sqlite3.OperationalError: - c.execute('ALTER TABLE tracks ADD COLUMN TrackNumber INTEGER') - - try: - c.execute('SELECT FolderName from snatched') - except sqlite3.OperationalError: - c.execute('ALTER TABLE snatched ADD COLUMN FolderName TEXT') - - try: - c.execute('SELECT Location from tracks') - except sqlite3.OperationalError: - c.execute('ALTER TABLE tracks ADD COLUMN Location TEXT') - - try: - c.execute('SELECT Location from have') - except sqlite3.OperationalError: - c.execute('ALTER TABLE have ADD COLUMN Location TEXT') - - try: - c.execute('SELECT BitRate from tracks') - except sqlite3.OperationalError: - c.execute('ALTER TABLE tracks ADD COLUMN BitRate INTEGER') - - try: - c.execute('SELECT CleanName from tracks') - except sqlite3.OperationalError: - c.execute('ALTER TABLE tracks ADD COLUMN CleanName TEXT') - - try: - c.execute('SELECT CleanName from have') - except sqlite3.OperationalError: - c.execute('ALTER TABLE have ADD COLUMN CleanName TEXT') - - # Add the Format column - try: - c.execute('SELECT Format from have') - except sqlite3.OperationalError: - c.execute('ALTER TABLE have ADD COLUMN Format TEXT DEFAULT NULL') - - try: - c.execute('SELECT Format from tracks') - except sqlite3.OperationalError: - c.execute('ALTER TABLE tracks ADD COLUMN Format TEXT DEFAULT NULL') - - try: - c.execute('SELECT LastUpdated from artists') - except sqlite3.OperationalError: - c.execute('ALTER TABLE artists ADD COLUMN LastUpdated TEXT DEFAULT NULL') - - try: - c.execute('SELECT ArtworkURL from artists') - except sqlite3.OperationalError: - c.execute('ALTER TABLE artists ADD COLUMN ArtworkURL TEXT DEFAULT NULL') - - try: - c.execute('SELECT ArtworkURL from albums') - except sqlite3.OperationalError: - c.execute('ALTER TABLE albums ADD COLUMN ArtworkURL TEXT DEFAULT NULL') - - try: - c.execute('SELECT ThumbURL from artists') - except sqlite3.OperationalError: - c.execute('ALTER TABLE artists ADD COLUMN ThumbURL TEXT DEFAULT NULL') - - try: - c.execute('SELECT ThumbURL from albums') - except sqlite3.OperationalError: - c.execute('ALTER TABLE albums ADD COLUMN ThumbURL TEXT DEFAULT NULL') - - try: - c.execute('SELECT ArtistID from descriptions') - except sqlite3.OperationalError: - c.execute('ALTER TABLE descriptions ADD COLUMN ArtistID TEXT DEFAULT NULL') - - try: - c.execute('SELECT LastUpdated from descriptions') - except sqlite3.OperationalError: - c.execute('ALTER TABLE descriptions ADD COLUMN LastUpdated TEXT DEFAULT NULL') - - try: - c.execute('SELECT ReleaseID from albums') - except sqlite3.OperationalError: - c.execute('ALTER TABLE albums ADD COLUMN ReleaseID TEXT DEFAULT NULL') - - try: - c.execute('SELECT ReleaseFormat from albums') - except sqlite3.OperationalError: - c.execute('ALTER TABLE albums ADD COLUMN ReleaseFormat TEXT DEFAULT NULL') - - try: - c.execute('SELECT ReleaseCountry from albums') - except sqlite3.OperationalError: - c.execute('ALTER TABLE albums ADD COLUMN ReleaseCountry TEXT DEFAULT NULL') - - try: - c.execute('SELECT ReleaseID from tracks') - except sqlite3.OperationalError: - c.execute('ALTER TABLE tracks ADD COLUMN ReleaseID TEXT DEFAULT NULL') - - try: - c.execute('SELECT Matched from have') - except sqlite3.OperationalError: - c.execute('ALTER TABLE have ADD COLUMN Matched TEXT DEFAULT NULL') - - try: - c.execute('SELECT Extras from artists') - except sqlite3.OperationalError: - c.execute('ALTER TABLE artists ADD COLUMN Extras TEXT DEFAULT NULL') - # Need to update some stuff when people are upgrading and have 'include extras' set globally/for an artist - if INCLUDE_EXTRAS: - EXTRAS = "1,2,3,4,5,6,7,8" - logger.info("Copying over current artist IncludeExtras information") - artists = c.execute('SELECT ArtistID, IncludeExtras from artists').fetchall() - for artist in artists: - if artist[1]: - c.execute('UPDATE artists SET Extras=? WHERE ArtistID=?', ("1,2,3,4,5,6,7,8", artist[0])) - - try: - c.execute('SELECT Kind from snatched') - except sqlite3.OperationalError: - c.execute('ALTER TABLE snatched ADD COLUMN Kind TEXT DEFAULT NULL') - - try: - c.execute('SELECT SearchTerm from albums') - except sqlite3.OperationalError: - c.execute('ALTER TABLE albums ADD COLUMN SearchTerm TEXT DEFAULT NULL') conn.commit() c.close() diff --git a/headphones/importer.py b/headphones/importer.py index d8fa677a..10c41abb 100644 --- a/headphones/importer.py +++ b/headphones/importer.py @@ -350,12 +350,13 @@ def addArtisttoDB(artistid, extrasonly=False, forcefull=False): newValueDict['Location'] = match['Location'] newValueDict['BitRate'] = match['BitRate'] newValueDict['Format'] = match['Format'] - myDB.action('UPDATE have SET Matched="True" WHERE Location=?', [match['Location']]) + #myDB.action('UPDATE have SET Matched="True" WHERE Location=?', [match['Location']]) + myDB.action('UPDATE have SET Matched=? WHERE Location=?', (rg['id'], match['Location'])) myDB.upsert("alltracks", newValueDict, controlValueDict) # Delete matched tracks from the have table - myDB.action('DELETE from have WHERE Matched="True"') + #myDB.action('DELETE from have WHERE Matched="True"') # If there's no release in the main albums tables, add the default (hybrid) # If there is a release, check the ReleaseID against the AlbumID to see if they differ (user updated) @@ -481,7 +482,8 @@ def addArtisttoDB(artistid, extrasonly=False, forcefull=False): latestalbum = myDB.action('SELECT AlbumTitle, ReleaseDate, AlbumID from albums WHERE ArtistID=? order by ReleaseDate DESC', [artistid]).fetchone() totaltracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=?', [artistid])) - havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=? AND Location IS NOT NULL', [artistid])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ?', [artist['artist_name']])) + #havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=? AND Location IS NOT NULL', [artistid])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ?', [artist['artist_name']])) + havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=? AND Location IS NOT NULL', [artistid])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ? AND Matched IS NULL', [artist['artist_name']])) controlValueDict = {"ArtistID": artistid} @@ -616,7 +618,7 @@ def addReleaseById(rid): newValueDict['Location'] = match['Location'] newValueDict['BitRate'] = match['BitRate'] newValueDict['Format'] = match['Format'] - myDB.action('DELETE from have WHERE Location=?', [match['Location']]) + #myDB.action('DELETE from have WHERE Location=?', [match['Location']]) myDB.upsert("tracks", newValueDict, controlValueDict) diff --git a/headphones/librarysync.py b/headphones/librarysync.py index 5ee57d8b..03925d7b 100644 --- a/headphones/librarysync.py +++ b/headphones/librarysync.py @@ -24,6 +24,7 @@ from headphones import db, logger, helpers, importer # You can scan a single directory and append it to the current library by specifying append=True, ArtistID & ArtistName def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, cron=False): + if cron and not headphones.LIBRARYSCAN: return @@ -46,13 +47,23 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, cron=Fal if not append: # Clean up bad filepaths - tracks = myDB.select('SELECT Location, TrackID from tracks WHERE Location IS NOT NULL') - - for track in tracks: - if not os.path.isfile(track['Location'].encode(headphones.SYS_ENCODING)): - myDB.action('UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE TrackID=?', [None, None, None, track['TrackID']]) + tracks = myDB.select('SELECT Location, TrackID from alltracks WHERE Location IS NOT NULL') - myDB.action('DELETE from have') + for track in tracks: + encoded_track_string = track['Location'].encode(headphones.SYS_ENCODING) + if not os.path.isfile(encoded_track_string): + myDB.action('UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE Location=?', [None, None, None, track['Location']]) + myDB.action('UPDATE alltracks SET Location=?, BitRate=?, Format=? WHERE Location=?', [None, None, None, track['Location']]) + + del_have_tracks = myDB.select('SELECT Location, Matched from have') + + for track in del_have_tracks: + encoded_track_string = track['Location'].encode(headphones.SYS_ENCODING) + if not os.path.isfile(encoded_track_string): + myDB.action('DELETE FROM have WHERE Location=?', [track['Location']]) + myDB.action('UPDATE have SET Matched=NULL WHERE Matched=?', [track['Matched']]) + logger.info('File %s removed from Headphones, as it is no longer on disk' % encoded_track_string.decode(headphones.SYS_ENCODING, 'replace')) + ###############myDB.action('DELETE from have') logger.info('Scanning music directory: %s' % dir.decode(headphones.SYS_ENCODING, 'replace')) @@ -60,6 +71,7 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, cron=Fal bitrates = [] song_list = [] + new_song_count = 0 for r,d,f in os.walk(dir): #need to abuse slicing to get a copy of the list, doing it directly will skip the element after a deleted one @@ -99,8 +111,15 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, cron=Fal # Add the song to our song list - # TODO: skip adding songs without the minimum requisite information (just a matter of putting together the right if statements) - song_dict = { 'TrackID' : f.mb_trackid, - 'ReleaseID' : f.mb_albumid, + if f_artist and f.album and f.title: + CleanName = helpers.cleanName(f_artist +' '+ f.album +' '+ f.title) + else: + CleanName = None + + controlValueDict = {'Location' : unicode_song_path} + + newValueDict = { 'TrackID' : f.mb_trackid, + #'ReleaseID' : f.mb_albumid, 'ArtistName' : f_artist, 'AlbumTitle' : f.album, 'TrackNumber': f.track, @@ -110,191 +129,117 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, cron=Fal 'TrackTitle' : f.title, 'BitRate' : f.bitrate, 'Format' : f.format, - 'Location' : unicode_song_path } + 'CleanName' : CleanName + } - song_list.append(song_dict) + #song_list.append(song_dict) + check_exist_song = myDB.action("SELECT * FROM have WHERE Location=?", [unicode_song_path]).fetchone() + #Only attempt to match songs that are new, haven't yet been matched, or metadata has changed. + if not check_exist_song: + myDB.upsert("have", newValueDict, controlValueDict) + new_song_count+=1 + #We're going to have to think about metadata changing, and setting Matched = None when/if we do + elif check_exist_song['CleanName'] != CleanName and check_exist_song['Matched'] != "Manual": + newValueDict['Matched'] = None + myDB.upsert("have", newValueDict, controlValueDict) + new_song_count+=1 + # Now we start track matching - total_number_of_songs = len(song_list) - logger.info("Found " + str(total_number_of_songs) + " tracks in: '" + dir.decode(headphones.SYS_ENCODING, 'replace') + "'. Matching tracks to the appropriate releases....") + logger.info("%s new/modified songs found and added to the database" % new_song_count) + song_list = myDB.action("SELECT * FROM have WHERE Matched IS NULL AND LOCATION LIKE ?", [dir+"%"]) + total_number_of_songs = myDB.action("SELECT COUNT(*) FROM have WHERE Matched IS NULL AND LOCATION LIKE ?", [dir+"%"]).fetchone()[0] + logger.info("Found " + str(total_number_of_songs) + " unmatched tracks in: '" + dir.decode(headphones.SYS_ENCODING, 'replace') + "'. Matching tracks to the appropriate releases....") # Sort the song_list by most vague (e.g. no trackid or releaseid) to most specific (both trackid & releaseid) # When we insert into the database, the tracks with the most specific information will overwrite the more general matches - song_list = helpers.multikeysort(song_list, ['ReleaseID', 'TrackID']) + ##############song_list = helpers.multikeysort(song_list, ['ReleaseID', 'TrackID']) + song_list = helpers.multikeysort(song_list, ['ArtistName', 'AlbumTitle']) # We'll use this to give a % completion, just because the track matching might take a while song_count = 0 + latest_artist = [] for song in song_list: + + latest_artist.append(song['ArtistName']) + if song_count == 0: + logger.info("Now matching songs by %s" % song['ArtistName']) + elif latest_artist[song_count] != latest_artist[song_count-1] and song_count !=0: + logger.info("Now matching songs by %s" % song['ArtistName']) + #print song['ArtistName']+' - '+song['AlbumTitle']+' - '+song['TrackTitle'] song_count += 1 completion_percentage = float(song_count)/total_number_of_songs * 100 if completion_percentage%10 == 0: logger.info("Track matching is " + str(completion_percentage) + "% complete") - # If the track has a trackid & releaseid (beets: albumid) that the most surefire way - # of identifying a track to a specific release so we'll use that first - if song['TrackID'] and song['ReleaseID']: + #THE "MORE-SPECIFIC" CLAUSES HERE HAVE ALL BEEN REMOVED. WHEN RUNNING A LIBRARY SCAN, THE ONLY CLAUSES THAT + #EVER GOT HIT WERE [ARTIST/ALBUM/TRACK] OR CLEANNAME. ARTISTID & RELEASEID ARE NEVER PASSED TO THIS FUNCTION, + #ARE NEVER FOUND, AND THE OTHER CLAUSES WERE NEVER HIT. FURTHERMORE, OTHER MATCHING FUNCTIONS IN THIS PROGRAM + #(IMPORTER.PY, MB.PY) SIMPLY DO A [ARTIST/ALBUM/TRACK] OR CLEANNAME MATCH, SO IT'S ALL CONSISTENT. - # Check both the tracks table & alltracks table in case they haven't populated the alltracks table yet - track = myDB.action('SELECT TrackID, ReleaseID, AlbumID from alltracks WHERE TrackID=? AND ReleaseID=?', [song['TrackID'], song['ReleaseID']]).fetchone() - - # It might be the case that the alltracks table isn't populated yet, so maybe we can only find a match in the tracks table - if not track: - track = myDB.action('SELECT TrackID, ReleaseID, AlbumID from tracks WHERE TrackID=? AND ReleaseID=?', [song['TrackID'], song['ReleaseID']]).fetchone() - - if track: - # Use TrackID & ReleaseID here since there can only be one possible match with a TrackID & ReleaseID query combo - controlValueDict = { 'TrackID' : track['TrackID'], - 'ReleaseID' : track['ReleaseID'] } - - # Insert it into the Headphones hybrid release (ReleaseID == AlbumID) - hybridControlValueDict = { 'TrackID' : track['TrackID'], - 'ReleaseID' : track['AlbumID'] } - - newValueDict = { 'Location' : song['Location'], - 'BitRate' : song['BitRate'], - 'Format' : song['Format'] } - - # Update both the tracks table and the alltracks table using the controlValueDict and hybridControlValueDict - myDB.upsert("alltracks", newValueDict, controlValueDict) - myDB.upsert("tracks", newValueDict, controlValueDict) - - myDB.upsert("alltracks", newValueDict, hybridControlValueDict) - myDB.upsert("tracks", newValueDict, hybridControlValueDict) - - # Matched. Move on to the next one: - continue - - # If we can't find it with TrackID & ReleaseID, next most specific will be - # releaseid + tracktitle, although perhaps less reliable due to a higher - # likelihood of variations in the song title (e.g. feat. artists) - if song['ReleaseID'] and song['TrackTitle']: - - track = myDB.action('SELECT TrackID, ReleaseID, AlbumID from alltracks WHERE ReleaseID=? AND TrackTitle=?', [song['ReleaseID'], song['TrackTitle']]).fetchone() - - if not track: - track = myDB.action('SELECT TrackID, ReleaseID, AlbumID from tracks WHERE ReleaseID=? AND TrackTitle=?', [song['ReleaseID'], song['TrackTitle']]).fetchone() - - if track: - # There can also only be one match for this query as well (although it might be on both the tracks and alltracks table) - # So use both TrackID & ReleaseID as the control values - controlValueDict = { 'TrackID' : track['TrackID'], - 'ReleaseID' : track['ReleaseID'] } - - hybridControlValueDict = { 'TrackID' : track['TrackID'], - 'ReleaseID' : track['AlbumID'] } - - newValueDict = { 'Location' : song['Location'], - 'BitRate' : song['BitRate'], - 'Format' : song['Format'] } - - # Update both tables here as well - myDB.upsert("alltracks", newValueDict, controlValueDict) - myDB.upsert("tracks", newValueDict, controlValueDict) - - myDB.upsert("alltracks", newValueDict, hybridControlValueDict) - myDB.upsert("tracks", newValueDict, hybridControlValueDict) - - # Done - continue - - # Next most specific will be the opposite: a TrackID and an AlbumTitle - # TrackIDs span multiple releases so if something is on an official album - # and a compilation, for example, this will match it to the right one - # However - there may be multiple matches here - if song['TrackID'] and song['AlbumTitle']: - - # Even though there might be multiple matches, we just need to grab one to confirm a match - track = myDB.action('SELECT TrackID, AlbumTitle from alltracks WHERE TrackID=? AND AlbumTitle LIKE ?', [song['TrackID'], song['AlbumTitle']]).fetchone() - - if not track: - track = myDB.action('SELECT TrackID, AlbumTitle from tracks WHERE TrackID=? AND AlbumTitle LIKE ?', [song['TrackID'], song['AlbumTitle']]).fetchone() - - if track: - # Don't need the hybridControlValueDict here since ReleaseID is not unique - controlValueDict = { 'TrackID' : track['TrackID'], - 'AlbumTitle' : track['AlbumTitle'] } - - newValueDict = { 'Location' : song['Location'], - 'BitRate' : song['BitRate'], - 'Format' : song['Format'] } - - myDB.upsert("alltracks", newValueDict, controlValueDict) - myDB.upsert("tracks", newValueDict, controlValueDict) - - continue - - # Next most specific is the ArtistName + AlbumTitle + TrackTitle combo (but probably - # even more unreliable than the previous queries, and might span multiple releases) if song['ArtistName'] and song['AlbumTitle'] and song['TrackTitle']: - track = myDB.action('SELECT ArtistName, AlbumTitle, TrackTitle from alltracks WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [song['ArtistName'], song['AlbumTitle'], song['TrackTitle']]).fetchone() - - if not track: - track = myDB.action('SELECT ArtistName, AlbumTitle, TrackTitle from tracks WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [song['ArtistName'], song['AlbumTitle'], song['TrackTitle']]).fetchone() - + track = myDB.action('SELECT ArtistName, AlbumTitle, TrackTitle, AlbumID from tracks WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [song['ArtistName'], song['AlbumTitle'], song['TrackTitle']]).fetchone() if track: controlValueDict = { 'ArtistName' : track['ArtistName'], 'AlbumTitle' : track['AlbumTitle'], - 'TrackTitle' : track['TrackTitle'] } - + 'TrackTitle' : track['TrackTitle'] } newValueDict = { 'Location' : song['Location'], 'BitRate' : song['BitRate'], 'Format' : song['Format'] } - - myDB.upsert("alltracks", newValueDict, controlValueDict) myDB.upsert("tracks", newValueDict, controlValueDict) - continue - - # Use the "CleanName" (ArtistName + AlbumTitle + TrackTitle stripped of punctuation, capitalization, etc) - # This is more reliable than the former but requires some string manipulation so we'll do it only - # if we can't find a match with the original data - if song['ArtistName'] and song['AlbumTitle'] and song['TrackTitle']: - - CleanName = helpers.cleanName(song['ArtistName'] +' '+ song['AlbumTitle'] +' '+song['TrackTitle']) - - track = myDB.action('SELECT CleanName from alltracks WHERE CleanName LIKE ?', [CleanName]).fetchone() - - if not track: - track = myDB.action('SELECT CleanName from tracks WHERE CleanName LIKE ?', [CleanName]).fetchone() - - if track: - controlValueDict = { 'CleanName' : track['CleanName'] } - + controlValueDict2 = { 'ArtistName' : song['ArtistName'], + 'AlbumTitle' : song['AlbumTitle'], + 'TrackTitle' : song['TrackTitle'] } + newValueDict2 = { 'Matched' : track['AlbumID']} + myDB.upsert("have", newValueDict2, controlValueDict2) + else: + track = myDB.action('SELECT CleanName, AlbumID from tracks WHERE CleanName LIKE ?', [song['CleanName']]).fetchone() + if track: + controlValueDict = { 'CleanName' : track['CleanName']} + newValueDict = { 'Location' : song['Location'], + 'BitRate' : song['BitRate'], + 'Format' : song['Format'] } + myDB.upsert("tracks", newValueDict, controlValueDict) + + controlValueDict2 = { 'CleanName' : song['CleanName']} + newValueDict2 = { 'Matched' : track['AlbumID']} + myDB.upsert("have", newValueDict2, controlValueDict2) + + + alltrack = myDB.action('SELECT ArtistName, AlbumTitle, TrackTitle, AlbumID from alltracks WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [song['ArtistName'], song['AlbumTitle'], song['TrackTitle']]).fetchone() + if alltrack: + controlValueDict = { 'ArtistName' : alltrack['ArtistName'], + 'AlbumTitle' : alltrack['AlbumTitle'], + 'TrackTitle' : alltrack['TrackTitle'] } newValueDict = { 'Location' : song['Location'], 'BitRate' : song['BitRate'], 'Format' : song['Format'] } - myDB.upsert("alltracks", newValueDict, controlValueDict) - myDB.upsert("tracks", newValueDict, controlValueDict) - continue - - # Match on TrackID alone if we can't find it using any of the above methods. This method is reliable - # but spans multiple releases - but that's why we're putting at the beginning as a last resort. If a track - # with more specific information exists in the library, it'll overwrite these values - if song['TrackID']: - - track = myDB.action('SELECT TrackID from alltracks WHERE TrackID=?', [song['TrackID']]).fetchone() - - if not track: - track = myDB.action('SELECT TrackID from tracks WHERE TrackID=?', [song['TrackID']]).fetchone() - - if track: - controlValueDict = { 'TrackID' : track['TrackID'] } - - newValueDict = { 'Location' : song['Location'], - 'BitRate' : song['BitRate'], - 'Format' : song['Format'] } + controlValueDict2 = { 'ArtistName' : song['ArtistName'], + 'AlbumTitle' : song['AlbumTitle'], + 'TrackTitle' : song['TrackTitle'] } + newValueDict2 = { 'Matched' : alltrack['AlbumID']} + myDB.upsert("have", newValueDict2, controlValueDict2) + else: + alltrack = myDB.action('SELECT CleanName, AlbumID from alltracks WHERE CleanName LIKE ?', [song['CleanName']]).fetchone() + if alltrack: + controlValueDict = { 'CleanName' : alltrack['CleanName']} + newValueDict = { 'Location' : song['Location'], + 'BitRate' : song['BitRate'], + 'Format' : song['Format'] } + myDB.upsert("alltracks", newValueDict, controlValueDict) - myDB.upsert("alltracks", newValueDict, controlValueDict) - myDB.upsert("tracks", newValueDict, controlValueDict) - - continue + controlValueDict2 = { 'CleanName' : song['CleanName']} + newValueDict2 = { 'Matched' : alltrack['AlbumID']} + myDB.upsert("have", newValueDict2, controlValueDict2) + # if we can't find a match in the database on a track level, it might be a new artist or it might be on a non-mb release if song['ArtistName']: @@ -302,23 +247,18 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, cron=Fal else: continue - # The have table will become the new database for unmatched tracks (i.e. tracks with no associated links in the database - if song['ArtistName'] and song['AlbumTitle'] and song['TrackTitle']: - CleanName = helpers.cleanName(song['ArtistName'] +' '+ song['AlbumTitle'] +' '+song['TrackTitle']) - else: - continue - - myDB.action('INSERT INTO have (ArtistName, AlbumTitle, TrackNumber, TrackTitle, TrackLength, BitRate, Genre, Date, TrackID, Location, CleanName, Format) VALUES( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', [song['ArtistName'], song['AlbumTitle'], song['TrackNumber'], song['TrackTitle'], song['TrackLength'], song['BitRate'], song['Genre'], song['Date'], song['TrackID'], song['Location'], CleanName, song['Format']]) + #######myDB.action('INSERT INTO have (ArtistName, AlbumTitle, TrackNumber, TrackTitle, TrackLength, BitRate, Genre, Date, TrackID, Location, CleanName, Format) VALUES( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', [song['ArtistName'], song['AlbumTitle'], song['TrackNumber'], song['TrackTitle'], song['TrackLength'], song['BitRate'], song['Genre'], song['Date'], song['TrackID'], song['Location'], CleanName, song['Format']]) logger.info('Completed matching tracks from directory: %s' % dir.decode(headphones.SYS_ENCODING, 'replace')) - + if not append: # Clean up the new artist list unique_artists = {}.fromkeys(new_artists).keys() current_artists = myDB.select('SELECT ArtistName, ArtistID from artists') - - artist_list = [f for f in unique_artists if f.lower() not in [x[0].lower() for x in current_artists]] + + #There was a bug where artists with special characters (-,') would show up in new artists. + artist_list = [f for f in unique_artists if helpers.cleanName(f).lower() not in [helpers.cleanName(x[0]).lower() for x in current_artists]] # Update track counts logger.info('Updating current artist track counts') @@ -326,7 +266,7 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, cron=Fal for artist in current_artists: # Have tracks are selected from tracks table and not all tracks because of duplicates # We update the track count upon an album switch to compliment this - havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=? AND Location IS NOT NULL', [artist['ArtistID']])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ?', [artist['ArtistName']])) + havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=? AND Location IS NOT NULL', [artist['ArtistID']])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ? AND Matched IS NULL', [artist['ArtistName']])) myDB.action('UPDATE artists SET HaveTracks=? WHERE ArtistID=?', [havetracks, artist['ArtistID']]) logger.info('Found %i new artists' % len(artist_list)) @@ -348,6 +288,41 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, cron=Fal # If we're appending a new album to the database, update the artists total track counts logger.info('Updating artist track counts') - havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=? AND Location IS NOT NULL', [ArtistID])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ?', [ArtistName])) + havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=? AND Location IS NOT NULL', [ArtistID])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ? AND Matched IS NULL', [ArtistName])) myDB.action('UPDATE artists SET HaveTracks=? WHERE ArtistID=?', [havetracks, ArtistID]) - + + update_album_status() + logger.info('Library scan complete') + + #ADDED THIS SECTION TO MARK ALBUMS AS DOWNLOADED IF ARTISTS ARE ADDED EN MASSE BEFORE LIBRARY IS SCANNED +def update_album_status(AlbumID=None): + myDB = db.DBConnection() + logger.info('Counting matched tracks to mark albums as skipped/downloaded') + if AlbumID: + album_status_updater = myDB.action('SELECT AlbumID, AlbumTitle, Status from albums WHERE AlbumID=?', [AlbumID]) + else: + album_status_updater = myDB.action('SELECT AlbumID, AlbumTitle, Status from albums') + for album in album_status_updater: + track_counter = myDB.action('SELECT Location from tracks where AlbumID=?', [album['AlbumID']]) + total_tracks = 0 + have_tracks = 0 + for track in track_counter: + total_tracks+=1 + if track['Location']: + have_tracks+=1 + if total_tracks != 0: + album_completion = float(float(have_tracks) / float(total_tracks)) * 100 + else: + album_completion = 0 + logger.info('Album %s does not have any tracks in database' % album['AlbumTitle']) + + if album['Status'] == "Downloaded" or album['Status'] == "Skipped": + if album_completion >= headphones.ALBUM_COMPLETION_PCT: + new_album_status = "Downloaded" + myDB.upsert("albums", {'Status' : "Downloaded"}, {'AlbumID' : album['AlbumID']}) + else: + new_album_status = "Skipped" + myDB.upsert("albums", {'Status' : "Skipped"}, {'AlbumID' : album['AlbumID']}) + if new_album_status != album['Status']: + logger.info('Album %s changed to %s' % (album['AlbumTitle'], new_album_status)) + logger.info('Album status update complete') \ No newline at end of file diff --git a/headphones/mb.py b/headphones/mb.py index 4f1d0e29..14404816 100644 --- a/headphones/mb.py +++ b/headphones/mb.py @@ -463,7 +463,7 @@ def get_new_releases(rgid,includeExtras=False,forcefull=False): newValueDict['Location'] = match['Location'] newValueDict['BitRate'] = match['BitRate'] newValueDict['Format'] = match['Format'] - myDB.action('UPDATE have SET Matched="True" WHERE Location=?', [match['Location']]) + #myDB.action('UPDATE have SET Matched="True" WHERE Location=?', [match['Location']]) myDB.upsert("alltracks", newValueDict, controlValueDict) num_new_releases = num_new_releases + 1 diff --git a/headphones/webserve.py b/headphones/webserve.py index 82fe8fa2..974777c9 100644 --- a/headphones/webserve.py +++ b/headphones/webserve.py @@ -24,11 +24,13 @@ from mako import exceptions import time import threading +import string +import json import headphones -from headphones import logger, searcher, db, importer, mb, lastfm, librarysync -from headphones.helpers import checked, radio,today +from headphones import logger, searcher, db, importer, mb, lastfm, librarysync, helpers +from headphones.helpers import checked, radio,today, cleanName import lib.simplejson as simplejson @@ -189,11 +191,13 @@ class WebInterface(object): def deleteArtist(self, ArtistID): logger.info(u"Deleting all traces of artist: " + ArtistID) myDB = db.DBConnection() + artistname = myDB.select('SELECT ArtistName from artists where ArtistID=?', [ArtistID]).fetchone() myDB.action('DELETE from artists WHERE ArtistID=?', [ArtistID]) myDB.action('DELETE from albums WHERE ArtistID=?', [ArtistID]) myDB.action('DELETE from tracks WHERE ArtistID=?', [ArtistID]) myDB.action('DELETE from allalbums WHERE ArtistID=?', [ArtistID]) myDB.action('DELETE from alltracks WHERE ArtistID=?', [ArtistID]) + myDB.action('UPDATE have SET Matched=NULL WHERE ArtistName=?', [artistname]) myDB.action('INSERT OR REPLACE into blacklist VALUES (?)', [ArtistID]) raise cherrypy.HTTPRedirect("home") deleteArtist.exposed = True @@ -213,7 +217,7 @@ class WebInterface(object): deleteEmptyArtists.exposed = True def refreshArtist(self, ArtistID): - threading.Thread(target=importer.addArtisttoDB, args=[ArtistID]).start() + threading.Thread(target=importer.addArtisttoDB, args=[ArtistID, False, True]).start() raise cherrypy.HTTPRedirect("artistPage?ArtistID=%s" % ArtistID) refreshArtist.exposed=True @@ -240,8 +244,15 @@ class WebInterface(object): raise cherrypy.HTTPRedirect("upcoming") markAlbums.exposed = True - def addArtists(self, **args): - threading.Thread(target=importer.artistlist_to_mbids, args=[args, True]).start() + def addArtists(self, action=None, **args): + if action == "add": + threading.Thread(target=importer.artistlist_to_mbids, args=[args, True]).start() + if action == "ignore": + myDB = db.DBConnection() + for artist in args: + myDB.action('DELETE FROM newartists WHERE ArtistName=?', [artist]) + myDB.action('UPDATE have SET Matched="Ignored" WHERE ArtistName=?', [artist]) + logger.info("Artist %s removed from new artist list and set to ignored" % artist) raise cherrypy.HTTPRedirect("home") addArtists.exposed = True @@ -338,6 +349,120 @@ class WebInterface(object): return serve_template(templatename="managenew.html", title="Manage New Artists", newartists=newartists) manageNew.exposed = True + def manageUnmatched(self): + myDB = db.DBConnection() + have_album_dictionary = [] + headphones_album_dictionary = [] + unmatched_albums = [] + have_albums = myDB.select('SELECT ArtistName, AlbumTitle, TrackTitle, CleanName from have WHERE Matched IS NULL GROUP BY AlbumTitle ORDER BY ArtistName') + for albums in have_albums: + #Have to skip over manually matched tracks + original_clean = helpers.cleanName(albums['ArtistName']+" "+albums['AlbumTitle']+" "+albums['TrackTitle']) + if original_clean == albums['CleanName']: + have_dict = { 'ArtistName' : albums['ArtistName'], 'AlbumTitle' : albums['AlbumTitle'] } + have_album_dictionary.append(have_dict) + headphones_albums = myDB.select('SELECT ArtistName, AlbumTitle from albums ORDER BY ArtistName') + for albums in headphones_albums: + headphones_dict = { 'ArtistName' : albums['ArtistName'], 'AlbumTitle' : albums['AlbumTitle'] } + headphones_album_dictionary.append(headphones_dict) + #unmatchedalbums = [f for f in have_album_dictionary if f not in [x for x in headphones_album_dictionary]] + + check = set([(cleanName(d['ArtistName']).lower(), cleanName(d['AlbumTitle']).lower()) for d in headphones_album_dictionary]) + unmatchedalbums = [d for d in have_album_dictionary if (cleanName(d['ArtistName']).lower(), cleanName(d['AlbumTitle']).lower()) not in check] + + + return serve_template(templatename="manageunmatched.html", title="Manage Unmatched Items", unmatchedalbums=unmatchedalbums) + manageUnmatched.exposed = True + + def markUnmatched(self, action=None, existing_artist=None, existing_album=None, new_artist=None, new_album=None): + myDB = db.DBConnection() + + if action == "ignoreArtist": + artist = existing_artist + myDB.action('UPDATE have SET Matched="Ignored" WHERE ArtistName=? AND Matched IS NULL', [artist]) + + elif action == "ignoreAlbum": + artist = existing_artist + album = existing_album + myDB.action('UPDATE have SET Matched="Ignored" WHERE ArtistName=? AND AlbumTitle=? AND Matched IS NULL', (artist, album)) + + elif action == "matchArtist": + existing_artist_clean = helpers.cleanName(existing_artist).lower() + new_artist_clean = helpers.cleanName(new_artist).lower() + if new_artist_clean != existing_artist_clean: + have_tracks = myDB.action('SELECT Matched, CleanName, Location, BitRate, Format FROM have WHERE ArtistName=?', [existing_artist]) + update_count = 0 + for entry in have_tracks: + old_clean_filename = entry['CleanName'] + if old_clean_filename.startswith(existing_artist_clean): + new_clean_filename = old_clean_filename.replace(existing_artist_clean, new_artist_clean, 1) + myDB.action('UPDATE have SET CleanName=? WHERE ArtistName=? AND CleanName=?', [new_clean_filename, existing_artist, old_clean_filename]) + controlValueDict = {"CleanName": new_clean_filename} + newValueDict = {"Location" : entry['Location'], + "BitRate" : entry['BitRate'], + "Format" : entry['Format'] + } + #Attempt to match tracks with new CleanName + match_alltracks = myDB.action('SELECT CleanName from alltracks WHERE CleanName=?', [new_clean_filename]).fetchone() + if match_alltracks: + myDB.upsert("alltracks", newValueDict, controlValueDict) + match_tracks = myDB.action('SELECT CleanName from tracks WHERE CleanName=?', [new_clean_filename]).fetchone() + if match_tracks: + myDB.upsert("tracks", newValueDict, controlValueDict) + myDB.action('UPDATE have SET Matched="Manual" WHERE CleanName=?', [new_clean_filename]) + update_count+=1 + #This was throwing errors and I don't know why, but it seems to be working fine. + #else: + #logger.info("There was an error modifying Artist %s. This should not have happened" % existing_artist) + logger.info("Manual matching yielded %s new matches for Artist %s" % (update_count, new_artist)) + if update_count > 0: + librarysync.update_album_status() + else: + logger.info("Artist %s already named appropriately; nothing to modify" % existing_artist) + + elif action == "matchAlbum": + existing_artist_clean = helpers.cleanName(existing_artist).lower() + new_artist_clean = helpers.cleanName(new_artist).lower() + existing_album_clean = helpers.cleanName(existing_album).lower() + new_album_clean = helpers.cleanName(new_album).lower() + existing_clean_string = existing_artist_clean+" "+existing_album_clean + new_clean_string = new_artist_clean+" "+new_album_clean + if existing_clean_string != new_clean_string: + have_tracks = myDB.action('SELECT Matched, CleanName, Location, BitRate, Format FROM have WHERE ArtistName=? AND AlbumTitle=?', (existing_artist, existing_album)) + update_count = 0 + for entry in have_tracks: + old_clean_filename = entry['CleanName'] + if old_clean_filename.startswith(existing_clean_string): + new_clean_filename = old_clean_filename.replace(existing_clean_string, new_clean_string, 1) + myDB.action('UPDATE have SET CleanName=? WHERE ArtistName=? AND AlbumTitle=? AND CleanName=?', [new_clean_filename, existing_artist, existing_album, old_clean_filename]) + controlValueDict = {"CleanName": new_clean_filename} + newValueDict = {"Location" : entry['Location'], + "BitRate" : entry['BitRate'], + "Format" : entry['Format'] + } + #Attempt to match tracks with new CleanName + match_alltracks = myDB.action('SELECT CleanName from alltracks WHERE CleanName=?', [new_clean_filename]).fetchone() + if match_alltracks: + myDB.upsert("alltracks", newValueDict, controlValueDict) + match_tracks = myDB.action('SELECT CleanName, AlbumID from tracks WHERE CleanName=?', [new_clean_filename]).fetchone() + if match_tracks: + myDB.upsert("tracks", newValueDict, controlValueDict) + myDB.action('UPDATE have SET Matched="Manual" WHERE CleanName=?', [new_clean_filename]) + update_count+=1 + #This was throwing errors and I don't know why, but it seems to be working fine. + #else: + #logger.info("There was an error modifying Artist %s / Album %s. This should not have happened" % (existing_artist, existing_album)) + logger.info("Manual matching yielded %s new matches for Artist %s / Album %s" % (update_count, new_artist, new_album)) + if update_count > 0: + librarysync.update_album_status() + else: + logger.info("Artist %s / Album %s already named appropriately; nothing to modify" % (existing_artist, existing_album)) + + + + raise cherrypy.HTTPRedirect('manageUnmatched') + markUnmatched.exposed = True + def markArtists(self, action=None, **args): myDB = db.DBConnection() artistsToAdd = [] @@ -545,6 +670,20 @@ class WebInterface(object): return s getArtists_json.exposed=True + def getAlbumsByArtist_json(self, artist=None): + myDB = db.DBConnection() + album_json = {} + counter = 0 + album_list = myDB.select("SELECT AlbumTitle from albums WHERE ArtistName=?", [artist]) + for album in album_list: + album_json[counter] = album['AlbumTitle'] + counter+=1 + json_albums = json.dumps(album_json) + + cherrypy.response.headers['Content-type'] = 'application/json' + return json_albums + getAlbumsByArtist_json.exposed=True + def clearhistory(self, type=None): myDB = db.DBConnection() if type == 'all': @@ -566,6 +705,26 @@ class WebInterface(object): generateAPI.exposed = True + def forceScan(self, keepmatched=None): + myDB = db.DBConnection() + ######################################### + #NEED TO MOVE THIS INTO A SEPARATE FUNCTION BEFORE RELEASE + myDB.select('DELETE from Have') + logger.info('Removed all entries in local library database') + myDB.select('UPDATE alltracks SET Location=NULL, BitRate=NULL, Format=NULL') + myDB.select('UPDATE tracks SET Location=NULL, BitRate=NULL, Format=NULL') + logger.info('All tracks in library unmatched') + myDB.action('UPDATE artists SET HaveTracks=NULL') + logger.info('Reset track counts for all artists') + myDB.action('UPDATE albums SET Status="Skipped" WHERE Status="Skipped" OR Status="Downloaded"') + logger.info('Marking all unwanted albums as Skipped') + try: + threading.Thread(target=librarysync.libraryScan).start() + except Exception, e: + logger.error('Unable to complete the scan: %s' % e) + raise cherrypy.HTTPRedirect("home") + forceScan.exposed = True + def config(self): interface_dir = os.path.join(headphones.PROG_DIR, 'data/interfaces/')