From 8ef1fb45c814986501121f3d6fa30496a1ee5f66 Mon Sep 17 00:00:00 2001 From: theguardian Date: Tue, 1 Oct 2013 12:25:09 -0700 Subject: [PATCH 01/47] Rehashed "update active artist" algorithm to address time/CPU load issues. --- headphones/importer.py | 430 ++++++++++++++++++++++------------------- headphones/mb.py | 201 +++++++++++++------ 2 files changed, 372 insertions(+), 259 deletions(-) diff --git a/headphones/importer.py b/headphones/importer.py index 0365a872..ede5c3b4 100644 --- a/headphones/importer.py +++ b/headphones/importer.py @@ -15,6 +15,7 @@ from lib.pyItunes import * import time +import threading import os from lib.beets.mediafile import MediaFile @@ -189,65 +190,142 @@ def addArtisttoDB(artistid, extrasonly=False): db_artist = myDB.action('SELECT IncludeExtras, Extras from artists WHERE ArtistID=?', [artistid]).fetchone() includeExtras = db_artist['IncludeExtras'] except IndexError: - includeExtras = False + includeExtras = False + + #Clean all references to release group in dB that are no longer referenced in musicbrainz + group_list = [] + force_repackage = 0 + #Don't nuke the database if there's a MusicBrainz error + if len(artist['releasegroups']) != 0: + for groups in artist['releasegroups']: + group_list.append(groups['id']) + remove_missing_groups_from_albums = myDB.action("SELECT ReleaseID FROM albums WHERE ArtistID=?", [artistid]) + for items in remove_missing_groups_from_albums: + if items['ReleaseID'] not in group_list: + # Remove all from albums/tracks that aren't in release groups + myDB.action("DELETE FROM albums WHERE ReleaseID=?", [items['ReleaseID']]) + myDB.action("DELETE FROM allalbums WHERE ReleaseID=?", [items['ReleaseID']]) + myDB.action("DELETE FROM tracks WHERE ReleaseID=?", [items['ReleaseID']]) + myDB.action("DELETE FROM alltracks WHERE ReleaseID=?", [items['ReleaseID']]) + logger.info("Removing all references to group %s to reflect MusicBrainz" % items['ReleaseID']) + force_repackage = 1 + else: + logger.info("Error pulling data from MusicBrainz: Maintaining dB") + + # Then search for releases within releasegroups, if releases don't exist, then remove from allalbums/alltracks + for rg in artist['releasegroups']: - logger.info("Now adding/updating: " + rg['title']) - + al_title = rg['title'] + today = helpers.today() rgid = rg['id'] - - # check if the album already exists - rg_exists = myDB.action("SELECT * from albums WHERE AlbumID=?", [rg['id']]).fetchone() - - releases = mb.get_all_releases(rgid,includeExtras) - if releases == []: - logger.info('No official releases in release group %s' % rg['title']) - continue - if not releases: - errors = True - logger.info('Unable to get release information for %s - there may not be any official releases in this release group' % rg['title']) - continue - - # This will be used later to build a hybrid release - fullreleaselist = [] + skip_log = 0 + #new_releases = 0 - for release in releases: - # What we're doing here now is first updating the allalbums & alltracks table to the most - # current info, then moving the appropriate release into the album table and its associated - # tracks into the tracks table - controlValueDict = {"ReleaseID" : release['ReleaseID']} + check_release_date = myDB.action("SELECT ReleaseDate from albums WHERE ArtistID=? AND AlbumTitle=?", (artistid, al_title)).fetchone() + if check_release_date: + if check_release_date[0] is None: + logger.info("Now updating: " + rg['title']) + new_releases = mb.get_new_releases(rgid,includeExtras) + elif len(check_release_date[0])!=10: + logger.info("Now updating: " + rg['title']) + new_releases = mb.get_new_releases(rgid,includeExtras) + else: + if helpers.get_age(today) - helpers.get_age(check_release_date[0]) < 365: + logger.info("Now updating: " + rg['title']) + new_releases = mb.get_new_releases(rgid,includeExtras) + else: + logger.info('%s is over a year old; not updating' % al_title) + skip_log = 1 + new_releases = 0 + else: + logger.info("Now adding/updating: " + rg['title']) + new_releases = mb.get_new_releases(rgid,includeExtras) - newValueDict = {"ArtistID": release['ArtistID'], - "ArtistName": release['ArtistName'], - "AlbumTitle": release['AlbumTitle'], - "AlbumID": release['AlbumID'], - "AlbumASIN": release['AlbumASIN'], - "ReleaseDate": release['ReleaseDate'], - "Type": release['Type'], - "ReleaseCountry": release['ReleaseCountry'], - "ReleaseFormat": release['ReleaseFormat'] + if force_repackage == 1: + new_releases = -1 + logger.info('Forcing repackage of %s, since dB references have been removed' % al_title) + else: + new_releases = new_releases + + #What this does is adds new releases per artist to the allalbums + alltracks databases + #new_releases = mb.get_new_releases(rgid,includeExtras) + #print al_title + #print new_releases + + if new_releases != 0: + # This will be used later to build a hybrid release + fullreleaselist = [] + #Search for releases within a release group + find_hybrid_releases = myDB.action("SELECT * from allalbums WHERE AlbumID=?", [rg['id']]) + # Build the dictionary for the fullreleaselist + for items in find_hybrid_releases: + hybrid_release_id = items['ReleaseID'] + newValueDict = {"ArtistID": items['ArtistID'], + "ArtistName": items['ArtistName'], + "AlbumTitle": items['AlbumTitle'], + "AlbumID": items['AlbumID'], + "AlbumASIN": items['AlbumASIN'], + "ReleaseDate": items['ReleaseDate'], + "Type": items['Type'], + "ReleaseCountry": items['ReleaseCountry'], + "ReleaseFormat": items['ReleaseFormat'] + } + find_hybrid_tracks = myDB.action("SELECT * from alltracks WHERE ReleaseID=?", [hybrid_release_id]) + totalTracks = 1 + hybrid_track_array = [] + for hybrid_tracks in find_hybrid_tracks: + hybrid_track_array.append({ + 'number': hybrid_tracks['TrackNumber'], + 'title': hybrid_tracks['TrackTitle'], + 'id': hybrid_tracks['TrackID'], + #'url': hybrid_tracks['TrackURL'], + 'duration': hybrid_tracks['TrackDuration'] + }) + totalTracks += 1 + newValueDict['ReleaseID'] = hybrid_release_id + newValueDict['Tracks'] = hybrid_track_array + fullreleaselist.append(newValueDict) + + # Basically just do the same thing again for the hybrid release + # This may end up being called with an empty fullreleaselist + try: + hybridrelease = getHybridRelease(fullreleaselist) + logger.info('Packaging %s releases into hybrid title' % rg['title']) + except Exception, e: + errors = True + logger.warn('Unable to get hybrid release information for %s: %s' % (rg['title'],e)) + continue + + # Use the ReleaseGroupID as the ReleaseID for the hybrid release to differentiate it + # We can then use the condition WHERE ReleaseID == ReleaseGroupID to select it + # The hybrid won't have a country or a format + controlValueDict = {"ReleaseID": rg['id']} + + newValueDict = {"ArtistID": artistid, + "ArtistName": artist['artist_name'], + "AlbumTitle": rg['title'], + "AlbumID": rg['id'], + "AlbumASIN": hybridrelease['AlbumASIN'], + "ReleaseDate": hybridrelease['ReleaseDate'], + "Type": rg['type'] } - + myDB.upsert("allalbums", newValueDict, controlValueDict) - # Build the dictionary for the fullreleaselist - newValueDict['ReleaseID'] = release['ReleaseID'] - newValueDict['Tracks'] = release['Tracks'] - fullreleaselist.append(newValueDict) - - for track in release['Tracks']: + for track in hybridrelease['Tracks']: cleanname = helpers.cleanName(artist['artist_name'] + ' ' + rg['title'] + ' ' + track['title']) controlValueDict = {"TrackID": track['id'], - "ReleaseID": release['ReleaseID']} + "ReleaseID": rg['id']} - newValueDict = {"ArtistID": release['ArtistID'], - "ArtistName": release['ArtistName'], - "AlbumTitle": release['AlbumTitle'], - "AlbumID": release['AlbumID'], - "AlbumASIN": release['AlbumASIN'], + newValueDict = {"ArtistID": artistid, + "ArtistName": artist['artist_name'], + "AlbumTitle": rg['title'], + "AlbumASIN": hybridrelease['AlbumASIN'], + "AlbumID": rg['id'], "TrackTitle": track['title'], "TrackDuration": track['duration'], "TrackNumber": track['number'], @@ -267,171 +345,119 @@ def addArtisttoDB(artistid, extrasonly=False): myDB.action('UPDATE have SET Matched="True" WHERE Location=?', [match['Location']]) myDB.upsert("alltracks", newValueDict, controlValueDict) - - # Basically just do the same thing again for the hybrid release - # This may end up being called with an empty fullreleaselist - try: - hybridrelease = getHybridRelease(fullreleaselist) - except Exception, e: - errors = True - logger.warn('Unable to get hybrid release information for %s: %s' % (rg['title'],e)) - continue - - # Use the ReleaseGroupID as the ReleaseID for the hybrid release to differentiate it - # We can then use the condition WHERE ReleaseID == ReleaseGroupID to select it - # The hybrid won't have a country or a format - controlValueDict = {"ReleaseID": rg['id']} - - newValueDict = {"ArtistID": artistid, - "ArtistName": artist['artist_name'], - "AlbumTitle": rg['title'], - "AlbumID": rg['id'], - "AlbumASIN": hybridrelease['AlbumASIN'], - "ReleaseDate": hybridrelease['ReleaseDate'], - "Type": rg['type'] - } - - myDB.upsert("allalbums", newValueDict, controlValueDict) - - for track in hybridrelease['Tracks']: - - cleanname = helpers.cleanName(artist['artist_name'] + ' ' + rg['title'] + ' ' + track['title']) - - controlValueDict = {"TrackID": track['id'], - "ReleaseID": rg['id']} - - newValueDict = {"ArtistID": artistid, - "ArtistName": artist['artist_name'], - "AlbumTitle": rg['title'], - "AlbumASIN": hybridrelease['AlbumASIN'], - "AlbumID": rg['id'], - "TrackTitle": track['title'], - "TrackDuration": track['duration'], - "TrackNumber": track['number'], - "CleanName": cleanname - } - - match = myDB.action('SELECT Location, BitRate, Format from have WHERE CleanName=?', [cleanname]).fetchone() - - if not match: - match = myDB.action('SELECT Location, BitRate, Format from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [artist['artist_name'], rg['title'], track['title']]).fetchone() - if not match: - match = myDB.action('SELECT Location, BitRate, Format from have WHERE TrackID=?', [track['id']]).fetchone() - if match: - newValueDict['Location'] = match['Location'] - newValueDict['BitRate'] = match['BitRate'] - newValueDict['Format'] = match['Format'] - myDB.action('UPDATE have SET Matched="True" WHERE Location=?', [match['Location']]) - - myDB.upsert("alltracks", newValueDict, controlValueDict) - - # Delete matched tracks from the have table - myDB.action('DELETE from have WHERE Matched="True"') - - # If there's no release in the main albums tables, add the default (hybrid) - # If there is a release, check the ReleaseID against the AlbumID to see if they differ (user updated) - if not rg_exists: - releaseid = rg['id'] - elif rg_exists and not rg_exists['ReleaseID']: - # Need to do some importing here - to transition the old format of using the release group - # only to using releasegroup & releaseid. These are the albums that are missing a ReleaseID - # so we'll need to move over the locations, bitrates & formats from the tracks table to the new - # alltracks table. Thankfully we can just use TrackIDs since they span releases/releasegroups - logger.info("Copying current track information to alternate releases") - tracks = myDB.action('SELECT * from tracks WHERE AlbumID=?', [rg['id']]).fetchall() - for track in tracks: - if track['Location']: - controlValueDict = {"TrackID": track['TrackID']} - newValueDict = {"Location": track['Location'], - "BitRate": track['BitRate'], - "Format": track['Format'], - } - myDB.upsert("alltracks", newValueDict, controlValueDict) - releaseid = rg['id'] - else: - releaseid = rg_exists['ReleaseID'] - - album = myDB.action('SELECT * from allalbums WHERE ReleaseID=?', [releaseid]).fetchone() - - controlValueDict = {"AlbumID": rg['id']} - - newValueDict = {"ArtistID": album['ArtistID'], - "ArtistName": album['ArtistName'], - "AlbumTitle": album['AlbumTitle'], - "ReleaseID": album['ReleaseID'], - "AlbumASIN": album['AlbumASIN'], - "ReleaseDate": album['ReleaseDate'], - "Type": album['Type'], - "ReleaseCountry": album['ReleaseCountry'], - "ReleaseFormat": album['ReleaseFormat'] - } - if not rg_exists: + # Delete matched tracks from the have table + myDB.action('DELETE from have WHERE Matched="True"') - today = helpers.today() - - newValueDict['DateAdded']= today - - if headphones.AUTOWANT_ALL: - newValueDict['Status'] = "Wanted" - elif album['ReleaseDate'] > today and headphones.AUTOWANT_UPCOMING: - newValueDict['Status'] = "Wanted" - # Sometimes "new" albums are added to musicbrainz after their release date, so let's try to catch these - # The first test just makes sure we have year-month-day - elif helpers.get_age(album['ReleaseDate']) and helpers.get_age(today) - helpers.get_age(album['ReleaseDate']) < 21 and headphones.AUTOWANT_UPCOMING: - newValueDict['Status'] = "Wanted" + # If there's no release in the main albums tables, add the default (hybrid) + # If there is a release, check the ReleaseID against the AlbumID to see if they differ (user updated) + # check if the album already exists + rg_exists = myDB.action("SELECT * from albums WHERE AlbumID=?", [rg['id']]).fetchone() + if not rg_exists: + releaseid = rg['id'] + elif rg_exists and not rg_exists['ReleaseID']: + # Need to do some importing here - to transition the old format of using the release group + # only to using releasegroup & releaseid. These are the albums that are missing a ReleaseID + # so we'll need to move over the locations, bitrates & formats from the tracks table to the new + # alltracks table. Thankfully we can just use TrackIDs since they span releases/releasegroups + logger.info("Copying current track information to alternate releases") + tracks = myDB.action('SELECT * from tracks WHERE AlbumID=?', [rg['id']]).fetchall() + for track in tracks: + if track['Location']: + controlValueDict = {"TrackID": track['TrackID']} + newValueDict = {"Location": track['Location'], + "BitRate": track['BitRate'], + "Format": track['Format'], + } + myDB.upsert("alltracks", newValueDict, controlValueDict) + releaseid = rg['id'] else: - newValueDict['Status'] = "Skipped" - - myDB.upsert("albums", newValueDict, controlValueDict) + releaseid = rg_exists['ReleaseID'] + + album = myDB.action('SELECT * from allalbums WHERE ReleaseID=?', [releaseid]).fetchone() - myDB.action('DELETE from tracks WHERE AlbumID=?', [rg['id']]) - tracks = myDB.action('SELECT * from alltracks WHERE ReleaseID=?', [releaseid]).fetchall() + controlValueDict = {"AlbumID": rg['id']} - # This is used to see how many tracks you have from an album - to mark it as downloaded. Default is 80%, can be set in config as ALBUM_COMPLETION_PCT - total_track_count = len(tracks) - - for track in tracks: - - controlValueDict = {"TrackID": track['TrackID'], - "AlbumID": rg['id']} - - newValueDict = {"ArtistID": track['ArtistID'], - "ArtistName": track['ArtistName'], - "AlbumTitle": track['AlbumTitle'], - "AlbumASIN": track['AlbumASIN'], - "ReleaseID": track['ReleaseID'], - "TrackTitle": track['TrackTitle'], - "TrackDuration": track['TrackDuration'], - "TrackNumber": track['TrackNumber'], - "CleanName": track['CleanName'], - "Location": track['Location'], - "Format": track['Format'], - "BitRate": track['BitRate'] + newValueDict = {"ArtistID": album['ArtistID'], + "ArtistName": album['ArtistName'], + "AlbumTitle": album['AlbumTitle'], + "ReleaseID": album['ReleaseID'], + "AlbumASIN": album['AlbumASIN'], + "ReleaseDate": album['ReleaseDate'], + "Type": album['Type'], + "ReleaseCountry": album['ReleaseCountry'], + "ReleaseFormat": album['ReleaseFormat'] } - - myDB.upsert("tracks", newValueDict, controlValueDict) + + if not rg_exists: + + today = helpers.today() + + newValueDict['DateAdded']= today + + if headphones.AUTOWANT_ALL: + newValueDict['Status'] = "Wanted" + elif album['ReleaseDate'] > today and headphones.AUTOWANT_UPCOMING: + newValueDict['Status'] = "Wanted" + # Sometimes "new" albums are added to musicbrainz after their release date, so let's try to catch these + # The first test just makes sure we have year-month-day + elif helpers.get_age(album['ReleaseDate']) and helpers.get_age(today) - helpers.get_age(album['ReleaseDate']) < 21 and headphones.AUTOWANT_UPCOMING: + newValueDict['Status'] = "Wanted" + else: + newValueDict['Status'] = "Skipped" + + myDB.upsert("albums", newValueDict, controlValueDict) - # Mark albums as downloaded if they have at least 80% (by default, configurable) of the album - have_track_count = len(myDB.select('SELECT * from tracks WHERE AlbumID=? AND Location IS NOT NULL', [rg['id']])) - marked_as_downloaded = False - - if rg_exists: - if rg_exists['Status'] == 'Skipped' and ((have_track_count/float(total_track_count)) >= (headphones.ALBUM_COMPLETION_PCT/100.0)): - myDB.action('UPDATE albums SET Status=? WHERE AlbumID=?', ['Downloaded', rg['id']]) - marked_as_downloaded = True + myDB.action('DELETE from tracks WHERE AlbumID=?', [rg['id']]) + tracks = myDB.action('SELECT * from alltracks WHERE ReleaseID=?', [releaseid]).fetchall() + + # This is used to see how many tracks you have from an album - to mark it as downloaded. Default is 80%, can be set in config as ALBUM_COMPLETION_PCT + total_track_count = len(tracks) + + for track in tracks: + + controlValueDict = {"TrackID": track['TrackID'], + "AlbumID": rg['id']} + + newValueDict = {"ArtistID": track['ArtistID'], + "ArtistName": track['ArtistName'], + "AlbumTitle": track['AlbumTitle'], + "AlbumASIN": track['AlbumASIN'], + "ReleaseID": track['ReleaseID'], + "TrackTitle": track['TrackTitle'], + "TrackDuration": track['TrackDuration'], + "TrackNumber": track['TrackNumber'], + "CleanName": track['CleanName'], + "Location": track['Location'], + "Format": track['Format'], + "BitRate": track['BitRate'] + } + + myDB.upsert("tracks", newValueDict, controlValueDict) + + # Mark albums as downloaded if they have at least 80% (by default, configurable) of the album + have_track_count = len(myDB.select('SELECT * from tracks WHERE AlbumID=? AND Location IS NOT NULL', [rg['id']])) + marked_as_downloaded = False + + if rg_exists: + if rg_exists['Status'] == 'Skipped' and ((have_track_count/float(total_track_count)) >= (headphones.ALBUM_COMPLETION_PCT/100.0)): + myDB.action('UPDATE albums SET Status=? WHERE AlbumID=?', ['Downloaded', rg['id']]) + marked_as_downloaded = True + else: + if ((have_track_count/float(total_track_count)) >= (headphones.ALBUM_COMPLETION_PCT/100.0)): + myDB.action('UPDATE albums SET Status=? WHERE AlbumID=?', ['Downloaded', rg['id']]) + marked_as_downloaded = True + + logger.info(u"Seeing if we need album art for " + rg['title']) + cache.getThumb(AlbumID=rg['id']) + + #start a search for the album if it's new, hasn't been marked as downloaded and autowant_all is selected: + if not rg_exists and not marked_as_downloaded and headphones.AUTOWANT_ALL: + from headphones import searcher + searcher.searchforalbum(albumid=rg['id']) else: - if ((have_track_count/float(total_track_count)) >= (headphones.ALBUM_COMPLETION_PCT/100.0)): - myDB.action('UPDATE albums SET Status=? WHERE AlbumID=?', ['Downloaded', rg['id']]) - marked_as_downloaded = True - - logger.info(u"Seeing if we need album art for " + rg['title']) - cache.getThumb(AlbumID=rg['id']) - - #start a search for the album if it's new, hasn't been marked as downloaded and autowant_all is selected: - if not rg_exists and not marked_as_downloaded and headphones.AUTOWANT_ALL: - from headphones import searcher - searcher.searchforalbum(albumid=rg['id']) + if skip_log == 0: + logger.info(u"No new releases, so no changes made to " + rg['title']) latestalbum = myDB.action('SELECT AlbumTitle, ReleaseDate, AlbumID from albums WHERE ArtistID=? order by ReleaseDate DESC', [artistid]).fetchone() totaltracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=?', [artistid])) diff --git a/headphones/mb.py b/headphones/mb.py index ce35d641..e725f047 100644 --- a/headphones/mb.py +++ b/headphones/mb.py @@ -19,7 +19,7 @@ import time import threading import headphones -from headphones import logger, db +from headphones import logger, db, helpers from headphones.helpers import multikeysort, replace_all import lib.musicbrainzngs as musicbrainzngs @@ -332,65 +332,152 @@ def getRelease(releaseid, include_artist_info=True): return release -def get_all_releases(rgid,includeExtras=False): - results = [] - try: - limit = 100 - newResults = None - while newResults == None or len(newResults) >= limit: - newResults = musicbrainzngs.browse_releases(release_group=rgid,includes=['artist-credits','labels','recordings','release-groups','media'],limit=limit,offset=len(results)) - if 'release-list' not in newResults: - break #may want to raise an exception here instead ? - newResults = newResults['release-list'] - results += newResults - - except WebServiceError, e: - logger.warn('Attempt to retrieve information from MusicBrainz for release group "%s" failed (%s)' % (rgid, str(e))) - time.sleep(5) - return False - - if not results or len(results) == 0: - return False +def get_new_releases(rgid,includeExtras=False): - - releases = [] - for releasedata in results: - #releasedata.get will return None if it doesn't have a status - #all official releases should have the Official status included - if not includeExtras and releasedata.get('status') != 'Official': - continue - - release = {} - release['AlbumTitle'] = unicode(releasedata['title']) - release['AlbumID'] = unicode(rgid) - release['AlbumASIN'] = unicode(releasedata['asin']) if 'asin' in releasedata else None - release['ReleaseDate'] = unicode(releasedata['date']) if 'date' in releasedata else None - release['ReleaseID'] = releasedata['id'] - if 'release-group' not in releasedata: - raise Exception('No release group associated with release id ' + releasedata['id'] + ' album id' + rgid) - release['Type'] = unicode(releasedata['release-group']['type']) - - - #making the assumption that the most important artist will be first in the list - if 'artist-credit' in releasedata: - release['ArtistID'] = unicode(releasedata['artist-credit'][0]['artist']['id']) - release['ArtistName'] = unicode(releasedata['artist-credit-phrase']) - else: - logger.warn('Release ' + releasedata['id'] + ' has no Artists associated.') - return False - - - release['ReleaseCountry'] = unicode(releasedata['country']) if 'country' in releasedata else u'Unknown' - #assuming that the list will contain media and that the format will be consistent + with mb_lock: + myDB = db.DBConnection() + results = [] try: - release['ReleaseFormat'] = unicode(releasedata['medium-list'][0]['format']) - except: - release['ReleaseFormat'] = u'Unknown' - - release['Tracks'] = getTracksFromRelease(releasedata) - releases.append(release) + limit = 100 + newResults = None + while newResults == None or len(newResults) >= limit: + newResults = musicbrainzngs.browse_releases(release_group=rgid,includes=['artist-credits','labels','recordings','release-groups','media'],limit=limit,offset=len(results)) + if 'release-list' not in newResults: + break #may want to raise an exception here instead ? + newResults = newResults['release-list'] + results += newResults + + except WebServiceError, e: + logger.warn('Attempt to retrieve information from MusicBrainz for release group "%s" failed (%s)' % (rgid, str(e))) + time.sleep(5) + return False + + if not results or len(results) == 0: + return False - return releases + #Clean all references to releases in dB that are no longer referenced in musicbrainz + release_list = [] + force_repackage = 0 + if len(results) != 0: + for release_mark in results: + release_list.append(unicode(release_mark['id'])) + release_title = release_mark['title'] + remove_missing_releases = myDB.action("SELECT ReleaseID FROM allalbums WHERE AlbumID=?", [rgid]) + for items in remove_missing_releases: + if items['ReleaseID'] not in release_list and items['ReleaseID'] != rgid: + # Remove all from albums/tracks that aren't in release + myDB.action("DELETE FROM albums WHERE ReleaseID=?", [items['ReleaseID']]) + myDB.action("DELETE FROM tracks WHERE ReleaseID=?", [items['ReleaseID']]) + myDB.action("DELETE FROM allalbums WHERE ReleaseID=?", [items['ReleaseID']]) + myDB.action("DELETE FROM alltracks WHERE ReleaseID=?", [items['ReleaseID']]) + logger.info("Removing all references to release %s to reflect MusicBrainz" % items['ReleaseID']) + force_repackage = 1 + else: + logger.info("Error pulling data from MusicBrainz: Maintaining dB") + + num_new_releases = 0 + + for releasedata in results: + #releasedata.get will return None if it doesn't have a status + #all official releases should have the Official status included + if not includeExtras and releasedata.get('status') != 'Official': + continue + + release = {} + rel_id_check = releasedata['id'] + artistid = unicode(releasedata['artist-credit'][0]['artist']['id']) + + album_checker = myDB.action('SELECT * from allalbums WHERE ReleaseID=?', [rel_id_check]).fetchone() + if not album_checker: + release['AlbumTitle'] = unicode(releasedata['title']) + release['AlbumID'] = unicode(rgid) + release['AlbumASIN'] = unicode(releasedata['asin']) if 'asin' in releasedata else None + release['ReleaseDate'] = unicode(releasedata['date']) if 'date' in releasedata else None + release['ReleaseID'] = releasedata['id'] + if 'release-group' not in releasedata: + raise Exception('No release group associated with release id ' + releasedata['id'] + ' album id' + rgid) + release['Type'] = unicode(releasedata['release-group']['type']) + + + #making the assumption that the most important artist will be first in the list + if 'artist-credit' in releasedata: + release['ArtistID'] = unicode(releasedata['artist-credit'][0]['artist']['id']) + release['ArtistName'] = unicode(releasedata['artist-credit-phrase']) + else: + logger.warn('Release ' + releasedata['id'] + ' has no Artists associated.') + return False + + + release['ReleaseCountry'] = unicode(releasedata['country']) if 'country' in releasedata else u'Unknown' + #assuming that the list will contain media and that the format will be consistent + try: + release['ReleaseFormat'] = unicode(releasedata['medium-list'][0]['format']) + except: + release['ReleaseFormat'] = u'Unknown' + + release['Tracks'] = getTracksFromRelease(releasedata) + + # What we're doing here now is first updating the allalbums & alltracks table to the most + # current info, then moving the appropriate release into the album table and its associated + # tracks into the tracks table + controlValueDict = {"ReleaseID" : release['ReleaseID']} + + newValueDict = {"ArtistID": release['ArtistID'], + "ArtistName": release['ArtistName'], + "AlbumTitle": release['AlbumTitle'], + "AlbumID": release['AlbumID'], + "AlbumASIN": release['AlbumASIN'], + "ReleaseDate": release['ReleaseDate'], + "Type": release['Type'], + "ReleaseCountry": release['ReleaseCountry'], + "ReleaseFormat": release['ReleaseFormat'] + } + + myDB.upsert("allalbums", newValueDict, controlValueDict) + + + for track in release['Tracks']: + + cleanname = helpers.cleanName(release['ArtistName'] + ' ' + release['AlbumTitle'] + ' ' + track['title']) + + controlValueDict = {"TrackID": track['id'], + "ReleaseID": release['ReleaseID']} + + newValueDict = {"ArtistID": release['ArtistID'], + "ArtistName": release['ArtistName'], + "AlbumTitle": release['AlbumTitle'], + "AlbumID": release['AlbumID'], + "AlbumASIN": release['AlbumASIN'], + "TrackTitle": track['title'], + "TrackDuration": track['duration'], + "TrackNumber": track['number'], + "CleanName": cleanname + } + + match = myDB.action('SELECT Location, BitRate, Format from have WHERE CleanName=?', [cleanname]).fetchone() + + if not match: + match = myDB.action('SELECT Location, BitRate, Format from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [release['ArtistName'], release['AlbumTitle'], track['title']]).fetchone() + if not match: + match = myDB.action('SELECT Location, BitRate, Format from have WHERE TrackID=?', [track['id']]).fetchone() + if match: + newValueDict['Location'] = match['Location'] + newValueDict['BitRate'] = match['BitRate'] + newValueDict['Format'] = match['Format'] + myDB.action('UPDATE have SET Matched="True" WHERE Location=?', [match['Location']]) + + myDB.upsert("alltracks", newValueDict, controlValueDict) + num_new_releases = num_new_releases + 1 + #print releasedata['title'] + #print num_new_releases + logger.info('New release %s (%s) added' % (release['AlbumTitle'], rel_id_check)) + if force_repackage == 1: + num_new_releases = -1 + logger.info('Forcing repackage of %s, since dB releases have been removed' % release_title) + else: + num_new_releases = num_new_releases + + return num_new_releases def getTracksFromRelease(release): totalTracks = 1 From 3639220a82c4d259ef084a2736fe0dbb4eef89e2 Mon Sep 17 00:00:00 2001 From: theguardian Date: Tue, 1 Oct 2013 17:27:00 -0700 Subject: [PATCH 02/47] Made a few minor syntax updates. --- headphones/importer.py | 58 ++++++++++++++++++++++-------------------- headphones/mb.py | 7 +++-- 2 files changed, 34 insertions(+), 31 deletions(-) diff --git a/headphones/importer.py b/headphones/importer.py index ede5c3b4..22478bc2 100644 --- a/headphones/importer.py +++ b/headphones/importer.py @@ -232,7 +232,7 @@ def addArtisttoDB(artistid, extrasonly=False): logger.info("Now updating: " + rg['title']) new_releases = mb.get_new_releases(rgid,includeExtras) else: - if helpers.get_age(today) - helpers.get_age(check_release_date[0]) < 365: + if helpers.get_age(today) - helpers.get_age(check_release_date[0]) < 36500: logger.info("Now updating: " + rg['title']) new_releases = mb.get_new_releases(rgid,includeExtras) else: @@ -261,32 +261,35 @@ def addArtisttoDB(artistid, extrasonly=False): find_hybrid_releases = myDB.action("SELECT * from allalbums WHERE AlbumID=?", [rg['id']]) # Build the dictionary for the fullreleaselist for items in find_hybrid_releases: - hybrid_release_id = items['ReleaseID'] - newValueDict = {"ArtistID": items['ArtistID'], - "ArtistName": items['ArtistName'], - "AlbumTitle": items['AlbumTitle'], - "AlbumID": items['AlbumID'], - "AlbumASIN": items['AlbumASIN'], - "ReleaseDate": items['ReleaseDate'], - "Type": items['Type'], - "ReleaseCountry": items['ReleaseCountry'], - "ReleaseFormat": items['ReleaseFormat'] - } - find_hybrid_tracks = myDB.action("SELECT * from alltracks WHERE ReleaseID=?", [hybrid_release_id]) - totalTracks = 1 - hybrid_track_array = [] - for hybrid_tracks in find_hybrid_tracks: - hybrid_track_array.append({ - 'number': hybrid_tracks['TrackNumber'], - 'title': hybrid_tracks['TrackTitle'], - 'id': hybrid_tracks['TrackID'], - #'url': hybrid_tracks['TrackURL'], - 'duration': hybrid_tracks['TrackDuration'] - }) - totalTracks += 1 - newValueDict['ReleaseID'] = hybrid_release_id - newValueDict['Tracks'] = hybrid_track_array - fullreleaselist.append(newValueDict) + if items['ReleaseID'] != rg['id']: #don't include hybrid information, since that's what we're replacing + hybrid_release_id = items['ReleaseID'] + newValueDict = {"ArtistID": items['ArtistID'], + "ArtistName": items['ArtistName'], + "AlbumTitle": items['AlbumTitle'], + "AlbumID": items['AlbumID'], + "AlbumASIN": items['AlbumASIN'], + "ReleaseDate": items['ReleaseDate'], + "Type": items['Type'], + "ReleaseCountry": items['ReleaseCountry'], + "ReleaseFormat": items['ReleaseFormat'] + } + find_hybrid_tracks = myDB.action("SELECT * from alltracks WHERE ReleaseID=?", [hybrid_release_id]) + totalTracks = 1 + hybrid_track_array = [] + for hybrid_tracks in find_hybrid_tracks: + hybrid_track_array.append({ + 'number': hybrid_tracks['TrackNumber'], + 'title': hybrid_tracks['TrackTitle'], + 'id': hybrid_tracks['TrackID'], + #'url': hybrid_tracks['TrackURL'], + 'duration': hybrid_tracks['TrackDuration'] + }) + totalTracks += 1 + newValueDict['ReleaseID'] = hybrid_release_id + newValueDict['Tracks'] = hybrid_track_array + fullreleaselist.append(newValueDict) + + #print fullreleaselist # Basically just do the same thing again for the hybrid release # This may end up being called with an empty fullreleaselist @@ -489,6 +492,7 @@ def addArtisttoDB(artistid, extrasonly=False): logger.info("Finished updating artist: " + artist['artist_name'] + " but with errors, so not marking it as updated in the database") else: logger.info(u"Updating complete for: " + artist['artist_name']) + def addReleaseById(rid): diff --git a/headphones/mb.py b/headphones/mb.py index e725f047..2ef7700e 100644 --- a/headphones/mb.py +++ b/headphones/mb.py @@ -357,7 +357,7 @@ def get_new_releases(rgid,includeExtras=False): #Clean all references to releases in dB that are no longer referenced in musicbrainz release_list = [] - force_repackage = 0 + force_repackage1 = 0 if len(results) != 0: for release_mark in results: release_list.append(unicode(release_mark['id'])) @@ -371,7 +371,7 @@ def get_new_releases(rgid,includeExtras=False): myDB.action("DELETE FROM allalbums WHERE ReleaseID=?", [items['ReleaseID']]) myDB.action("DELETE FROM alltracks WHERE ReleaseID=?", [items['ReleaseID']]) logger.info("Removing all references to release %s to reflect MusicBrainz" % items['ReleaseID']) - force_repackage = 1 + force_repackage1 = 1 else: logger.info("Error pulling data from MusicBrainz: Maintaining dB") @@ -435,7 +435,6 @@ def get_new_releases(rgid,includeExtras=False): myDB.upsert("allalbums", newValueDict, controlValueDict) - for track in release['Tracks']: cleanname = helpers.cleanName(release['ArtistName'] + ' ' + release['AlbumTitle'] + ' ' + track['title']) @@ -471,7 +470,7 @@ def get_new_releases(rgid,includeExtras=False): #print releasedata['title'] #print num_new_releases logger.info('New release %s (%s) added' % (release['AlbumTitle'], rel_id_check)) - if force_repackage == 1: + if force_repackage1 == 1: num_new_releases = -1 logger.info('Forcing repackage of %s, since dB releases have been removed' % release_title) else: From 239c9841a1f40391cf690b3e3306f8edffd136d7 Mon Sep 17 00:00:00 2001 From: theguardian Date: Tue, 1 Oct 2013 18:06:27 -0700 Subject: [PATCH 03/47] Reverted ignore date back to 365 days - seems good for now. --- headphones/importer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/headphones/importer.py b/headphones/importer.py index 22478bc2..94ec7b37 100644 --- a/headphones/importer.py +++ b/headphones/importer.py @@ -232,7 +232,7 @@ def addArtisttoDB(artistid, extrasonly=False): logger.info("Now updating: " + rg['title']) new_releases = mb.get_new_releases(rgid,includeExtras) else: - if helpers.get_age(today) - helpers.get_age(check_release_date[0]) < 36500: + if helpers.get_age(today) - helpers.get_age(check_release_date[0]) < 365: logger.info("Now updating: " + rg['title']) new_releases = mb.get_new_releases(rgid,includeExtras) else: From 79f69f4e6eba84af440e6a433634b2c0c420c3a4 Mon Sep 17 00:00:00 2001 From: theguardian Date: Tue, 1 Oct 2013 23:59:47 -0700 Subject: [PATCH 04/47] Minor modification to make logger reflect eventual user input variable --- headphones/importer.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/headphones/importer.py b/headphones/importer.py index 94ec7b37..0ea32d93 100644 --- a/headphones/importer.py +++ b/headphones/importer.py @@ -221,7 +221,8 @@ def addArtisttoDB(artistid, extrasonly=False): today = helpers.today() rgid = rg['id'] skip_log = 0 - #new_releases = 0 + #Make a user configurable variable to skip update of albums with release dates older than this date (in days) + pause_delta = 365 check_release_date = myDB.action("SELECT ReleaseDate from albums WHERE ArtistID=? AND AlbumTitle=?", (artistid, al_title)).fetchone() if check_release_date: @@ -232,11 +233,11 @@ def addArtisttoDB(artistid, extrasonly=False): logger.info("Now updating: " + rg['title']) new_releases = mb.get_new_releases(rgid,includeExtras) else: - if helpers.get_age(today) - helpers.get_age(check_release_date[0]) < 365: + if helpers.get_age(today) - helpers.get_age(check_release_date[0]) < pause_delta: logger.info("Now updating: " + rg['title']) new_releases = mb.get_new_releases(rgid,includeExtras) else: - logger.info('%s is over a year old; not updating' % al_title) + logger.info('%s is over %s days old; not updating' % (al_title, pause_delta)) skip_log = 1 new_releases = 0 else: From 4bbf067776fe947d934e5594aa06cbb94618abdb Mon Sep 17 00:00:00 2001 From: theguardian Date: Wed, 2 Oct 2013 09:26:38 -0700 Subject: [PATCH 05/47] Fixed the bug where "Get Extras" caused albums to be deleted. Updated webserve.py to remove all traces of deleted items: necessary for this new algorithm to function globally. --- headphones/importer.py | 4 ++-- headphones/mb.py | 19 ++++++++++--------- headphones/webserve.py | 6 ++++++ 3 files changed, 18 insertions(+), 11 deletions(-) diff --git a/headphones/importer.py b/headphones/importer.py index 0ea32d93..d71914f8 100644 --- a/headphones/importer.py +++ b/headphones/importer.py @@ -196,7 +196,7 @@ def addArtisttoDB(artistid, extrasonly=False): group_list = [] force_repackage = 0 #Don't nuke the database if there's a MusicBrainz error - if len(artist['releasegroups']) != 0: + if len(artist['releasegroups']) != 0 and not extrasonly: for groups in artist['releasegroups']: group_list.append(groups['id']) remove_missing_groups_from_albums = myDB.action("SELECT ReleaseID FROM albums WHERE ArtistID=?", [artistid]) @@ -246,7 +246,7 @@ def addArtisttoDB(artistid, extrasonly=False): if force_repackage == 1: new_releases = -1 - logger.info('Forcing repackage of %s, since dB references have been removed' % al_title) + logger.info('Forcing repackage of %s, since dB groups have been removed' % al_title) else: new_releases = new_releases diff --git a/headphones/mb.py b/headphones/mb.py index 2ef7700e..8cec8873 100644 --- a/headphones/mb.py +++ b/headphones/mb.py @@ -363,15 +363,16 @@ def get_new_releases(rgid,includeExtras=False): release_list.append(unicode(release_mark['id'])) release_title = release_mark['title'] remove_missing_releases = myDB.action("SELECT ReleaseID FROM allalbums WHERE AlbumID=?", [rgid]) - for items in remove_missing_releases: - if items['ReleaseID'] not in release_list and items['ReleaseID'] != rgid: - # Remove all from albums/tracks that aren't in release - myDB.action("DELETE FROM albums WHERE ReleaseID=?", [items['ReleaseID']]) - myDB.action("DELETE FROM tracks WHERE ReleaseID=?", [items['ReleaseID']]) - myDB.action("DELETE FROM allalbums WHERE ReleaseID=?", [items['ReleaseID']]) - myDB.action("DELETE FROM alltracks WHERE ReleaseID=?", [items['ReleaseID']]) - logger.info("Removing all references to release %s to reflect MusicBrainz" % items['ReleaseID']) - force_repackage1 = 1 + if remove_missing_releases: + for items in remove_missing_releases: + if items['ReleaseID'] not in release_list and items['ReleaseID'] != rgid: + # Remove all from albums/tracks that aren't in release + myDB.action("DELETE FROM albums WHERE ReleaseID=?", [items['ReleaseID']]) + myDB.action("DELETE FROM tracks WHERE ReleaseID=?", [items['ReleaseID']]) + myDB.action("DELETE FROM allalbums WHERE ReleaseID=?", [items['ReleaseID']]) + myDB.action("DELETE FROM alltracks WHERE ReleaseID=?", [items['ReleaseID']]) + logger.info("Removing all references to release %s to reflect MusicBrainz" % items['ReleaseID']) + force_repackage1 = 1 else: logger.info("Error pulling data from MusicBrainz: Maintaining dB") diff --git a/headphones/webserve.py b/headphones/webserve.py index 1807409f..20dcc873 100644 --- a/headphones/webserve.py +++ b/headphones/webserve.py @@ -163,6 +163,8 @@ class WebInterface(object): for album in extraalbums: myDB.action('DELETE from tracks WHERE ArtistID=? AND AlbumID=?', [ArtistID, album['AlbumID']]) myDB.action('DELETE from albums WHERE ArtistID=? AND AlbumID=?', [ArtistID, album['AlbumID']]) + myDB.action('DELETE from allalbums WHERE ArtistID=? AND AlbumID=?', [ArtistID, album['AlbumID']]) + myDB.action('DELETE from alltracks WHERE ArtistID=? AND AlbumID=?', [ArtistID, album['AlbumID']]) raise cherrypy.HTTPRedirect("artistPage?ArtistID=%s" % ArtistID) removeExtras.exposed = True @@ -274,6 +276,8 @@ class WebInterface(object): myDB = db.DBConnection() myDB.action('DELETE from albums WHERE AlbumID=?', [AlbumID]) myDB.action('DELETE from tracks WHERE AlbumID=?', [AlbumID]) + myDB.action('DELETE from allalbums WHERE AlbumID=?', [AlbumID]) + myDB.action('DELETE from alltracks WHERE AlbumID=?', [AlbumID]) if ArtistID: raise cherrypy.HTTPRedirect("artistPage?ArtistID=%s" % ArtistID) else: @@ -342,6 +346,8 @@ class WebInterface(object): myDB.action('DELETE from artists WHERE ArtistID=?', [ArtistID]) myDB.action('DELETE from albums WHERE ArtistID=?', [ArtistID]) myDB.action('DELETE from tracks WHERE ArtistID=?', [ArtistID]) + myDB.action('DELETE from allalbums WHERE AlbumID=?', [AlbumID]) + myDB.action('DELETE from alltracks WHERE AlbumID=?', [AlbumID]) myDB.action('INSERT OR REPLACE into blacklist VALUES (?)', [ArtistID]) elif action == 'pause': controlValueDict = {'ArtistID': ArtistID} From 9aa8bb8a72ac56ac1ec7095e6b45ff641d78f926 Mon Sep 17 00:00:00 2001 From: theguardian Date: Wed, 2 Oct 2013 09:34:00 -0700 Subject: [PATCH 06/47] Removed mb_lock from get_new_releases (which is how it was originally) - should fix database locking errors. --- headphones/mb.py | 255 +++++++++++++++++++++++------------------------ 1 file changed, 127 insertions(+), 128 deletions(-) diff --git a/headphones/mb.py b/headphones/mb.py index 8cec8873..742f628f 100644 --- a/headphones/mb.py +++ b/headphones/mb.py @@ -334,150 +334,149 @@ def getRelease(releaseid, include_artist_info=True): def get_new_releases(rgid,includeExtras=False): - with mb_lock: - myDB = db.DBConnection() - results = [] - try: - limit = 100 - newResults = None - while newResults == None or len(newResults) >= limit: - newResults = musicbrainzngs.browse_releases(release_group=rgid,includes=['artist-credits','labels','recordings','release-groups','media'],limit=limit,offset=len(results)) - if 'release-list' not in newResults: - break #may want to raise an exception here instead ? - newResults = newResults['release-list'] - results += newResults - - except WebServiceError, e: - logger.warn('Attempt to retrieve information from MusicBrainz for release group "%s" failed (%s)' % (rgid, str(e))) - time.sleep(5) - return False + myDB = db.DBConnection() + results = [] + try: + limit = 100 + newResults = None + while newResults == None or len(newResults) >= limit: + newResults = musicbrainzngs.browse_releases(release_group=rgid,includes=['artist-credits','labels','recordings','release-groups','media'],limit=limit,offset=len(results)) + if 'release-list' not in newResults: + break #may want to raise an exception here instead ? + newResults = newResults['release-list'] + results += newResults - if not results or len(results) == 0: - return False + except WebServiceError, e: + logger.warn('Attempt to retrieve information from MusicBrainz for release group "%s" failed (%s)' % (rgid, str(e))) + time.sleep(5) + return False + + if not results or len(results) == 0: + return False - #Clean all references to releases in dB that are no longer referenced in musicbrainz - release_list = [] - force_repackage1 = 0 - if len(results) != 0: - for release_mark in results: - release_list.append(unicode(release_mark['id'])) - release_title = release_mark['title'] - remove_missing_releases = myDB.action("SELECT ReleaseID FROM allalbums WHERE AlbumID=?", [rgid]) - if remove_missing_releases: - for items in remove_missing_releases: - if items['ReleaseID'] not in release_list and items['ReleaseID'] != rgid: - # Remove all from albums/tracks that aren't in release - myDB.action("DELETE FROM albums WHERE ReleaseID=?", [items['ReleaseID']]) - myDB.action("DELETE FROM tracks WHERE ReleaseID=?", [items['ReleaseID']]) - myDB.action("DELETE FROM allalbums WHERE ReleaseID=?", [items['ReleaseID']]) - myDB.action("DELETE FROM alltracks WHERE ReleaseID=?", [items['ReleaseID']]) - logger.info("Removing all references to release %s to reflect MusicBrainz" % items['ReleaseID']) - force_repackage1 = 1 - else: - logger.info("Error pulling data from MusicBrainz: Maintaining dB") + #Clean all references to releases in dB that are no longer referenced in musicbrainz + release_list = [] + force_repackage1 = 0 + if len(results) != 0: + for release_mark in results: + release_list.append(unicode(release_mark['id'])) + release_title = release_mark['title'] + remove_missing_releases = myDB.action("SELECT ReleaseID FROM allalbums WHERE AlbumID=?", [rgid]) + if remove_missing_releases: + for items in remove_missing_releases: + if items['ReleaseID'] not in release_list and items['ReleaseID'] != rgid: + # Remove all from albums/tracks that aren't in release + myDB.action("DELETE FROM albums WHERE ReleaseID=?", [items['ReleaseID']]) + myDB.action("DELETE FROM tracks WHERE ReleaseID=?", [items['ReleaseID']]) + myDB.action("DELETE FROM allalbums WHERE ReleaseID=?", [items['ReleaseID']]) + myDB.action("DELETE FROM alltracks WHERE ReleaseID=?", [items['ReleaseID']]) + logger.info("Removing all references to release %s to reflect MusicBrainz" % items['ReleaseID']) + force_repackage1 = 1 + else: + logger.info("Error pulling data from MusicBrainz: Maintaining dB") - num_new_releases = 0 + num_new_releases = 0 - for releasedata in results: - #releasedata.get will return None if it doesn't have a status - #all official releases should have the Official status included - if not includeExtras and releasedata.get('status') != 'Official': - continue + for releasedata in results: + #releasedata.get will return None if it doesn't have a status + #all official releases should have the Official status included + if not includeExtras and releasedata.get('status') != 'Official': + continue + + release = {} + rel_id_check = releasedata['id'] + artistid = unicode(releasedata['artist-credit'][0]['artist']['id']) + + album_checker = myDB.action('SELECT * from allalbums WHERE ReleaseID=?', [rel_id_check]).fetchone() + if not album_checker: + release['AlbumTitle'] = unicode(releasedata['title']) + release['AlbumID'] = unicode(rgid) + release['AlbumASIN'] = unicode(releasedata['asin']) if 'asin' in releasedata else None + release['ReleaseDate'] = unicode(releasedata['date']) if 'date' in releasedata else None + release['ReleaseID'] = releasedata['id'] + if 'release-group' not in releasedata: + raise Exception('No release group associated with release id ' + releasedata['id'] + ' album id' + rgid) + release['Type'] = unicode(releasedata['release-group']['type']) + + + #making the assumption that the most important artist will be first in the list + if 'artist-credit' in releasedata: + release['ArtistID'] = unicode(releasedata['artist-credit'][0]['artist']['id']) + release['ArtistName'] = unicode(releasedata['artist-credit-phrase']) + else: + logger.warn('Release ' + releasedata['id'] + ' has no Artists associated.') + return False + + + release['ReleaseCountry'] = unicode(releasedata['country']) if 'country' in releasedata else u'Unknown' + #assuming that the list will contain media and that the format will be consistent + try: + release['ReleaseFormat'] = unicode(releasedata['medium-list'][0]['format']) + except: + release['ReleaseFormat'] = u'Unknown' + + release['Tracks'] = getTracksFromRelease(releasedata) + + # What we're doing here now is first updating the allalbums & alltracks table to the most + # current info, then moving the appropriate release into the album table and its associated + # tracks into the tracks table + controlValueDict = {"ReleaseID" : release['ReleaseID']} + + newValueDict = {"ArtistID": release['ArtistID'], + "ArtistName": release['ArtistName'], + "AlbumTitle": release['AlbumTitle'], + "AlbumID": release['AlbumID'], + "AlbumASIN": release['AlbumASIN'], + "ReleaseDate": release['ReleaseDate'], + "Type": release['Type'], + "ReleaseCountry": release['ReleaseCountry'], + "ReleaseFormat": release['ReleaseFormat'] + } + + myDB.upsert("allalbums", newValueDict, controlValueDict) - release = {} - rel_id_check = releasedata['id'] - artistid = unicode(releasedata['artist-credit'][0]['artist']['id']) + for track in release['Tracks']: - album_checker = myDB.action('SELECT * from allalbums WHERE ReleaseID=?', [rel_id_check]).fetchone() - if not album_checker: - release['AlbumTitle'] = unicode(releasedata['title']) - release['AlbumID'] = unicode(rgid) - release['AlbumASIN'] = unicode(releasedata['asin']) if 'asin' in releasedata else None - release['ReleaseDate'] = unicode(releasedata['date']) if 'date' in releasedata else None - release['ReleaseID'] = releasedata['id'] - if 'release-group' not in releasedata: - raise Exception('No release group associated with release id ' + releasedata['id'] + ' album id' + rgid) - release['Type'] = unicode(releasedata['release-group']['type']) - - - #making the assumption that the most important artist will be first in the list - if 'artist-credit' in releasedata: - release['ArtistID'] = unicode(releasedata['artist-credit'][0]['artist']['id']) - release['ArtistName'] = unicode(releasedata['artist-credit-phrase']) - else: - logger.warn('Release ' + releasedata['id'] + ' has no Artists associated.') - return False - - - release['ReleaseCountry'] = unicode(releasedata['country']) if 'country' in releasedata else u'Unknown' - #assuming that the list will contain media and that the format will be consistent - try: - release['ReleaseFormat'] = unicode(releasedata['medium-list'][0]['format']) - except: - release['ReleaseFormat'] = u'Unknown' - - release['Tracks'] = getTracksFromRelease(releasedata) - - # What we're doing here now is first updating the allalbums & alltracks table to the most - # current info, then moving the appropriate release into the album table and its associated - # tracks into the tracks table - controlValueDict = {"ReleaseID" : release['ReleaseID']} + cleanname = helpers.cleanName(release['ArtistName'] + ' ' + release['AlbumTitle'] + ' ' + track['title']) + + controlValueDict = {"TrackID": track['id'], + "ReleaseID": release['ReleaseID']} newValueDict = {"ArtistID": release['ArtistID'], "ArtistName": release['ArtistName'], "AlbumTitle": release['AlbumTitle'], "AlbumID": release['AlbumID'], "AlbumASIN": release['AlbumASIN'], - "ReleaseDate": release['ReleaseDate'], - "Type": release['Type'], - "ReleaseCountry": release['ReleaseCountry'], - "ReleaseFormat": release['ReleaseFormat'] + "TrackTitle": track['title'], + "TrackDuration": track['duration'], + "TrackNumber": track['number'], + "CleanName": cleanname } - - myDB.upsert("allalbums", newValueDict, controlValueDict) - - for track in release['Tracks']: - - cleanname = helpers.cleanName(release['ArtistName'] + ' ' + release['AlbumTitle'] + ' ' + track['title']) + + match = myDB.action('SELECT Location, BitRate, Format from have WHERE CleanName=?', [cleanname]).fetchone() - controlValueDict = {"TrackID": track['id'], - "ReleaseID": release['ReleaseID']} - - newValueDict = {"ArtistID": release['ArtistID'], - "ArtistName": release['ArtistName'], - "AlbumTitle": release['AlbumTitle'], - "AlbumID": release['AlbumID'], - "AlbumASIN": release['AlbumASIN'], - "TrackTitle": track['title'], - "TrackDuration": track['duration'], - "TrackNumber": track['number'], - "CleanName": cleanname - } + if not match: + match = myDB.action('SELECT Location, BitRate, Format from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [release['ArtistName'], release['AlbumTitle'], track['title']]).fetchone() + if not match: + match = myDB.action('SELECT Location, BitRate, Format from have WHERE TrackID=?', [track['id']]).fetchone() + if match: + newValueDict['Location'] = match['Location'] + newValueDict['BitRate'] = match['BitRate'] + newValueDict['Format'] = match['Format'] + myDB.action('UPDATE have SET Matched="True" WHERE Location=?', [match['Location']]) - match = myDB.action('SELECT Location, BitRate, Format from have WHERE CleanName=?', [cleanname]).fetchone() - - if not match: - match = myDB.action('SELECT Location, BitRate, Format from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [release['ArtistName'], release['AlbumTitle'], track['title']]).fetchone() - if not match: - match = myDB.action('SELECT Location, BitRate, Format from have WHERE TrackID=?', [track['id']]).fetchone() - if match: - newValueDict['Location'] = match['Location'] - newValueDict['BitRate'] = match['BitRate'] - newValueDict['Format'] = match['Format'] - myDB.action('UPDATE have SET Matched="True" WHERE Location=?', [match['Location']]) - - myDB.upsert("alltracks", newValueDict, controlValueDict) - num_new_releases = num_new_releases + 1 - #print releasedata['title'] - #print num_new_releases - logger.info('New release %s (%s) added' % (release['AlbumTitle'], rel_id_check)) - if force_repackage1 == 1: - num_new_releases = -1 - logger.info('Forcing repackage of %s, since dB releases have been removed' % release_title) - else: - num_new_releases = num_new_releases + myDB.upsert("alltracks", newValueDict, controlValueDict) + num_new_releases = num_new_releases + 1 + #print releasedata['title'] + #print num_new_releases + logger.info('New release %s (%s) added' % (release['AlbumTitle'], rel_id_check)) + if force_repackage1 == 1: + num_new_releases = -1 + logger.info('Forcing repackage of %s, since dB releases have been removed' % release_title) + else: + num_new_releases = num_new_releases - return num_new_releases + return num_new_releases def getTracksFromRelease(release): totalTracks = 1 From 75a5f86cded2671386b0004a5cccd96da9ce52fd Mon Sep 17 00:00:00 2001 From: theguardian Date: Wed, 2 Oct 2013 10:18:14 -0700 Subject: [PATCH 07/47] Added user inputs for MusicBrainz Update Interval & Album Age to ignore. --- data/interfaces/default/config.html | 8 ++++++++ headphones/__init__.py | 10 ++++++++-- headphones/importer.py | 2 +- headphones/webserve.py | 6 +++++- 4 files changed, 22 insertions(+), 4 deletions(-) diff --git a/data/interfaces/default/config.html b/data/interfaces/default/config.html index a087b2af..2379ad7d 100644 --- a/data/interfaces/default/config.html +++ b/data/interfaces/default/config.html @@ -97,6 +97,14 @@ mins +
+ + hours +
+
+ + days +
diff --git a/headphones/__init__.py b/headphones/__init__.py index 78803803..8c9af893 100644 --- a/headphones/__init__.py +++ b/headphones/__init__.py @@ -130,6 +130,8 @@ SEARCH_INTERVAL = 360 LIBRARYSCAN = False LIBRARYSCAN_INTERVAL = 300 DOWNLOAD_SCAN_INTERVAL = 5 +UPDATE_DB_INTERVAL = 24 +MB_IGNORE_AGE = 365 SAB_HOST = None SAB_USERNAME = None @@ -300,7 +302,7 @@ def initialize(): ADD_ALBUM_ART, ALBUM_ART_FORMAT, EMBED_ALBUM_ART, EMBED_LYRICS, DOWNLOAD_DIR, BLACKHOLE, BLACKHOLE_DIR, USENET_RETENTION, SEARCH_INTERVAL, \ TORRENTBLACKHOLE_DIR, NUMBEROFSEEDERS, ISOHUNT, KAT, PIRATEBAY, MININOVA, WAFFLES, WAFFLES_UID, WAFFLES_PASSKEY, \ RUTRACKER, RUTRACKER_USER, RUTRACKER_PASSWORD, WHATCD, WHATCD_USERNAME, WHATCD_PASSWORD, DOWNLOAD_TORRENT_DIR, \ - LIBRARYSCAN, LIBRARYSCAN_INTERVAL, DOWNLOAD_SCAN_INTERVAL, SAB_HOST, SAB_USERNAME, SAB_PASSWORD, SAB_APIKEY, SAB_CATEGORY, \ + LIBRARYSCAN, LIBRARYSCAN_INTERVAL, DOWNLOAD_SCAN_INTERVAL, UPDATE_DB_INTERVAL, MB_IGNORE_AGE, SAB_HOST, SAB_USERNAME, SAB_PASSWORD, SAB_APIKEY, SAB_CATEGORY, \ NZBGET_USERNAME, NZBGET_PASSWORD, NZBGET_CATEGORY, NZBGET_HOST, HEADPHONES_INDEXER, NZBMATRIX, TRANSMISSION_HOST, TRANSMISSION_USERNAME, TRANSMISSION_PASSWORD, \ UTORRENT_HOST, UTORRENT_USERNAME, UTORRENT_PASSWORD, NEWZNAB, NEWZNAB_HOST, NEWZNAB_APIKEY, NEWZNAB_ENABLED, EXTRA_NEWZNABS, \ NZBSORG, NZBSORG_UID, NZBSORG_HASH, NZBSRUS, NZBSRUS_UID, NZBSRUS_APIKEY, NZB_DOWNLOADER, TORRENT_DOWNLOADER, PREFERRED_WORDS, REQUIRED_WORDS, IGNORED_WORDS, \ @@ -403,6 +405,8 @@ def initialize(): LIBRARYSCAN = bool(check_setting_int(CFG, 'General', 'libraryscan', 1)) LIBRARYSCAN_INTERVAL = check_setting_int(CFG, 'General', 'libraryscan_interval', 300) DOWNLOAD_SCAN_INTERVAL = check_setting_int(CFG, 'General', 'download_scan_interval', 5) + UPDATE_DB_INTERVAL = check_setting_int(CFG, 'General', 'update_db_interval', 24) + MB_IGNORE_AGE = check_setting_int(CFG, 'General', 'mb_ignore_age', 365) TORRENTBLACKHOLE_DIR = check_setting_str(CFG, 'General', 'torrentblackhole_dir', '') NUMBEROFSEEDERS = check_setting_str(CFG, 'General', 'numberofseeders', '10') @@ -787,6 +791,8 @@ def config_write(): new_config['General']['libraryscan'] = int(LIBRARYSCAN) new_config['General']['libraryscan_interval'] = LIBRARYSCAN_INTERVAL new_config['General']['download_scan_interval'] = DOWNLOAD_SCAN_INTERVAL + new_config['General']['update_db_interval'] = UPDATE_DB_INTERVAL + new_config['General']['mb_ignore_age'] = MB_IGNORE_AGE new_config['SABnzbd'] = {} new_config['SABnzbd']['sab_host'] = SAB_HOST @@ -911,7 +917,7 @@ def start(): # Start our scheduled background tasks from headphones import updater, searcher, librarysync, postprocessor - SCHED.add_interval_job(updater.dbUpdate, hours=24) + SCHED.add_interval_job(updater.dbUpdate, hours=UPDATE_DB_INTERVAL) SCHED.add_interval_job(searcher.searchforalbum, minutes=SEARCH_INTERVAL) SCHED.add_interval_job(librarysync.libraryScan, minutes=LIBRARYSCAN_INTERVAL, kwargs={'cron':True}) diff --git a/headphones/importer.py b/headphones/importer.py index d71914f8..b4874b79 100644 --- a/headphones/importer.py +++ b/headphones/importer.py @@ -222,7 +222,7 @@ def addArtisttoDB(artistid, extrasonly=False): rgid = rg['id'] skip_log = 0 #Make a user configurable variable to skip update of albums with release dates older than this date (in days) - pause_delta = 365 + pause_delta = headphones.MB_IGNORE_AGE check_release_date = myDB.action("SELECT ReleaseDate from albums WHERE ArtistID=? AND AlbumTitle=?", (artistid, al_title)).fetchone() if check_release_date: diff --git a/headphones/webserve.py b/headphones/webserve.py index 20dcc873..0811e68d 100644 --- a/headphones/webserve.py +++ b/headphones/webserve.py @@ -577,6 +577,8 @@ class WebInterface(object): "api_enabled" : checked(headphones.API_ENABLED), "api_key" : headphones.API_KEY, "download_scan_interval" : headphones.DOWNLOAD_SCAN_INTERVAL, + "update_db_interval" : headphones.UPDATE_DB_INTERVAL, + "mb_ignore_age" : headphones.MB_IGNORE_AGE, "nzb_search_interval" : headphones.SEARCH_INTERVAL, "libraryscan_interval" : headphones.LIBRARYSCAN_INTERVAL, "sab_host" : headphones.SAB_HOST, @@ -723,7 +725,7 @@ class WebInterface(object): config.exposed = True def configUpdate(self, http_host='0.0.0.0', http_username=None, http_port=8181, http_password=None, launch_browser=0, api_enabled=0, api_key=None, - download_scan_interval=None, nzb_search_interval=None, libraryscan_interval=None, sab_host=None, sab_username=None, sab_apikey=None, sab_password=None, + download_scan_interval=None, update_db_interval=None, mb_ignore_age=None, nzb_search_interval=None, libraryscan_interval=None, sab_host=None, sab_username=None, sab_apikey=None, sab_password=None, sab_category=None, nzbget_host=None, nzbget_username=None, nzbget_password=None, nzbget_category=None, transmission_host=None, transmission_username=None, transmission_password=None, utorrent_host=None, utorrent_username=None, utorrent_password=None, nzb_downloader=0, torrent_downloader=0, download_dir=None, blackhole_dir=None, usenet_retention=None, use_headphones_indexer=0, newznab=0, newznab_host=None, newznab_apikey=None, newznab_enabled=0, nzbsorg=0, nzbsorg_uid=None, nzbsorg_hash=None, nzbsrus=0, nzbsrus_uid=None, nzbsrus_apikey=None, @@ -750,6 +752,8 @@ class WebInterface(object): headphones.API_ENABLED = api_enabled headphones.API_KEY = api_key headphones.DOWNLOAD_SCAN_INTERVAL = download_scan_interval + headphones.UPDATE_DB_INTERVAL = update_db_interval + headphones.MB_IGNORE_AGE = mb_ignore_age headphones.SEARCH_INTERVAL = nzb_search_interval headphones.LIBRARYSCAN_INTERVAL = libraryscan_interval headphones.SAB_HOST = sab_host From af97f16c258134106bd40b21125ffbaa5dd1ffa9 Mon Sep 17 00:00:00 2001 From: theguardian Date: Wed, 2 Oct 2013 11:26:06 -0700 Subject: [PATCH 08/47] Fixed bug where user-selected album release was overwritten by headphones hybrid release. --- headphones/importer.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/headphones/importer.py b/headphones/importer.py index b4874b79..a808d548 100644 --- a/headphones/importer.py +++ b/headphones/importer.py @@ -199,9 +199,9 @@ def addArtisttoDB(artistid, extrasonly=False): if len(artist['releasegroups']) != 0 and not extrasonly: for groups in artist['releasegroups']: group_list.append(groups['id']) - remove_missing_groups_from_albums = myDB.action("SELECT ReleaseID FROM albums WHERE ArtistID=?", [artistid]) + remove_missing_groups_from_albums = myDB.action("SELECT AlbumID, ReleaseID FROM albums WHERE ArtistID=?", [artistid]) for items in remove_missing_groups_from_albums: - if items['ReleaseID'] not in group_list: + if items['ReleaseID'] not in group_list and items['AlbumID']==items['ReleaseID']: #added 2nd clause for when user picks alternate release # Remove all from albums/tracks that aren't in release groups myDB.action("DELETE FROM albums WHERE ReleaseID=?", [items['ReleaseID']]) myDB.action("DELETE FROM allalbums WHERE ReleaseID=?", [items['ReleaseID']]) @@ -410,7 +410,10 @@ def addArtisttoDB(artistid, extrasonly=False): else: newValueDict['Status'] = "Skipped" - myDB.upsert("albums", newValueDict, controlValueDict) + #Only update albums table with hybrid release if user didn't choose an alternate release + check_alternate_release = myDB.action("SELECT AlbumID, ReleaseID FROM albums WHERE ArtistID=? AND AlbumID=?", ([artistid], [rg['id']])).fetchone() + if check_alternate_release[0] == check_alternate_release[1]: + myDB.upsert("albums", newValueDict, controlValueDict) myDB.action('DELETE from tracks WHERE AlbumID=?', [rg['id']]) tracks = myDB.action('SELECT * from alltracks WHERE ReleaseID=?', [releaseid]).fetchall() @@ -437,7 +440,10 @@ def addArtisttoDB(artistid, extrasonly=False): "BitRate": track['BitRate'] } - myDB.upsert("tracks", newValueDict, controlValueDict) + #Only update tracks table with hybrid release if user didn't choose an alternate release + check_alternate_release = myDB.action("SELECT AlbumID, ReleaseID FROM albums WHERE ArtistID=? AND AlbumID=?", ([artistid], [rg['id']])).fetchone() + if check_alternate_release[0] == check_alternate_release[1]: + myDB.upsert("tracks", newValueDict, controlValueDict) # Mark albums as downloaded if they have at least 80% (by default, configurable) of the album have_track_count = len(myDB.select('SELECT * from tracks WHERE AlbumID=? AND Location IS NOT NULL', [rg['id']])) From 7a19d08ab2ac5f7b0f247cc88f2cb14a5600d035 Mon Sep 17 00:00:00 2001 From: theguardian Date: Wed, 2 Oct 2013 15:03:25 -0700 Subject: [PATCH 09/47] Added ability to do a "comprehensive" MusicBrainz update to emulate the original update process. Fixed a bug when adding albums introduced in last commit. --- data/interfaces/default/manage.html | 3 +- headphones/importer.py | 66 +++++++++++++++++------------ headphones/mb.py | 4 +- headphones/updater.py | 4 +- headphones/webserve.py | 10 ++++- 5 files changed, 52 insertions(+), 35 deletions(-) diff --git a/data/interfaces/default/manage.html b/data/interfaces/default/manage.html index 66405aab..4c0de24c 100644 --- a/data/interfaces/default/manage.html +++ b/data/interfaces/default/manage.html @@ -119,7 +119,8 @@ Force Search
Force Check for Wanted Albums - Force Update Active Artists + Force Update Active Artists [Fast] + Force Update Active Artists [Comprehensive] Force Post-Process Albums in Download Folder Check for Headphones Updates Delete empty Artists diff --git a/headphones/importer.py b/headphones/importer.py index a808d548..d8fa677a 100644 --- a/headphones/importer.py +++ b/headphones/importer.py @@ -109,7 +109,7 @@ def addArtistIDListToDB(artistidlist): for artistid in artistidlist: addArtisttoDB(artistid) -def addArtisttoDB(artistid, extrasonly=False): +def addArtisttoDB(artistid, extrasonly=False, forcefull=False): # Putting this here to get around the circular import. We're using this to update thumbnails for artist/albums from headphones import cache @@ -224,31 +224,35 @@ def addArtisttoDB(artistid, extrasonly=False): #Make a user configurable variable to skip update of albums with release dates older than this date (in days) pause_delta = headphones.MB_IGNORE_AGE - check_release_date = myDB.action("SELECT ReleaseDate from albums WHERE ArtistID=? AND AlbumTitle=?", (artistid, al_title)).fetchone() - if check_release_date: - if check_release_date[0] is None: - logger.info("Now updating: " + rg['title']) - new_releases = mb.get_new_releases(rgid,includeExtras) - elif len(check_release_date[0])!=10: - logger.info("Now updating: " + rg['title']) - new_releases = mb.get_new_releases(rgid,includeExtras) - else: - if helpers.get_age(today) - helpers.get_age(check_release_date[0]) < pause_delta: + if not forcefull: + check_release_date = myDB.action("SELECT ReleaseDate from albums WHERE ArtistID=? AND AlbumTitle=?", (artistid, al_title)).fetchone() + if check_release_date: + if check_release_date[0] is None: logger.info("Now updating: " + rg['title']) - new_releases = mb.get_new_releases(rgid,includeExtras) + new_releases = mb.get_new_releases(rgid,includeExtras) + elif len(check_release_date[0])!=10: + logger.info("Now updating: " + rg['title']) + new_releases = mb.get_new_releases(rgid,includeExtras) else: - logger.info('%s is over %s days old; not updating' % (al_title, pause_delta)) - skip_log = 1 - new_releases = 0 + if helpers.get_age(today) - helpers.get_age(check_release_date[0]) < pause_delta: + logger.info("Now updating: " + rg['title']) + new_releases = mb.get_new_releases(rgid,includeExtras) + else: + logger.info('%s is over %s days old; not updating' % (al_title, pause_delta)) + skip_log = 1 + new_releases = 0 + else: + logger.info("Now adding/updating: " + rg['title']) + new_releases = mb.get_new_releases(rgid,includeExtras) + + if force_repackage == 1: + new_releases = -1 + logger.info('Forcing repackage of %s, since dB groups have been removed' % al_title) + else: + new_releases = new_releases else: logger.info("Now adding/updating: " + rg['title']) - new_releases = mb.get_new_releases(rgid,includeExtras) - - if force_repackage == 1: - new_releases = -1 - logger.info('Forcing repackage of %s, since dB groups have been removed' % al_title) - else: - new_releases = new_releases + new_releases = mb.get_new_releases(rgid,includeExtras,forcefull) #What this does is adds new releases per artist to the allalbums + alltracks databases #new_releases = mb.get_new_releases(rgid,includeExtras) @@ -411,9 +415,12 @@ def addArtisttoDB(artistid, extrasonly=False): newValueDict['Status'] = "Skipped" #Only update albums table with hybrid release if user didn't choose an alternate release - check_alternate_release = myDB.action("SELECT AlbumID, ReleaseID FROM albums WHERE ArtistID=? AND AlbumID=?", ([artistid], [rg['id']])).fetchone() - if check_alternate_release[0] == check_alternate_release[1]: - myDB.upsert("albums", newValueDict, controlValueDict) + check_alternate_release = myDB.action("SELECT AlbumID, ReleaseID FROM albums WHERE ArtistID=? AND AlbumID=?", (artistid, rg['id'])).fetchone() + if check_alternate_release: + if check_alternate_release[0] == check_alternate_release[1]: + myDB.upsert("albums", newValueDict, controlValueDict) + else: + myDB.upsert("albums", newValueDict, controlValueDict) myDB.action('DELETE from tracks WHERE AlbumID=?', [rg['id']]) tracks = myDB.action('SELECT * from alltracks WHERE ReleaseID=?', [releaseid]).fetchall() @@ -441,9 +448,12 @@ def addArtisttoDB(artistid, extrasonly=False): } #Only update tracks table with hybrid release if user didn't choose an alternate release - check_alternate_release = myDB.action("SELECT AlbumID, ReleaseID FROM albums WHERE ArtistID=? AND AlbumID=?", ([artistid], [rg['id']])).fetchone() - if check_alternate_release[0] == check_alternate_release[1]: - myDB.upsert("tracks", newValueDict, controlValueDict) + check_alternate_release = myDB.action("SELECT AlbumID, ReleaseID FROM albums WHERE ArtistID=? AND AlbumID=?", (artistid, rg['id'])).fetchone() + if check_alternate_release: + if check_alternate_release[0] == check_alternate_release[1]: + myDB.upsert("tracks", newValueDict, controlValueDict) + else: + myDB.upsert("tracks", newValueDict, controlValueDict) # Mark albums as downloaded if they have at least 80% (by default, configurable) of the album have_track_count = len(myDB.select('SELECT * from tracks WHERE AlbumID=? AND Location IS NOT NULL', [rg['id']])) diff --git a/headphones/mb.py b/headphones/mb.py index 742f628f..4f1d0e29 100644 --- a/headphones/mb.py +++ b/headphones/mb.py @@ -332,7 +332,7 @@ def getRelease(releaseid, include_artist_info=True): return release -def get_new_releases(rgid,includeExtras=False): +def get_new_releases(rgid,includeExtras=False,forcefull=False): myDB = db.DBConnection() results = [] @@ -388,7 +388,7 @@ def get_new_releases(rgid,includeExtras=False): artistid = unicode(releasedata['artist-credit'][0]['artist']['id']) album_checker = myDB.action('SELECT * from allalbums WHERE ReleaseID=?', [rel_id_check]).fetchone() - if not album_checker: + if not album_checker or forcefull: release['AlbumTitle'] = unicode(releasedata['title']) release['AlbumID'] = unicode(rgid) release['AlbumASIN'] = unicode(releasedata['asin']) if 'asin' in releasedata else None diff --git a/headphones/updater.py b/headphones/updater.py index c80eec0b..1088af58 100644 --- a/headphones/updater.py +++ b/headphones/updater.py @@ -17,7 +17,7 @@ import headphones from headphones import logger, db, importer -def dbUpdate(): +def dbUpdate(forcefull=False): myDB = db.DBConnection() @@ -28,6 +28,6 @@ def dbUpdate(): for artist in activeartists: artistid = artist[0] - importer.addArtisttoDB(artistid) + importer.addArtisttoDB(artistid=artistid, extrasonly=False, forcefull=forcefull) logger.info('Update complete') diff --git a/headphones/webserve.py b/headphones/webserve.py index 0811e68d..82fe8fa2 100644 --- a/headphones/webserve.py +++ b/headphones/webserve.py @@ -150,7 +150,7 @@ class WebInterface(object): newValueDict = {'IncludeExtras': 1, 'Extras': extras} myDB.upsert("artists", newValueDict, controlValueDict) - threading.Thread(target=importer.addArtisttoDB, args=[ArtistID, True]).start() + threading.Thread(target=importer.addArtisttoDB, args=[ArtistID, True, False]).start() raise cherrypy.HTTPRedirect("artistPage?ArtistID=%s" % ArtistID) getExtras.exposed = True @@ -403,10 +403,16 @@ class WebInterface(object): def forceUpdate(self): from headphones import updater - threading.Thread(target=updater.dbUpdate).start() + threading.Thread(target=updater.dbUpdate, args=[False]).start() raise cherrypy.HTTPRedirect("home") forceUpdate.exposed = True + def forceFullUpdate(self): + from headphones import updater + threading.Thread(target=updater.dbUpdate, args=[True]).start() + raise cherrypy.HTTPRedirect("home") + forceFullUpdate.exposed = True + def forceSearch(self): from headphones import searcher threading.Thread(target=searcher.searchforalbum).start() From fc00567191e9f34eb4b1e2f8111ad24edc6a3940 Mon Sep 17 00:00:00 2001 From: theguardian Date: Tue, 8 Oct 2013 23:32:18 -0700 Subject: [PATCH 10/47] First significant cut at refactoring the library sync function to optimize time/CPU usage. Headphones now keeps all local file information persistent, and affords many new opportunities that weren't available before. There may be a bug here and there. Please report. --- data/interfaces/default/album.html | 2 +- data/interfaces/default/artist.html | 2 +- data/interfaces/default/index.html | 2 +- data/interfaces/default/manage.html | 20 +- data/interfaces/default/managenew.html | 5 +- data/interfaces/default/manageunmatched.html | 157 +++++++++ headphones/__init__.py | 162 ---------- headphones/importer.py | 10 +- headphones/librarysync.py | 315 +++++++++---------- headphones/mb.py | 2 +- headphones/webserve.py | 169 +++++++++- 11 files changed, 499 insertions(+), 347 deletions(-) create mode 100644 data/interfaces/default/manageunmatched.html diff --git a/data/interfaces/default/album.html b/data/interfaces/default/album.html index bec42cfb..2496bd4c 100644 --- a/data/interfaces/default/album.html +++ b/data/interfaces/default/album.html @@ -138,7 +138,7 @@ %endfor <% - unmatched = myDB.select('SELECT * from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ?', [album['ArtistName'], album['AlbumTitle']]) + unmatched_temp = myDB.select('SELECT * from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND Matched is null ORDER BY CAST(TrackNumber AS INTEGER)', [album['ArtistName'], album['AlbumTitle']]) %> %if unmatched: %for track in unmatched: diff --git a/data/interfaces/default/artist.html b/data/interfaces/default/artist.html index e891b1fa..845f6719 100644 --- a/data/interfaces/default/artist.html +++ b/data/interfaces/default/artist.html @@ -93,7 +93,7 @@ myDB = db.DBConnection() totaltracks = len(myDB.select('SELECT TrackTitle from tracks WHERE AlbumID=?', [album['AlbumID']])) - havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE AlbumID=? AND Location IS NOT NULL', [album['AlbumID']])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ? AND AlbumTitle LIKE ?', [album['ArtistName'], album['AlbumTitle']])) + havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE AlbumID=? AND Location IS NOT NULL', [album['AlbumID']])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ? AND AlbumTitle LIKE ? AND Matched IS NULL', [album['ArtistName'], album['AlbumTitle']])) try: percent = (havetracks*100.0)/totaltracks diff --git a/data/interfaces/default/index.html b/data/interfaces/default/index.html index b67aaf7d..e0201252 100644 --- a/data/interfaces/default/index.html +++ b/data/interfaces/default/index.html @@ -69,7 +69,7 @@ } if (artist['ReleaseInFuture'] === 'True') { - grade = 'gradeA'; + grade = 'gradeA';6666666666666666 } else { diff --git a/data/interfaces/default/manage.html b/data/interfaces/default/manage.html index 4c0de24c..b4ebc824 100644 --- a/data/interfaces/default/manage.html +++ b/data/interfaces/default/manage.html @@ -22,6 +22,7 @@ %if not headphones.ADD_ARTISTS: Manage New Artists %endif + Manage Unmatched
@@ -35,6 +36,7 @@
  • Scan Music Library
  • Imports
  • Force Actions
    • +
  • Force Legacy
    • + + +
    +
    + Force Legacy +

    Comprehensive Updating

    +

    Please note that these functions will take a significant amount of time to complete.

    +
    + Force Update Active Artists [Comprehensive] + Force Re-scan Library [Comprehensive] +
    +
    + +
    + + + <%def name="javascriptIncludes()"> diff --git a/data/interfaces/default/managenew.html b/data/interfaces/default/managenew.html index 17d567f7..41f9263d 100644 --- a/data/interfaces/default/managenew.html +++ b/data/interfaces/default/managenew.html @@ -20,7 +20,10 @@
    - Add selected artists +
    diff --git a/data/interfaces/default/manageunmatched.html b/data/interfaces/default/manageunmatched.html new file mode 100644 index 00000000..2054382c --- /dev/null +++ b/data/interfaces/default/manageunmatched.html @@ -0,0 +1,157 @@ +<%inherit file="base.html" /> +<%! + import headphones + import json + from headphones import db, helpers + myDB = db.DBConnection() + artist_json = {} + counter = 0 + artist_list = myDB.action("SELECT ArtistName from artists ORDER BY ArtistName COLLATE NOCASE") + for artist in artist_list: + artist_json[counter] = artist['ArtistName'] + counter+=1 + json_artists = json.dumps(artist_json) +%> + +<%def name="headerIncludes()"> +
    +
    +
    +
    + « Back to manage overview + + + +<%def name="body()"> +
    +
    +

    manageManage Unmatched Albums

    +
    + +
    + + + + + + + + + + <% count_albums=0 %> + %for album in unmatchedalbums: + + + + + + + <% count_albums+=1 %> + %endfor + +
    Local ArtistLocal AlbumMatch ArtistMatch Album
    ${album['ArtistName']} + + + + + + ${album['AlbumTitle']} +
    + + + + +
    +     		+ +
    + + + + +
    +     		+ +
    + + + + + + +
    +
    + + + +<%def name="headIncludes()"> + + + +<%def name="javascriptIncludes()"> + + + diff --git a/headphones/__init__.py b/headphones/__init__.py index 8c9af893..758b28dc 100644 --- a/headphones/__init__.py +++ b/headphones/__init__.py @@ -954,169 +954,7 @@ def dbcheck(): c.execute('CREATE INDEX IF NOT EXISTS tracks_albumid ON tracks(AlbumID ASC)') c.execute('CREATE INDEX IF NOT EXISTS album_artistid_reldate ON albums(ArtistID ASC, ReleaseDate DESC)') - try: - c.execute('SELECT IncludeExtras from artists') - except sqlite3.OperationalError: - c.execute('ALTER TABLE artists ADD COLUMN IncludeExtras INTEGER DEFAULT 0') - try: - c.execute('SELECT LatestAlbum from artists') - except sqlite3.OperationalError: - c.execute('ALTER TABLE artists ADD COLUMN LatestAlbum TEXT') - - try: - c.execute('SELECT ReleaseDate from artists') - except sqlite3.OperationalError: - c.execute('ALTER TABLE artists ADD COLUMN ReleaseDate TEXT') - - try: - c.execute('SELECT AlbumID from artists') - except sqlite3.OperationalError: - c.execute('ALTER TABLE artists ADD COLUMN AlbumID TEXT') - - try: - c.execute('SELECT HaveTracks from artists') - except sqlite3.OperationalError: - c.execute('ALTER TABLE artists ADD COLUMN HaveTracks INTEGER DEFAULT 0') - - try: - c.execute('SELECT TotalTracks from artists') - except sqlite3.OperationalError: - c.execute('ALTER TABLE artists ADD COLUMN TotalTracks INTEGER DEFAULT 0') - - try: - c.execute('SELECT Type from albums') - except sqlite3.OperationalError: - c.execute('ALTER TABLE albums ADD COLUMN Type TEXT DEFAULT "Album"') - - try: - c.execute('SELECT TrackNumber from tracks') - except sqlite3.OperationalError: - c.execute('ALTER TABLE tracks ADD COLUMN TrackNumber INTEGER') - - try: - c.execute('SELECT FolderName from snatched') - except sqlite3.OperationalError: - c.execute('ALTER TABLE snatched ADD COLUMN FolderName TEXT') - - try: - c.execute('SELECT Location from tracks') - except sqlite3.OperationalError: - c.execute('ALTER TABLE tracks ADD COLUMN Location TEXT') - - try: - c.execute('SELECT Location from have') - except sqlite3.OperationalError: - c.execute('ALTER TABLE have ADD COLUMN Location TEXT') - - try: - c.execute('SELECT BitRate from tracks') - except sqlite3.OperationalError: - c.execute('ALTER TABLE tracks ADD COLUMN BitRate INTEGER') - - try: - c.execute('SELECT CleanName from tracks') - except sqlite3.OperationalError: - c.execute('ALTER TABLE tracks ADD COLUMN CleanName TEXT') - - try: - c.execute('SELECT CleanName from have') - except sqlite3.OperationalError: - c.execute('ALTER TABLE have ADD COLUMN CleanName TEXT') - - # Add the Format column - try: - c.execute('SELECT Format from have') - except sqlite3.OperationalError: - c.execute('ALTER TABLE have ADD COLUMN Format TEXT DEFAULT NULL') - - try: - c.execute('SELECT Format from tracks') - except sqlite3.OperationalError: - c.execute('ALTER TABLE tracks ADD COLUMN Format TEXT DEFAULT NULL') - - try: - c.execute('SELECT LastUpdated from artists') - except sqlite3.OperationalError: - c.execute('ALTER TABLE artists ADD COLUMN LastUpdated TEXT DEFAULT NULL') - - try: - c.execute('SELECT ArtworkURL from artists') - except sqlite3.OperationalError: - c.execute('ALTER TABLE artists ADD COLUMN ArtworkURL TEXT DEFAULT NULL') - - try: - c.execute('SELECT ArtworkURL from albums') - except sqlite3.OperationalError: - c.execute('ALTER TABLE albums ADD COLUMN ArtworkURL TEXT DEFAULT NULL') - - try: - c.execute('SELECT ThumbURL from artists') - except sqlite3.OperationalError: - c.execute('ALTER TABLE artists ADD COLUMN ThumbURL TEXT DEFAULT NULL') - - try: - c.execute('SELECT ThumbURL from albums') - except sqlite3.OperationalError: - c.execute('ALTER TABLE albums ADD COLUMN ThumbURL TEXT DEFAULT NULL') - - try: - c.execute('SELECT ArtistID from descriptions') - except sqlite3.OperationalError: - c.execute('ALTER TABLE descriptions ADD COLUMN ArtistID TEXT DEFAULT NULL') - - try: - c.execute('SELECT LastUpdated from descriptions') - except sqlite3.OperationalError: - c.execute('ALTER TABLE descriptions ADD COLUMN LastUpdated TEXT DEFAULT NULL') - - try: - c.execute('SELECT ReleaseID from albums') - except sqlite3.OperationalError: - c.execute('ALTER TABLE albums ADD COLUMN ReleaseID TEXT DEFAULT NULL') - - try: - c.execute('SELECT ReleaseFormat from albums') - except sqlite3.OperationalError: - c.execute('ALTER TABLE albums ADD COLUMN ReleaseFormat TEXT DEFAULT NULL') - - try: - c.execute('SELECT ReleaseCountry from albums') - except sqlite3.OperationalError: - c.execute('ALTER TABLE albums ADD COLUMN ReleaseCountry TEXT DEFAULT NULL') - - try: - c.execute('SELECT ReleaseID from tracks') - except sqlite3.OperationalError: - c.execute('ALTER TABLE tracks ADD COLUMN ReleaseID TEXT DEFAULT NULL') - - try: - c.execute('SELECT Matched from have') - except sqlite3.OperationalError: - c.execute('ALTER TABLE have ADD COLUMN Matched TEXT DEFAULT NULL') - - try: - c.execute('SELECT Extras from artists') - except sqlite3.OperationalError: - c.execute('ALTER TABLE artists ADD COLUMN Extras TEXT DEFAULT NULL') - # Need to update some stuff when people are upgrading and have 'include extras' set globally/for an artist - if INCLUDE_EXTRAS: - EXTRAS = "1,2,3,4,5,6,7,8" - logger.info("Copying over current artist IncludeExtras information") - artists = c.execute('SELECT ArtistID, IncludeExtras from artists').fetchall() - for artist in artists: - if artist[1]: - c.execute('UPDATE artists SET Extras=? WHERE ArtistID=?', ("1,2,3,4,5,6,7,8", artist[0])) - - try: - c.execute('SELECT Kind from snatched') - except sqlite3.OperationalError: - c.execute('ALTER TABLE snatched ADD COLUMN Kind TEXT DEFAULT NULL') - - try: - c.execute('SELECT SearchTerm from albums') - except sqlite3.OperationalError: - c.execute('ALTER TABLE albums ADD COLUMN SearchTerm TEXT DEFAULT NULL') conn.commit() c.close() diff --git a/headphones/importer.py b/headphones/importer.py index d8fa677a..10c41abb 100644 --- a/headphones/importer.py +++ b/headphones/importer.py @@ -350,12 +350,13 @@ def addArtisttoDB(artistid, extrasonly=False, forcefull=False): newValueDict['Location'] = match['Location'] newValueDict['BitRate'] = match['BitRate'] newValueDict['Format'] = match['Format'] - myDB.action('UPDATE have SET Matched="True" WHERE Location=?', [match['Location']]) + #myDB.action('UPDATE have SET Matched="True" WHERE Location=?', [match['Location']]) + myDB.action('UPDATE have SET Matched=? WHERE Location=?', (rg['id'], match['Location'])) myDB.upsert("alltracks", newValueDict, controlValueDict) # Delete matched tracks from the have table - myDB.action('DELETE from have WHERE Matched="True"') + #myDB.action('DELETE from have WHERE Matched="True"') # If there's no release in the main albums tables, add the default (hybrid) # If there is a release, check the ReleaseID against the AlbumID to see if they differ (user updated) @@ -481,7 +482,8 @@ def addArtisttoDB(artistid, extrasonly=False, forcefull=False): latestalbum = myDB.action('SELECT AlbumTitle, ReleaseDate, AlbumID from albums WHERE ArtistID=? order by ReleaseDate DESC', [artistid]).fetchone() totaltracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=?', [artistid])) - havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=? AND Location IS NOT NULL', [artistid])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ?', [artist['artist_name']])) + #havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=? AND Location IS NOT NULL', [artistid])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ?', [artist['artist_name']])) + havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=? AND Location IS NOT NULL', [artistid])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ? AND Matched IS NULL', [artist['artist_name']])) controlValueDict = {"ArtistID": artistid} @@ -616,7 +618,7 @@ def addReleaseById(rid): newValueDict['Location'] = match['Location'] newValueDict['BitRate'] = match['BitRate'] newValueDict['Format'] = match['Format'] - myDB.action('DELETE from have WHERE Location=?', [match['Location']]) + #myDB.action('DELETE from have WHERE Location=?', [match['Location']]) myDB.upsert("tracks", newValueDict, controlValueDict) diff --git a/headphones/librarysync.py b/headphones/librarysync.py index 5ee57d8b..03925d7b 100644 --- a/headphones/librarysync.py +++ b/headphones/librarysync.py @@ -24,6 +24,7 @@ from headphones import db, logger, helpers, importer # You can scan a single directory and append it to the current library by specifying append=True, ArtistID & ArtistName def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, cron=False): + if cron and not headphones.LIBRARYSCAN: return @@ -46,13 +47,23 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, cron=Fal if not append: # Clean up bad filepaths - tracks = myDB.select('SELECT Location, TrackID from tracks WHERE Location IS NOT NULL') - - for track in tracks: - if not os.path.isfile(track['Location'].encode(headphones.SYS_ENCODING)): - myDB.action('UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE TrackID=?', [None, None, None, track['TrackID']]) + tracks = myDB.select('SELECT Location, TrackID from alltracks WHERE Location IS NOT NULL') - myDB.action('DELETE from have') + for track in tracks: + encoded_track_string = track['Location'].encode(headphones.SYS_ENCODING) + if not os.path.isfile(encoded_track_string): + myDB.action('UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE Location=?', [None, None, None, track['Location']]) + myDB.action('UPDATE alltracks SET Location=?, BitRate=?, Format=? WHERE Location=?', [None, None, None, track['Location']]) + + del_have_tracks = myDB.select('SELECT Location, Matched from have') + + for track in del_have_tracks: + encoded_track_string = track['Location'].encode(headphones.SYS_ENCODING) + if not os.path.isfile(encoded_track_string): + myDB.action('DELETE FROM have WHERE Location=?', [track['Location']]) + myDB.action('UPDATE have SET Matched=NULL WHERE Matched=?', [track['Matched']]) + logger.info('File %s removed from Headphones, as it is no longer on disk' % encoded_track_string.decode(headphones.SYS_ENCODING, 'replace')) + ###############myDB.action('DELETE from have') logger.info('Scanning music directory: %s' % dir.decode(headphones.SYS_ENCODING, 'replace')) @@ -60,6 +71,7 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, cron=Fal bitrates = [] song_list = [] + new_song_count = 0 for r,d,f in os.walk(dir): #need to abuse slicing to get a copy of the list, doing it directly will skip the element after a deleted one @@ -99,8 +111,15 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, cron=Fal # Add the song to our song list - # TODO: skip adding songs without the minimum requisite information (just a matter of putting together the right if statements) - song_dict = { 'TrackID' : f.mb_trackid, - 'ReleaseID' : f.mb_albumid, + if f_artist and f.album and f.title: + CleanName = helpers.cleanName(f_artist +' '+ f.album +' '+ f.title) + else: + CleanName = None + + controlValueDict = {'Location' : unicode_song_path} + + newValueDict = { 'TrackID' : f.mb_trackid, + #'ReleaseID' : f.mb_albumid, 'ArtistName' : f_artist, 'AlbumTitle' : f.album, 'TrackNumber': f.track, @@ -110,191 +129,117 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, cron=Fal 'TrackTitle' : f.title, 'BitRate' : f.bitrate, 'Format' : f.format, - 'Location' : unicode_song_path } + 'CleanName' : CleanName + } - song_list.append(song_dict) + #song_list.append(song_dict) + check_exist_song = myDB.action("SELECT * FROM have WHERE Location=?", [unicode_song_path]).fetchone() + #Only attempt to match songs that are new, haven't yet been matched, or metadata has changed. + if not check_exist_song: + myDB.upsert("have", newValueDict, controlValueDict) + new_song_count+=1 + #We're going to have to think about metadata changing, and setting Matched = None when/if we do + elif check_exist_song['CleanName'] != CleanName and check_exist_song['Matched'] != "Manual": + newValueDict['Matched'] = None + myDB.upsert("have", newValueDict, controlValueDict) + new_song_count+=1 + # Now we start track matching - total_number_of_songs = len(song_list) - logger.info("Found " + str(total_number_of_songs) + " tracks in: '" + dir.decode(headphones.SYS_ENCODING, 'replace') + "'. Matching tracks to the appropriate releases....") + logger.info("%s new/modified songs found and added to the database" % new_song_count) + song_list = myDB.action("SELECT * FROM have WHERE Matched IS NULL AND LOCATION LIKE ?", [dir+"%"]) + total_number_of_songs = myDB.action("SELECT COUNT(*) FROM have WHERE Matched IS NULL AND LOCATION LIKE ?", [dir+"%"]).fetchone()[0] + logger.info("Found " + str(total_number_of_songs) + " unmatched tracks in: '" + dir.decode(headphones.SYS_ENCODING, 'replace') + "'. Matching tracks to the appropriate releases....") # Sort the song_list by most vague (e.g. no trackid or releaseid) to most specific (both trackid & releaseid) # When we insert into the database, the tracks with the most specific information will overwrite the more general matches - song_list = helpers.multikeysort(song_list, ['ReleaseID', 'TrackID']) + ##############song_list = helpers.multikeysort(song_list, ['ReleaseID', 'TrackID']) + song_list = helpers.multikeysort(song_list, ['ArtistName', 'AlbumTitle']) # We'll use this to give a % completion, just because the track matching might take a while song_count = 0 + latest_artist = [] for song in song_list: + + latest_artist.append(song['ArtistName']) + if song_count == 0: + logger.info("Now matching songs by %s" % song['ArtistName']) + elif latest_artist[song_count] != latest_artist[song_count-1] and song_count !=0: + logger.info("Now matching songs by %s" % song['ArtistName']) + #print song['ArtistName']+' - '+song['AlbumTitle']+' - '+song['TrackTitle'] song_count += 1 completion_percentage = float(song_count)/total_number_of_songs * 100 if completion_percentage%10 == 0: logger.info("Track matching is " + str(completion_percentage) + "% complete") - # If the track has a trackid & releaseid (beets: albumid) that the most surefire way - # of identifying a track to a specific release so we'll use that first - if song['TrackID'] and song['ReleaseID']: + #THE "MORE-SPECIFIC" CLAUSES HERE HAVE ALL BEEN REMOVED. WHEN RUNNING A LIBRARY SCAN, THE ONLY CLAUSES THAT + #EVER GOT HIT WERE [ARTIST/ALBUM/TRACK] OR CLEANNAME. ARTISTID & RELEASEID ARE NEVER PASSED TO THIS FUNCTION, + #ARE NEVER FOUND, AND THE OTHER CLAUSES WERE NEVER HIT. FURTHERMORE, OTHER MATCHING FUNCTIONS IN THIS PROGRAM + #(IMPORTER.PY, MB.PY) SIMPLY DO A [ARTIST/ALBUM/TRACK] OR CLEANNAME MATCH, SO IT'S ALL CONSISTENT. - # Check both the tracks table & alltracks table in case they haven't populated the alltracks table yet - track = myDB.action('SELECT TrackID, ReleaseID, AlbumID from alltracks WHERE TrackID=? AND ReleaseID=?', [song['TrackID'], song['ReleaseID']]).fetchone() - - # It might be the case that the alltracks table isn't populated yet, so maybe we can only find a match in the tracks table - if not track: - track = myDB.action('SELECT TrackID, ReleaseID, AlbumID from tracks WHERE TrackID=? AND ReleaseID=?', [song['TrackID'], song['ReleaseID']]).fetchone() - - if track: - # Use TrackID & ReleaseID here since there can only be one possible match with a TrackID & ReleaseID query combo - controlValueDict = { 'TrackID' : track['TrackID'], - 'ReleaseID' : track['ReleaseID'] } - - # Insert it into the Headphones hybrid release (ReleaseID == AlbumID) - hybridControlValueDict = { 'TrackID' : track['TrackID'], - 'ReleaseID' : track['AlbumID'] } - - newValueDict = { 'Location' : song['Location'], - 'BitRate' : song['BitRate'], - 'Format' : song['Format'] } - - # Update both the tracks table and the alltracks table using the controlValueDict and hybridControlValueDict - myDB.upsert("alltracks", newValueDict, controlValueDict) - myDB.upsert("tracks", newValueDict, controlValueDict) - - myDB.upsert("alltracks", newValueDict, hybridControlValueDict) - myDB.upsert("tracks", newValueDict, hybridControlValueDict) - - # Matched. Move on to the next one: - continue - - # If we can't find it with TrackID & ReleaseID, next most specific will be - # releaseid + tracktitle, although perhaps less reliable due to a higher - # likelihood of variations in the song title (e.g. feat. artists) - if song['ReleaseID'] and song['TrackTitle']: - - track = myDB.action('SELECT TrackID, ReleaseID, AlbumID from alltracks WHERE ReleaseID=? AND TrackTitle=?', [song['ReleaseID'], song['TrackTitle']]).fetchone() - - if not track: - track = myDB.action('SELECT TrackID, ReleaseID, AlbumID from tracks WHERE ReleaseID=? AND TrackTitle=?', [song['ReleaseID'], song['TrackTitle']]).fetchone() - - if track: - # There can also only be one match for this query as well (although it might be on both the tracks and alltracks table) - # So use both TrackID & ReleaseID as the control values - controlValueDict = { 'TrackID' : track['TrackID'], - 'ReleaseID' : track['ReleaseID'] } - - hybridControlValueDict = { 'TrackID' : track['TrackID'], - 'ReleaseID' : track['AlbumID'] } - - newValueDict = { 'Location' : song['Location'], - 'BitRate' : song['BitRate'], - 'Format' : song['Format'] } - - # Update both tables here as well - myDB.upsert("alltracks", newValueDict, controlValueDict) - myDB.upsert("tracks", newValueDict, controlValueDict) - - myDB.upsert("alltracks", newValueDict, hybridControlValueDict) - myDB.upsert("tracks", newValueDict, hybridControlValueDict) - - # Done - continue - - # Next most specific will be the opposite: a TrackID and an AlbumTitle - # TrackIDs span multiple releases so if something is on an official album - # and a compilation, for example, this will match it to the right one - # However - there may be multiple matches here - if song['TrackID'] and song['AlbumTitle']: - - # Even though there might be multiple matches, we just need to grab one to confirm a match - track = myDB.action('SELECT TrackID, AlbumTitle from alltracks WHERE TrackID=? AND AlbumTitle LIKE ?', [song['TrackID'], song['AlbumTitle']]).fetchone() - - if not track: - track = myDB.action('SELECT TrackID, AlbumTitle from tracks WHERE TrackID=? AND AlbumTitle LIKE ?', [song['TrackID'], song['AlbumTitle']]).fetchone() - - if track: - # Don't need the hybridControlValueDict here since ReleaseID is not unique - controlValueDict = { 'TrackID' : track['TrackID'], - 'AlbumTitle' : track['AlbumTitle'] } - - newValueDict = { 'Location' : song['Location'], - 'BitRate' : song['BitRate'], - 'Format' : song['Format'] } - - myDB.upsert("alltracks", newValueDict, controlValueDict) - myDB.upsert("tracks", newValueDict, controlValueDict) - - continue - - # Next most specific is the ArtistName + AlbumTitle + TrackTitle combo (but probably - # even more unreliable than the previous queries, and might span multiple releases) if song['ArtistName'] and song['AlbumTitle'] and song['TrackTitle']: - track = myDB.action('SELECT ArtistName, AlbumTitle, TrackTitle from alltracks WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [song['ArtistName'], song['AlbumTitle'], song['TrackTitle']]).fetchone() - - if not track: - track = myDB.action('SELECT ArtistName, AlbumTitle, TrackTitle from tracks WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [song['ArtistName'], song['AlbumTitle'], song['TrackTitle']]).fetchone() - + track = myDB.action('SELECT ArtistName, AlbumTitle, TrackTitle, AlbumID from tracks WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [song['ArtistName'], song['AlbumTitle'], song['TrackTitle']]).fetchone() if track: controlValueDict = { 'ArtistName' : track['ArtistName'], 'AlbumTitle' : track['AlbumTitle'], - 'TrackTitle' : track['TrackTitle'] } - + 'TrackTitle' : track['TrackTitle'] } newValueDict = { 'Location' : song['Location'], 'BitRate' : song['BitRate'], 'Format' : song['Format'] } - - myDB.upsert("alltracks", newValueDict, controlValueDict) myDB.upsert("tracks", newValueDict, controlValueDict) - continue - - # Use the "CleanName" (ArtistName + AlbumTitle + TrackTitle stripped of punctuation, capitalization, etc) - # This is more reliable than the former but requires some string manipulation so we'll do it only - # if we can't find a match with the original data - if song['ArtistName'] and song['AlbumTitle'] and song['TrackTitle']: - - CleanName = helpers.cleanName(song['ArtistName'] +' '+ song['AlbumTitle'] +' '+song['TrackTitle']) - - track = myDB.action('SELECT CleanName from alltracks WHERE CleanName LIKE ?', [CleanName]).fetchone() - - if not track: - track = myDB.action('SELECT CleanName from tracks WHERE CleanName LIKE ?', [CleanName]).fetchone() - - if track: - controlValueDict = { 'CleanName' : track['CleanName'] } - + controlValueDict2 = { 'ArtistName' : song['ArtistName'], + 'AlbumTitle' : song['AlbumTitle'], + 'TrackTitle' : song['TrackTitle'] } + newValueDict2 = { 'Matched' : track['AlbumID']} + myDB.upsert("have", newValueDict2, controlValueDict2) + else: + track = myDB.action('SELECT CleanName, AlbumID from tracks WHERE CleanName LIKE ?', [song['CleanName']]).fetchone() + if track: + controlValueDict = { 'CleanName' : track['CleanName']} + newValueDict = { 'Location' : song['Location'], + 'BitRate' : song['BitRate'], + 'Format' : song['Format'] } + myDB.upsert("tracks", newValueDict, controlValueDict) + + controlValueDict2 = { 'CleanName' : song['CleanName']} + newValueDict2 = { 'Matched' : track['AlbumID']} + myDB.upsert("have", newValueDict2, controlValueDict2) + + + alltrack = myDB.action('SELECT ArtistName, AlbumTitle, TrackTitle, AlbumID from alltracks WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [song['ArtistName'], song['AlbumTitle'], song['TrackTitle']]).fetchone() + if alltrack: + controlValueDict = { 'ArtistName' : alltrack['ArtistName'], + 'AlbumTitle' : alltrack['AlbumTitle'], + 'TrackTitle' : alltrack['TrackTitle'] } newValueDict = { 'Location' : song['Location'], 'BitRate' : song['BitRate'], 'Format' : song['Format'] } - myDB.upsert("alltracks", newValueDict, controlValueDict) - myDB.upsert("tracks", newValueDict, controlValueDict) - continue - - # Match on TrackID alone if we can't find it using any of the above methods. This method is reliable - # but spans multiple releases - but that's why we're putting at the beginning as a last resort. If a track - # with more specific information exists in the library, it'll overwrite these values - if song['TrackID']: - - track = myDB.action('SELECT TrackID from alltracks WHERE TrackID=?', [song['TrackID']]).fetchone() - - if not track: - track = myDB.action('SELECT TrackID from tracks WHERE TrackID=?', [song['TrackID']]).fetchone() - - if track: - controlValueDict = { 'TrackID' : track['TrackID'] } - - newValueDict = { 'Location' : song['Location'], - 'BitRate' : song['BitRate'], - 'Format' : song['Format'] } + controlValueDict2 = { 'ArtistName' : song['ArtistName'], + 'AlbumTitle' : song['AlbumTitle'], + 'TrackTitle' : song['TrackTitle'] } + newValueDict2 = { 'Matched' : alltrack['AlbumID']} + myDB.upsert("have", newValueDict2, controlValueDict2) + else: + alltrack = myDB.action('SELECT CleanName, AlbumID from alltracks WHERE CleanName LIKE ?', [song['CleanName']]).fetchone() + if alltrack: + controlValueDict = { 'CleanName' : alltrack['CleanName']} + newValueDict = { 'Location' : song['Location'], + 'BitRate' : song['BitRate'], + 'Format' : song['Format'] } + myDB.upsert("alltracks", newValueDict, controlValueDict) - myDB.upsert("alltracks", newValueDict, controlValueDict) - myDB.upsert("tracks", newValueDict, controlValueDict) - - continue + controlValueDict2 = { 'CleanName' : song['CleanName']} + newValueDict2 = { 'Matched' : alltrack['AlbumID']} + myDB.upsert("have", newValueDict2, controlValueDict2) + # if we can't find a match in the database on a track level, it might be a new artist or it might be on a non-mb release if song['ArtistName']: @@ -302,23 +247,18 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, cron=Fal else: continue - # The have table will become the new database for unmatched tracks (i.e. tracks with no associated links in the database - if song['ArtistName'] and song['AlbumTitle'] and song['TrackTitle']: - CleanName = helpers.cleanName(song['ArtistName'] +' '+ song['AlbumTitle'] +' '+song['TrackTitle']) - else: - continue - - myDB.action('INSERT INTO have (ArtistName, AlbumTitle, TrackNumber, TrackTitle, TrackLength, BitRate, Genre, Date, TrackID, Location, CleanName, Format) VALUES( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', [song['ArtistName'], song['AlbumTitle'], song['TrackNumber'], song['TrackTitle'], song['TrackLength'], song['BitRate'], song['Genre'], song['Date'], song['TrackID'], song['Location'], CleanName, song['Format']]) + #######myDB.action('INSERT INTO have (ArtistName, AlbumTitle, TrackNumber, TrackTitle, TrackLength, BitRate, Genre, Date, TrackID, Location, CleanName, Format) VALUES( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', [song['ArtistName'], song['AlbumTitle'], song['TrackNumber'], song['TrackTitle'], song['TrackLength'], song['BitRate'], song['Genre'], song['Date'], song['TrackID'], song['Location'], CleanName, song['Format']]) logger.info('Completed matching tracks from directory: %s' % dir.decode(headphones.SYS_ENCODING, 'replace')) - + if not append: # Clean up the new artist list unique_artists = {}.fromkeys(new_artists).keys() current_artists = myDB.select('SELECT ArtistName, ArtistID from artists') - - artist_list = [f for f in unique_artists if f.lower() not in [x[0].lower() for x in current_artists]] + + #There was a bug where artists with special characters (-,') would show up in new artists. + artist_list = [f for f in unique_artists if helpers.cleanName(f).lower() not in [helpers.cleanName(x[0]).lower() for x in current_artists]] # Update track counts logger.info('Updating current artist track counts') @@ -326,7 +266,7 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, cron=Fal for artist in current_artists: # Have tracks are selected from tracks table and not all tracks because of duplicates # We update the track count upon an album switch to compliment this - havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=? AND Location IS NOT NULL', [artist['ArtistID']])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ?', [artist['ArtistName']])) + havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=? AND Location IS NOT NULL', [artist['ArtistID']])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ? AND Matched IS NULL', [artist['ArtistName']])) myDB.action('UPDATE artists SET HaveTracks=? WHERE ArtistID=?', [havetracks, artist['ArtistID']]) logger.info('Found %i new artists' % len(artist_list)) @@ -348,6 +288,41 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, cron=Fal # If we're appending a new album to the database, update the artists total track counts logger.info('Updating artist track counts') - havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=? AND Location IS NOT NULL', [ArtistID])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ?', [ArtistName])) + havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=? AND Location IS NOT NULL', [ArtistID])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ? AND Matched IS NULL', [ArtistName])) myDB.action('UPDATE artists SET HaveTracks=? WHERE ArtistID=?', [havetracks, ArtistID]) - + + update_album_status() + logger.info('Library scan complete') + + #ADDED THIS SECTION TO MARK ALBUMS AS DOWNLOADED IF ARTISTS ARE ADDED EN MASSE BEFORE LIBRARY IS SCANNED +def update_album_status(AlbumID=None): + myDB = db.DBConnection() + logger.info('Counting matched tracks to mark albums as skipped/downloaded') + if AlbumID: + album_status_updater = myDB.action('SELECT AlbumID, AlbumTitle, Status from albums WHERE AlbumID=?', [AlbumID]) + else: + album_status_updater = myDB.action('SELECT AlbumID, AlbumTitle, Status from albums') + for album in album_status_updater: + track_counter = myDB.action('SELECT Location from tracks where AlbumID=?', [album['AlbumID']]) + total_tracks = 0 + have_tracks = 0 + for track in track_counter: + total_tracks+=1 + if track['Location']: + have_tracks+=1 + if total_tracks != 0: + album_completion = float(float(have_tracks) / float(total_tracks)) * 100 + else: + album_completion = 0 + logger.info('Album %s does not have any tracks in database' % album['AlbumTitle']) + + if album['Status'] == "Downloaded" or album['Status'] == "Skipped": + if album_completion >= headphones.ALBUM_COMPLETION_PCT: + new_album_status = "Downloaded" + myDB.upsert("albums", {'Status' : "Downloaded"}, {'AlbumID' : album['AlbumID']}) + else: + new_album_status = "Skipped" + myDB.upsert("albums", {'Status' : "Skipped"}, {'AlbumID' : album['AlbumID']}) + if new_album_status != album['Status']: + logger.info('Album %s changed to %s' % (album['AlbumTitle'], new_album_status)) + logger.info('Album status update complete') \ No newline at end of file diff --git a/headphones/mb.py b/headphones/mb.py index 4f1d0e29..14404816 100644 --- a/headphones/mb.py +++ b/headphones/mb.py @@ -463,7 +463,7 @@ def get_new_releases(rgid,includeExtras=False,forcefull=False): newValueDict['Location'] = match['Location'] newValueDict['BitRate'] = match['BitRate'] newValueDict['Format'] = match['Format'] - myDB.action('UPDATE have SET Matched="True" WHERE Location=?', [match['Location']]) + #myDB.action('UPDATE have SET Matched="True" WHERE Location=?', [match['Location']]) myDB.upsert("alltracks", newValueDict, controlValueDict) num_new_releases = num_new_releases + 1 diff --git a/headphones/webserve.py b/headphones/webserve.py index 82fe8fa2..974777c9 100644 --- a/headphones/webserve.py +++ b/headphones/webserve.py @@ -24,11 +24,13 @@ from mako import exceptions import time import threading +import string +import json import headphones -from headphones import logger, searcher, db, importer, mb, lastfm, librarysync -from headphones.helpers import checked, radio,today +from headphones import logger, searcher, db, importer, mb, lastfm, librarysync, helpers +from headphones.helpers import checked, radio,today, cleanName import lib.simplejson as simplejson @@ -189,11 +191,13 @@ class WebInterface(object): def deleteArtist(self, ArtistID): logger.info(u"Deleting all traces of artist: " + ArtistID) myDB = db.DBConnection() + artistname = myDB.select('SELECT ArtistName from artists where ArtistID=?', [ArtistID]).fetchone() myDB.action('DELETE from artists WHERE ArtistID=?', [ArtistID]) myDB.action('DELETE from albums WHERE ArtistID=?', [ArtistID]) myDB.action('DELETE from tracks WHERE ArtistID=?', [ArtistID]) myDB.action('DELETE from allalbums WHERE ArtistID=?', [ArtistID]) myDB.action('DELETE from alltracks WHERE ArtistID=?', [ArtistID]) + myDB.action('UPDATE have SET Matched=NULL WHERE ArtistName=?', [artistname]) myDB.action('INSERT OR REPLACE into blacklist VALUES (?)', [ArtistID]) raise cherrypy.HTTPRedirect("home") deleteArtist.exposed = True @@ -213,7 +217,7 @@ class WebInterface(object): deleteEmptyArtists.exposed = True def refreshArtist(self, ArtistID): - threading.Thread(target=importer.addArtisttoDB, args=[ArtistID]).start() + threading.Thread(target=importer.addArtisttoDB, args=[ArtistID, False, True]).start() raise cherrypy.HTTPRedirect("artistPage?ArtistID=%s" % ArtistID) refreshArtist.exposed=True @@ -240,8 +244,15 @@ class WebInterface(object): raise cherrypy.HTTPRedirect("upcoming") markAlbums.exposed = True - def addArtists(self, **args): - threading.Thread(target=importer.artistlist_to_mbids, args=[args, True]).start() + def addArtists(self, action=None, **args): + if action == "add": + threading.Thread(target=importer.artistlist_to_mbids, args=[args, True]).start() + if action == "ignore": + myDB = db.DBConnection() + for artist in args: + myDB.action('DELETE FROM newartists WHERE ArtistName=?', [artist]) + myDB.action('UPDATE have SET Matched="Ignored" WHERE ArtistName=?', [artist]) + logger.info("Artist %s removed from new artist list and set to ignored" % artist) raise cherrypy.HTTPRedirect("home") addArtists.exposed = True @@ -338,6 +349,120 @@ class WebInterface(object): return serve_template(templatename="managenew.html", title="Manage New Artists", newartists=newartists) manageNew.exposed = True + def manageUnmatched(self): + myDB = db.DBConnection() + have_album_dictionary = [] + headphones_album_dictionary = [] + unmatched_albums = [] + have_albums = myDB.select('SELECT ArtistName, AlbumTitle, TrackTitle, CleanName from have WHERE Matched IS NULL GROUP BY AlbumTitle ORDER BY ArtistName') + for albums in have_albums: + #Have to skip over manually matched tracks + original_clean = helpers.cleanName(albums['ArtistName']+" "+albums['AlbumTitle']+" "+albums['TrackTitle']) + if original_clean == albums['CleanName']: + have_dict = { 'ArtistName' : albums['ArtistName'], 'AlbumTitle' : albums['AlbumTitle'] } + have_album_dictionary.append(have_dict) + headphones_albums = myDB.select('SELECT ArtistName, AlbumTitle from albums ORDER BY ArtistName') + for albums in headphones_albums: + headphones_dict = { 'ArtistName' : albums['ArtistName'], 'AlbumTitle' : albums['AlbumTitle'] } + headphones_album_dictionary.append(headphones_dict) + #unmatchedalbums = [f for f in have_album_dictionary if f not in [x for x in headphones_album_dictionary]] + + check = set([(cleanName(d['ArtistName']).lower(), cleanName(d['AlbumTitle']).lower()) for d in headphones_album_dictionary]) + unmatchedalbums = [d for d in have_album_dictionary if (cleanName(d['ArtistName']).lower(), cleanName(d['AlbumTitle']).lower()) not in check] + + + return serve_template(templatename="manageunmatched.html", title="Manage Unmatched Items", unmatchedalbums=unmatchedalbums) + manageUnmatched.exposed = True + + def markUnmatched(self, action=None, existing_artist=None, existing_album=None, new_artist=None, new_album=None): + myDB = db.DBConnection() + + if action == "ignoreArtist": + artist = existing_artist + myDB.action('UPDATE have SET Matched="Ignored" WHERE ArtistName=? AND Matched IS NULL', [artist]) + + elif action == "ignoreAlbum": + artist = existing_artist + album = existing_album + myDB.action('UPDATE have SET Matched="Ignored" WHERE ArtistName=? AND AlbumTitle=? AND Matched IS NULL', (artist, album)) + + elif action == "matchArtist": + existing_artist_clean = helpers.cleanName(existing_artist).lower() + new_artist_clean = helpers.cleanName(new_artist).lower() + if new_artist_clean != existing_artist_clean: + have_tracks = myDB.action('SELECT Matched, CleanName, Location, BitRate, Format FROM have WHERE ArtistName=?', [existing_artist]) + update_count = 0 + for entry in have_tracks: + old_clean_filename = entry['CleanName'] + if old_clean_filename.startswith(existing_artist_clean): + new_clean_filename = old_clean_filename.replace(existing_artist_clean, new_artist_clean, 1) + myDB.action('UPDATE have SET CleanName=? WHERE ArtistName=? AND CleanName=?', [new_clean_filename, existing_artist, old_clean_filename]) + controlValueDict = {"CleanName": new_clean_filename} + newValueDict = {"Location" : entry['Location'], + "BitRate" : entry['BitRate'], + "Format" : entry['Format'] + } + #Attempt to match tracks with new CleanName + match_alltracks = myDB.action('SELECT CleanName from alltracks WHERE CleanName=?', [new_clean_filename]).fetchone() + if match_alltracks: + myDB.upsert("alltracks", newValueDict, controlValueDict) + match_tracks = myDB.action('SELECT CleanName from tracks WHERE CleanName=?', [new_clean_filename]).fetchone() + if match_tracks: + myDB.upsert("tracks", newValueDict, controlValueDict) + myDB.action('UPDATE have SET Matched="Manual" WHERE CleanName=?', [new_clean_filename]) + update_count+=1 + #This was throwing errors and I don't know why, but it seems to be working fine. + #else: + #logger.info("There was an error modifying Artist %s. This should not have happened" % existing_artist) + logger.info("Manual matching yielded %s new matches for Artist %s" % (update_count, new_artist)) + if update_count > 0: + librarysync.update_album_status() + else: + logger.info("Artist %s already named appropriately; nothing to modify" % existing_artist) + + elif action == "matchAlbum": + existing_artist_clean = helpers.cleanName(existing_artist).lower() + new_artist_clean = helpers.cleanName(new_artist).lower() + existing_album_clean = helpers.cleanName(existing_album).lower() + new_album_clean = helpers.cleanName(new_album).lower() + existing_clean_string = existing_artist_clean+" "+existing_album_clean + new_clean_string = new_artist_clean+" "+new_album_clean + if existing_clean_string != new_clean_string: + have_tracks = myDB.action('SELECT Matched, CleanName, Location, BitRate, Format FROM have WHERE ArtistName=? AND AlbumTitle=?', (existing_artist, existing_album)) + update_count = 0 + for entry in have_tracks: + old_clean_filename = entry['CleanName'] + if old_clean_filename.startswith(existing_clean_string): + new_clean_filename = old_clean_filename.replace(existing_clean_string, new_clean_string, 1) + myDB.action('UPDATE have SET CleanName=? WHERE ArtistName=? AND AlbumTitle=? AND CleanName=?', [new_clean_filename, existing_artist, existing_album, old_clean_filename]) + controlValueDict = {"CleanName": new_clean_filename} + newValueDict = {"Location" : entry['Location'], + "BitRate" : entry['BitRate'], + "Format" : entry['Format'] + } + #Attempt to match tracks with new CleanName + match_alltracks = myDB.action('SELECT CleanName from alltracks WHERE CleanName=?', [new_clean_filename]).fetchone() + if match_alltracks: + myDB.upsert("alltracks", newValueDict, controlValueDict) + match_tracks = myDB.action('SELECT CleanName, AlbumID from tracks WHERE CleanName=?', [new_clean_filename]).fetchone() + if match_tracks: + myDB.upsert("tracks", newValueDict, controlValueDict) + myDB.action('UPDATE have SET Matched="Manual" WHERE CleanName=?', [new_clean_filename]) + update_count+=1 + #This was throwing errors and I don't know why, but it seems to be working fine. + #else: + #logger.info("There was an error modifying Artist %s / Album %s. This should not have happened" % (existing_artist, existing_album)) + logger.info("Manual matching yielded %s new matches for Artist %s / Album %s" % (update_count, new_artist, new_album)) + if update_count > 0: + librarysync.update_album_status() + else: + logger.info("Artist %s / Album %s already named appropriately; nothing to modify" % (existing_artist, existing_album)) + + + + raise cherrypy.HTTPRedirect('manageUnmatched') + markUnmatched.exposed = True + def markArtists(self, action=None, **args): myDB = db.DBConnection() artistsToAdd = [] @@ -545,6 +670,20 @@ class WebInterface(object): return s getArtists_json.exposed=True + def getAlbumsByArtist_json(self, artist=None): + myDB = db.DBConnection() + album_json = {} + counter = 0 + album_list = myDB.select("SELECT AlbumTitle from albums WHERE ArtistName=?", [artist]) + for album in album_list: + album_json[counter] = album['AlbumTitle'] + counter+=1 + json_albums = json.dumps(album_json) + + cherrypy.response.headers['Content-type'] = 'application/json' + return json_albums + getAlbumsByArtist_json.exposed=True + def clearhistory(self, type=None): myDB = db.DBConnection() if type == 'all': @@ -566,6 +705,26 @@ class WebInterface(object): generateAPI.exposed = True + def forceScan(self, keepmatched=None): + myDB = db.DBConnection() + ######################################### + #NEED TO MOVE THIS INTO A SEPARATE FUNCTION BEFORE RELEASE + myDB.select('DELETE from Have') + logger.info('Removed all entries in local library database') + myDB.select('UPDATE alltracks SET Location=NULL, BitRate=NULL, Format=NULL') + myDB.select('UPDATE tracks SET Location=NULL, BitRate=NULL, Format=NULL') + logger.info('All tracks in library unmatched') + myDB.action('UPDATE artists SET HaveTracks=NULL') + logger.info('Reset track counts for all artists') + myDB.action('UPDATE albums SET Status="Skipped" WHERE Status="Skipped" OR Status="Downloaded"') + logger.info('Marking all unwanted albums as Skipped') + try: + threading.Thread(target=librarysync.libraryScan).start() + except Exception, e: + logger.error('Unable to complete the scan: %s' % e) + raise cherrypy.HTTPRedirect("home") + forceScan.exposed = True + def config(self): interface_dir = os.path.join(headphones.PROG_DIR, 'data/interfaces/') From 71607d4cf7ce60388c827a6c17841ec21ec5c2d8 Mon Sep 17 00:00:00 2001 From: theguardian Date: Wed, 9 Oct 2013 00:03:53 -0700 Subject: [PATCH 11/47] Stupid error. --- data/interfaces/default/album.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/interfaces/default/album.html b/data/interfaces/default/album.html index 2496bd4c..687e4540 100644 --- a/data/interfaces/default/album.html +++ b/data/interfaces/default/album.html @@ -138,7 +138,7 @@ %endfor <% - unmatched_temp = myDB.select('SELECT * from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND Matched is null ORDER BY CAST(TrackNumber AS INTEGER)', [album['ArtistName'], album['AlbumTitle']]) + unmatched = myDB.select('SELECT * from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND Matched is null ORDER BY CAST(TrackNumber AS INTEGER)', [album['ArtistName'], album['AlbumTitle']]) %> %if unmatched: %for track in unmatched: From 9151a986560add9b93b91f2d82d7299a12c13001 Mon Sep 17 00:00:00 2001 From: theguardian Date: Wed, 9 Oct 2013 00:08:30 -0700 Subject: [PATCH 12/47] I blame my cat for these oops-ies --- data/interfaces/default/index.html | 2 +- headphones/__init__.py | 163 +++++++++++++++++++++++++++++ 2 files changed, 164 insertions(+), 1 deletion(-) diff --git a/data/interfaces/default/index.html b/data/interfaces/default/index.html index e0201252..b67aaf7d 100644 --- a/data/interfaces/default/index.html +++ b/data/interfaces/default/index.html @@ -69,7 +69,7 @@ } if (artist['ReleaseInFuture'] === 'True') { - grade = 'gradeA';6666666666666666 + grade = 'gradeA'; } else { diff --git a/headphones/__init__.py b/headphones/__init__.py index 758b28dc..d99bf819 100644 --- a/headphones/__init__.py +++ b/headphones/__init__.py @@ -954,6 +954,169 @@ def dbcheck(): c.execute('CREATE INDEX IF NOT EXISTS tracks_albumid ON tracks(AlbumID ASC)') c.execute('CREATE INDEX IF NOT EXISTS album_artistid_reldate ON albums(ArtistID ASC, ReleaseDate DESC)') + try: + c.execute('SELECT IncludeExtras from artists') + except sqlite3.OperationalError: + c.execute('ALTER TABLE artists ADD COLUMN IncludeExtras INTEGER DEFAULT 0') + + try: + c.execute('SELECT LatestAlbum from artists') + except sqlite3.OperationalError: + c.execute('ALTER TABLE artists ADD COLUMN LatestAlbum TEXT') + + try: + c.execute('SELECT ReleaseDate from artists') + except sqlite3.OperationalError: + c.execute('ALTER TABLE artists ADD COLUMN ReleaseDate TEXT') + + try: + c.execute('SELECT AlbumID from artists') + except sqlite3.OperationalError: + c.execute('ALTER TABLE artists ADD COLUMN AlbumID TEXT') + + try: + c.execute('SELECT HaveTracks from artists') + except sqlite3.OperationalError: + c.execute('ALTER TABLE artists ADD COLUMN HaveTracks INTEGER DEFAULT 0') + + try: + c.execute('SELECT TotalTracks from artists') + except sqlite3.OperationalError: + c.execute('ALTER TABLE artists ADD COLUMN TotalTracks INTEGER DEFAULT 0') + + try: + c.execute('SELECT Type from albums') + except sqlite3.OperationalError: + c.execute('ALTER TABLE albums ADD COLUMN Type TEXT DEFAULT "Album"') + + try: + c.execute('SELECT TrackNumber from tracks') + except sqlite3.OperationalError: + c.execute('ALTER TABLE tracks ADD COLUMN TrackNumber INTEGER') + + try: + c.execute('SELECT FolderName from snatched') + except sqlite3.OperationalError: + c.execute('ALTER TABLE snatched ADD COLUMN FolderName TEXT') + + try: + c.execute('SELECT Location from tracks') + except sqlite3.OperationalError: + c.execute('ALTER TABLE tracks ADD COLUMN Location TEXT') + + try: + c.execute('SELECT Location from have') + except sqlite3.OperationalError: + c.execute('ALTER TABLE have ADD COLUMN Location TEXT') + + try: + c.execute('SELECT BitRate from tracks') + except sqlite3.OperationalError: + c.execute('ALTER TABLE tracks ADD COLUMN BitRate INTEGER') + + try: + c.execute('SELECT CleanName from tracks') + except sqlite3.OperationalError: + c.execute('ALTER TABLE tracks ADD COLUMN CleanName TEXT') + + try: + c.execute('SELECT CleanName from have') + except sqlite3.OperationalError: + c.execute('ALTER TABLE have ADD COLUMN CleanName TEXT') + + # Add the Format column + try: + c.execute('SELECT Format from have') + except sqlite3.OperationalError: + c.execute('ALTER TABLE have ADD COLUMN Format TEXT DEFAULT NULL') + + try: + c.execute('SELECT Format from tracks') + except sqlite3.OperationalError: + c.execute('ALTER TABLE tracks ADD COLUMN Format TEXT DEFAULT NULL') + + try: + c.execute('SELECT LastUpdated from artists') + except sqlite3.OperationalError: + c.execute('ALTER TABLE artists ADD COLUMN LastUpdated TEXT DEFAULT NULL') + + try: + c.execute('SELECT ArtworkURL from artists') + except sqlite3.OperationalError: + c.execute('ALTER TABLE artists ADD COLUMN ArtworkURL TEXT DEFAULT NULL') + + try: + c.execute('SELECT ArtworkURL from albums') + except sqlite3.OperationalError: + c.execute('ALTER TABLE albums ADD COLUMN ArtworkURL TEXT DEFAULT NULL') + + try: + c.execute('SELECT ThumbURL from artists') + except sqlite3.OperationalError: + c.execute('ALTER TABLE artists ADD COLUMN ThumbURL TEXT DEFAULT NULL') + + try: + c.execute('SELECT ThumbURL from albums') + except sqlite3.OperationalError: + c.execute('ALTER TABLE albums ADD COLUMN ThumbURL TEXT DEFAULT NULL') + + try: + c.execute('SELECT ArtistID from descriptions') + except sqlite3.OperationalError: + c.execute('ALTER TABLE descriptions ADD COLUMN ArtistID TEXT DEFAULT NULL') + + try: + c.execute('SELECT LastUpdated from descriptions') + except sqlite3.OperationalError: + c.execute('ALTER TABLE descriptions ADD COLUMN LastUpdated TEXT DEFAULT NULL') + + try: + c.execute('SELECT ReleaseID from albums') + except sqlite3.OperationalError: + c.execute('ALTER TABLE albums ADD COLUMN ReleaseID TEXT DEFAULT NULL') + + try: + c.execute('SELECT ReleaseFormat from albums') + except sqlite3.OperationalError: + c.execute('ALTER TABLE albums ADD COLUMN ReleaseFormat TEXT DEFAULT NULL') + + try: + c.execute('SELECT ReleaseCountry from albums') + except sqlite3.OperationalError: + c.execute('ALTER TABLE albums ADD COLUMN ReleaseCountry TEXT DEFAULT NULL') + + try: + c.execute('SELECT ReleaseID from tracks') + except sqlite3.OperationalError: + c.execute('ALTER TABLE tracks ADD COLUMN ReleaseID TEXT DEFAULT NULL') + + try: + c.execute('SELECT Matched from have') + except sqlite3.OperationalError: + c.execute('ALTER TABLE have ADD COLUMN Matched TEXT DEFAULT NULL') + + try: + c.execute('SELECT Extras from artists') + except sqlite3.OperationalError: + c.execute('ALTER TABLE artists ADD COLUMN Extras TEXT DEFAULT NULL') + # Need to update some stuff when people are upgrading and have 'include extras' set globally/for an artist + if INCLUDE_EXTRAS: + EXTRAS = "1,2,3,4,5,6,7,8" + logger.info("Copying over current artist IncludeExtras information") + artists = c.execute('SELECT ArtistID, IncludeExtras from artists').fetchall() + for artist in artists: + if artist[1]: + c.execute('UPDATE artists SET Extras=? WHERE ArtistID=?', ("1,2,3,4,5,6,7,8", artist[0])) + + try: + c.execute('SELECT Kind from snatched') + except sqlite3.OperationalError: + c.execute('ALTER TABLE snatched ADD COLUMN Kind TEXT DEFAULT NULL') + + try: + c.execute('SELECT SearchTerm from albums') + except sqlite3.OperationalError: + c.execute('ALTER TABLE albums ADD COLUMN SearchTerm TEXT DEFAULT NULL') conn.commit() From 237e429669733c3db8011a56f53bae1a525f4011 Mon Sep 17 00:00:00 2001 From: theguardian Date: Wed, 9 Oct 2013 11:55:29 -0700 Subject: [PATCH 13/47] Fixed a few minor bugs where library scan would overwrite cleanname of manually matched track & delete artist would fail. --- headphones/librarysync.py | 3 +-- headphones/webserve.py | 6 ++++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/headphones/librarysync.py b/headphones/librarysync.py index 03925d7b..6af6c4fb 100644 --- a/headphones/librarysync.py +++ b/headphones/librarysync.py @@ -138,8 +138,7 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, cron=Fal if not check_exist_song: myDB.upsert("have", newValueDict, controlValueDict) new_song_count+=1 - #We're going to have to think about metadata changing, and setting Matched = None when/if we do - elif check_exist_song['CleanName'] != CleanName and check_exist_song['Matched'] != "Manual": + elif check_exist_song['ArtistName'] != f_artist or check_exist_song['AlbumTitle'] != f.album or check_exist_song['TrackTitle'] != f.title: newValueDict['Matched'] = None myDB.upsert("have", newValueDict, controlValueDict) new_song_count+=1 diff --git a/headphones/webserve.py b/headphones/webserve.py index 974777c9..784ef20e 100644 --- a/headphones/webserve.py +++ b/headphones/webserve.py @@ -191,7 +191,9 @@ class WebInterface(object): def deleteArtist(self, ArtistID): logger.info(u"Deleting all traces of artist: " + ArtistID) myDB = db.DBConnection() - artistname = myDB.select('SELECT ArtistName from artists where ArtistID=?', [ArtistID]).fetchone() + namecheck = myDB.select('SELECT ArtistName from artists where ArtistID=?', [ArtistID]) + for name in namecheck: + artistname=name['ArtistName'] myDB.action('DELETE from artists WHERE ArtistID=?', [ArtistID]) myDB.action('DELETE from albums WHERE ArtistID=?', [ArtistID]) myDB.action('DELETE from tracks WHERE ArtistID=?', [ArtistID]) @@ -451,7 +453,7 @@ class WebInterface(object): update_count+=1 #This was throwing errors and I don't know why, but it seems to be working fine. #else: - #logger.info("There was an error modifying Artist %s / Album %s. This should not have happened" % (existing_artist, existing_album)) + #logger.info("There was an error modifying Artist %s / Album %s with clean name %s" % (existing_artist, existing_album, existing_clean_string)) logger.info("Manual matching yielded %s new matches for Artist %s / Album %s" % (update_count, new_artist, new_album)) if update_count > 0: librarysync.update_album_status() From 4ea867836e59ba3fa581d55f1618f31a4cc28928 Mon Sep 17 00:00:00 2001 From: theguardian Date: Wed, 9 Oct 2013 16:11:25 -0700 Subject: [PATCH 14/47] Optimized a few scan functions. --- headphones/librarysync.py | 17 +++++++++-------- headphones/webserve.py | 5 +++-- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/headphones/librarysync.py b/headphones/librarysync.py index 6af6c4fb..b60ab590 100644 --- a/headphones/librarysync.py +++ b/headphones/librarysync.py @@ -315,13 +315,14 @@ def update_album_status(AlbumID=None): album_completion = 0 logger.info('Album %s does not have any tracks in database' % album['AlbumTitle']) - if album['Status'] == "Downloaded" or album['Status'] == "Skipped": - if album_completion >= headphones.ALBUM_COMPLETION_PCT: - new_album_status = "Downloaded" - myDB.upsert("albums", {'Status' : "Downloaded"}, {'AlbumID' : album['AlbumID']}) - else: + if album_completion >= headphones.ALBUM_COMPLETION_PCT: + new_album_status = "Downloaded" + else: + if album['Status'] == "Skipped" or album['Status'] == "Downloaded": new_album_status = "Skipped" - myDB.upsert("albums", {'Status' : "Skipped"}, {'AlbumID' : album['AlbumID']}) - if new_album_status != album['Status']: - logger.info('Album %s changed to %s' % (album['AlbumTitle'], new_album_status)) + else: + new_album_status = album['Status'] + myDB.upsert("albums", {'Status' : new_album_status}, {'AlbumID' : album['AlbumID']}) + if new_album_status != album['Status']: + logger.info('Album %s changed to %s' % (album['AlbumTitle'], new_album_status)) logger.info('Album status update complete') \ No newline at end of file diff --git a/headphones/webserve.py b/headphones/webserve.py index 784ef20e..d3838bf0 100644 --- a/headphones/webserve.py +++ b/headphones/webserve.py @@ -408,17 +408,18 @@ class WebInterface(object): match_alltracks = myDB.action('SELECT CleanName from alltracks WHERE CleanName=?', [new_clean_filename]).fetchone() if match_alltracks: myDB.upsert("alltracks", newValueDict, controlValueDict) - match_tracks = myDB.action('SELECT CleanName from tracks WHERE CleanName=?', [new_clean_filename]).fetchone() + match_tracks = myDB.action('SELECT CleanName, AlbumID from tracks WHERE CleanName=?', [new_clean_filename]).fetchone() if match_tracks: myDB.upsert("tracks", newValueDict, controlValueDict) myDB.action('UPDATE have SET Matched="Manual" WHERE CleanName=?', [new_clean_filename]) + album_id = match_tracks['AlbumID'] update_count+=1 #This was throwing errors and I don't know why, but it seems to be working fine. #else: #logger.info("There was an error modifying Artist %s. This should not have happened" % existing_artist) logger.info("Manual matching yielded %s new matches for Artist %s" % (update_count, new_artist)) if update_count > 0: - librarysync.update_album_status() + librarysync.update_album_status(album_id) else: logger.info("Artist %s already named appropriately; nothing to modify" % existing_artist) From b0d75b46ecc82754688f5ca54ba3b130519fa116 Mon Sep 17 00:00:00 2001 From: theguardian Date: Fri, 11 Oct 2013 20:08:36 -0700 Subject: [PATCH 15/47] Revamped "Manage Unmatched" page, added "Manage Manual" page. --- data/interfaces/default/js/script.js | 1 + data/interfaces/default/managemanual.html | 121 ++++++++++++ data/interfaces/default/manageunmatched.html | 198 ++++++++++++++----- headphones/webserve.py | 89 ++++++++- 4 files changed, 351 insertions(+), 58 deletions(-) create mode 100644 data/interfaces/default/managemanual.html diff --git a/data/interfaces/default/js/script.js b/data/interfaces/default/js/script.js index e021074b..a0bb860e 100644 --- a/data/interfaces/default/js/script.js +++ b/data/interfaces/default/js/script.js @@ -315,6 +315,7 @@ function doAjaxCall(url,elem,reload,form) { console.log('refresh'); refreshTable(); } if ( reload == "tabs") refreshTab(); + if ( reload == "page") location.reload(); if ( form ) { // Change the option to 'choose...' $(formID + " select").children('option[disabled=disabled]').attr('selected','selected'); diff --git a/data/interfaces/default/managemanual.html b/data/interfaces/default/managemanual.html new file mode 100644 index 00000000..e20882ca --- /dev/null +++ b/data/interfaces/default/managemanual.html @@ -0,0 +1,121 @@ +<%inherit file="base.html" /> +<%! + import headphones + from headphones import db, helpers + myDB = db.DBConnection() +%> + +<%def name="headerIncludes()"> +
    +
    +
    +
    + « Back to Unmatched Albums + + + +<%def name="body()"> +
    +
    +

    manageManage Manually Changed Albums

    +
    + + + + + + + + + + + <% count_albums=0 %> + %for album in manualalbums: + + <% + old_artist_clean = album['ArtistName'].replace('&','%26').replace('+', '%2B').replace("'","%27") + old_album_clean = album['AlbumTitle'].replace('&','%26').replace('+', '%2B').replace("'","%27") + %> + + + + + <% count_albums+=1 %> + %endfor + +
    Local ArtistLocal AlbumPrevious Action
    ${album['ArtistName']}
    + + +     		${album['AlbumTitle']}
    + + + +     		${album['AlbumStatus']} + +
    +
    + + +<%def name="headIncludes()"> + + + +<%def name="javascriptIncludes()"> + + + diff --git a/data/interfaces/default/manageunmatched.html b/data/interfaces/default/manageunmatched.html index 2054382c..3fdc7eaa 100644 --- a/data/interfaces/default/manageunmatched.html +++ b/data/interfaces/default/manageunmatched.html @@ -16,6 +16,7 @@ <%def name="headerIncludes()"> « Back to manage overview @@ -31,56 +32,76 @@ - - - - + + <% count_albums=0 %> %for album in unmatchedalbums: - - - - <% count_albums+=1 %> @@ -116,8 +137,25 @@ initActions(); }); - function click_Artist(clicked_id) { - n=clicked_id.replace("LoadArtists",""); + + function ignore_Artist(clicked_id) { + n=clicked_id.replace("ignore_artists",""); + $("#ignore_artist_dialog"+n).dialog(); + return false; + } + + function ignore_Album(clicked_id) { + n=clicked_id.replace("ignore_albums",""); + $("#ignore_album_dialog"+n).dialog(); + return false; + } + + + function load_Artist(clicked_id) { + n=clicked_id.replace("match_artists",""); + var d = $("#artist_dialog"+n).dialog(); + d.dialog("option", "width", 450); + d.dialog("option", "position", "center"); $('#artist_options'+n).html(''); $.each(${json_artists}, function(key, value) { $('#artist_options'+n).append($(" %endfor <% - unmatched = myDB.select('SELECT * from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND Matched is null ORDER BY CAST(TrackNumber AS INTEGER)', [album['ArtistName'], album['AlbumTitle']]) + unmatched = myDB.select('SELECT * from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND Matched = "Failed" ORDER BY CAST(TrackNumber AS INTEGER)', [album['ArtistName'], album['AlbumTitle']]) %> %if unmatched: %for track in unmatched: diff --git a/data/interfaces/default/artist.html b/data/interfaces/default/artist.html index 845f6719..a53e2029 100644 --- a/data/interfaces/default/artist.html +++ b/data/interfaces/default/artist.html @@ -93,7 +93,7 @@ myDB = db.DBConnection() totaltracks = len(myDB.select('SELECT TrackTitle from tracks WHERE AlbumID=?', [album['AlbumID']])) - havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE AlbumID=? AND Location IS NOT NULL', [album['AlbumID']])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ? AND AlbumTitle LIKE ? AND Matched IS NULL', [album['ArtistName'], album['AlbumTitle']])) + havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE AlbumID=? AND Location IS NOT NULL', [album['AlbumID']])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ? AND AlbumTitle LIKE ? AND Matched = "Failed"', [album['ArtistName'], album['AlbumTitle']])) try: percent = (havetracks*100.0)/totaltracks diff --git a/data/interfaces/default/config.html b/data/interfaces/default/config.html index 2379ad7d..37b48175 100644 --- a/data/interfaces/default/config.html +++ b/data/interfaces/default/config.html @@ -95,7 +95,7 @@
    - mins + hours
    diff --git a/data/interfaces/default/managealbums.html b/data/interfaces/default/managealbums.html index 487028d6..02201131 100644 --- a/data/interfaces/default/managealbums.html +++ b/data/interfaces/default/managealbums.html @@ -57,7 +57,7 @@ myDB = db.DBConnection() totaltracks = len(myDB.select('SELECT TrackTitle from tracks WHERE AlbumID=?', [album['AlbumID']])) - havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE AlbumID=? AND Location IS NOT NULL', [album['AlbumID']])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ? AND AlbumTitle LIKE ? AND Matched IS NULL', [album['ArtistName'], album['AlbumTitle']])) + havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE AlbumID=? AND Location IS NOT NULL', [album['AlbumID']])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ? AND AlbumTitle LIKE ? AND Matched = "Failed"', [album['ArtistName'], album['AlbumTitle']])) try: percent = (havetracks*100.0)/totaltracks diff --git a/headphones/__init__.py b/headphones/__init__.py index ab0674c3..b35bd19a 100644 --- a/headphones/__init__.py +++ b/headphones/__init__.py @@ -919,7 +919,7 @@ def start(): SCHED.add_interval_job(updater.dbUpdate, hours=UPDATE_DB_INTERVAL) SCHED.add_interval_job(searcher.searchforalbum, minutes=SEARCH_INTERVAL) - SCHED.add_interval_job(librarysync.libraryScan, minutes=LIBRARYSCAN_INTERVAL, kwargs={'cron':True}) + SCHED.add_interval_job(librarysync.libraryScan, hours=LIBRARYSCAN_INTERVAL, kwargs={'cron':True}) if CHECK_GITHUB: SCHED.add_interval_job(versioncheck.checkGithub, minutes=CHECK_GITHUB_INTERVAL) @@ -964,6 +964,8 @@ def dbcheck(): c.execute('CREATE INDEX IF NOT EXISTS tracks_CleanName ON tracks(CleanName ASC)') c.execute('CREATE INDEX IF NOT EXISTS alltracks_Metadata ON alltracks(ArtistName ASC, AlbumTitle ASC, TrackTitle ASC)') c.execute('CREATE INDEX IF NOT EXISTS alltracks_CleanName ON alltracks(CleanName ASC)') + c.execute('CREATE INDEX IF NOT EXISTS tracks_Location ON tracks(Location ASC)') + c.execute('CREATE INDEX IF NOT EXISTS alltracks_Location ON alltracks(Location ASC)') try: c.execute('SELECT IncludeExtras from artists') diff --git a/headphones/importer.py b/headphones/importer.py index 78120a2e..b78c69ab 100644 --- a/headphones/importer.py +++ b/headphones/importer.py @@ -464,7 +464,7 @@ def addArtisttoDB(artistid, extrasonly=False, forcefull=False): latestalbum = myDB.action('SELECT AlbumTitle, ReleaseDate, AlbumID from albums WHERE ArtistID=? order by ReleaseDate DESC', [artistid]).fetchone() totaltracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=?', [artistid])) #havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=? AND Location IS NOT NULL', [artistid])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ?', [artist['artist_name']])) - havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=? AND Location IS NOT NULL', [artistid])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ? AND Matched IS NULL', [artist['artist_name']])) + havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=? AND Location IS NOT NULL', [artistid])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ? AND Matched = "Failed"', [artist['artist_name']])) controlValueDict = {"ArtistID": artistid} @@ -491,6 +491,7 @@ def addArtisttoDB(artistid, extrasonly=False, forcefull=False): if errors: logger.info("[%s] Finished updating artist: %s but with errors, so not marking it as updated in the database" % (artist['artist_name'], artist['artist_name'])) else: + myDB.action('DELETE FROM newartists WHERE ArtistName = ?', [artist['artist_name']]) logger.info(u"Updating complete for: %s" % artist['artist_name']) diff --git a/headphones/librarysync.py b/headphones/librarysync.py index cda014e6..fdb981a4 100644 --- a/headphones/librarysync.py +++ b/headphones/librarysync.py @@ -44,6 +44,9 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, cron=Fal return myDB = db.DBConnection() + new_artists = [] + + logger.info('Scanning music directory: %s' % dir.decode(headphones.SYS_ENCODING, 'replace')) if not append: # Clean up bad filepaths @@ -55,22 +58,25 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, cron=Fal myDB.action('UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE Location=?', [None, None, None, track['Location']]) myDB.action('UPDATE alltracks SET Location=?, BitRate=?, Format=? WHERE Location=?', [None, None, None, track['Location']]) - del_have_tracks = myDB.select('SELECT Location, Matched from have') + del_have_tracks = myDB.select('SELECT Location, Matched, ArtistName from have') for track in del_have_tracks: encoded_track_string = track['Location'].encode(headphones.SYS_ENCODING) if not os.path.isfile(encoded_track_string): + if track['ArtistName']: + #Make sure deleted files get accounted for when updating artist track counts + new_artists.append(track['ArtistName']) myDB.action('DELETE FROM have WHERE Location=?', [track['Location']]) logger.info('File %s removed from Headphones, as it is no longer on disk' % encoded_track_string.decode(headphones.SYS_ENCODING, 'replace')) ###############myDB.action('DELETE from have') - logger.info('Scanning music directory: %s' % dir.decode(headphones.SYS_ENCODING, 'replace')) - - new_artists = [] bitrates = [] song_list = [] new_song_count = 0 + file_count = 0 + + latest_subdirectory = [] for r,d,f in os.walk(dir): #need to abuse slicing to get a copy of the list, doing it directly will skip the element after a deleted one @@ -79,9 +85,16 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, cron=Fal if directory.startswith("."): d.remove(directory) for files in f: + # MEDIA_FORMATS = music file extensions, e.g. mp3, flac, etc if any(files.lower().endswith('.' + x.lower()) for x in headphones.MEDIA_FORMATS): + latest_subdirectory.append(r.replace(dir,'')) + if file_count == 0 and r.replace(dir,'') !='': + logger.info("[%s] Now scanning subdirectory %s" % (dir, r.replace(dir,''))) + elif latest_subdirectory[file_count] != latest_subdirectory[file_count-1] and file_count !=0: + logger.info("[%s] Now scanning subdirectory %s" % (dir, r.replace(dir,''))) + song = os.path.join(r, files) # We need the unicode path to use for logging, inserting into database @@ -135,19 +148,39 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, cron=Fal check_exist_song = myDB.action("SELECT * FROM have WHERE Location=?", [unicode_song_path]).fetchone() #Only attempt to match songs that are new, haven't yet been matched, or metadata has changed. if not check_exist_song: + #This is a new track + if f_artist: + new_artists.append(f_artist) myDB.upsert("have", newValueDict, controlValueDict) new_song_count+=1 - elif check_exist_song['ArtistName'] != f_artist or check_exist_song['AlbumTitle'] != f.album or check_exist_song['TrackTitle'] != f.title: - newValueDict['Matched'] = None - myDB.upsert("have", newValueDict, controlValueDict) - new_song_count+=1 + else: + if check_exist_song['ArtistName'] != f_artist or check_exist_song['AlbumTitle'] != f.album or check_exist_song['TrackTitle'] != f.title: + #Important track metadata has been modified, need to run matcher again + if f_artist and f_artist != check_exist_song['ArtistName']: + new_artists.append(f_artist) + elif f_artist and f_artist == check_exist_song['ArtistName'] and check_exist_song['Matched'] != "Ignored": + new_artists.append(f_artist) + else: + continue + + newValueDict['Matched'] = None + myDB.upsert("have", newValueDict, controlValueDict) + myDB.action('UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE Location=?', [None, None, None, unicode_song_path]) + myDB.action('UPDATE alltracks SET Location=?, BitRate=?, Format=? WHERE Location=?', [None, None, None, unicode_song_path]) + new_song_count+=1 + else: + #This track information hasn't changed + if f_artist and check_exist_song['Matched'] != "Ignored": + new_artists.append(f_artist) + + file_count+=1 # Now we start track matching logger.info("%s new/modified songs found and added to the database" % new_song_count) - song_list = myDB.action("SELECT * FROM have WHERE Matched IS NULL AND CleanName IS NOT NULL AND LOCATION LIKE ?", [dir+"%"]) + song_list = myDB.action("SELECT * FROM have WHERE Matched IS NULL AND LOCATION LIKE ?", [dir+"%"]) total_number_of_songs = myDB.action("SELECT COUNT(*) FROM have WHERE Matched IS NULL AND LOCATION LIKE ?", [dir+"%"]).fetchone()[0] - logger.info("Found " + str(total_number_of_songs) + " unmatched tracks in: '" + dir.decode(headphones.SYS_ENCODING, 'replace') + "'. Matching tracks to the appropriate releases....") + logger.info("Found " + str(total_number_of_songs) + " new/modified tracks in: '" + dir.decode(headphones.SYS_ENCODING, 'replace') + "'. Matching tracks to the appropriate releases....") # Sort the song_list by most vague (e.g. no trackid or releaseid) to most specific (both trackid & releaseid) # When we insert into the database, the tracks with the most specific information will overwrite the more general matches @@ -191,9 +224,7 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, cron=Fal 'Format' : song['Format'] } myDB.upsert("tracks", newValueDict, controlValueDict) - controlValueDict2 = { 'ArtistName' : song['ArtistName'], - 'AlbumTitle' : song['AlbumTitle'], - 'TrackTitle' : song['TrackTitle'] } + controlValueDict2 = { 'Location' : song['Location']} newValueDict2 = { 'Matched' : track['AlbumID']} myDB.upsert("have", newValueDict2, controlValueDict2) else: @@ -205,9 +236,13 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, cron=Fal 'Format' : song['Format'] } myDB.upsert("tracks", newValueDict, controlValueDict) - controlValueDict2 = { 'CleanName' : song['CleanName']} + controlValueDict2 = { 'Location' : song['Location']} newValueDict2 = { 'Matched' : track['AlbumID']} myDB.upsert("have", newValueDict2, controlValueDict2) + else: + controlValueDict2 = { 'Location' : song['Location']} + newValueDict2 = { 'Matched' : "Failed"} + myDB.upsert("have", newValueDict2, controlValueDict2) alltrack = myDB.action('SELECT ArtistName, AlbumTitle, TrackTitle, AlbumID from alltracks WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [song['ArtistName'], song['AlbumTitle'], song['TrackTitle']]).fetchone() @@ -220,9 +255,7 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, cron=Fal 'Format' : song['Format'] } myDB.upsert("alltracks", newValueDict, controlValueDict) - controlValueDict2 = { 'ArtistName' : song['ArtistName'], - 'AlbumTitle' : song['AlbumTitle'], - 'TrackTitle' : song['TrackTitle'] } + controlValueDict2 = { 'Location' : song['Location']} newValueDict2 = { 'Matched' : alltrack['AlbumID']} myDB.upsert("have", newValueDict2, controlValueDict2) else: @@ -234,16 +267,17 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, cron=Fal 'Format' : song['Format'] } myDB.upsert("alltracks", newValueDict, controlValueDict) - controlValueDict2 = { 'CleanName' : song['CleanName']} + controlValueDict2 = { 'Location' : song['Location']} newValueDict2 = { 'Matched' : alltrack['AlbumID']} myDB.upsert("have", newValueDict2, controlValueDict2) - - - # if we can't find a match in the database on a track level, it might be a new artist or it might be on a non-mb release - if song['ArtistName']: - new_artists.append(song['ArtistName']) + else: + controlValueDict2 = { 'Location' : song['Location']} + newValueDict2 = { 'Matched' : "Failed"} + myDB.upsert("have", newValueDict2, controlValueDict2) else: - continue + controlValueDict2 = { 'Location' : song['Location']} + newValueDict2 = { 'Matched' : "Failed"} + myDB.upsert("have", newValueDict2, controlValueDict2) #######myDB.action('INSERT INTO have (ArtistName, AlbumTitle, TrackNumber, TrackTitle, TrackLength, BitRate, Genre, Date, TrackID, Location, CleanName, Format) VALUES( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', [song['ArtistName'], song['AlbumTitle'], song['TrackNumber'], song['TrackTitle'], song['TrackLength'], song['BitRate'], song['Genre'], song['Date'], song['TrackID'], song['Location'], CleanName, song['Format']]) @@ -251,21 +285,25 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, cron=Fal if not append: + logger.info('Updating scanned artist track counts') + # Clean up the new artist list unique_artists = {}.fromkeys(new_artists).keys() current_artists = myDB.select('SELECT ArtistName, ArtistID from artists') #There was a bug where artists with special characters (-,') would show up in new artists. artist_list = [f for f in unique_artists if helpers.cleanName(f).lower() not in [helpers.cleanName(x[0]).lower() for x in current_artists]] - + artists_checked = [f for f in unique_artists if helpers.cleanName(f).lower() in [helpers.cleanName(x[0]).lower() for x in current_artists]] + # Update track counts - logger.info('Updating current artist track counts') - for artist in current_artists: + for artist in artists_checked: # Have tracks are selected from tracks table and not all tracks because of duplicates # We update the track count upon an album switch to compliment this - havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=? AND Location IS NOT NULL', [artist['ArtistID']])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ? AND Matched IS NULL', [artist['ArtistName']])) - myDB.action('UPDATE artists SET HaveTracks=? WHERE ArtistID=?', [havetracks, artist['ArtistID']]) + havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistName like ? AND Location IS NOT NULL', [artist])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ? AND Matched = "Failed"', [artist])) + #Note, some people complain about having "artist have tracks" > # of tracks total in artist official releases + # (can fix by getting rid of second len statement) + myDB.action('UPDATE artists SET HaveTracks=? WHERE ArtistName=?', [havetracks, artist]) logger.info('Found %i new artists' % len(artist_list)) @@ -275,9 +313,9 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, cron=Fal importer.artistlist_to_mbids(artist_list) else: logger.info('To add these artists, go to Manage->Manage New Artists') - myDB.action('DELETE from newartists') + #myDB.action('DELETE from newartists') for artist in artist_list: - myDB.action('INSERT into newartists VALUES (?)', [artist]) + myDB.action('INSERT OR IGNORE INTO newartists VALUES (?)', [artist]) if headphones.DETECT_BITRATE: headphones.PREFERRED_BITRATE = sum(bitrates)/len(bitrates)/1000 @@ -286,7 +324,7 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, cron=Fal # If we're appending a new album to the database, update the artists total track counts logger.info('Updating artist track counts') - havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=? AND Location IS NOT NULL', [ArtistID])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ? AND Matched IS NULL', [ArtistName])) + havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=? AND Location IS NOT NULL', [ArtistID])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ? AND Matched = "Failed"', [ArtistName])) myDB.action('UPDATE artists SET HaveTracks=? WHERE ArtistID=?', [havetracks, ArtistID]) update_album_status() diff --git a/headphones/mb.py b/headphones/mb.py index 6e5d4a3b..e7629bba 100644 --- a/headphones/mb.py +++ b/headphones/mb.py @@ -478,6 +478,7 @@ def get_new_releases(rgid,includeExtras=False,forcefull=False): newValueDict['BitRate'] = match['BitRate'] newValueDict['Format'] = match['Format'] #myDB.action('UPDATE have SET Matched="True" WHERE Location=?', [match['Location']]) + myDB.action('UPDATE have SET Matched=? WHERE Location=?', (release['AlbumID'], match['Location'])) myDB.upsert("alltracks", newValueDict, controlValueDict) num_new_releases = num_new_releases + 1 diff --git a/headphones/webserve.py b/headphones/webserve.py index ed79a809..c2bae283 100644 --- a/headphones/webserve.py +++ b/headphones/webserve.py @@ -357,7 +357,7 @@ class WebInterface(object): have_album_dictionary = [] headphones_album_dictionary = [] unmatched_albums = [] - have_albums = myDB.select('SELECT ArtistName, AlbumTitle, TrackTitle, CleanName from have WHERE Matched IS NULL GROUP BY AlbumTitle ORDER BY ArtistName') + have_albums = myDB.select('SELECT ArtistName, AlbumTitle, TrackTitle, CleanName from have WHERE Matched = "Failed" GROUP BY AlbumTitle ORDER BY ArtistName') for albums in have_albums: #Have to skip over manually matched tracks if albums['ArtistName'] and albums['AlbumTitle'] and albums['TrackTitle']: @@ -385,12 +385,12 @@ class WebInterface(object): if action == "ignoreArtist": artist = existing_artist - myDB.action('UPDATE have SET Matched="Ignored" WHERE ArtistName=? AND Matched IS NULL', [artist]) + myDB.action('UPDATE have SET Matched="Ignored" WHERE ArtistName=? AND Matched = "Failed"', [artist]) elif action == "ignoreAlbum": artist = existing_artist album = existing_album - myDB.action('UPDATE have SET Matched="Ignored" WHERE ArtistName=? AND AlbumTitle=? AND Matched IS NULL', (artist, album)) + myDB.action('UPDATE have SET Matched="Ignored" WHERE ArtistName=? AND AlbumTitle=? AND Matched = "Failed"', (artist, album)) elif action == "matchArtist": existing_artist_clean = helpers.cleanName(existing_artist).lower() @@ -491,13 +491,13 @@ class WebInterface(object): myDB = db.DBConnection() if action == "unignoreArtist": artist = existing_artist - myDB.action('UPDATE have SET Matched=NULL WHERE ArtistName=? AND Matched="Ignored"', [artist]) + myDB.action('UPDATE have SET Matched="Failed" WHERE ArtistName=? AND Matched="Ignored"', [artist]) logger.info("Artist: %s successfully restored to unmatched list" % artist) elif action == "unignoreAlbum": artist = existing_artist album = existing_album - myDB.action('UPDATE have SET Matched=NULL WHERE ArtistName=? AND AlbumTitle=? AND Matched="Ignored"', (artist, album)) + myDB.action('UPDATE have SET Matched="Failed" WHERE ArtistName=? AND AlbumTitle=? AND Matched="Ignored"', (artist, album)) logger.info("Album: %s successfully restored to unmatched list" % album) elif action == "unmatchArtist": @@ -511,7 +511,7 @@ class WebInterface(object): if tracks['CleanName'] != original_clean: myDB.action('UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE CleanName=?', [None, None, None, tracks['CleanName']]) myDB.action('UPDATE alltracks SET Location=?, BitRate=?, Format=? WHERE CleanName=?', [None, None, None, tracks['CleanName']]) - myDB.action('UPDATE have SET CleanName=?, Matched=NULL WHERE ArtistName=? AND AlbumTitle=? AND TrackTitle=?', (original_clean, artist, album, track_title)) + myDB.action('UPDATE have SET CleanName=?, Matched="Failed" WHERE ArtistName=? AND AlbumTitle=? AND TrackTitle=?', (original_clean, artist, album, track_title)) update_count+=1 if update_count > 0: librarysync.update_album_status() @@ -531,7 +531,7 @@ class WebInterface(object): album_id = album_id_check[0] myDB.action('UPDATE tracks SET Location=?, BitRate=?, Format=? WHERE CleanName=?', [None, None, None, tracks['CleanName']]) myDB.action('UPDATE alltracks SET Location=?, BitRate=?, Format=? WHERE CleanName=?', [None, None, None, tracks['CleanName']]) - myDB.action('UPDATE have SET CleanName=?, Matched=NULL WHERE ArtistName=? AND AlbumTitle=? AND TrackTitle=?', (original_clean, artist, album, track_title)) + myDB.action('UPDATE have SET CleanName=?, Matched="Failed" WHERE ArtistName=? AND AlbumTitle=? AND TrackTitle=?', (original_clean, artist, album, track_title)) update_count+=1 if update_count > 0: librarysync.update_album_status(album_id) From ecb402e10e377baba7783b4240093ea77a782c7d Mon Sep 17 00:00:00 2001 From: theguardian Date: Sat, 14 Dec 2013 09:03:14 -0800 Subject: [PATCH 41/47] Fixed bug where logger search froze on librarysync information --- headphones/librarysync.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/headphones/librarysync.py b/headphones/librarysync.py index fdb981a4..843eaf11 100644 --- a/headphones/librarysync.py +++ b/headphones/librarysync.py @@ -89,11 +89,12 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, cron=Fal # MEDIA_FORMATS = music file extensions, e.g. mp3, flac, etc if any(files.lower().endswith('.' + x.lower()) for x in headphones.MEDIA_FORMATS): - latest_subdirectory.append(r.replace(dir,'')) + subdirectory = r.replace(dir,'') + latest_subdirectory.append(subdirectory) if file_count == 0 and r.replace(dir,'') !='': - logger.info("[%s] Now scanning subdirectory %s" % (dir, r.replace(dir,''))) + logger.info("[%s] Now scanning subdirectory %s" % (dir.decode(headphones.SYS_ENCODING, 'replace'), subdirectory.decode(headphones.SYS_ENCODING, 'replace'))) elif latest_subdirectory[file_count] != latest_subdirectory[file_count-1] and file_count !=0: - logger.info("[%s] Now scanning subdirectory %s" % (dir, r.replace(dir,''))) + logger.info("[%s] Now scanning subdirectory %s" % (dir.decode(headphones.SYS_ENCODING, 'replace'), subdirectory.decode(headphones.SYS_ENCODING, 'replace'))) song = os.path.join(r, files) From 435cd780b2045d66550483101a603b0467e0406c Mon Sep 17 00:00:00 2001 From: rembo10 Date: Thu, 19 Dec 2013 17:15:34 +0000 Subject: [PATCH 42/47] Fixed bug eliminating valid torrents due to maxsize with waffles --- headphones/searcher.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/headphones/searcher.py b/headphones/searcher.py index 6dfe0640..108feae4 100644 --- a/headphones/searcher.py +++ b/headphones/searcher.py @@ -1005,7 +1005,7 @@ def searchTorrent(albumid=None, new=False, losslessOnly=False): try: title = item.title desc_match = re.search(r"Size: (\d+)<", item.description) - size = desc_match.group(1) + size = int(desc_match.group(1)) url = item.link resultlist.append((title, size, url, provider)) logger.info('Found %s. Size: %s' % (title, helpers.bytes_to_mb(size))) From 62f294a51dc4de81713e34d511677a014d36b1b7 Mon Sep 17 00:00:00 2001 From: Quentin Date: Sat, 11 Jan 2014 13:19:15 +0100 Subject: [PATCH 43/47] The transmission url detection now : - is executed only if the url is not already ending with rpc - detects ports up to 6 digits long (changed from 4) --- headphones/transmission.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/headphones/transmission.py b/headphones/transmission.py index 72b60ff1..f0b477e3 100644 --- a/headphones/transmission.py +++ b/headphones/transmission.py @@ -85,10 +85,12 @@ def torrentAction(method, arguments): if host.endswith('/'): host = host[:-1] - if host.endswith(':\d{2,4}'): - host = host + "/transmission/rpc" - else - host = host + "/rpc" + if not host.endswith('/rpc'): + if host.endswith(':\d{2,6}'): + host = host + "/transmission/rpc" + else: + host = host + "/rpc" + request = urllib2.Request(host) if username and password: base64string = base64.encodestring('%s:%s' % (username, password)).replace('\n', '') From b6965b3db39c1c643a5a63711056b7529136e0c6 Mon Sep 17 00:00:00 2001 From: Joseph Lombrozo Date: Sat, 11 Jan 2014 21:26:14 -0800 Subject: [PATCH 44/47] Add description to the getArtist API response --- headphones/api.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/headphones/api.py b/headphones/api.py index 16a22229..e6892e82 100644 --- a/headphones/api.py +++ b/headphones/api.py @@ -123,8 +123,9 @@ class Api(object): artist = self._dic_from_query('SELECT * from artists WHERE ArtistID="' + self.id + '"') albums = self._dic_from_query('SELECT * from albums WHERE ArtistID="' + self.id + '" order by ReleaseDate DESC') + description = self._dic_from_query('SELECT * from descriptions WHERE ArtistID="' + self.id + '"') - self.data = { 'artist': artist, 'albums': albums } + self.data = { 'artist': artist, 'albums': albums, 'description' : description } return def _getAlbum(self, **kwargs): From 594dc6acaa51690eeb778c1de492a4a50e727c66 Mon Sep 17 00:00:00 2001 From: Joseph Lombrozo Date: Sat, 11 Jan 2014 21:33:17 -0800 Subject: [PATCH 45/47] Tabs to spaces. Whoops! --- headphones/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/headphones/api.py b/headphones/api.py index e6892e82..7754f3f6 100644 --- a/headphones/api.py +++ b/headphones/api.py @@ -123,7 +123,7 @@ class Api(object): artist = self._dic_from_query('SELECT * from artists WHERE ArtistID="' + self.id + '"') albums = self._dic_from_query('SELECT * from albums WHERE ArtistID="' + self.id + '" order by ReleaseDate DESC') - description = self._dic_from_query('SELECT * from descriptions WHERE ArtistID="' + self.id + '"') + description = self._dic_from_query('SELECT * from descriptions WHERE ArtistID="' + self.id + '"') self.data = { 'artist': artist, 'albums': albums, 'description' : description } return From 31cdb095dc4ab4401167bc43b1d3a50c1018d68c Mon Sep 17 00:00:00 2001 From: rembo10 Date: Mon, 13 Jan 2014 10:32:32 +0000 Subject: [PATCH 46/47] Took off a little of the transmission_host description text --- data/interfaces/default/config.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/interfaces/default/config.html b/data/interfaces/default/config.html index 5351047e..56b14ddf 100644 --- a/data/interfaces/default/config.html +++ b/data/interfaces/default/config.html @@ -204,7 +204,7 @@
    - usually http://localhost:9091, add transmission folder if you changed it, ex : http://localhost:9091/torrent + usually http://localhost:9091
    From 3ccfc9cea9bb160ee7b3cef5516f4fcf9110d0ed Mon Sep 17 00:00:00 2001 From: rembo10 Date: Mon, 13 Jan 2014 10:39:58 +0000 Subject: [PATCH 47/47] Added AdeHub's changes from here: https://github.com/rembo10/headphones/pull/1330 --- headphones/music_encoder.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/headphones/music_encoder.py b/headphones/music_encoder.py index 2b00bfc8..e37f2d43 100644 --- a/headphones/music_encoder.py +++ b/headphones/music_encoder.py @@ -244,8 +244,15 @@ def command(encoder,musicSource,musicDest,albumPath): logger.info('Encoding %s...' % (musicSource.decode(headphones.SYS_ENCODING, 'replace'))) logger.debug(subprocess.list2cmdline(cmd)) - startupinfo = subprocess.STARTUPINFO() - startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW + # stop windows opening the cmd + startupinfo = None + if headphones.SYS_PLATFORM == "win32": + startupinfo = subprocess.STARTUPINFO() + try: + startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW + except AttributeError: + startupinfo.dwFlags |= subprocess._subprocess.STARTF_USESHOWWINDOW + p = subprocess.Popen(cmd, startupinfo=startupinfo, stdin=open(os.devnull, 'rb'), stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = p.communicate(headphones.ENCODER)
    Local ArtistLocal AlbumMatch ArtistMatch AlbumLocal ArtistLocal Album
    ${album['ArtistName']} -
    - - - -
    + <% + old_artist_clean = album['ArtistName'].replace('&','%26').replace("'","%27") + old_album_clean = album['AlbumTitle'].replace('&','%26').replace("'","%27") + old_artist_js = album['ArtistName'].replace("'","\\'").replace('"','\\"') + old_album_js = album['AlbumTitle'].replace("'","\\'").replace('"','\\"') + %> +     		${album['ArtistName']}
    + + + + + +     		${album['AlbumTitle']} -
    - - - - -
    -     		- -
    - - - - -
    -     		- -
    - - - +
    		${album['AlbumTitle']}
    + + + + + +
    @@ -1259,6 +1275,7 @@ initConfigCheckbox("#usenewznab"); initConfigCheckbox("#usenzbsrus"); initConfigCheckbox("#usenzbsorg"); + initConfigCheckbox("#useomgwtfnzbs"); initConfigCheckbox("#usepiratebay"); initConfigCheckbox("#usewaffles"); initConfigCheckbox("#userutracker"); diff --git a/headphones/__init__.py b/headphones/__init__.py index 234d2cea..639a558b 100644 --- a/headphones/__init__.py +++ b/headphones/__init__.py @@ -166,6 +166,10 @@ NZBSRUS = False NZBSRUS_UID = None NZBSRUS_APIKEY = None +OMGWTFNZBS = False +OMGWTFNZBS_UID = None +OMGWTFNZBS_APIKEY = None + PREFERRED_WORDS = None IGNORED_WORDS = None REQUIRED_WORDS = None @@ -305,8 +309,9 @@ def initialize(): LIBRARYSCAN, LIBRARYSCAN_INTERVAL, DOWNLOAD_SCAN_INTERVAL, SAB_HOST, SAB_USERNAME, SAB_PASSWORD, SAB_APIKEY, SAB_CATEGORY, \ NZBGET_USERNAME, NZBGET_PASSWORD, NZBGET_CATEGORY, NZBGET_HOST, HEADPHONES_INDEXER, NZBMATRIX, TRANSMISSION_HOST, TRANSMISSION_USERNAME, TRANSMISSION_PASSWORD, \ UTORRENT_HOST, UTORRENT_USERNAME, UTORRENT_PASSWORD, NEWZNAB, NEWZNAB_HOST, NEWZNAB_APIKEY, NEWZNAB_ENABLED, EXTRA_NEWZNABS, \ - NZBSORG, NZBSORG_UID, NZBSORG_HASH, NZBSRUS, NZBSRUS_UID, NZBSRUS_APIKEY, NZB_DOWNLOADER, TORRENT_DOWNLOADER, PREFERRED_WORDS, REQUIRED_WORDS, IGNORED_WORDS, \ - LASTFM_USERNAME, INTERFACE, FOLDER_PERMISSIONS, FILE_PERMISSIONS, ENCODERFOLDER, ENCODER_PATH, ENCODER, XLDPROFILE, BITRATE, SAMPLINGFREQUENCY, \ + NZBSORG, NZBSORG_UID, NZBSORG_HASH, NZBSRUS, NZBSRUS_UID, NZBSRUS_APIKEY, OMGWTFNZBS, OMGWTFNZBS_UID, OMGWTFNZBS_APIKEY, \ + NZB_DOWNLOADER, TORRENT_DOWNLOADER, PREFERRED_WORDS, REQUIRED_WORDS, IGNORED_WORDS, LASTFM_USERNAME, \ + INTERFACE, FOLDER_PERMISSIONS, FILE_PERMISSIONS, ENCODERFOLDER, ENCODER_PATH, ENCODER, XLDPROFILE, BITRATE, SAMPLINGFREQUENCY, \ MUSIC_ENCODER, ADVANCEDENCODER, ENCODEROUTPUTFORMAT, ENCODERQUALITY, ENCODERVBRCBR, ENCODERLOSSLESS, DELETE_LOSSLESS_FILES, \ PROWL_ENABLED, PROWL_PRIORITY, PROWL_KEYS, PROWL_ONSNATCH, PUSHOVER_ENABLED, PUSHOVER_PRIORITY, PUSHOVER_KEYS, PUSHOVER_ONSNATCH, MIRRORLIST, \ MIRROR, CUSTOMHOST, CUSTOMPORT, CUSTOMSLEEP, HPUSER, HPPASS, XBMC_ENABLED, XBMC_HOST, XBMC_USERNAME, XBMC_PASSWORD, XBMC_UPDATE, \ @@ -326,6 +331,7 @@ def initialize(): CheckSection('Newznab') CheckSection('NZBsorg') CheckSection('NZBsRus') + CheckSection('omgwtfnzbs') CheckSection('Waffles') CheckSection('Rutracker') CheckSection('What.cd') @@ -465,6 +471,10 @@ def initialize(): NZBSRUS_UID = check_setting_str(CFG, 'NZBsRus', 'nzbsrus_uid', '') NZBSRUS_APIKEY = check_setting_str(CFG, 'NZBsRus', 'nzbsrus_apikey', '') + OMGWTFNZBS = bool(check_setting_int(CFG, 'omgwtfnzbs', 'omgwtfnzbs', 0)) + OMGWTFNZBS_UID = check_setting_str(CFG, 'omgwtfnzbs', 'omgwtfnzbs_uid', '') + OMGWTFNZBS_APIKEY = check_setting_str(CFG, 'omgwtfnzbs', 'omgwtfnzbs_apikey', '') + PREFERRED_WORDS = check_setting_str(CFG, 'General', 'preferred_words', '') IGNORED_WORDS = check_setting_str(CFG, 'General', 'ignored_words', '') REQUIRED_WORDS = check_setting_str(CFG, 'General', 'required_words', '') @@ -842,6 +852,11 @@ def config_write(): new_config['NZBsRus']['nzbsrus_uid'] = NZBSRUS_UID new_config['NZBsRus']['nzbsrus_apikey'] = NZBSRUS_APIKEY + new_config['omgwtfnzbs'] = {} + new_config['omgwtfnzbs']['omgwtfnzbs'] = int(OMGWTFNZBS) + new_config['omgwtfnzbs']['omgwtfnzbs_uid'] = OMGWTFNZBS_UID + new_config['omgwtfnzbs']['omgwtfnzbs_apikey'] = OMGWTFNZBS_APIKEY + new_config['General']['preferred_words'] = PREFERRED_WORDS new_config['General']['ignored_words'] = IGNORED_WORDS new_config['General']['required_words'] = REQUIRED_WORDS diff --git a/headphones/searcher.py b/headphones/searcher.py index eb227622..40b0395b 100644 --- a/headphones/searcher.py +++ b/headphones/searcher.py @@ -114,7 +114,7 @@ def searchforalbum(albumid=None, new=False, lossless=False): for result in results: foundNZB = "none" - if (headphones.HEADPHONES_INDEXER or headphones.NEWZNAB or headphones.NZBSORG or headphones.NZBSRUS) and (headphones.SAB_HOST or headphones.BLACKHOLE_DIR or headphones.NZBGET_HOST): + if (headphones.HEADPHONES_INDEXER or headphones.NEWZNAB or headphones.NZBSORG or headphones.NZBSRUS or headphones.OMGWTFNZBS) and (headphones.SAB_HOST or headphones.BLACKHOLE_DIR or headphones.NZBGET_HOST): if result['Status'] == "Wanted Lossless": foundNZB = searchNZB(result['AlbumID'], new, losslessOnly=True) else: @@ -131,7 +131,7 @@ def searchforalbum(albumid=None, new=False, lossless=False): foundNZB = "none" - if (headphones.HEADPHONES_INDEXER or headphones.NEWZNAB or headphones.NZBSORG or headphones.NZBSRUS) and (headphones.SAB_HOST or headphones.BLACKHOLE_DIR or headphones.NZBGET_HOST): + if (headphones.HEADPHONES_INDEXER or headphones.NEWZNAB or headphones.NZBSORG or headphones.NZBSRUS or headphones.OMGWTFNZBS) and (headphones.SAB_HOST or headphones.BLACKHOLE_DIR or headphones.NZBGET_HOST): foundNZB = searchNZB(albumid, new, lossless) if (headphones.KAT or headphones.PIRATEBAY or headphones.ISOHUNT or headphones.MININOVA or headphones.WAFFLES or headphones.RUTRACKER or headphones.WHATCD) and foundNZB == "none": @@ -433,6 +433,65 @@ def searchNZB(albumid=None, new=False, losslessOnly=False): except Exception, e: logger.error(u"An unknown error occurred trying to parse the feed: %s" % e) + + if headphones.OMGWTFNZBS: + + provider = "omgwtfnzbs" + + if headphones.PREFERRED_QUALITY == 3 or losslessOnly: + categories = "22" + elif headphones.PREFERRED_QUALITY: + categories = "22,7" + else: + categories = "7" + + if albums['Type'] == 'Other': + categories = "29" + logger.info("Album type is audiobook/spokenword. Searching all music categories") + + params = { "user": headphones.OMGWTFNZBS_UID, + "api": headphones.OMGWTFNZBS_APIKEY, + "catid": categories, + "retention": headphones.USENET_RETENTION, + "search": term + } + + searchURL = 'http://api.omgwtfnzbs.org/json/?' + urllib.urlencode(params) + + # Add a user-agent + request = urllib2.Request(searchURL) + request.add_header('User-Agent', 'headphones/0.0 +https://github.com/rembo10/headphones') + opener = urllib2.build_opener() + + logger.info(u'Parsing results from omgwtfnzbs' % searchURL) + + try: + data = opener.open(request).read() + except Exception, e: + logger.warn('Error fetching data from omgwtfnzbs: %s' % e) + data = False + + if data: + + d = json.loads(data) + + if 'notice' in data: + logger.info(u"No results returned from omgwtfnzbs: %s" % d['notice']) + pass + + else: + for item in d: + try: + url = item['getnzb'] + title = item['release'] + size = int(item['sizebytes']) + + resultlist.append((title, size, url, provider)) + logger.info('Found %s. Size: %s' % (title, helpers.bytes_to_mb(size))) + + except Exception, e: + logger.error(u"An unknown error occurred trying to parse the results: %s" % e) + # attempt to verify that this isn't a substring result # when looking for "Foo - Foo" we don't want "Foobar" # this should be less of an issue when it isn't a self-titled album so we'll only check vs artist diff --git a/headphones/webserve.py b/headphones/webserve.py index 3477c293..21fdb979 100644 --- a/headphones/webserve.py +++ b/headphones/webserve.py @@ -610,6 +610,9 @@ class WebInterface(object): "use_nzbsrus" : checked(headphones.NZBSRUS), "nzbsrus_uid" : headphones.NZBSRUS_UID, "nzbsrus_apikey" : headphones.NZBSRUS_APIKEY, + "use_omgwtfnzbs" : checked(headphones.OMGWTFNZBS), + "omgwtfnzbs_uid" : headphones.OMGWTFNZBS_UID, + "omgwtfnzbs_apikey" : headphones.OMGWTFNZBS_APIKEY, "preferred_words" : headphones.PREFERRED_WORDS, "ignored_words" : headphones.IGNORED_WORDS, "required_words" : headphones.REQUIRED_WORDS, @@ -723,7 +726,7 @@ class WebInterface(object): download_scan_interval=None, nzb_search_interval=None, libraryscan_interval=None, sab_host=None, sab_username=None, sab_apikey=None, sab_password=None, sab_category=None, nzbget_host=None, nzbget_username=None, nzbget_password=None, nzbget_category=None, transmission_host=None, transmission_username=None, transmission_password=None, utorrent_host=None, utorrent_username=None, utorrent_password=None, nzb_downloader=0, torrent_downloader=0, download_dir=None, blackhole_dir=None, usenet_retention=None, - use_headphones_indexer=0, newznab=0, newznab_host=None, newznab_apikey=None, newznab_enabled=0, nzbsorg=0, nzbsorg_uid=None, nzbsorg_hash=None, nzbsrus=0, nzbsrus_uid=None, nzbsrus_apikey=None, + use_headphones_indexer=0, newznab=0, newznab_host=None, newznab_apikey=None, newznab_enabled=0, nzbsorg=0, nzbsorg_uid=None, nzbsorg_hash=None, nzbsrus=0, nzbsrus_uid=None, nzbsrus_apikey=None, omgwtfnzbs=0, omgwtfnzbs_uid=None, omgwtfnzbs_apikey=None, preferred_words=None, required_words=None, ignored_words=None, preferred_quality=0, preferred_bitrate=None, detect_bitrate=0, move_files=0, torrentblackhole_dir=None, download_torrent_dir=None, numberofseeders=None, use_piratebay=0, piratebay_proxy_url=None, use_isohunt=0, use_kat=0, use_mininova=0, waffles=0, waffles_uid=None, waffles_passkey=None, whatcd=0, whatcd_username=None, whatcd_password=None, rutracker=0, rutracker_user=None, rutracker_password=None, rename_files=0, correct_metadata=0, cleanup_files=0, add_album_art=0, album_art_format=None, embed_album_art=0, embed_lyrics=0, @@ -780,6 +783,9 @@ class WebInterface(object): headphones.NZBSRUS = nzbsrus headphones.NZBSRUS_UID = nzbsrus_uid headphones.NZBSRUS_APIKEY = nzbsrus_apikey + headphones.OMGWTFNZBS = omgwtfnzbs + headphones.OMGWTFNZBS_UID = omgwtfnzbs_uid + headphones.OMGWTFNZBS_APIKEY = omgwtfnzbs_apikey headphones.PREFERRED_WORDS = preferred_words headphones.IGNORED_WORDS = ignored_words headphones.REQUIRED_WORDS = required_words From f75d9bd50ecb40a875cdf11069d6231995ba6cc2 Mon Sep 17 00:00:00 2001 From: theguardian Date: Sun, 13 Oct 2013 09:43:20 -0700 Subject: [PATCH 20/47] Fixed havetrack counts in "Manage Albums" view with new system. --- data/interfaces/default/managealbums.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/interfaces/default/managealbums.html b/data/interfaces/default/managealbums.html index a30dde6a..487028d6 100644 --- a/data/interfaces/default/managealbums.html +++ b/data/interfaces/default/managealbums.html @@ -57,7 +57,7 @@ myDB = db.DBConnection() totaltracks = len(myDB.select('SELECT TrackTitle from tracks WHERE AlbumID=?', [album['AlbumID']])) - havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE AlbumID=? AND Location IS NOT NULL', [album['AlbumID']])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ? AND AlbumTitle LIKE ?', [album['ArtistName'], album['AlbumTitle']])) + havetracks = len(myDB.select('SELECT TrackTitle from tracks WHERE AlbumID=? AND Location IS NOT NULL', [album['AlbumID']])) + len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ? AND AlbumTitle LIKE ? AND Matched IS NULL', [album['ArtistName'], album['AlbumTitle']])) try: percent = (havetracks*100.0)/totaltracks From 8fd631f96b4077a92ae440db4440f810bcf654b1 Mon Sep 17 00:00:00 2001 From: theguardian Date: Sun, 13 Oct 2013 13:45:10 -0700 Subject: [PATCH 21/47] Cleaned up unnecessary code. No longer suggested that users run a comprehensive active artist update on first-run. --- data/interfaces/default/manage.html | 3 -- headphones/importer.py | 54 ++++++++--------------------- headphones/mb.py | 3 ++ headphones/searcher.py | 2 ++ headphones/updater.py | 12 +------ 5 files changed, 21 insertions(+), 53 deletions(-) diff --git a/data/interfaces/default/manage.html b/data/interfaces/default/manage.html index 7d680c1d..6d98bb53 100644 --- a/data/interfaces/default/manage.html +++ b/data/interfaces/default/manage.html @@ -147,12 +147,9 @@
    Force Legacy

    Please note that these functions will take a significant amount of time to complete.

    -
    Force Update Active Artists [Comprehensive]
    - *Warning: If you choose [Force Update Active Artists], artist "extras" will be reset, manual album release matching will be reset, "paused" artists will lose ALL information, and "wanted" albums will be reset to include only recent/future albums. It's recommended you activate all paused artists before running. -

    Force Re-scan Library [Comprehensive]
    *Warning: If you choose [Force Re-scan Library], any manually ignored/matched artists/albums will be reset to "unmatched". diff --git a/headphones/importer.py b/headphones/importer.py index 10c41abb..45204464 100644 --- a/headphones/importer.py +++ b/headphones/importer.py @@ -199,15 +199,15 @@ def addArtisttoDB(artistid, extrasonly=False, forcefull=False): if len(artist['releasegroups']) != 0 and not extrasonly: for groups in artist['releasegroups']: group_list.append(groups['id']) - remove_missing_groups_from_albums = myDB.action("SELECT AlbumID, ReleaseID FROM albums WHERE ArtistID=?", [artistid]) + remove_missing_groups_from_albums = myDB.action("SELECT AlbumID FROM albums WHERE ArtistID=?", [artistid]) for items in remove_missing_groups_from_albums: - if items['ReleaseID'] not in group_list and items['AlbumID']==items['ReleaseID']: #added 2nd clause for when user picks alternate release + if items['AlbumID'] not in group_list: # Remove all from albums/tracks that aren't in release groups - myDB.action("DELETE FROM albums WHERE ReleaseID=?", [items['ReleaseID']]) - myDB.action("DELETE FROM allalbums WHERE ReleaseID=?", [items['ReleaseID']]) - myDB.action("DELETE FROM tracks WHERE ReleaseID=?", [items['ReleaseID']]) - myDB.action("DELETE FROM alltracks WHERE ReleaseID=?", [items['ReleaseID']]) - logger.info("Removing all references to group %s to reflect MusicBrainz" % items['ReleaseID']) + myDB.action("DELETE FROM albums WHERE AlbumID=?", [items['AlbumID']]) + myDB.action("DELETE FROM allalbums WHERE AlbumID=?", [items['AlbumID']]) + myDB.action("DELETE FROM tracks WHERE AlbumID=?", [items['AlbumID']]) + myDB.action("DELETE FROM alltracks WHERE AlbumID=?", [items['AlbumID']]) + logger.info("Removing all references to group %s to reflect MusicBrainz" % items['AlbumID']) force_repackage = 1 else: logger.info("Error pulling data from MusicBrainz: Maintaining dB") @@ -247,7 +247,7 @@ def addArtisttoDB(artistid, extrasonly=False, forcefull=False): if force_repackage == 1: new_releases = -1 - logger.info('Forcing repackage of %s, since dB groups have been removed' % al_title) + logger.info('Forcing repackage of %s, since release groups have been removed' % al_title) else: new_releases = new_releases else: @@ -260,6 +260,11 @@ def addArtisttoDB(artistid, extrasonly=False, forcefull=False): #print new_releases if new_releases != 0: + #Dump existing hybrid release since we're repackaging/replacing it + myDB.action("DELETE from albums WHERE ReleaseID=?", [rg['id']]) + myDB.action("DELETE from allalbums WHERE ReleaseID=?", [rg['id']]) + myDB.action("DELETE from tracks WHERE ReleaseID=?", [rg['id']]) + myDB.action("DELETE from alltracks WHERE ReleaseID=?", [rg['id']]) # This will be used later to build a hybrid release fullreleaselist = [] #Search for releases within a release group @@ -364,22 +369,6 @@ def addArtisttoDB(artistid, extrasonly=False, forcefull=False): rg_exists = myDB.action("SELECT * from albums WHERE AlbumID=?", [rg['id']]).fetchone() if not rg_exists: releaseid = rg['id'] - elif rg_exists and not rg_exists['ReleaseID']: - # Need to do some importing here - to transition the old format of using the release group - # only to using releasegroup & releaseid. These are the albums that are missing a ReleaseID - # so we'll need to move over the locations, bitrates & formats from the tracks table to the new - # alltracks table. Thankfully we can just use TrackIDs since they span releases/releasegroups - logger.info("Copying current track information to alternate releases") - tracks = myDB.action('SELECT * from tracks WHERE AlbumID=?', [rg['id']]).fetchall() - for track in tracks: - if track['Location']: - controlValueDict = {"TrackID": track['TrackID']} - newValueDict = {"Location": track['Location'], - "BitRate": track['BitRate'], - "Format": track['Format'], - } - myDB.upsert("alltracks", newValueDict, controlValueDict) - releaseid = rg['id'] else: releaseid = rg_exists['ReleaseID'] @@ -415,15 +404,8 @@ def addArtisttoDB(artistid, extrasonly=False, forcefull=False): else: newValueDict['Status'] = "Skipped" - #Only update albums table with hybrid release if user didn't choose an alternate release - check_alternate_release = myDB.action("SELECT AlbumID, ReleaseID FROM albums WHERE ArtistID=? AND AlbumID=?", (artistid, rg['id'])).fetchone() - if check_alternate_release: - if check_alternate_release[0] == check_alternate_release[1]: - myDB.upsert("albums", newValueDict, controlValueDict) - else: - myDB.upsert("albums", newValueDict, controlValueDict) + myDB.upsert("albums", newValueDict, controlValueDict) - myDB.action('DELETE from tracks WHERE AlbumID=?', [rg['id']]) tracks = myDB.action('SELECT * from alltracks WHERE ReleaseID=?', [releaseid]).fetchall() # This is used to see how many tracks you have from an album - to mark it as downloaded. Default is 80%, can be set in config as ALBUM_COMPLETION_PCT @@ -448,13 +430,7 @@ def addArtisttoDB(artistid, extrasonly=False, forcefull=False): "BitRate": track['BitRate'] } - #Only update tracks table with hybrid release if user didn't choose an alternate release - check_alternate_release = myDB.action("SELECT AlbumID, ReleaseID FROM albums WHERE ArtistID=? AND AlbumID=?", (artistid, rg['id'])).fetchone() - if check_alternate_release: - if check_alternate_release[0] == check_alternate_release[1]: - myDB.upsert("tracks", newValueDict, controlValueDict) - else: - myDB.upsert("tracks", newValueDict, controlValueDict) + myDB.upsert("tracks", newValueDict, controlValueDict) # Mark albums as downloaded if they have at least 80% (by default, configurable) of the album have_track_count = len(myDB.select('SELECT * from tracks WHERE AlbumID=? AND Location IS NOT NULL', [rg['id']])) diff --git a/headphones/mb.py b/headphones/mb.py index 14404816..89578e18 100644 --- a/headphones/mb.py +++ b/headphones/mb.py @@ -389,6 +389,9 @@ def get_new_releases(rgid,includeExtras=False,forcefull=False): album_checker = myDB.action('SELECT * from allalbums WHERE ReleaseID=?', [rel_id_check]).fetchone() if not album_checker or forcefull: + #DELETE all references to this release since we're updating it anyway. + myDB.action('DELETE from allalbums WHERE ReleaseID=?', [rel_id_check]) + myDB.action('DELETE from alltracks WHERE ReleaseID=?', [rel_id_check]) release['AlbumTitle'] = unicode(releasedata['title']) release['AlbumID'] = unicode(rgid) release['AlbumASIN'] = unicode(releasedata['asin']) if 'asin' in releasedata else None diff --git a/headphones/searcher.py b/headphones/searcher.py index 3472499a..69f2fe1f 100644 --- a/headphones/searcher.py +++ b/headphones/searcher.py @@ -137,6 +137,8 @@ def searchforalbum(albumid=None, new=False, lossless=False): if (headphones.KAT or headphones.PIRATEBAY or headphones.ISOHUNT or headphones.MININOVA or headphones.WAFFLES or headphones.RUTRACKER or headphones.WHATCD) and foundNZB == "none": searchTorrent(albumid, new, lossless) + logger.info('Search for Wanted albums complete') + def searchNZB(albumid=None, new=False, losslessOnly=False): myDB = db.DBConnection() diff --git a/headphones/updater.py b/headphones/updater.py index 49fdadac..6bd0450f 100644 --- a/headphones/updater.py +++ b/headphones/updater.py @@ -21,16 +21,6 @@ def dbUpdate(forcefull=False): myDB = db.DBConnection() - #This can be updated to NOT include: paused artists, artists with extras enabled, wanted albums, albums matched to specific releases, etc - #But it absolutely FLIES if these dB's are destroyed in their entirety. With the new system, there's really no need to pause artists. - if forcefull==True: - myDB.select('DELETE from albums') - myDB.select('DELETE from allalbums') - myDB.select('DELETE from tracks') - myDB.select('DELETE from alltracks') - myDB.select('DELETE from descriptions') - myDB.select('UPDATE artists SET LatestAlbum=?, ReleaseDate=?, AlbumID=?, HaveTracks=?, TotalTracks=?', [None, None, None, None, None]) - activeartists = myDB.select('SELECT ArtistID, ArtistName from artists WHERE Status="Active" or Status="Loading" order by LastUpdated ASC') logger.info('Starting update for %i active artists' % len(activeartists)) @@ -39,4 +29,4 @@ def dbUpdate(forcefull=False): artistid = artist[0] importer.addArtisttoDB(artistid=artistid, extrasonly=False, forcefull=forcefull) - logger.info('Update complete') + logger.info('Active artist update complete') From 94c404e7bde298b2a17fbbff6c9aef0d9550940f Mon Sep 17 00:00:00 2001 From: theguardian Date: Tue, 15 Oct 2013 09:07:54 -0700 Subject: [PATCH 22/47] Fixed bug where untagged tracks would be passed to unmatched. These qualifiers should be treated separately. --- headphones/webserve.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/headphones/webserve.py b/headphones/webserve.py index a4069136..c6fea169 100644 --- a/headphones/webserve.py +++ b/headphones/webserve.py @@ -360,10 +360,13 @@ class WebInterface(object): have_albums = myDB.select('SELECT ArtistName, AlbumTitle, TrackTitle, CleanName from have WHERE Matched IS NULL GROUP BY AlbumTitle ORDER BY ArtistName') for albums in have_albums: #Have to skip over manually matched tracks - original_clean = helpers.cleanName(albums['ArtistName']+" "+albums['AlbumTitle']+" "+albums['TrackTitle']) - if original_clean == albums['CleanName']: - have_dict = { 'ArtistName' : albums['ArtistName'], 'AlbumTitle' : albums['AlbumTitle'] } - have_album_dictionary.append(have_dict) + if albums['ArtistName'] and albums['AlbumTitle'] and albums['TrackTitle']: + original_clean = helpers.cleanName(albums['ArtistName']+" "+albums['AlbumTitle']+" "+albums['TrackTitle']) + # else: + # original_clean = None + if original_clean == albums['CleanName']: + have_dict = { 'ArtistName' : albums['ArtistName'], 'AlbumTitle' : albums['AlbumTitle'] } + have_album_dictionary.append(have_dict) headphones_albums = myDB.select('SELECT ArtistName, AlbumTitle from albums ORDER BY ArtistName') for albums in headphones_albums: headphones_dict = { 'ArtistName' : albums['ArtistName'], 'AlbumTitle' : albums['AlbumTitle'] } From 467947d48ec43edbc996f2b35dce77fed204888b Mon Sep 17 00:00:00 2001 From: theguardian Date: Tue, 15 Oct 2013 10:20:08 -0700 Subject: [PATCH 23/47] Don't attempt to match tracks that don't have enough information to make matching possible. --- headphones/librarysync.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/headphones/librarysync.py b/headphones/librarysync.py index b60ab590..a9f76d11 100644 --- a/headphones/librarysync.py +++ b/headphones/librarysync.py @@ -146,7 +146,7 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, cron=Fal # Now we start track matching logger.info("%s new/modified songs found and added to the database" % new_song_count) - song_list = myDB.action("SELECT * FROM have WHERE Matched IS NULL AND LOCATION LIKE ?", [dir+"%"]) + song_list = myDB.action("SELECT * FROM have WHERE Matched IS NULL AND CleanName IS NOT NULL AND LOCATION LIKE ?", [dir+"%"]) total_number_of_songs = myDB.action("SELECT COUNT(*) FROM have WHERE Matched IS NULL AND LOCATION LIKE ?", [dir+"%"]).fetchone()[0] logger.info("Found " + str(total_number_of_songs) + " unmatched tracks in: '" + dir.decode(headphones.SYS_ENCODING, 'replace') + "'. Matching tracks to the appropriate releases....") From 67dc54f01b7abd20db7d6edeadd97cff31379fad Mon Sep 17 00:00:00 2001 From: Bonham84 Date: Tue, 15 Oct 2013 23:53:31 +0000 Subject: [PATCH 24/47] Handle FLAC albumart correctly. --- lib/beets/mediafile.py | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/lib/beets/mediafile.py b/lib/beets/mediafile.py index 785ce2a9..905d70e0 100644 --- a/lib/beets/mediafile.py +++ b/lib/beets/mediafile.py @@ -572,9 +572,23 @@ class ImageField(object): # No cover found. return None + elif obj.type == 'flac': + if 'metadata_block_picture' not in obj.mgfile: + return None + + for data in obj.mgfile['metadata_block_picture']: + try: + pic = lib.mutagen.flac.Picture(data) + break + except TypeError: + pass + else: + return None + + return pic.data else: - # Here we're assuming everything but MP3 and MPEG-4 uses + # Here we're assuming everything but MP3, FLAC and MPEG-4 use # the Xiph/Vorbis Comments standard. This may not be valid. # http://wiki.xiph.org/VorbisComment#Cover_art @@ -623,6 +637,13 @@ class ImageField(object): else: cover = lib.mutagen.mp4.MP4Cover(val, self._mp4kind(val)) obj.mgfile['covr'] = [cover] + + elif obj.type == 'flac': + if val is None: + pic = lib.mutagen.flac.Picture() + pic.data = val + pic.mime = self._mime(val) + obj.mgfile['metadata_block_picture'] = [pic.write()] else: # Again, assuming Vorbis Comments standard. From 8cd08401f5bd1a78dac562c707a666b18f4b1e84 Mon Sep 17 00:00:00 2001 From: theguardian Date: Tue, 15 Oct 2013 21:07:44 -0700 Subject: [PATCH 25/47] Fixed Error '500' for Managed Manually Matched & Ignored --- headphones/webserve.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/headphones/webserve.py b/headphones/webserve.py index c6fea169..bdc33b73 100644 --- a/headphones/webserve.py +++ b/headphones/webserve.py @@ -472,15 +472,16 @@ class WebInterface(object): manual_albums = [] manualalbums = myDB.select('SELECT ArtistName, AlbumTitle, TrackTitle, CleanName, Matched from have') for albums in manualalbums: - original_clean = helpers.cleanName(albums['ArtistName']+" "+albums['AlbumTitle']+" "+albums['TrackTitle']) - if albums['Matched'] == "Ignored" or albums['Matched'] == "Manual" or albums['CleanName'] != original_clean: - if albums['Matched'] == "Ignored": - album_status = "Ignored" - elif albums['Matched'] == "Manual" or albums['CleanName'] != original_clean: - album_status = "Matched" - manual_dict = { 'ArtistName' : albums['ArtistName'], 'AlbumTitle' : albums['AlbumTitle'], 'AlbumStatus' : album_status } - if manual_dict not in manual_albums: - manual_albums.append(manual_dict) + if albums['ArtistName'] and albums['AlbumTitle'] and albums['TrackTitle']: + original_clean = helpers.cleanName(albums['ArtistName']+" "+albums['AlbumTitle']+" "+albums['TrackTitle']) + if albums['Matched'] == "Ignored" or albums['Matched'] == "Manual" or albums['CleanName'] != original_clean: + if albums['Matched'] == "Ignored": + album_status = "Ignored" + elif albums['Matched'] == "Manual" or albums['CleanName'] != original_clean: + album_status = "Matched" + manual_dict = { 'ArtistName' : albums['ArtistName'], 'AlbumTitle' : albums['AlbumTitle'], 'AlbumStatus' : album_status } + if manual_dict not in manual_albums: + manual_albums.append(manual_dict) manual_albums_sorted = sorted(manual_albums, key=itemgetter('ArtistName', 'AlbumTitle')) return serve_template(templatename="managemanual.html", title="Manage Manual Items", manualalbums=manual_albums_sorted) From 6ebd26ecfd2a83557f176fb5dd559ed389775f14 Mon Sep 17 00:00:00 2001 From: theguardian Date: Wed, 23 Oct 2013 17:31:21 -0700 Subject: [PATCH 26/47] Fixed minor issue where manual 'have' tracks would be unmatched if a local file was deleted. --- headphones/librarysync.py | 1 - 1 file changed, 1 deletion(-) diff --git a/headphones/librarysync.py b/headphones/librarysync.py index a9f76d11..cda014e6 100644 --- a/headphones/librarysync.py +++ b/headphones/librarysync.py @@ -61,7 +61,6 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, cron=Fal encoded_track_string = track['Location'].encode(headphones.SYS_ENCODING) if not os.path.isfile(encoded_track_string): myDB.action('DELETE FROM have WHERE Location=?', [track['Location']]) - myDB.action('UPDATE have SET Matched=NULL WHERE Matched=?', [track['Matched']]) logger.info('File %s removed from Headphones, as it is no longer on disk' % encoded_track_string.decode(headphones.SYS_ENCODING, 'replace')) ###############myDB.action('DELETE from have') From 4c8f11555e6ef7369f9771cf2c59c46e33e07e14 Mon Sep 17 00:00:00 2001 From: theguardian Date: Wed, 23 Oct 2013 21:34:12 -0700 Subject: [PATCH 27/47] Added database indices on allalbums/alltracks tables. Active artist update FLIES throughout the process when starting from scratch. --- headphones/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/headphones/__init__.py b/headphones/__init__.py index d99bf819..2dd83f7b 100644 --- a/headphones/__init__.py +++ b/headphones/__init__.py @@ -953,6 +953,9 @@ def dbcheck(): c.execute('CREATE TABLE IF NOT EXISTS releases (ReleaseID TEXT, ReleaseGroupID TEXT, UNIQUE(ReleaseID, ReleaseGroupID))') c.execute('CREATE INDEX IF NOT EXISTS tracks_albumid ON tracks(AlbumID ASC)') c.execute('CREATE INDEX IF NOT EXISTS album_artistid_reldate ON albums(ArtistID ASC, ReleaseDate DESC)') + c.execute('CREATE INDEX IF NOT EXISTS alltracks_relid ON alltracks(ReleaseID ASC, TrackID ASC)') + c.execute('CREATE INDEX IF NOT EXISTS allalbums_relid ON allalbums(ReleaseID ASC)') + c.execute('CREATE INDEX IF NOT EXISTS have_location ON have(Location ASC)') try: c.execute('SELECT IncludeExtras from artists') From af0e570b4411fcbda810f6ef9867d1817a18a0d3 Mon Sep 17 00:00:00 2001 From: theguardian Date: Thu, 24 Oct 2013 09:20:18 -0700 Subject: [PATCH 28/47] Fixed minor bug where track numbers were out of order on album page (due to new index) --- headphones/webserve.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/headphones/webserve.py b/headphones/webserve.py index bdc33b73..ed79a809 100644 --- a/headphones/webserve.py +++ b/headphones/webserve.py @@ -108,7 +108,7 @@ class WebInterface(object): def albumPage(self, AlbumID): myDB = db.DBConnection() album = myDB.action('SELECT * from albums WHERE AlbumID=?', [AlbumID]).fetchone() - tracks = myDB.select('SELECT * from tracks WHERE AlbumID=?', [AlbumID]) + tracks = myDB.select('SELECT * from tracks WHERE AlbumID=? ORDER BY CAST(TrackNumber AS INTEGER)', [AlbumID]) description = myDB.action('SELECT * from descriptions WHERE ReleaseGroupID=?', [AlbumID]).fetchone() title = album['ArtistName'] + ' - ' + album['AlbumTitle'] return serve_template(templatename="album.html", title=title, album=album, tracks=tracks, description=description) From 17fb7938331e14c2e73dbf73382265ccf71f295a Mon Sep 17 00:00:00 2001 From: theguardian Date: Thu, 24 Oct 2013 14:48:04 -0700 Subject: [PATCH 29/47] Added indices on tracks/alltracks/have tables to speed up library sync & improve overall speed of browsing headphones. --- headphones/__init__.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/headphones/__init__.py b/headphones/__init__.py index 2dd83f7b..ab0674c3 100644 --- a/headphones/__init__.py +++ b/headphones/__init__.py @@ -953,9 +953,17 @@ def dbcheck(): c.execute('CREATE TABLE IF NOT EXISTS releases (ReleaseID TEXT, ReleaseGroupID TEXT, UNIQUE(ReleaseID, ReleaseGroupID))') c.execute('CREATE INDEX IF NOT EXISTS tracks_albumid ON tracks(AlbumID ASC)') c.execute('CREATE INDEX IF NOT EXISTS album_artistid_reldate ON albums(ArtistID ASC, ReleaseDate DESC)') + #Below creates indices to speed up Active Artist updating c.execute('CREATE INDEX IF NOT EXISTS alltracks_relid ON alltracks(ReleaseID ASC, TrackID ASC)') c.execute('CREATE INDEX IF NOT EXISTS allalbums_relid ON allalbums(ReleaseID ASC)') c.execute('CREATE INDEX IF NOT EXISTS have_location ON have(Location ASC)') + #Below creates indices to speed up library scanning & matching + c.execute('CREATE INDEX IF NOT EXISTS have_Metadata ON have(ArtistName ASC, AlbumTitle ASC, TrackTitle ASC)') + c.execute('CREATE INDEX IF NOT EXISTS have_CleanName ON have(CleanName ASC)') + c.execute('CREATE INDEX IF NOT EXISTS tracks_Metadata ON tracks(ArtistName ASC, AlbumTitle ASC, TrackTitle ASC)') + c.execute('CREATE INDEX IF NOT EXISTS tracks_CleanName ON tracks(CleanName ASC)') + c.execute('CREATE INDEX IF NOT EXISTS alltracks_Metadata ON alltracks(ArtistName ASC, AlbumTitle ASC, TrackTitle ASC)') + c.execute('CREATE INDEX IF NOT EXISTS alltracks_CleanName ON alltracks(CleanName ASC)') try: c.execute('SELECT IncludeExtras from artists') From b8729660110eab63b034b58ac985c9391651bd81 Mon Sep 17 00:00:00 2001 From: quentin Date: Tue, 5 Nov 2013 14:50:34 +0100 Subject: [PATCH 30/47] Added the possibility to make it work with transmission in a different subfolder --- data/interfaces/default/config.html | 2 +- headphones/transmission.py | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/data/interfaces/default/config.html b/data/interfaces/default/config.html index b4246cfe..8d0de436 100644 --- a/data/interfaces/default/config.html +++ b/data/interfaces/default/config.html @@ -196,7 +196,7 @@
    - usually http://localhost:9091 + usually http://localhost:9091, add transmission folder if you changed it, ex : http://localhost:9091/torrent
    diff --git a/headphones/transmission.py b/headphones/transmission.py index 9b31bd96..72b60ff1 100644 --- a/headphones/transmission.py +++ b/headphones/transmission.py @@ -20,6 +20,7 @@ import urllib2 import lib.simplejson as json import base64 import time +import re # This is just a simple script to send torrents to transmission. The # intention is to turn this into a class where we can check the state @@ -83,8 +84,11 @@ def torrentAction(method, arguments): if host.endswith('/'): host = host[:-1] - - host = host + "/transmission/rpc" + + if host.endswith(':\d{2,4}'): + host = host + "/transmission/rpc" + else + host = host + "/rpc" request = urllib2.Request(host) if username and password: base64string = base64.encodestring('%s:%s' % (username, password)).replace('\n', '') From 16dbe9a1cb038db70871c446863436bbbf07a10b Mon Sep 17 00:00:00 2001 From: stereoa Date: Fri, 22 Nov 2013 00:13:45 -0500 Subject: [PATCH 31/47] made postprocessing not pop up subprocess window --- headphones/music_encoder.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/headphones/music_encoder.py b/headphones/music_encoder.py index 2c54974c..2b00bfc8 100644 --- a/headphones/music_encoder.py +++ b/headphones/music_encoder.py @@ -244,7 +244,9 @@ def command(encoder,musicSource,musicDest,albumPath): logger.info('Encoding %s...' % (musicSource.decode(headphones.SYS_ENCODING, 'replace'))) logger.debug(subprocess.list2cmdline(cmd)) - p = subprocess.Popen(cmd, stdin=open(os.devnull, 'rb'), stdout=subprocess.PIPE, stderr=subprocess.PIPE) + startupinfo = subprocess.STARTUPINFO() + startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW + p = subprocess.Popen(cmd, startupinfo=startupinfo, stdin=open(os.devnull, 'rb'), stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = p.communicate(headphones.ENCODER) From 162c3cb6043b3d24b560e06bfef20051aa91739d Mon Sep 17 00:00:00 2001 From: cohena Date: Sun, 1 Dec 2013 19:43:00 -0800 Subject: [PATCH 32/47] Fix search hang in pygazelle if server didn't respond, timeout is now 30 secs --- lib/pygazelle/api.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/pygazelle/api.py b/lib/pygazelle/api.py index 75272cd5..4e93b47d 100644 --- a/lib/pygazelle/api.py +++ b/lib/pygazelle/api.py @@ -51,6 +51,7 @@ class GazelleAPI(object): self.passkey = None self.userid = None self.logged_in_user = None + self.default_timeout = 30 self.cached_users = {} self.cached_artists = {} self.cached_tags = {} @@ -97,7 +98,8 @@ class GazelleAPI(object): loginpage = 'https://what.cd/login.php' data = {'username': self.username, 'password': self.password} - r = self.session.post(loginpage, data=data) + r = self.session.post(loginpage, data=data, timeout=self.default_timeout) + self.past_request_timestamps.append(time.time()) if r.status_code != 200: raise LoginException("Login returned status code %s" % r.status_code) @@ -112,7 +114,6 @@ class GazelleAPI(object): self.passkey = accountinfo['passkey'] self.logged_in_user = User(self.userid, self) self.logged_in_user.set_index_data(accountinfo) - self.past_request_timestamps.append(time.time()) def request(self, action, autologin=True, **kwargs): """ @@ -153,7 +154,7 @@ class GazelleAPI(object): if self.authkey: params['auth'] = self.authkey params.update(kwargs) - r = self.session.get(url, params=params, allow_redirects=False) + r = self.session.get(url, params=params, allow_redirects=False, timeout=self.default_timeout) self.past_request_timestamps.append(time.time()) return r.content From b30b472e58529b3ede8470a9ac3d6f743ed65812 Mon Sep 17 00:00:00 2001 From: cohena Date: Sun, 1 Dec 2013 20:11:01 -0800 Subject: [PATCH 33/47] Fix what.cd lossless searching --- headphones/searcher.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/headphones/searcher.py b/headphones/searcher.py index 478de86d..31690dc2 100644 --- a/headphones/searcher.py +++ b/headphones/searcher.py @@ -1011,11 +1011,11 @@ def searchTorrent(albumid=None, new=False, losslessOnly=False): bitrate = None bitrate_string = bitrate - if headphones.PREFERRED_QUALITY == 2 or losslessOnly: # Lossless Only mode + if headphones.PREFERRED_QUALITY == 3 or losslessOnly: # Lossless Only mode search_formats = [gazelleformat.FLAC] maxsize = 10000000000 - elif headphones.PREFERRED_QUALITY == 3: # Preferred quality mode - search_formats=[None] # should return all + elif headphones.PREFERRED_QUALITY == 2: # Preferred quality mode + search_formats = [None] # should return all bitrate = headphones.PREFERRED_BITRATE if bitrate: for encoding_string in gazelleencoding.ALL_ENCODINGS: From bb36c914b7d897d6a08cc4ffc93440a802cc30ab Mon Sep 17 00:00:00 2001 From: davidrot Date: Sun, 8 Dec 2013 11:58:12 +0100 Subject: [PATCH 34/47] Bug fix * character in filename --- headphones/postprocessor.py | 1 + 1 file changed, 1 insertion(+) diff --git a/headphones/postprocessor.py b/headphones/postprocessor.py index 368bd770..8f5bd73f 100644 --- a/headphones/postprocessor.py +++ b/headphones/postprocessor.py @@ -843,6 +843,7 @@ def renameFiles(albumpath, downloaded_track_list, release): new_file_name = new_file_name.replace('?','_').replace(':', '_').encode(headphones.SYS_ENCODING, 'replace') + new_file_name = new_file_name.replace('*','_') if headphones.FILE_UNDERSCORES: new_file_name = new_file_name.replace(' ', '_') From 418f93ed9627fd8862311207376e6181b5fcc4a0 Mon Sep 17 00:00:00 2001 From: PHiAX Date: Sun, 8 Dec 2013 22:11:12 +0100 Subject: [PATCH 35/47] Added | character to replacelist for folders. I got this error, so tried to fix it. 08-dec-2013 21:38:53 - ERROR :: Thread-143 : Could not create folder for Minor Earth | Major Sky. Not moving: [Error 123] The filename, directory name, or volume label syntax is incorrect: 'Z:\\Music\\Album\\A\\a-ha\\minor earth | major sky [2000]' --- headphones/postprocessor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/headphones/postprocessor.py b/headphones/postprocessor.py index 368bd770..c090f3b3 100644 --- a/headphones/postprocessor.py +++ b/headphones/postprocessor.py @@ -533,7 +533,7 @@ def moveFiles(albumpath, release, tracks): } folder = helpers.replace_all(headphones.FOLDER_FORMAT.strip(), values) - folder = folder.replace('./', '_/').replace(':','_').replace('?','_').replace('/.','/_').replace('<','_').replace('>','_') + folder = folder.replace('./', '_/').replace(':','_').replace('?','_').replace('/.','/_').replace('<','_').replace('>','_').replace('|','_') if folder.endswith('.'): folder = folder.replace(folder[len(folder)-1], '_') From cc61cc5c9f72ec2338d5f6e8276404137505a313 Mon Sep 17 00:00:00 2001 From: theguardian Date: Sun, 8 Dec 2013 15:20:32 -0800 Subject: [PATCH 36/47] Fixed issue where matching introduced false positives if metadata has MusicBrainz Track ID set. --- headphones/importer.py | 8 ++++---- headphones/mb.py | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/headphones/importer.py b/headphones/importer.py index 45204464..e352bf61 100644 --- a/headphones/importer.py +++ b/headphones/importer.py @@ -349,8 +349,8 @@ def addArtisttoDB(artistid, extrasonly=False, forcefull=False): if not match: match = myDB.action('SELECT Location, BitRate, Format from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [artist['artist_name'], rg['title'], track['title']]).fetchone() - if not match: - match = myDB.action('SELECT Location, BitRate, Format from have WHERE TrackID=?', [track['id']]).fetchone() + #if not match: + #match = myDB.action('SELECT Location, BitRate, Format from have WHERE TrackID=?', [track['id']]).fetchone() if match: newValueDict['Location'] = match['Location'] newValueDict['BitRate'] = match['BitRate'] @@ -587,8 +587,8 @@ def addReleaseById(rid): if not match: match = myDB.action('SELECT Location, BitRate, Format from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [release_dict['artist_name'], release_dict['rg_title'], track['title']]).fetchone() - if not match: - match = myDB.action('SELECT Location, BitRate, Format from have WHERE TrackID=?', [track['id']]).fetchone() + #if not match: + #match = myDB.action('SELECT Location, BitRate, Format from have WHERE TrackID=?', [track['id']]).fetchone() if match: newValueDict['Location'] = match['Location'] diff --git a/headphones/mb.py b/headphones/mb.py index 89578e18..8356769b 100644 --- a/headphones/mb.py +++ b/headphones/mb.py @@ -460,8 +460,8 @@ def get_new_releases(rgid,includeExtras=False,forcefull=False): if not match: match = myDB.action('SELECT Location, BitRate, Format from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [release['ArtistName'], release['AlbumTitle'], track['title']]).fetchone() - if not match: - match = myDB.action('SELECT Location, BitRate, Format from have WHERE TrackID=?', [track['id']]).fetchone() + #if not match: + #match = myDB.action('SELECT Location, BitRate, Format from have WHERE TrackID=?', [track['id']]).fetchone() if match: newValueDict['Location'] = match['Location'] newValueDict['BitRate'] = match['BitRate'] From fbb641328766a599f01f0dce4935922bd871dd57 Mon Sep 17 00:00:00 2001 From: theguardian Date: Tue, 10 Dec 2013 13:53:55 -0800 Subject: [PATCH 37/47] Fixed bug where specific releases would not show disambiguation in track title. Releases should now be WYSIWYG reflection of musicbrainz.org (with the exception of multiple disc releases to be handled next). --- headphones/mb.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/headphones/mb.py b/headphones/mb.py index 8356769b..ca66ecc0 100644 --- a/headphones/mb.py +++ b/headphones/mb.py @@ -486,9 +486,13 @@ def getTracksFromRelease(release): tracks = [] for medium in release['medium-list']: for track in medium['track-list']: + try: + track_title = unicode(track['title']) + except: + track_title = unicode(track['recording']['title']) tracks.append({ 'number': totalTracks, - 'title': unicode(track['recording']['title']), + 'title': track_title, 'id': unicode(track['recording']['id']), 'url': u"http://musicbrainz.org/track/" + track['recording']['id'], 'duration': int(track['length']) if 'length' in track else 0 From 4c8f933900783764e53396b22642f4da508a552d Mon Sep 17 00:00:00 2001 From: theguardian Date: Tue, 10 Dec 2013 17:46:14 -0800 Subject: [PATCH 38/47] Added more release information to release database and "alternate albums" on album page. --- data/interfaces/default/album.html | 6 +++--- headphones/mb.py | 13 ++++++++++++- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/data/interfaces/default/album.html b/data/interfaces/default/album.html index 687e4540..1acee990 100644 --- a/data/interfaces/default/album.html +++ b/data/interfaces/default/album.html @@ -21,7 +21,7 @@