From 9aa8bb8a72ac56ac1ec7095e6b45ff641d78f926 Mon Sep 17 00:00:00 2001 From: theguardian Date: Wed, 2 Oct 2013 09:34:00 -0700 Subject: [PATCH] Removed mb_lock from get_new_releases (which is how it was originally) - should fix database locking errors. --- headphones/mb.py | 255 +++++++++++++++++++++++------------------------ 1 file changed, 127 insertions(+), 128 deletions(-) diff --git a/headphones/mb.py b/headphones/mb.py index 8cec8873..742f628f 100644 --- a/headphones/mb.py +++ b/headphones/mb.py @@ -334,150 +334,149 @@ def getRelease(releaseid, include_artist_info=True): def get_new_releases(rgid,includeExtras=False): - with mb_lock: - myDB = db.DBConnection() - results = [] - try: - limit = 100 - newResults = None - while newResults == None or len(newResults) >= limit: - newResults = musicbrainzngs.browse_releases(release_group=rgid,includes=['artist-credits','labels','recordings','release-groups','media'],limit=limit,offset=len(results)) - if 'release-list' not in newResults: - break #may want to raise an exception here instead ? - newResults = newResults['release-list'] - results += newResults - - except WebServiceError, e: - logger.warn('Attempt to retrieve information from MusicBrainz for release group "%s" failed (%s)' % (rgid, str(e))) - time.sleep(5) - return False + myDB = db.DBConnection() + results = [] + try: + limit = 100 + newResults = None + while newResults == None or len(newResults) >= limit: + newResults = musicbrainzngs.browse_releases(release_group=rgid,includes=['artist-credits','labels','recordings','release-groups','media'],limit=limit,offset=len(results)) + if 'release-list' not in newResults: + break #may want to raise an exception here instead ? + newResults = newResults['release-list'] + results += newResults - if not results or len(results) == 0: - return False + except WebServiceError, e: + logger.warn('Attempt to retrieve information from MusicBrainz for release group "%s" failed (%s)' % (rgid, str(e))) + time.sleep(5) + return False + + if not results or len(results) == 0: + return False - #Clean all references to releases in dB that are no longer referenced in musicbrainz - release_list = [] - force_repackage1 = 0 - if len(results) != 0: - for release_mark in results: - release_list.append(unicode(release_mark['id'])) - release_title = release_mark['title'] - remove_missing_releases = myDB.action("SELECT ReleaseID FROM allalbums WHERE AlbumID=?", [rgid]) - if remove_missing_releases: - for items in remove_missing_releases: - if items['ReleaseID'] not in release_list and items['ReleaseID'] != rgid: - # Remove all from albums/tracks that aren't in release - myDB.action("DELETE FROM albums WHERE ReleaseID=?", [items['ReleaseID']]) - myDB.action("DELETE FROM tracks WHERE ReleaseID=?", [items['ReleaseID']]) - myDB.action("DELETE FROM allalbums WHERE ReleaseID=?", [items['ReleaseID']]) - myDB.action("DELETE FROM alltracks WHERE ReleaseID=?", [items['ReleaseID']]) - logger.info("Removing all references to release %s to reflect MusicBrainz" % items['ReleaseID']) - force_repackage1 = 1 - else: - logger.info("Error pulling data from MusicBrainz: Maintaining dB") + #Clean all references to releases in dB that are no longer referenced in musicbrainz + release_list = [] + force_repackage1 = 0 + if len(results) != 0: + for release_mark in results: + release_list.append(unicode(release_mark['id'])) + release_title = release_mark['title'] + remove_missing_releases = myDB.action("SELECT ReleaseID FROM allalbums WHERE AlbumID=?", [rgid]) + if remove_missing_releases: + for items in remove_missing_releases: + if items['ReleaseID'] not in release_list and items['ReleaseID'] != rgid: + # Remove all from albums/tracks that aren't in release + myDB.action("DELETE FROM albums WHERE ReleaseID=?", [items['ReleaseID']]) + myDB.action("DELETE FROM tracks WHERE ReleaseID=?", [items['ReleaseID']]) + myDB.action("DELETE FROM allalbums WHERE ReleaseID=?", [items['ReleaseID']]) + myDB.action("DELETE FROM alltracks WHERE ReleaseID=?", [items['ReleaseID']]) + logger.info("Removing all references to release %s to reflect MusicBrainz" % items['ReleaseID']) + force_repackage1 = 1 + else: + logger.info("Error pulling data from MusicBrainz: Maintaining dB") - num_new_releases = 0 + num_new_releases = 0 - for releasedata in results: - #releasedata.get will return None if it doesn't have a status - #all official releases should have the Official status included - if not includeExtras and releasedata.get('status') != 'Official': - continue + for releasedata in results: + #releasedata.get will return None if it doesn't have a status + #all official releases should have the Official status included + if not includeExtras and releasedata.get('status') != 'Official': + continue + + release = {} + rel_id_check = releasedata['id'] + artistid = unicode(releasedata['artist-credit'][0]['artist']['id']) + + album_checker = myDB.action('SELECT * from allalbums WHERE ReleaseID=?', [rel_id_check]).fetchone() + if not album_checker: + release['AlbumTitle'] = unicode(releasedata['title']) + release['AlbumID'] = unicode(rgid) + release['AlbumASIN'] = unicode(releasedata['asin']) if 'asin' in releasedata else None + release['ReleaseDate'] = unicode(releasedata['date']) if 'date' in releasedata else None + release['ReleaseID'] = releasedata['id'] + if 'release-group' not in releasedata: + raise Exception('No release group associated with release id ' + releasedata['id'] + ' album id' + rgid) + release['Type'] = unicode(releasedata['release-group']['type']) + + + #making the assumption that the most important artist will be first in the list + if 'artist-credit' in releasedata: + release['ArtistID'] = unicode(releasedata['artist-credit'][0]['artist']['id']) + release['ArtistName'] = unicode(releasedata['artist-credit-phrase']) + else: + logger.warn('Release ' + releasedata['id'] + ' has no Artists associated.') + return False + + + release['ReleaseCountry'] = unicode(releasedata['country']) if 'country' in releasedata else u'Unknown' + #assuming that the list will contain media and that the format will be consistent + try: + release['ReleaseFormat'] = unicode(releasedata['medium-list'][0]['format']) + except: + release['ReleaseFormat'] = u'Unknown' + + release['Tracks'] = getTracksFromRelease(releasedata) + + # What we're doing here now is first updating the allalbums & alltracks table to the most + # current info, then moving the appropriate release into the album table and its associated + # tracks into the tracks table + controlValueDict = {"ReleaseID" : release['ReleaseID']} + + newValueDict = {"ArtistID": release['ArtistID'], + "ArtistName": release['ArtistName'], + "AlbumTitle": release['AlbumTitle'], + "AlbumID": release['AlbumID'], + "AlbumASIN": release['AlbumASIN'], + "ReleaseDate": release['ReleaseDate'], + "Type": release['Type'], + "ReleaseCountry": release['ReleaseCountry'], + "ReleaseFormat": release['ReleaseFormat'] + } + + myDB.upsert("allalbums", newValueDict, controlValueDict) - release = {} - rel_id_check = releasedata['id'] - artistid = unicode(releasedata['artist-credit'][0]['artist']['id']) + for track in release['Tracks']: - album_checker = myDB.action('SELECT * from allalbums WHERE ReleaseID=?', [rel_id_check]).fetchone() - if not album_checker: - release['AlbumTitle'] = unicode(releasedata['title']) - release['AlbumID'] = unicode(rgid) - release['AlbumASIN'] = unicode(releasedata['asin']) if 'asin' in releasedata else None - release['ReleaseDate'] = unicode(releasedata['date']) if 'date' in releasedata else None - release['ReleaseID'] = releasedata['id'] - if 'release-group' not in releasedata: - raise Exception('No release group associated with release id ' + releasedata['id'] + ' album id' + rgid) - release['Type'] = unicode(releasedata['release-group']['type']) - - - #making the assumption that the most important artist will be first in the list - if 'artist-credit' in releasedata: - release['ArtistID'] = unicode(releasedata['artist-credit'][0]['artist']['id']) - release['ArtistName'] = unicode(releasedata['artist-credit-phrase']) - else: - logger.warn('Release ' + releasedata['id'] + ' has no Artists associated.') - return False - - - release['ReleaseCountry'] = unicode(releasedata['country']) if 'country' in releasedata else u'Unknown' - #assuming that the list will contain media and that the format will be consistent - try: - release['ReleaseFormat'] = unicode(releasedata['medium-list'][0]['format']) - except: - release['ReleaseFormat'] = u'Unknown' - - release['Tracks'] = getTracksFromRelease(releasedata) - - # What we're doing here now is first updating the allalbums & alltracks table to the most - # current info, then moving the appropriate release into the album table and its associated - # tracks into the tracks table - controlValueDict = {"ReleaseID" : release['ReleaseID']} + cleanname = helpers.cleanName(release['ArtistName'] + ' ' + release['AlbumTitle'] + ' ' + track['title']) + + controlValueDict = {"TrackID": track['id'], + "ReleaseID": release['ReleaseID']} newValueDict = {"ArtistID": release['ArtistID'], "ArtistName": release['ArtistName'], "AlbumTitle": release['AlbumTitle'], "AlbumID": release['AlbumID'], "AlbumASIN": release['AlbumASIN'], - "ReleaseDate": release['ReleaseDate'], - "Type": release['Type'], - "ReleaseCountry": release['ReleaseCountry'], - "ReleaseFormat": release['ReleaseFormat'] + "TrackTitle": track['title'], + "TrackDuration": track['duration'], + "TrackNumber": track['number'], + "CleanName": cleanname } - - myDB.upsert("allalbums", newValueDict, controlValueDict) - - for track in release['Tracks']: - - cleanname = helpers.cleanName(release['ArtistName'] + ' ' + release['AlbumTitle'] + ' ' + track['title']) + + match = myDB.action('SELECT Location, BitRate, Format from have WHERE CleanName=?', [cleanname]).fetchone() - controlValueDict = {"TrackID": track['id'], - "ReleaseID": release['ReleaseID']} - - newValueDict = {"ArtistID": release['ArtistID'], - "ArtistName": release['ArtistName'], - "AlbumTitle": release['AlbumTitle'], - "AlbumID": release['AlbumID'], - "AlbumASIN": release['AlbumASIN'], - "TrackTitle": track['title'], - "TrackDuration": track['duration'], - "TrackNumber": track['number'], - "CleanName": cleanname - } + if not match: + match = myDB.action('SELECT Location, BitRate, Format from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [release['ArtistName'], release['AlbumTitle'], track['title']]).fetchone() + if not match: + match = myDB.action('SELECT Location, BitRate, Format from have WHERE TrackID=?', [track['id']]).fetchone() + if match: + newValueDict['Location'] = match['Location'] + newValueDict['BitRate'] = match['BitRate'] + newValueDict['Format'] = match['Format'] + myDB.action('UPDATE have SET Matched="True" WHERE Location=?', [match['Location']]) - match = myDB.action('SELECT Location, BitRate, Format from have WHERE CleanName=?', [cleanname]).fetchone() - - if not match: - match = myDB.action('SELECT Location, BitRate, Format from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [release['ArtistName'], release['AlbumTitle'], track['title']]).fetchone() - if not match: - match = myDB.action('SELECT Location, BitRate, Format from have WHERE TrackID=?', [track['id']]).fetchone() - if match: - newValueDict['Location'] = match['Location'] - newValueDict['BitRate'] = match['BitRate'] - newValueDict['Format'] = match['Format'] - myDB.action('UPDATE have SET Matched="True" WHERE Location=?', [match['Location']]) - - myDB.upsert("alltracks", newValueDict, controlValueDict) - num_new_releases = num_new_releases + 1 - #print releasedata['title'] - #print num_new_releases - logger.info('New release %s (%s) added' % (release['AlbumTitle'], rel_id_check)) - if force_repackage1 == 1: - num_new_releases = -1 - logger.info('Forcing repackage of %s, since dB releases have been removed' % release_title) - else: - num_new_releases = num_new_releases + myDB.upsert("alltracks", newValueDict, controlValueDict) + num_new_releases = num_new_releases + 1 + #print releasedata['title'] + #print num_new_releases + logger.info('New release %s (%s) added' % (release['AlbumTitle'], rel_id_check)) + if force_repackage1 == 1: + num_new_releases = -1 + logger.info('Forcing repackage of %s, since dB releases have been removed' % release_title) + else: + num_new_releases = num_new_releases - return num_new_releases + return num_new_releases def getTracksFromRelease(release): totalTracks = 1