diff --git a/headphones/importer.py b/headphones/importer.py index 0365a872..ede5c3b4 100644 --- a/headphones/importer.py +++ b/headphones/importer.py @@ -15,6 +15,7 @@ from lib.pyItunes import * import time +import threading import os from lib.beets.mediafile import MediaFile @@ -189,65 +190,142 @@ def addArtisttoDB(artistid, extrasonly=False): db_artist = myDB.action('SELECT IncludeExtras, Extras from artists WHERE ArtistID=?', [artistid]).fetchone() includeExtras = db_artist['IncludeExtras'] except IndexError: - includeExtras = False + includeExtras = False + + #Clean all references to release group in dB that are no longer referenced in musicbrainz + group_list = [] + force_repackage = 0 + #Don't nuke the database if there's a MusicBrainz error + if len(artist['releasegroups']) != 0: + for groups in artist['releasegroups']: + group_list.append(groups['id']) + remove_missing_groups_from_albums = myDB.action("SELECT ReleaseID FROM albums WHERE ArtistID=?", [artistid]) + for items in remove_missing_groups_from_albums: + if items['ReleaseID'] not in group_list: + # Remove all from albums/tracks that aren't in release groups + myDB.action("DELETE FROM albums WHERE ReleaseID=?", [items['ReleaseID']]) + myDB.action("DELETE FROM allalbums WHERE ReleaseID=?", [items['ReleaseID']]) + myDB.action("DELETE FROM tracks WHERE ReleaseID=?", [items['ReleaseID']]) + myDB.action("DELETE FROM alltracks WHERE ReleaseID=?", [items['ReleaseID']]) + logger.info("Removing all references to group %s to reflect MusicBrainz" % items['ReleaseID']) + force_repackage = 1 + else: + logger.info("Error pulling data from MusicBrainz: Maintaining dB") + + # Then search for releases within releasegroups, if releases don't exist, then remove from allalbums/alltracks + for rg in artist['releasegroups']: - logger.info("Now adding/updating: " + rg['title']) - + al_title = rg['title'] + today = helpers.today() rgid = rg['id'] - - # check if the album already exists - rg_exists = myDB.action("SELECT * from albums WHERE AlbumID=?", [rg['id']]).fetchone() - - releases = mb.get_all_releases(rgid,includeExtras) - if releases == []: - logger.info('No official releases in release group %s' % rg['title']) - continue - if not releases: - errors = True - logger.info('Unable to get release information for %s - there may not be any official releases in this release group' % rg['title']) - continue - - # This will be used later to build a hybrid release - fullreleaselist = [] + skip_log = 0 + #new_releases = 0 - for release in releases: - # What we're doing here now is first updating the allalbums & alltracks table to the most - # current info, then moving the appropriate release into the album table and its associated - # tracks into the tracks table - controlValueDict = {"ReleaseID" : release['ReleaseID']} + check_release_date = myDB.action("SELECT ReleaseDate from albums WHERE ArtistID=? AND AlbumTitle=?", (artistid, al_title)).fetchone() + if check_release_date: + if check_release_date[0] is None: + logger.info("Now updating: " + rg['title']) + new_releases = mb.get_new_releases(rgid,includeExtras) + elif len(check_release_date[0])!=10: + logger.info("Now updating: " + rg['title']) + new_releases = mb.get_new_releases(rgid,includeExtras) + else: + if helpers.get_age(today) - helpers.get_age(check_release_date[0]) < 365: + logger.info("Now updating: " + rg['title']) + new_releases = mb.get_new_releases(rgid,includeExtras) + else: + logger.info('%s is over a year old; not updating' % al_title) + skip_log = 1 + new_releases = 0 + else: + logger.info("Now adding/updating: " + rg['title']) + new_releases = mb.get_new_releases(rgid,includeExtras) - newValueDict = {"ArtistID": release['ArtistID'], - "ArtistName": release['ArtistName'], - "AlbumTitle": release['AlbumTitle'], - "AlbumID": release['AlbumID'], - "AlbumASIN": release['AlbumASIN'], - "ReleaseDate": release['ReleaseDate'], - "Type": release['Type'], - "ReleaseCountry": release['ReleaseCountry'], - "ReleaseFormat": release['ReleaseFormat'] + if force_repackage == 1: + new_releases = -1 + logger.info('Forcing repackage of %s, since dB references have been removed' % al_title) + else: + new_releases = new_releases + + #What this does is adds new releases per artist to the allalbums + alltracks databases + #new_releases = mb.get_new_releases(rgid,includeExtras) + #print al_title + #print new_releases + + if new_releases != 0: + # This will be used later to build a hybrid release + fullreleaselist = [] + #Search for releases within a release group + find_hybrid_releases = myDB.action("SELECT * from allalbums WHERE AlbumID=?", [rg['id']]) + # Build the dictionary for the fullreleaselist + for items in find_hybrid_releases: + hybrid_release_id = items['ReleaseID'] + newValueDict = {"ArtistID": items['ArtistID'], + "ArtistName": items['ArtistName'], + "AlbumTitle": items['AlbumTitle'], + "AlbumID": items['AlbumID'], + "AlbumASIN": items['AlbumASIN'], + "ReleaseDate": items['ReleaseDate'], + "Type": items['Type'], + "ReleaseCountry": items['ReleaseCountry'], + "ReleaseFormat": items['ReleaseFormat'] + } + find_hybrid_tracks = myDB.action("SELECT * from alltracks WHERE ReleaseID=?", [hybrid_release_id]) + totalTracks = 1 + hybrid_track_array = [] + for hybrid_tracks in find_hybrid_tracks: + hybrid_track_array.append({ + 'number': hybrid_tracks['TrackNumber'], + 'title': hybrid_tracks['TrackTitle'], + 'id': hybrid_tracks['TrackID'], + #'url': hybrid_tracks['TrackURL'], + 'duration': hybrid_tracks['TrackDuration'] + }) + totalTracks += 1 + newValueDict['ReleaseID'] = hybrid_release_id + newValueDict['Tracks'] = hybrid_track_array + fullreleaselist.append(newValueDict) + + # Basically just do the same thing again for the hybrid release + # This may end up being called with an empty fullreleaselist + try: + hybridrelease = getHybridRelease(fullreleaselist) + logger.info('Packaging %s releases into hybrid title' % rg['title']) + except Exception, e: + errors = True + logger.warn('Unable to get hybrid release information for %s: %s' % (rg['title'],e)) + continue + + # Use the ReleaseGroupID as the ReleaseID for the hybrid release to differentiate it + # We can then use the condition WHERE ReleaseID == ReleaseGroupID to select it + # The hybrid won't have a country or a format + controlValueDict = {"ReleaseID": rg['id']} + + newValueDict = {"ArtistID": artistid, + "ArtistName": artist['artist_name'], + "AlbumTitle": rg['title'], + "AlbumID": rg['id'], + "AlbumASIN": hybridrelease['AlbumASIN'], + "ReleaseDate": hybridrelease['ReleaseDate'], + "Type": rg['type'] } - + myDB.upsert("allalbums", newValueDict, controlValueDict) - # Build the dictionary for the fullreleaselist - newValueDict['ReleaseID'] = release['ReleaseID'] - newValueDict['Tracks'] = release['Tracks'] - fullreleaselist.append(newValueDict) - - for track in release['Tracks']: + for track in hybridrelease['Tracks']: cleanname = helpers.cleanName(artist['artist_name'] + ' ' + rg['title'] + ' ' + track['title']) controlValueDict = {"TrackID": track['id'], - "ReleaseID": release['ReleaseID']} + "ReleaseID": rg['id']} - newValueDict = {"ArtistID": release['ArtistID'], - "ArtistName": release['ArtistName'], - "AlbumTitle": release['AlbumTitle'], - "AlbumID": release['AlbumID'], - "AlbumASIN": release['AlbumASIN'], + newValueDict = {"ArtistID": artistid, + "ArtistName": artist['artist_name'], + "AlbumTitle": rg['title'], + "AlbumASIN": hybridrelease['AlbumASIN'], + "AlbumID": rg['id'], "TrackTitle": track['title'], "TrackDuration": track['duration'], "TrackNumber": track['number'], @@ -267,171 +345,119 @@ def addArtisttoDB(artistid, extrasonly=False): myDB.action('UPDATE have SET Matched="True" WHERE Location=?', [match['Location']]) myDB.upsert("alltracks", newValueDict, controlValueDict) - - # Basically just do the same thing again for the hybrid release - # This may end up being called with an empty fullreleaselist - try: - hybridrelease = getHybridRelease(fullreleaselist) - except Exception, e: - errors = True - logger.warn('Unable to get hybrid release information for %s: %s' % (rg['title'],e)) - continue - - # Use the ReleaseGroupID as the ReleaseID for the hybrid release to differentiate it - # We can then use the condition WHERE ReleaseID == ReleaseGroupID to select it - # The hybrid won't have a country or a format - controlValueDict = {"ReleaseID": rg['id']} - - newValueDict = {"ArtistID": artistid, - "ArtistName": artist['artist_name'], - "AlbumTitle": rg['title'], - "AlbumID": rg['id'], - "AlbumASIN": hybridrelease['AlbumASIN'], - "ReleaseDate": hybridrelease['ReleaseDate'], - "Type": rg['type'] - } - - myDB.upsert("allalbums", newValueDict, controlValueDict) - - for track in hybridrelease['Tracks']: - - cleanname = helpers.cleanName(artist['artist_name'] + ' ' + rg['title'] + ' ' + track['title']) - - controlValueDict = {"TrackID": track['id'], - "ReleaseID": rg['id']} - - newValueDict = {"ArtistID": artistid, - "ArtistName": artist['artist_name'], - "AlbumTitle": rg['title'], - "AlbumASIN": hybridrelease['AlbumASIN'], - "AlbumID": rg['id'], - "TrackTitle": track['title'], - "TrackDuration": track['duration'], - "TrackNumber": track['number'], - "CleanName": cleanname - } - - match = myDB.action('SELECT Location, BitRate, Format from have WHERE CleanName=?', [cleanname]).fetchone() - - if not match: - match = myDB.action('SELECT Location, BitRate, Format from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [artist['artist_name'], rg['title'], track['title']]).fetchone() - if not match: - match = myDB.action('SELECT Location, BitRate, Format from have WHERE TrackID=?', [track['id']]).fetchone() - if match: - newValueDict['Location'] = match['Location'] - newValueDict['BitRate'] = match['BitRate'] - newValueDict['Format'] = match['Format'] - myDB.action('UPDATE have SET Matched="True" WHERE Location=?', [match['Location']]) - - myDB.upsert("alltracks", newValueDict, controlValueDict) - - # Delete matched tracks from the have table - myDB.action('DELETE from have WHERE Matched="True"') - - # If there's no release in the main albums tables, add the default (hybrid) - # If there is a release, check the ReleaseID against the AlbumID to see if they differ (user updated) - if not rg_exists: - releaseid = rg['id'] - elif rg_exists and not rg_exists['ReleaseID']: - # Need to do some importing here - to transition the old format of using the release group - # only to using releasegroup & releaseid. These are the albums that are missing a ReleaseID - # so we'll need to move over the locations, bitrates & formats from the tracks table to the new - # alltracks table. Thankfully we can just use TrackIDs since they span releases/releasegroups - logger.info("Copying current track information to alternate releases") - tracks = myDB.action('SELECT * from tracks WHERE AlbumID=?', [rg['id']]).fetchall() - for track in tracks: - if track['Location']: - controlValueDict = {"TrackID": track['TrackID']} - newValueDict = {"Location": track['Location'], - "BitRate": track['BitRate'], - "Format": track['Format'], - } - myDB.upsert("alltracks", newValueDict, controlValueDict) - releaseid = rg['id'] - else: - releaseid = rg_exists['ReleaseID'] - - album = myDB.action('SELECT * from allalbums WHERE ReleaseID=?', [releaseid]).fetchone() - - controlValueDict = {"AlbumID": rg['id']} - - newValueDict = {"ArtistID": album['ArtistID'], - "ArtistName": album['ArtistName'], - "AlbumTitle": album['AlbumTitle'], - "ReleaseID": album['ReleaseID'], - "AlbumASIN": album['AlbumASIN'], - "ReleaseDate": album['ReleaseDate'], - "Type": album['Type'], - "ReleaseCountry": album['ReleaseCountry'], - "ReleaseFormat": album['ReleaseFormat'] - } - if not rg_exists: + # Delete matched tracks from the have table + myDB.action('DELETE from have WHERE Matched="True"') - today = helpers.today() - - newValueDict['DateAdded']= today - - if headphones.AUTOWANT_ALL: - newValueDict['Status'] = "Wanted" - elif album['ReleaseDate'] > today and headphones.AUTOWANT_UPCOMING: - newValueDict['Status'] = "Wanted" - # Sometimes "new" albums are added to musicbrainz after their release date, so let's try to catch these - # The first test just makes sure we have year-month-day - elif helpers.get_age(album['ReleaseDate']) and helpers.get_age(today) - helpers.get_age(album['ReleaseDate']) < 21 and headphones.AUTOWANT_UPCOMING: - newValueDict['Status'] = "Wanted" + # If there's no release in the main albums tables, add the default (hybrid) + # If there is a release, check the ReleaseID against the AlbumID to see if they differ (user updated) + # check if the album already exists + rg_exists = myDB.action("SELECT * from albums WHERE AlbumID=?", [rg['id']]).fetchone() + if not rg_exists: + releaseid = rg['id'] + elif rg_exists and not rg_exists['ReleaseID']: + # Need to do some importing here - to transition the old format of using the release group + # only to using releasegroup & releaseid. These are the albums that are missing a ReleaseID + # so we'll need to move over the locations, bitrates & formats from the tracks table to the new + # alltracks table. Thankfully we can just use TrackIDs since they span releases/releasegroups + logger.info("Copying current track information to alternate releases") + tracks = myDB.action('SELECT * from tracks WHERE AlbumID=?', [rg['id']]).fetchall() + for track in tracks: + if track['Location']: + controlValueDict = {"TrackID": track['TrackID']} + newValueDict = {"Location": track['Location'], + "BitRate": track['BitRate'], + "Format": track['Format'], + } + myDB.upsert("alltracks", newValueDict, controlValueDict) + releaseid = rg['id'] else: - newValueDict['Status'] = "Skipped" - - myDB.upsert("albums", newValueDict, controlValueDict) + releaseid = rg_exists['ReleaseID'] + + album = myDB.action('SELECT * from allalbums WHERE ReleaseID=?', [releaseid]).fetchone() - myDB.action('DELETE from tracks WHERE AlbumID=?', [rg['id']]) - tracks = myDB.action('SELECT * from alltracks WHERE ReleaseID=?', [releaseid]).fetchall() + controlValueDict = {"AlbumID": rg['id']} - # This is used to see how many tracks you have from an album - to mark it as downloaded. Default is 80%, can be set in config as ALBUM_COMPLETION_PCT - total_track_count = len(tracks) - - for track in tracks: - - controlValueDict = {"TrackID": track['TrackID'], - "AlbumID": rg['id']} - - newValueDict = {"ArtistID": track['ArtistID'], - "ArtistName": track['ArtistName'], - "AlbumTitle": track['AlbumTitle'], - "AlbumASIN": track['AlbumASIN'], - "ReleaseID": track['ReleaseID'], - "TrackTitle": track['TrackTitle'], - "TrackDuration": track['TrackDuration'], - "TrackNumber": track['TrackNumber'], - "CleanName": track['CleanName'], - "Location": track['Location'], - "Format": track['Format'], - "BitRate": track['BitRate'] + newValueDict = {"ArtistID": album['ArtistID'], + "ArtistName": album['ArtistName'], + "AlbumTitle": album['AlbumTitle'], + "ReleaseID": album['ReleaseID'], + "AlbumASIN": album['AlbumASIN'], + "ReleaseDate": album['ReleaseDate'], + "Type": album['Type'], + "ReleaseCountry": album['ReleaseCountry'], + "ReleaseFormat": album['ReleaseFormat'] } - - myDB.upsert("tracks", newValueDict, controlValueDict) + + if not rg_exists: + + today = helpers.today() + + newValueDict['DateAdded']= today + + if headphones.AUTOWANT_ALL: + newValueDict['Status'] = "Wanted" + elif album['ReleaseDate'] > today and headphones.AUTOWANT_UPCOMING: + newValueDict['Status'] = "Wanted" + # Sometimes "new" albums are added to musicbrainz after their release date, so let's try to catch these + # The first test just makes sure we have year-month-day + elif helpers.get_age(album['ReleaseDate']) and helpers.get_age(today) - helpers.get_age(album['ReleaseDate']) < 21 and headphones.AUTOWANT_UPCOMING: + newValueDict['Status'] = "Wanted" + else: + newValueDict['Status'] = "Skipped" + + myDB.upsert("albums", newValueDict, controlValueDict) - # Mark albums as downloaded if they have at least 80% (by default, configurable) of the album - have_track_count = len(myDB.select('SELECT * from tracks WHERE AlbumID=? AND Location IS NOT NULL', [rg['id']])) - marked_as_downloaded = False - - if rg_exists: - if rg_exists['Status'] == 'Skipped' and ((have_track_count/float(total_track_count)) >= (headphones.ALBUM_COMPLETION_PCT/100.0)): - myDB.action('UPDATE albums SET Status=? WHERE AlbumID=?', ['Downloaded', rg['id']]) - marked_as_downloaded = True + myDB.action('DELETE from tracks WHERE AlbumID=?', [rg['id']]) + tracks = myDB.action('SELECT * from alltracks WHERE ReleaseID=?', [releaseid]).fetchall() + + # This is used to see how many tracks you have from an album - to mark it as downloaded. Default is 80%, can be set in config as ALBUM_COMPLETION_PCT + total_track_count = len(tracks) + + for track in tracks: + + controlValueDict = {"TrackID": track['TrackID'], + "AlbumID": rg['id']} + + newValueDict = {"ArtistID": track['ArtistID'], + "ArtistName": track['ArtistName'], + "AlbumTitle": track['AlbumTitle'], + "AlbumASIN": track['AlbumASIN'], + "ReleaseID": track['ReleaseID'], + "TrackTitle": track['TrackTitle'], + "TrackDuration": track['TrackDuration'], + "TrackNumber": track['TrackNumber'], + "CleanName": track['CleanName'], + "Location": track['Location'], + "Format": track['Format'], + "BitRate": track['BitRate'] + } + + myDB.upsert("tracks", newValueDict, controlValueDict) + + # Mark albums as downloaded if they have at least 80% (by default, configurable) of the album + have_track_count = len(myDB.select('SELECT * from tracks WHERE AlbumID=? AND Location IS NOT NULL', [rg['id']])) + marked_as_downloaded = False + + if rg_exists: + if rg_exists['Status'] == 'Skipped' and ((have_track_count/float(total_track_count)) >= (headphones.ALBUM_COMPLETION_PCT/100.0)): + myDB.action('UPDATE albums SET Status=? WHERE AlbumID=?', ['Downloaded', rg['id']]) + marked_as_downloaded = True + else: + if ((have_track_count/float(total_track_count)) >= (headphones.ALBUM_COMPLETION_PCT/100.0)): + myDB.action('UPDATE albums SET Status=? WHERE AlbumID=?', ['Downloaded', rg['id']]) + marked_as_downloaded = True + + logger.info(u"Seeing if we need album art for " + rg['title']) + cache.getThumb(AlbumID=rg['id']) + + #start a search for the album if it's new, hasn't been marked as downloaded and autowant_all is selected: + if not rg_exists and not marked_as_downloaded and headphones.AUTOWANT_ALL: + from headphones import searcher + searcher.searchforalbum(albumid=rg['id']) else: - if ((have_track_count/float(total_track_count)) >= (headphones.ALBUM_COMPLETION_PCT/100.0)): - myDB.action('UPDATE albums SET Status=? WHERE AlbumID=?', ['Downloaded', rg['id']]) - marked_as_downloaded = True - - logger.info(u"Seeing if we need album art for " + rg['title']) - cache.getThumb(AlbumID=rg['id']) - - #start a search for the album if it's new, hasn't been marked as downloaded and autowant_all is selected: - if not rg_exists and not marked_as_downloaded and headphones.AUTOWANT_ALL: - from headphones import searcher - searcher.searchforalbum(albumid=rg['id']) + if skip_log == 0: + logger.info(u"No new releases, so no changes made to " + rg['title']) latestalbum = myDB.action('SELECT AlbumTitle, ReleaseDate, AlbumID from albums WHERE ArtistID=? order by ReleaseDate DESC', [artistid]).fetchone() totaltracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=?', [artistid])) diff --git a/headphones/mb.py b/headphones/mb.py index ce35d641..e725f047 100644 --- a/headphones/mb.py +++ b/headphones/mb.py @@ -19,7 +19,7 @@ import time import threading import headphones -from headphones import logger, db +from headphones import logger, db, helpers from headphones.helpers import multikeysort, replace_all import lib.musicbrainzngs as musicbrainzngs @@ -332,65 +332,152 @@ def getRelease(releaseid, include_artist_info=True): return release -def get_all_releases(rgid,includeExtras=False): - results = [] - try: - limit = 100 - newResults = None - while newResults == None or len(newResults) >= limit: - newResults = musicbrainzngs.browse_releases(release_group=rgid,includes=['artist-credits','labels','recordings','release-groups','media'],limit=limit,offset=len(results)) - if 'release-list' not in newResults: - break #may want to raise an exception here instead ? - newResults = newResults['release-list'] - results += newResults - - except WebServiceError, e: - logger.warn('Attempt to retrieve information from MusicBrainz for release group "%s" failed (%s)' % (rgid, str(e))) - time.sleep(5) - return False - - if not results or len(results) == 0: - return False +def get_new_releases(rgid,includeExtras=False): - - releases = [] - for releasedata in results: - #releasedata.get will return None if it doesn't have a status - #all official releases should have the Official status included - if not includeExtras and releasedata.get('status') != 'Official': - continue - - release = {} - release['AlbumTitle'] = unicode(releasedata['title']) - release['AlbumID'] = unicode(rgid) - release['AlbumASIN'] = unicode(releasedata['asin']) if 'asin' in releasedata else None - release['ReleaseDate'] = unicode(releasedata['date']) if 'date' in releasedata else None - release['ReleaseID'] = releasedata['id'] - if 'release-group' not in releasedata: - raise Exception('No release group associated with release id ' + releasedata['id'] + ' album id' + rgid) - release['Type'] = unicode(releasedata['release-group']['type']) - - - #making the assumption that the most important artist will be first in the list - if 'artist-credit' in releasedata: - release['ArtistID'] = unicode(releasedata['artist-credit'][0]['artist']['id']) - release['ArtistName'] = unicode(releasedata['artist-credit-phrase']) - else: - logger.warn('Release ' + releasedata['id'] + ' has no Artists associated.') - return False - - - release['ReleaseCountry'] = unicode(releasedata['country']) if 'country' in releasedata else u'Unknown' - #assuming that the list will contain media and that the format will be consistent + with mb_lock: + myDB = db.DBConnection() + results = [] try: - release['ReleaseFormat'] = unicode(releasedata['medium-list'][0]['format']) - except: - release['ReleaseFormat'] = u'Unknown' - - release['Tracks'] = getTracksFromRelease(releasedata) - releases.append(release) + limit = 100 + newResults = None + while newResults == None or len(newResults) >= limit: + newResults = musicbrainzngs.browse_releases(release_group=rgid,includes=['artist-credits','labels','recordings','release-groups','media'],limit=limit,offset=len(results)) + if 'release-list' not in newResults: + break #may want to raise an exception here instead ? + newResults = newResults['release-list'] + results += newResults + + except WebServiceError, e: + logger.warn('Attempt to retrieve information from MusicBrainz for release group "%s" failed (%s)' % (rgid, str(e))) + time.sleep(5) + return False + + if not results or len(results) == 0: + return False - return releases + #Clean all references to releases in dB that are no longer referenced in musicbrainz + release_list = [] + force_repackage = 0 + if len(results) != 0: + for release_mark in results: + release_list.append(unicode(release_mark['id'])) + release_title = release_mark['title'] + remove_missing_releases = myDB.action("SELECT ReleaseID FROM allalbums WHERE AlbumID=?", [rgid]) + for items in remove_missing_releases: + if items['ReleaseID'] not in release_list and items['ReleaseID'] != rgid: + # Remove all from albums/tracks that aren't in release + myDB.action("DELETE FROM albums WHERE ReleaseID=?", [items['ReleaseID']]) + myDB.action("DELETE FROM tracks WHERE ReleaseID=?", [items['ReleaseID']]) + myDB.action("DELETE FROM allalbums WHERE ReleaseID=?", [items['ReleaseID']]) + myDB.action("DELETE FROM alltracks WHERE ReleaseID=?", [items['ReleaseID']]) + logger.info("Removing all references to release %s to reflect MusicBrainz" % items['ReleaseID']) + force_repackage = 1 + else: + logger.info("Error pulling data from MusicBrainz: Maintaining dB") + + num_new_releases = 0 + + for releasedata in results: + #releasedata.get will return None if it doesn't have a status + #all official releases should have the Official status included + if not includeExtras and releasedata.get('status') != 'Official': + continue + + release = {} + rel_id_check = releasedata['id'] + artistid = unicode(releasedata['artist-credit'][0]['artist']['id']) + + album_checker = myDB.action('SELECT * from allalbums WHERE ReleaseID=?', [rel_id_check]).fetchone() + if not album_checker: + release['AlbumTitle'] = unicode(releasedata['title']) + release['AlbumID'] = unicode(rgid) + release['AlbumASIN'] = unicode(releasedata['asin']) if 'asin' in releasedata else None + release['ReleaseDate'] = unicode(releasedata['date']) if 'date' in releasedata else None + release['ReleaseID'] = releasedata['id'] + if 'release-group' not in releasedata: + raise Exception('No release group associated with release id ' + releasedata['id'] + ' album id' + rgid) + release['Type'] = unicode(releasedata['release-group']['type']) + + + #making the assumption that the most important artist will be first in the list + if 'artist-credit' in releasedata: + release['ArtistID'] = unicode(releasedata['artist-credit'][0]['artist']['id']) + release['ArtistName'] = unicode(releasedata['artist-credit-phrase']) + else: + logger.warn('Release ' + releasedata['id'] + ' has no Artists associated.') + return False + + + release['ReleaseCountry'] = unicode(releasedata['country']) if 'country' in releasedata else u'Unknown' + #assuming that the list will contain media and that the format will be consistent + try: + release['ReleaseFormat'] = unicode(releasedata['medium-list'][0]['format']) + except: + release['ReleaseFormat'] = u'Unknown' + + release['Tracks'] = getTracksFromRelease(releasedata) + + # What we're doing here now is first updating the allalbums & alltracks table to the most + # current info, then moving the appropriate release into the album table and its associated + # tracks into the tracks table + controlValueDict = {"ReleaseID" : release['ReleaseID']} + + newValueDict = {"ArtistID": release['ArtistID'], + "ArtistName": release['ArtistName'], + "AlbumTitle": release['AlbumTitle'], + "AlbumID": release['AlbumID'], + "AlbumASIN": release['AlbumASIN'], + "ReleaseDate": release['ReleaseDate'], + "Type": release['Type'], + "ReleaseCountry": release['ReleaseCountry'], + "ReleaseFormat": release['ReleaseFormat'] + } + + myDB.upsert("allalbums", newValueDict, controlValueDict) + + + for track in release['Tracks']: + + cleanname = helpers.cleanName(release['ArtistName'] + ' ' + release['AlbumTitle'] + ' ' + track['title']) + + controlValueDict = {"TrackID": track['id'], + "ReleaseID": release['ReleaseID']} + + newValueDict = {"ArtistID": release['ArtistID'], + "ArtistName": release['ArtistName'], + "AlbumTitle": release['AlbumTitle'], + "AlbumID": release['AlbumID'], + "AlbumASIN": release['AlbumASIN'], + "TrackTitle": track['title'], + "TrackDuration": track['duration'], + "TrackNumber": track['number'], + "CleanName": cleanname + } + + match = myDB.action('SELECT Location, BitRate, Format from have WHERE CleanName=?', [cleanname]).fetchone() + + if not match: + match = myDB.action('SELECT Location, BitRate, Format from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [release['ArtistName'], release['AlbumTitle'], track['title']]).fetchone() + if not match: + match = myDB.action('SELECT Location, BitRate, Format from have WHERE TrackID=?', [track['id']]).fetchone() + if match: + newValueDict['Location'] = match['Location'] + newValueDict['BitRate'] = match['BitRate'] + newValueDict['Format'] = match['Format'] + myDB.action('UPDATE have SET Matched="True" WHERE Location=?', [match['Location']]) + + myDB.upsert("alltracks", newValueDict, controlValueDict) + num_new_releases = num_new_releases + 1 + #print releasedata['title'] + #print num_new_releases + logger.info('New release %s (%s) added' % (release['AlbumTitle'], rel_id_check)) + if force_repackage == 1: + num_new_releases = -1 + logger.info('Forcing repackage of %s, since dB releases have been removed' % release_title) + else: + num_new_releases = num_new_releases + + return num_new_releases def getTracksFromRelease(release): totalTracks = 1