mirror of
https://github.com/rembo10/headphones.git
synced 2026-05-14 23:59:30 +01:00
Rehashed "update active artist" algorithm to address time/CPU load issues.
This commit is contained in:
theguardian
@@ -15,6 +15,7 @@
|
||||
|
||||
from lib.pyItunes import *
|
||||
import time
|
||||
import threading
|
||||
import os
|
||||
from lib.beets.mediafile import MediaFile
|
||||
|
||||
@@ -189,65 +190,142 @@ def addArtisttoDB(artistid, extrasonly=False):
|
||||
db_artist = myDB.action('SELECT IncludeExtras, Extras from artists WHERE ArtistID=?', [artistid]).fetchone()
|
||||
includeExtras = db_artist['IncludeExtras']
|
||||
except IndexError:
|
||||
includeExtras = False
|
||||
includeExtras = False
|
||||
|
||||
#Clean all references to release group in dB that are no longer referenced in musicbrainz
|
||||
group_list = []
|
||||
force_repackage = 0
|
||||
#Don't nuke the database if there's a MusicBrainz error
|
||||
if len(artist['releasegroups']) != 0:
|
||||
for groups in artist['releasegroups']:
|
||||
group_list.append(groups['id'])
|
||||
remove_missing_groups_from_albums = myDB.action("SELECT ReleaseID FROM albums WHERE ArtistID=?", [artistid])
|
||||
for items in remove_missing_groups_from_albums:
|
||||
if items['ReleaseID'] not in group_list:
|
||||
# Remove all from albums/tracks that aren't in release groups
|
||||
myDB.action("DELETE FROM albums WHERE ReleaseID=?", [items['ReleaseID']])
|
||||
myDB.action("DELETE FROM allalbums WHERE ReleaseID=?", [items['ReleaseID']])
|
||||
myDB.action("DELETE FROM tracks WHERE ReleaseID=?", [items['ReleaseID']])
|
||||
myDB.action("DELETE FROM alltracks WHERE ReleaseID=?", [items['ReleaseID']])
|
||||
logger.info("Removing all references to group %s to reflect MusicBrainz" % items['ReleaseID'])
|
||||
force_repackage = 1
|
||||
else:
|
||||
logger.info("Error pulling data from MusicBrainz: Maintaining dB")
|
||||
|
||||
# Then search for releases within releasegroups, if releases don't exist, then remove from allalbums/alltracks
|
||||
|
||||
|
||||
for rg in artist['releasegroups']:
|
||||
|
||||
logger.info("Now adding/updating: " + rg['title'])
|
||||
|
||||
al_title = rg['title']
|
||||
today = helpers.today()
|
||||
rgid = rg['id']
|
||||
|
||||
# check if the album already exists
|
||||
rg_exists = myDB.action("SELECT * from albums WHERE AlbumID=?", [rg['id']]).fetchone()
|
||||
|
||||
releases = mb.get_all_releases(rgid,includeExtras)
|
||||
if releases == []:
|
||||
logger.info('No official releases in release group %s' % rg['title'])
|
||||
continue
|
||||
if not releases:
|
||||
errors = True
|
||||
logger.info('Unable to get release information for %s - there may not be any official releases in this release group' % rg['title'])
|
||||
continue
|
||||
|
||||
# This will be used later to build a hybrid release
|
||||
fullreleaselist = []
|
||||
skip_log = 0
|
||||
#new_releases = 0
|
||||
|
||||
for release in releases:
|
||||
# What we're doing here now is first updating the allalbums & alltracks table to the most
|
||||
# current info, then moving the appropriate release into the album table and its associated
|
||||
# tracks into the tracks table
|
||||
controlValueDict = {"ReleaseID" : release['ReleaseID']}
|
||||
check_release_date = myDB.action("SELECT ReleaseDate from albums WHERE ArtistID=? AND AlbumTitle=?", (artistid, al_title)).fetchone()
|
||||
if check_release_date:
|
||||
if check_release_date[0] is None:
|
||||
logger.info("Now updating: " + rg['title'])
|
||||
new_releases = mb.get_new_releases(rgid,includeExtras)
|
||||
elif len(check_release_date[0])!=10:
|
||||
logger.info("Now updating: " + rg['title'])
|
||||
new_releases = mb.get_new_releases(rgid,includeExtras)
|
||||
else:
|
||||
if helpers.get_age(today) - helpers.get_age(check_release_date[0]) < 365:
|
||||
logger.info("Now updating: " + rg['title'])
|
||||
new_releases = mb.get_new_releases(rgid,includeExtras)
|
||||
else:
|
||||
logger.info('%s is over a year old; not updating' % al_title)
|
||||
skip_log = 1
|
||||
new_releases = 0
|
||||
else:
|
||||
logger.info("Now adding/updating: " + rg['title'])
|
||||
new_releases = mb.get_new_releases(rgid,includeExtras)
|
||||
|
||||
newValueDict = {"ArtistID": release['ArtistID'],
|
||||
"ArtistName": release['ArtistName'],
|
||||
"AlbumTitle": release['AlbumTitle'],
|
||||
"AlbumID": release['AlbumID'],
|
||||
"AlbumASIN": release['AlbumASIN'],
|
||||
"ReleaseDate": release['ReleaseDate'],
|
||||
"Type": release['Type'],
|
||||
"ReleaseCountry": release['ReleaseCountry'],
|
||||
"ReleaseFormat": release['ReleaseFormat']
|
||||
if force_repackage == 1:
|
||||
new_releases = -1
|
||||
logger.info('Forcing repackage of %s, since dB references have been removed' % al_title)
|
||||
else:
|
||||
new_releases = new_releases
|
||||
|
||||
#What this does is adds new releases per artist to the allalbums + alltracks databases
|
||||
#new_releases = mb.get_new_releases(rgid,includeExtras)
|
||||
#print al_title
|
||||
#print new_releases
|
||||
|
||||
if new_releases != 0:
|
||||
# This will be used later to build a hybrid release
|
||||
fullreleaselist = []
|
||||
#Search for releases within a release group
|
||||
find_hybrid_releases = myDB.action("SELECT * from allalbums WHERE AlbumID=?", [rg['id']])
|
||||
# Build the dictionary for the fullreleaselist
|
||||
for items in find_hybrid_releases:
|
||||
hybrid_release_id = items['ReleaseID']
|
||||
newValueDict = {"ArtistID": items['ArtistID'],
|
||||
"ArtistName": items['ArtistName'],
|
||||
"AlbumTitle": items['AlbumTitle'],
|
||||
"AlbumID": items['AlbumID'],
|
||||
"AlbumASIN": items['AlbumASIN'],
|
||||
"ReleaseDate": items['ReleaseDate'],
|
||||
"Type": items['Type'],
|
||||
"ReleaseCountry": items['ReleaseCountry'],
|
||||
"ReleaseFormat": items['ReleaseFormat']
|
||||
}
|
||||
find_hybrid_tracks = myDB.action("SELECT * from alltracks WHERE ReleaseID=?", [hybrid_release_id])
|
||||
totalTracks = 1
|
||||
hybrid_track_array = []
|
||||
for hybrid_tracks in find_hybrid_tracks:
|
||||
hybrid_track_array.append({
|
||||
'number': hybrid_tracks['TrackNumber'],
|
||||
'title': hybrid_tracks['TrackTitle'],
|
||||
'id': hybrid_tracks['TrackID'],
|
||||
#'url': hybrid_tracks['TrackURL'],
|
||||
'duration': hybrid_tracks['TrackDuration']
|
||||
})
|
||||
totalTracks += 1
|
||||
newValueDict['ReleaseID'] = hybrid_release_id
|
||||
newValueDict['Tracks'] = hybrid_track_array
|
||||
fullreleaselist.append(newValueDict)
|
||||
|
||||
# Basically just do the same thing again for the hybrid release
|
||||
# This may end up being called with an empty fullreleaselist
|
||||
try:
|
||||
hybridrelease = getHybridRelease(fullreleaselist)
|
||||
logger.info('Packaging %s releases into hybrid title' % rg['title'])
|
||||
except Exception, e:
|
||||
errors = True
|
||||
logger.warn('Unable to get hybrid release information for %s: %s' % (rg['title'],e))
|
||||
continue
|
||||
|
||||
# Use the ReleaseGroupID as the ReleaseID for the hybrid release to differentiate it
|
||||
# We can then use the condition WHERE ReleaseID == ReleaseGroupID to select it
|
||||
# The hybrid won't have a country or a format
|
||||
controlValueDict = {"ReleaseID": rg['id']}
|
||||
|
||||
newValueDict = {"ArtistID": artistid,
|
||||
"ArtistName": artist['artist_name'],
|
||||
"AlbumTitle": rg['title'],
|
||||
"AlbumID": rg['id'],
|
||||
"AlbumASIN": hybridrelease['AlbumASIN'],
|
||||
"ReleaseDate": hybridrelease['ReleaseDate'],
|
||||
"Type": rg['type']
|
||||
}
|
||||
|
||||
|
||||
myDB.upsert("allalbums", newValueDict, controlValueDict)
|
||||
|
||||
# Build the dictionary for the fullreleaselist
|
||||
newValueDict['ReleaseID'] = release['ReleaseID']
|
||||
newValueDict['Tracks'] = release['Tracks']
|
||||
fullreleaselist.append(newValueDict)
|
||||
|
||||
for track in release['Tracks']:
|
||||
for track in hybridrelease['Tracks']:
|
||||
|
||||
cleanname = helpers.cleanName(artist['artist_name'] + ' ' + rg['title'] + ' ' + track['title'])
|
||||
|
||||
controlValueDict = {"TrackID": track['id'],
|
||||
"ReleaseID": release['ReleaseID']}
|
||||
"ReleaseID": rg['id']}
|
||||
|
||||
newValueDict = {"ArtistID": release['ArtistID'],
|
||||
"ArtistName": release['ArtistName'],
|
||||
"AlbumTitle": release['AlbumTitle'],
|
||||
"AlbumID": release['AlbumID'],
|
||||
"AlbumASIN": release['AlbumASIN'],
|
||||
newValueDict = {"ArtistID": artistid,
|
||||
"ArtistName": artist['artist_name'],
|
||||
"AlbumTitle": rg['title'],
|
||||
"AlbumASIN": hybridrelease['AlbumASIN'],
|
||||
"AlbumID": rg['id'],
|
||||
"TrackTitle": track['title'],
|
||||
"TrackDuration": track['duration'],
|
||||
"TrackNumber": track['number'],
|
||||
@@ -267,171 +345,119 @@ def addArtisttoDB(artistid, extrasonly=False):
|
||||
myDB.action('UPDATE have SET Matched="True" WHERE Location=?', [match['Location']])
|
||||
|
||||
myDB.upsert("alltracks", newValueDict, controlValueDict)
|
||||
|
||||
# Basically just do the same thing again for the hybrid release
|
||||
# This may end up being called with an empty fullreleaselist
|
||||
try:
|
||||
hybridrelease = getHybridRelease(fullreleaselist)
|
||||
except Exception, e:
|
||||
errors = True
|
||||
logger.warn('Unable to get hybrid release information for %s: %s' % (rg['title'],e))
|
||||
continue
|
||||
|
||||
# Use the ReleaseGroupID as the ReleaseID for the hybrid release to differentiate it
|
||||
# We can then use the condition WHERE ReleaseID == ReleaseGroupID to select it
|
||||
# The hybrid won't have a country or a format
|
||||
controlValueDict = {"ReleaseID": rg['id']}
|
||||
|
||||
newValueDict = {"ArtistID": artistid,
|
||||
"ArtistName": artist['artist_name'],
|
||||
"AlbumTitle": rg['title'],
|
||||
"AlbumID": rg['id'],
|
||||
"AlbumASIN": hybridrelease['AlbumASIN'],
|
||||
"ReleaseDate": hybridrelease['ReleaseDate'],
|
||||
"Type": rg['type']
|
||||
}
|
||||
|
||||
myDB.upsert("allalbums", newValueDict, controlValueDict)
|
||||
|
||||
for track in hybridrelease['Tracks']:
|
||||
|
||||
cleanname = helpers.cleanName(artist['artist_name'] + ' ' + rg['title'] + ' ' + track['title'])
|
||||
|
||||
controlValueDict = {"TrackID": track['id'],
|
||||
"ReleaseID": rg['id']}
|
||||
|
||||
newValueDict = {"ArtistID": artistid,
|
||||
"ArtistName": artist['artist_name'],
|
||||
"AlbumTitle": rg['title'],
|
||||
"AlbumASIN": hybridrelease['AlbumASIN'],
|
||||
"AlbumID": rg['id'],
|
||||
"TrackTitle": track['title'],
|
||||
"TrackDuration": track['duration'],
|
||||
"TrackNumber": track['number'],
|
||||
"CleanName": cleanname
|
||||
}
|
||||
|
||||
match = myDB.action('SELECT Location, BitRate, Format from have WHERE CleanName=?', [cleanname]).fetchone()
|
||||
|
||||
if not match:
|
||||
match = myDB.action('SELECT Location, BitRate, Format from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [artist['artist_name'], rg['title'], track['title']]).fetchone()
|
||||
if not match:
|
||||
match = myDB.action('SELECT Location, BitRate, Format from have WHERE TrackID=?', [track['id']]).fetchone()
|
||||
if match:
|
||||
newValueDict['Location'] = match['Location']
|
||||
newValueDict['BitRate'] = match['BitRate']
|
||||
newValueDict['Format'] = match['Format']
|
||||
myDB.action('UPDATE have SET Matched="True" WHERE Location=?', [match['Location']])
|
||||
|
||||
myDB.upsert("alltracks", newValueDict, controlValueDict)
|
||||
|
||||
# Delete matched tracks from the have table
|
||||
myDB.action('DELETE from have WHERE Matched="True"')
|
||||
|
||||
# If there's no release in the main albums tables, add the default (hybrid)
|
||||
# If there is a release, check the ReleaseID against the AlbumID to see if they differ (user updated)
|
||||
if not rg_exists:
|
||||
releaseid = rg['id']
|
||||
elif rg_exists and not rg_exists['ReleaseID']:
|
||||
# Need to do some importing here - to transition the old format of using the release group
|
||||
# only to using releasegroup & releaseid. These are the albums that are missing a ReleaseID
|
||||
# so we'll need to move over the locations, bitrates & formats from the tracks table to the new
|
||||
# alltracks table. Thankfully we can just use TrackIDs since they span releases/releasegroups
|
||||
logger.info("Copying current track information to alternate releases")
|
||||
tracks = myDB.action('SELECT * from tracks WHERE AlbumID=?', [rg['id']]).fetchall()
|
||||
for track in tracks:
|
||||
if track['Location']:
|
||||
controlValueDict = {"TrackID": track['TrackID']}
|
||||
newValueDict = {"Location": track['Location'],
|
||||
"BitRate": track['BitRate'],
|
||||
"Format": track['Format'],
|
||||
}
|
||||
myDB.upsert("alltracks", newValueDict, controlValueDict)
|
||||
releaseid = rg['id']
|
||||
else:
|
||||
releaseid = rg_exists['ReleaseID']
|
||||
|
||||
album = myDB.action('SELECT * from allalbums WHERE ReleaseID=?', [releaseid]).fetchone()
|
||||
|
||||
controlValueDict = {"AlbumID": rg['id']}
|
||||
|
||||
newValueDict = {"ArtistID": album['ArtistID'],
|
||||
"ArtistName": album['ArtistName'],
|
||||
"AlbumTitle": album['AlbumTitle'],
|
||||
"ReleaseID": album['ReleaseID'],
|
||||
"AlbumASIN": album['AlbumASIN'],
|
||||
"ReleaseDate": album['ReleaseDate'],
|
||||
"Type": album['Type'],
|
||||
"ReleaseCountry": album['ReleaseCountry'],
|
||||
"ReleaseFormat": album['ReleaseFormat']
|
||||
}
|
||||
|
||||
if not rg_exists:
|
||||
# Delete matched tracks from the have table
|
||||
myDB.action('DELETE from have WHERE Matched="True"')
|
||||
|
||||
today = helpers.today()
|
||||
|
||||
newValueDict['DateAdded']= today
|
||||
|
||||
if headphones.AUTOWANT_ALL:
|
||||
newValueDict['Status'] = "Wanted"
|
||||
elif album['ReleaseDate'] > today and headphones.AUTOWANT_UPCOMING:
|
||||
newValueDict['Status'] = "Wanted"
|
||||
# Sometimes "new" albums are added to musicbrainz after their release date, so let's try to catch these
|
||||
# The first test just makes sure we have year-month-day
|
||||
elif helpers.get_age(album['ReleaseDate']) and helpers.get_age(today) - helpers.get_age(album['ReleaseDate']) < 21 and headphones.AUTOWANT_UPCOMING:
|
||||
newValueDict['Status'] = "Wanted"
|
||||
# If there's no release in the main albums tables, add the default (hybrid)
|
||||
# If there is a release, check the ReleaseID against the AlbumID to see if they differ (user updated)
|
||||
# check if the album already exists
|
||||
rg_exists = myDB.action("SELECT * from albums WHERE AlbumID=?", [rg['id']]).fetchone()
|
||||
if not rg_exists:
|
||||
releaseid = rg['id']
|
||||
elif rg_exists and not rg_exists['ReleaseID']:
|
||||
# Need to do some importing here - to transition the old format of using the release group
|
||||
# only to using releasegroup & releaseid. These are the albums that are missing a ReleaseID
|
||||
# so we'll need to move over the locations, bitrates & formats from the tracks table to the new
|
||||
# alltracks table. Thankfully we can just use TrackIDs since they span releases/releasegroups
|
||||
logger.info("Copying current track information to alternate releases")
|
||||
tracks = myDB.action('SELECT * from tracks WHERE AlbumID=?', [rg['id']]).fetchall()
|
||||
for track in tracks:
|
||||
if track['Location']:
|
||||
controlValueDict = {"TrackID": track['TrackID']}
|
||||
newValueDict = {"Location": track['Location'],
|
||||
"BitRate": track['BitRate'],
|
||||
"Format": track['Format'],
|
||||
}
|
||||
myDB.upsert("alltracks", newValueDict, controlValueDict)
|
||||
releaseid = rg['id']
|
||||
else:
|
||||
newValueDict['Status'] = "Skipped"
|
||||
|
||||
myDB.upsert("albums", newValueDict, controlValueDict)
|
||||
releaseid = rg_exists['ReleaseID']
|
||||
|
||||
album = myDB.action('SELECT * from allalbums WHERE ReleaseID=?', [releaseid]).fetchone()
|
||||
|
||||
myDB.action('DELETE from tracks WHERE AlbumID=?', [rg['id']])
|
||||
tracks = myDB.action('SELECT * from alltracks WHERE ReleaseID=?', [releaseid]).fetchall()
|
||||
controlValueDict = {"AlbumID": rg['id']}
|
||||
|
||||
# This is used to see how many tracks you have from an album - to mark it as downloaded. Default is 80%, can be set in config as ALBUM_COMPLETION_PCT
|
||||
total_track_count = len(tracks)
|
||||
|
||||
for track in tracks:
|
||||
|
||||
controlValueDict = {"TrackID": track['TrackID'],
|
||||
"AlbumID": rg['id']}
|
||||
|
||||
newValueDict = {"ArtistID": track['ArtistID'],
|
||||
"ArtistName": track['ArtistName'],
|
||||
"AlbumTitle": track['AlbumTitle'],
|
||||
"AlbumASIN": track['AlbumASIN'],
|
||||
"ReleaseID": track['ReleaseID'],
|
||||
"TrackTitle": track['TrackTitle'],
|
||||
"TrackDuration": track['TrackDuration'],
|
||||
"TrackNumber": track['TrackNumber'],
|
||||
"CleanName": track['CleanName'],
|
||||
"Location": track['Location'],
|
||||
"Format": track['Format'],
|
||||
"BitRate": track['BitRate']
|
||||
newValueDict = {"ArtistID": album['ArtistID'],
|
||||
"ArtistName": album['ArtistName'],
|
||||
"AlbumTitle": album['AlbumTitle'],
|
||||
"ReleaseID": album['ReleaseID'],
|
||||
"AlbumASIN": album['AlbumASIN'],
|
||||
"ReleaseDate": album['ReleaseDate'],
|
||||
"Type": album['Type'],
|
||||
"ReleaseCountry": album['ReleaseCountry'],
|
||||
"ReleaseFormat": album['ReleaseFormat']
|
||||
}
|
||||
|
||||
myDB.upsert("tracks", newValueDict, controlValueDict)
|
||||
|
||||
if not rg_exists:
|
||||
|
||||
today = helpers.today()
|
||||
|
||||
newValueDict['DateAdded']= today
|
||||
|
||||
if headphones.AUTOWANT_ALL:
|
||||
newValueDict['Status'] = "Wanted"
|
||||
elif album['ReleaseDate'] > today and headphones.AUTOWANT_UPCOMING:
|
||||
newValueDict['Status'] = "Wanted"
|
||||
# Sometimes "new" albums are added to musicbrainz after their release date, so let's try to catch these
|
||||
# The first test just makes sure we have year-month-day
|
||||
elif helpers.get_age(album['ReleaseDate']) and helpers.get_age(today) - helpers.get_age(album['ReleaseDate']) < 21 and headphones.AUTOWANT_UPCOMING:
|
||||
newValueDict['Status'] = "Wanted"
|
||||
else:
|
||||
newValueDict['Status'] = "Skipped"
|
||||
|
||||
myDB.upsert("albums", newValueDict, controlValueDict)
|
||||
|
||||
# Mark albums as downloaded if they have at least 80% (by default, configurable) of the album
|
||||
have_track_count = len(myDB.select('SELECT * from tracks WHERE AlbumID=? AND Location IS NOT NULL', [rg['id']]))
|
||||
marked_as_downloaded = False
|
||||
|
||||
if rg_exists:
|
||||
if rg_exists['Status'] == 'Skipped' and ((have_track_count/float(total_track_count)) >= (headphones.ALBUM_COMPLETION_PCT/100.0)):
|
||||
myDB.action('UPDATE albums SET Status=? WHERE AlbumID=?', ['Downloaded', rg['id']])
|
||||
marked_as_downloaded = True
|
||||
myDB.action('DELETE from tracks WHERE AlbumID=?', [rg['id']])
|
||||
tracks = myDB.action('SELECT * from alltracks WHERE ReleaseID=?', [releaseid]).fetchall()
|
||||
|
||||
# This is used to see how many tracks you have from an album - to mark it as downloaded. Default is 80%, can be set in config as ALBUM_COMPLETION_PCT
|
||||
total_track_count = len(tracks)
|
||||
|
||||
for track in tracks:
|
||||
|
||||
controlValueDict = {"TrackID": track['TrackID'],
|
||||
"AlbumID": rg['id']}
|
||||
|
||||
newValueDict = {"ArtistID": track['ArtistID'],
|
||||
"ArtistName": track['ArtistName'],
|
||||
"AlbumTitle": track['AlbumTitle'],
|
||||
"AlbumASIN": track['AlbumASIN'],
|
||||
"ReleaseID": track['ReleaseID'],
|
||||
"TrackTitle": track['TrackTitle'],
|
||||
"TrackDuration": track['TrackDuration'],
|
||||
"TrackNumber": track['TrackNumber'],
|
||||
"CleanName": track['CleanName'],
|
||||
"Location": track['Location'],
|
||||
"Format": track['Format'],
|
||||
"BitRate": track['BitRate']
|
||||
}
|
||||
|
||||
myDB.upsert("tracks", newValueDict, controlValueDict)
|
||||
|
||||
# Mark albums as downloaded if they have at least 80% (by default, configurable) of the album
|
||||
have_track_count = len(myDB.select('SELECT * from tracks WHERE AlbumID=? AND Location IS NOT NULL', [rg['id']]))
|
||||
marked_as_downloaded = False
|
||||
|
||||
if rg_exists:
|
||||
if rg_exists['Status'] == 'Skipped' and ((have_track_count/float(total_track_count)) >= (headphones.ALBUM_COMPLETION_PCT/100.0)):
|
||||
myDB.action('UPDATE albums SET Status=? WHERE AlbumID=?', ['Downloaded', rg['id']])
|
||||
marked_as_downloaded = True
|
||||
else:
|
||||
if ((have_track_count/float(total_track_count)) >= (headphones.ALBUM_COMPLETION_PCT/100.0)):
|
||||
myDB.action('UPDATE albums SET Status=? WHERE AlbumID=?', ['Downloaded', rg['id']])
|
||||
marked_as_downloaded = True
|
||||
|
||||
logger.info(u"Seeing if we need album art for " + rg['title'])
|
||||
cache.getThumb(AlbumID=rg['id'])
|
||||
|
||||
#start a search for the album if it's new, hasn't been marked as downloaded and autowant_all is selected:
|
||||
if not rg_exists and not marked_as_downloaded and headphones.AUTOWANT_ALL:
|
||||
from headphones import searcher
|
||||
searcher.searchforalbum(albumid=rg['id'])
|
||||
else:
|
||||
if ((have_track_count/float(total_track_count)) >= (headphones.ALBUM_COMPLETION_PCT/100.0)):
|
||||
myDB.action('UPDATE albums SET Status=? WHERE AlbumID=?', ['Downloaded', rg['id']])
|
||||
marked_as_downloaded = True
|
||||
|
||||
logger.info(u"Seeing if we need album art for " + rg['title'])
|
||||
cache.getThumb(AlbumID=rg['id'])
|
||||
|
||||
#start a search for the album if it's new, hasn't been marked as downloaded and autowant_all is selected:
|
||||
if not rg_exists and not marked_as_downloaded and headphones.AUTOWANT_ALL:
|
||||
from headphones import searcher
|
||||
searcher.searchforalbum(albumid=rg['id'])
|
||||
if skip_log == 0:
|
||||
logger.info(u"No new releases, so no changes made to " + rg['title'])
|
||||
|
||||
latestalbum = myDB.action('SELECT AlbumTitle, ReleaseDate, AlbumID from albums WHERE ArtistID=? order by ReleaseDate DESC', [artistid]).fetchone()
|
||||
totaltracks = len(myDB.select('SELECT TrackTitle from tracks WHERE ArtistID=?', [artistid]))
|
||||
|
||||
201
headphones/mb.py
201
headphones/mb.py
@@ -19,7 +19,7 @@ import time
|
||||
import threading
|
||||
|
||||
import headphones
|
||||
from headphones import logger, db
|
||||
from headphones import logger, db, helpers
|
||||
from headphones.helpers import multikeysort, replace_all
|
||||
|
||||
import lib.musicbrainzngs as musicbrainzngs
|
||||
@@ -332,65 +332,152 @@ def getRelease(releaseid, include_artist_info=True):
|
||||
|
||||
return release
|
||||
|
||||
def get_all_releases(rgid,includeExtras=False):
|
||||
results = []
|
||||
try:
|
||||
limit = 100
|
||||
newResults = None
|
||||
while newResults == None or len(newResults) >= limit:
|
||||
newResults = musicbrainzngs.browse_releases(release_group=rgid,includes=['artist-credits','labels','recordings','release-groups','media'],limit=limit,offset=len(results))
|
||||
if 'release-list' not in newResults:
|
||||
break #may want to raise an exception here instead ?
|
||||
newResults = newResults['release-list']
|
||||
results += newResults
|
||||
|
||||
except WebServiceError, e:
|
||||
logger.warn('Attempt to retrieve information from MusicBrainz for release group "%s" failed (%s)' % (rgid, str(e)))
|
||||
time.sleep(5)
|
||||
return False
|
||||
|
||||
if not results or len(results) == 0:
|
||||
return False
|
||||
def get_new_releases(rgid,includeExtras=False):
|
||||
|
||||
|
||||
releases = []
|
||||
for releasedata in results:
|
||||
#releasedata.get will return None if it doesn't have a status
|
||||
#all official releases should have the Official status included
|
||||
if not includeExtras and releasedata.get('status') != 'Official':
|
||||
continue
|
||||
|
||||
release = {}
|
||||
release['AlbumTitle'] = unicode(releasedata['title'])
|
||||
release['AlbumID'] = unicode(rgid)
|
||||
release['AlbumASIN'] = unicode(releasedata['asin']) if 'asin' in releasedata else None
|
||||
release['ReleaseDate'] = unicode(releasedata['date']) if 'date' in releasedata else None
|
||||
release['ReleaseID'] = releasedata['id']
|
||||
if 'release-group' not in releasedata:
|
||||
raise Exception('No release group associated with release id ' + releasedata['id'] + ' album id' + rgid)
|
||||
release['Type'] = unicode(releasedata['release-group']['type'])
|
||||
|
||||
|
||||
#making the assumption that the most important artist will be first in the list
|
||||
if 'artist-credit' in releasedata:
|
||||
release['ArtistID'] = unicode(releasedata['artist-credit'][0]['artist']['id'])
|
||||
release['ArtistName'] = unicode(releasedata['artist-credit-phrase'])
|
||||
else:
|
||||
logger.warn('Release ' + releasedata['id'] + ' has no Artists associated.')
|
||||
return False
|
||||
|
||||
|
||||
release['ReleaseCountry'] = unicode(releasedata['country']) if 'country' in releasedata else u'Unknown'
|
||||
#assuming that the list will contain media and that the format will be consistent
|
||||
with mb_lock:
|
||||
myDB = db.DBConnection()
|
||||
results = []
|
||||
try:
|
||||
release['ReleaseFormat'] = unicode(releasedata['medium-list'][0]['format'])
|
||||
except:
|
||||
release['ReleaseFormat'] = u'Unknown'
|
||||
|
||||
release['Tracks'] = getTracksFromRelease(releasedata)
|
||||
releases.append(release)
|
||||
limit = 100
|
||||
newResults = None
|
||||
while newResults == None or len(newResults) >= limit:
|
||||
newResults = musicbrainzngs.browse_releases(release_group=rgid,includes=['artist-credits','labels','recordings','release-groups','media'],limit=limit,offset=len(results))
|
||||
if 'release-list' not in newResults:
|
||||
break #may want to raise an exception here instead ?
|
||||
newResults = newResults['release-list']
|
||||
results += newResults
|
||||
|
||||
except WebServiceError, e:
|
||||
logger.warn('Attempt to retrieve information from MusicBrainz for release group "%s" failed (%s)' % (rgid, str(e)))
|
||||
time.sleep(5)
|
||||
return False
|
||||
|
||||
if not results or len(results) == 0:
|
||||
return False
|
||||
|
||||
return releases
|
||||
#Clean all references to releases in dB that are no longer referenced in musicbrainz
|
||||
release_list = []
|
||||
force_repackage = 0
|
||||
if len(results) != 0:
|
||||
for release_mark in results:
|
||||
release_list.append(unicode(release_mark['id']))
|
||||
release_title = release_mark['title']
|
||||
remove_missing_releases = myDB.action("SELECT ReleaseID FROM allalbums WHERE AlbumID=?", [rgid])
|
||||
for items in remove_missing_releases:
|
||||
if items['ReleaseID'] not in release_list and items['ReleaseID'] != rgid:
|
||||
# Remove all from albums/tracks that aren't in release
|
||||
myDB.action("DELETE FROM albums WHERE ReleaseID=?", [items['ReleaseID']])
|
||||
myDB.action("DELETE FROM tracks WHERE ReleaseID=?", [items['ReleaseID']])
|
||||
myDB.action("DELETE FROM allalbums WHERE ReleaseID=?", [items['ReleaseID']])
|
||||
myDB.action("DELETE FROM alltracks WHERE ReleaseID=?", [items['ReleaseID']])
|
||||
logger.info("Removing all references to release %s to reflect MusicBrainz" % items['ReleaseID'])
|
||||
force_repackage = 1
|
||||
else:
|
||||
logger.info("Error pulling data from MusicBrainz: Maintaining dB")
|
||||
|
||||
num_new_releases = 0
|
||||
|
||||
for releasedata in results:
|
||||
#releasedata.get will return None if it doesn't have a status
|
||||
#all official releases should have the Official status included
|
||||
if not includeExtras and releasedata.get('status') != 'Official':
|
||||
continue
|
||||
|
||||
release = {}
|
||||
rel_id_check = releasedata['id']
|
||||
artistid = unicode(releasedata['artist-credit'][0]['artist']['id'])
|
||||
|
||||
album_checker = myDB.action('SELECT * from allalbums WHERE ReleaseID=?', [rel_id_check]).fetchone()
|
||||
if not album_checker:
|
||||
release['AlbumTitle'] = unicode(releasedata['title'])
|
||||
release['AlbumID'] = unicode(rgid)
|
||||
release['AlbumASIN'] = unicode(releasedata['asin']) if 'asin' in releasedata else None
|
||||
release['ReleaseDate'] = unicode(releasedata['date']) if 'date' in releasedata else None
|
||||
release['ReleaseID'] = releasedata['id']
|
||||
if 'release-group' not in releasedata:
|
||||
raise Exception('No release group associated with release id ' + releasedata['id'] + ' album id' + rgid)
|
||||
release['Type'] = unicode(releasedata['release-group']['type'])
|
||||
|
||||
|
||||
#making the assumption that the most important artist will be first in the list
|
||||
if 'artist-credit' in releasedata:
|
||||
release['ArtistID'] = unicode(releasedata['artist-credit'][0]['artist']['id'])
|
||||
release['ArtistName'] = unicode(releasedata['artist-credit-phrase'])
|
||||
else:
|
||||
logger.warn('Release ' + releasedata['id'] + ' has no Artists associated.')
|
||||
return False
|
||||
|
||||
|
||||
release['ReleaseCountry'] = unicode(releasedata['country']) if 'country' in releasedata else u'Unknown'
|
||||
#assuming that the list will contain media and that the format will be consistent
|
||||
try:
|
||||
release['ReleaseFormat'] = unicode(releasedata['medium-list'][0]['format'])
|
||||
except:
|
||||
release['ReleaseFormat'] = u'Unknown'
|
||||
|
||||
release['Tracks'] = getTracksFromRelease(releasedata)
|
||||
|
||||
# What we're doing here now is first updating the allalbums & alltracks table to the most
|
||||
# current info, then moving the appropriate release into the album table and its associated
|
||||
# tracks into the tracks table
|
||||
controlValueDict = {"ReleaseID" : release['ReleaseID']}
|
||||
|
||||
newValueDict = {"ArtistID": release['ArtistID'],
|
||||
"ArtistName": release['ArtistName'],
|
||||
"AlbumTitle": release['AlbumTitle'],
|
||||
"AlbumID": release['AlbumID'],
|
||||
"AlbumASIN": release['AlbumASIN'],
|
||||
"ReleaseDate": release['ReleaseDate'],
|
||||
"Type": release['Type'],
|
||||
"ReleaseCountry": release['ReleaseCountry'],
|
||||
"ReleaseFormat": release['ReleaseFormat']
|
||||
}
|
||||
|
||||
myDB.upsert("allalbums", newValueDict, controlValueDict)
|
||||
|
||||
|
||||
for track in release['Tracks']:
|
||||
|
||||
cleanname = helpers.cleanName(release['ArtistName'] + ' ' + release['AlbumTitle'] + ' ' + track['title'])
|
||||
|
||||
controlValueDict = {"TrackID": track['id'],
|
||||
"ReleaseID": release['ReleaseID']}
|
||||
|
||||
newValueDict = {"ArtistID": release['ArtistID'],
|
||||
"ArtistName": release['ArtistName'],
|
||||
"AlbumTitle": release['AlbumTitle'],
|
||||
"AlbumID": release['AlbumID'],
|
||||
"AlbumASIN": release['AlbumASIN'],
|
||||
"TrackTitle": track['title'],
|
||||
"TrackDuration": track['duration'],
|
||||
"TrackNumber": track['number'],
|
||||
"CleanName": cleanname
|
||||
}
|
||||
|
||||
match = myDB.action('SELECT Location, BitRate, Format from have WHERE CleanName=?', [cleanname]).fetchone()
|
||||
|
||||
if not match:
|
||||
match = myDB.action('SELECT Location, BitRate, Format from have WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [release['ArtistName'], release['AlbumTitle'], track['title']]).fetchone()
|
||||
if not match:
|
||||
match = myDB.action('SELECT Location, BitRate, Format from have WHERE TrackID=?', [track['id']]).fetchone()
|
||||
if match:
|
||||
newValueDict['Location'] = match['Location']
|
||||
newValueDict['BitRate'] = match['BitRate']
|
||||
newValueDict['Format'] = match['Format']
|
||||
myDB.action('UPDATE have SET Matched="True" WHERE Location=?', [match['Location']])
|
||||
|
||||
myDB.upsert("alltracks", newValueDict, controlValueDict)
|
||||
num_new_releases = num_new_releases + 1
|
||||
#print releasedata['title']
|
||||
#print num_new_releases
|
||||
logger.info('New release %s (%s) added' % (release['AlbumTitle'], rel_id_check))
|
||||
if force_repackage == 1:
|
||||
num_new_releases = -1
|
||||
logger.info('Forcing repackage of %s, since dB releases have been removed' % release_title)
|
||||
else:
|
||||
num_new_releases = num_new_releases
|
||||
|
||||
return num_new_releases
|
||||
|
||||
def getTracksFromRelease(release):
|
||||
totalTracks = 1
|
||||
|
||||
Reference in New Issue
Block a user