From 8655eecd734d6bfe011e115a924693ee753a585b Mon Sep 17 00:00:00 2001 From: Remy Date: Sat, 16 Jul 2011 16:36:21 -0700 Subject: [PATCH] Changes: Now uses a release group id to identify albums (to prevent duplicates down the road), updater now just uses the same function as importer --- headphones/__init__.py | 14 +++++++- headphones/importer.py | 62 +++++++++++++++++++++++++---------- headphones/searcher.py | 8 ++--- headphones/updater.py | 73 +++++++++--------------------------------- headphones/webserve.py | 10 +++--- 5 files changed, 83 insertions(+), 84 deletions(-) diff --git a/headphones/__init__.py b/headphones/__init__.py index 2827ba2a..49ffc8bf 100644 --- a/headphones/__init__.py +++ b/headphones/__init__.py @@ -36,6 +36,7 @@ CFG = None DB_FILE = None LOG_DIR = None +CACHE_DIR = None HTTP_PORT = None HTTP_HOST = None @@ -136,7 +137,7 @@ def initialize(): with INIT_LOCK: - global __INITIALIZED__, FULL_PATH, PROG_DIR, QUIET, DAEMON, DATA_DIR, CONFIG_FILE, CFG, LOG_DIR, \ + global __INITIALIZED__, FULL_PATH, PROG_DIR, QUIET, DAEMON, DATA_DIR, CONFIG_FILE, CFG, LOG_DIR, CACHE_DIR, \ HTTP_PORT, HTTP_HOST, HTTP_USERNAME, HTTP_PASSWORD, HTTP_ROOT, LAUNCH_BROWSER, GIT_PATH, \ CURRENT_VERSION, LATEST_VERSION,\ MUSIC_DIR, PREFER_LOSSLESS, FLAC_TO_MP3, MOVE_FILES, RENAME_FILES, FOLDER_FORMAT, \ @@ -220,6 +221,17 @@ def initialize(): # Start the logger, silence console logging if we need to logger.headphones_log.initLogger(quiet=QUIET) + + # Put the cache dir in the data dir for now + CACHE_DIR = os.path.join(DATA_DIR, 'cache') + if not os.path.exists(CACHE_DIR): + try: + os.makedirs(CACHE_DIR) + except OSError: + logger.error('Could not create cache dir. Check permissions of datadir: ' + DATA_DIR) + + + # Initialize the database logger.info('Checking to see if the database has all tables....') try: diff --git a/headphones/importer.py b/headphones/importer.py index 67f6f606..1eff8c8c 100644 --- a/headphones/importer.py +++ b/headphones/importer.py @@ -18,9 +18,13 @@ def scanMusic(dir=None): for r,d,f in os.walk(unicode(dir)): for files in f: - if any(files.endswith(x) for x in (".mp3", ".flac", ".aac", ".ogg", ".ape")): - results.append(os.path.join(r,files)) - + try: + if any(files.endswith(x) for x in (".mp3", ".flac", ".aac", ".ogg", ".ape")): + results.append(os.path.join(r,files)) + except Exception, e: + logger.warn('Can not decode file %s. Error: ' % (files, e)) + continue + logger.info(u'%i music files found' % len(results)) if results: @@ -48,6 +52,7 @@ def scanMusic(dir=None): artistlist = {}.fromkeys(lst).keys() logger.info(u"Preparing to import %i artists" % len(artistlist)) + artistlist_to_mbids(artistlist) def itunesImport(pathtoxml): @@ -68,33 +73,43 @@ def itunesImport(pathtoxml): exclude = ['.ds_store', 'various artists', 'untitled folder', 'va'] artistlist = [f for f in rawlist if f.lower() not in exclude] + logger.info('Starting directory/xml import...') artistlist_to_mbids(artistlist) + +def is_exists(artistid): + + myDB = db.DBConnection() + # See if the artist is already in the database + artistlist = myDB.select('SELECT ArtistID, ArtistName from artists WHERE ArtistID=?', [artistid]) + + if any(artistid in x for x in artistlist): + logger.info(artistlist[0][1] + u" is already in the database, skipping") + return True + else: + return False + + def artistlist_to_mbids(artistlist): for artist in artistlist: results = mb.findArtist(artist, limit=1) artistid = results[0]['id'] - if artistid != various_artists_mbid: + if artistid != various_artists_mbid and not is_exists(artistid): addArtisttoDB(artistid) def addArtisttoDB(artistid): - + + # Can't add various artists - throws an error from MB if artistid == various_artists_mbid: logger.warn('Cannot import Various Artists.') return myDB = db.DBConnection() - - artistlist = myDB.select('SELECT ArtistID, ArtistName from artists WHERE ArtistID=?', [artistid]) - - if any(artistid in x for x in artistlist): - logger.info(artistlist[0][1] + u" is already in the database, skipping") - return - + artist = mb.getArtist(artistid) if artist['artist_name'].startswith('The '): @@ -103,7 +118,7 @@ def addArtisttoDB(artistid): sortname = artist['artist_name'] - + logger.info(u"Now adding/updating: " + artist['artist_name']) controlValueDict = {"ArtistID": artistid} newValueDict = {"ArtistName": artist['artist_name'], "ArtistSortName": sortname, @@ -124,8 +139,8 @@ def addArtisttoDB(artistid): release = mb.getRelease(releaseid) - logger.info(u"Now adding album: " + release['title']+ " to the database") - controlValueDict = {"AlbumID": release['id']} + logger.info(u"Now adding/updating album: " + rg['title']) + controlValueDict = {"AlbumID": rg['id']} newValueDict = {"ArtistID": artistid, "ArtistName": artist['artist_name'], "AlbumTitle": rg['title'], @@ -136,8 +151,11 @@ def addArtisttoDB(artistid): } myDB.upsert("albums", newValueDict, controlValueDict) + + # I changed the albumid from releaseid -> rgid, so might need to delete albums that have a releaseid + myDB.action('DELETE from albums WHERE AlbumID=?', [release['id']]) - latestrelease = myDB.select("SELECT ReleaseDate, DateAdded from albums WHERE AlbumID=?", [release['id']]) + latestrelease = myDB.select("SELECT ReleaseDate, DateAdded from albums WHERE AlbumID=?", [rg['id']]) if latestrelease[0][0] > latestrelease[0][1]: logger.info(release['title'] + u" is an upcoming album. Setting its status to 'Wanted'...") @@ -147,7 +165,17 @@ def addArtisttoDB(artistid): for track in release['tracks']: - myDB.action('INSERT INTO tracks VALUES( ?, ?, ?, ?, ?, ?, ?, ?)', [artistid, artist['artist_name'], rg['title'], release['asin'], release['id'], track['title'], track['duration'], track['id']]) + controlValueDict = {"TrackID": track['id']} + newValueDict = {"ArtistID": artistid, + "ArtistName": artist['artist_name'], + "AlbumTitle": rg['title'], + "AlbumASIN": release['asin'], + "AlbumID": rg['id'], + "TrackTitle": track['title'], + "TrackDuration": track['duration'], + } + + myDB.upsert("tracks", newValueDict, controlValueDict) controlValueDict = {"ArtistID": artistid} newValueDict = {"Status": "Active"} diff --git a/headphones/searcher.py b/headphones/searcher.py index 33abe979..1cadb55a 100644 --- a/headphones/searcher.py +++ b/headphones/searcher.py @@ -4,7 +4,7 @@ import lib.feedparser as feedparser import os, re import headphones -from headphones import logger, db +from headphones import logger, db, helpers def searchNZB(albumid=None): @@ -19,12 +19,12 @@ def searchNZB(albumid=None): reldate = albums[3] year = reldate[:4] - clname = string.replace(albums[0], ' & ', ' ') - clalbum = string.replace(albums[1], ' & ', ' ') + clname = string.replace(helpers.latinToAscii(albums[0]), ' & ', ' ') + clalbum = string.replace(helpers.latinToAscii(albums[1]), ' & ', ' ') term1 = re.sub('[\.\-]', ' ', '%s %s %s' % (clname, clalbum, year)).encode('utf-8') term = string.replace(term1, '"', '') - logger.info(u"Searching for "+term+" since it was marked as wanted") + logger.info("Searching for %s since it was marked as wanted" % term) resultlist = [] diff --git a/headphones/updater.py b/headphones/updater.py index 14bd270c..5a817859 100644 --- a/headphones/updater.py +++ b/headphones/updater.py @@ -1,67 +1,24 @@ -import lib.musicbrainz2.webservice as ws -import lib.musicbrainz2.model as m -import lib.musicbrainz2.utils as u -from headphones.mb import getReleaseGroup -import time -import os - import headphones -from headphones import logger, db, mb + +from headphones import logger, db, mb, importer def dbUpdate(): myDB = db.DBConnection() activeartists = myDB.select('SELECT ArtistID, ArtistName from artists WHERE Status="Active"') - - i = 0 - - while i < len(activeartists): - - artistid = activeartists[i][0] - artistname = activeartists[i][1] - logger.info(u"Updating album information for artist: " + artistname) - - artist = mb.getArtist(artistid) - - for rg in artist['releasegroups']: - - rgid = rg['id'] - - releaseid = mb.getReleaseGroup(rgid) - - results = mb.getRelease(releaseid) - - albumlist = myDB.select('SELECT AlbumID from albums WHERE ArtistID=?', [artistid]) - - if any(releaseid in x for x in albumlist): - - logger.info(results['title'] + " already exists in the database. Updating ASIN, Release Date, Tracks") - - myDB.action('UPDATE albums SET AlbumASIN=?, ReleaseDate=? WHERE AlbumID=?', [results['asin'], results['date'], results['id']]) - - for track in results['tracks']: - - myDB.action('UPDATE tracks SET TrackDuration=? WHERE AlbumID=? AND TrackID=?', [track['duration'], results['id'], track['id']]) - - else: - - logger.info(u"New album found! Adding "+results['title']+"to the database...") - - myDB.action('INSERT INTO albums VALUES( ?, ?, ?, ?, ?, CURRENT_DATE, ?, ?)', [artistid, artist['artist_name'], rg['title'], results['asin'], results['date'], results['id'], 'Skipped']) - - latestrelease = myDB.select('SELECT ReleaseDate, DateAdded from albums WHERE AlbumID=?', [results['id']]) - - if latestrelease[0][0] > latestrelease[0][1]: - - myDB.action('UPDATE albums SET Status = "Wanted" WHERE AlbumID=?', results['id']) - - else: - pass - - for track in results['tracks']: - - myDB.action('INSERT INTO tracks VALUES( ?, ?, ?, ?, ?, ?, ?, ?)', [artistid, artist['artist_name'], rg['title'], results['asin'], results['id'], track['title'], track['duration'], track['id']]) - i += 1 + logger.info('Starting update for %i active artists' % len(activeartists)) + + for artist in activeartists: + + artistid = artist[0] + importer.addArtisttoDB(artistid) + + logger.info('Update complete') + + + + + diff --git a/headphones/webserve.py b/headphones/webserve.py index 3f0149b7..a3097c8c 100644 --- a/headphones/webserve.py +++ b/headphones/webserve.py @@ -93,8 +93,9 @@ class WebInterface(object): page.append(templates._nav) myDB = db.DBConnection() + artist = myDB.select('SELECT ArtistName from artists WHERE ArtistID=?', [ArtistID]) results = myDB.select('SELECT AlbumTitle, ReleaseDate, AlbumID, Status, ArtistName, AlbumASIN from albums WHERE ArtistID=? order by ReleaseDate DESC', [ArtistID]) - + i = 0 page.append('''

%s

@@ -104,7 +105,7 @@ class WebInterface(object): - ''' % (results[0][4])) + ''' % artist[0][0]) while i < len(results): totaltracks = len(myDB.select('SELECT TrackTitle from tracks WHERE AlbumID=?', [results[i][2]])) havetracks = len(myDB.select('SELECT TrackTitle from have WHERE ArtistName like ? AND AlbumTitle like ?', [results[i][4], results[i][0]])) @@ -126,7 +127,7 @@ class WebInterface(object): newStatus = '%s' % (results[i][3]) page.append(''' + (link)''' % (results[i][5], results[i][2], results[i][0], results[i][2], results[i][1], newStatus, percent)) @@ -143,6 +144,7 @@ class WebInterface(object): page.append(templates._logobar) page.append(templates._nav) myDB = db.DBConnection() + results = myDB.select('SELECT ArtistID, ArtistName, AlbumTitle, TrackTitle, TrackDuration, TrackID, AlbumASIN from tracks WHERE AlbumID=?', [AlbumID]) if results[0][6]: @@ -226,7 +228,7 @@ class WebInterface(object): def addArtist(self, artistid): threading.Thread(target=importer.addArtisttoDB, args=[artistid]).start() - time.sleep(2) + time.sleep(5) raise cherrypy.HTTPRedirect("home") addArtist.exposed = True
Release Date Status Have
%s - (link) %s %s