From 58859ae2a65ee70b68d18a9009b88bc40067497c Mon Sep 17 00:00:00 2001 From: Remy Date: Wed, 27 Jul 2011 15:04:23 -0700 Subject: [PATCH] Fixed extra tracks bug, recursively reoving directories on post processing, better post processing logs --- headphones/__init__.py | 7 ++++++- headphones/helpers.py | 2 +- headphones/importer.py | 21 +++++++++------------ headphones/mb.py | 32 +++++++++++++++++++++++--------- headphones/postprocessor.py | 8 ++++---- headphones/searcher.py | 6 +++++- headphones/updater.py | 2 +- 7 files changed, 49 insertions(+), 29 deletions(-) diff --git a/headphones/__init__.py b/headphones/__init__.py index 4eaa640e..c931b444 100644 --- a/headphones/__init__.py +++ b/headphones/__init__.py @@ -400,7 +400,7 @@ def dbcheck(): c=conn.cursor() c.execute('CREATE TABLE IF NOT EXISTS artists (ArtistID TEXT UNIQUE, ArtistName TEXT, ArtistSortName TEXT, DateAdded TEXT, Status TEXT, IncludeExtras INTEGER, LatestAlbum TEXT, ReleaseDate TEXT, AlbumID TEXT, HaveTracks INTEGER, TotalTracks INTEGER)') c.execute('CREATE TABLE IF NOT EXISTS albums (ArtistID TEXT, ArtistName TEXT, AlbumTitle TEXT, AlbumASIN TEXT, ReleaseDate TEXT, DateAdded TEXT, AlbumID TEXT UNIQUE, Status TEXT, Type TEXT)') - c.execute('CREATE TABLE IF NOT EXISTS tracks (ArtistID TEXT, ArtistName TEXT, AlbumTitle TEXT, AlbumASIN TEXT, AlbumID TEXT, TrackTitle TEXT, TrackDuration, TrackID TEXT)') + c.execute('CREATE TABLE IF NOT EXISTS tracks (ArtistID TEXT, ArtistName TEXT, AlbumTitle TEXT, AlbumASIN TEXT, AlbumID TEXT, TrackTitle TEXT, TrackDuration, TrackID TEXT, TrackNumber INTEGER)') c.execute('CREATE TABLE IF NOT EXISTS snatched (AlbumID TEXT, Title TEXT, Size INTEGER, URL TEXT, DateAdded TEXT, Status TEXT, FolderName TEXT)') c.execute('CREATE TABLE IF NOT EXISTS have (ArtistName TEXT, AlbumTitle TEXT, TrackNumber TEXT, TrackTitle TEXT, TrackLength TEXT, BitRate TEXT, Genre TEXT, Date TEXT, TrackID TEXT)') @@ -438,6 +438,11 @@ def dbcheck(): c.execute('SELECT Type from albums') except sqlite3.OperationalError: c.execute('ALTER TABLE albums ADD COLUMN Type TEXT DEFAULT "Album"') + + try: + c.execute('SELECT TrackNumber from tracks') + except sqlite3.OperationalError: + c.execute('ALTER TABLE tracks ADD COLUMN TrackNumber INTEGER') try: c.execute('SELECT FolderName from snatched') diff --git a/headphones/helpers.py b/headphones/helpers.py index 461eaf5f..6ea9c923 100644 --- a/headphones/helpers.py +++ b/headphones/helpers.py @@ -8,7 +8,7 @@ from headphones import logger def multikeysort(items, columns): - comparers = [ ((itemgetter(col[1:].strip()), -1) if col.startswith('-') else (itemgetter(col.strip()), 1)) for col in columns] + comparers = [ ((itemgetter(col[1:].strip()), -1) if col.startswith('-') else (itemgetter(col.strip()), 1)) for col in columns] def comparer(left, right): for fn, mult in comparers: diff --git a/headphones/importer.py b/headphones/importer.py index f9a85d48..cf7456c7 100644 --- a/headphones/importer.py +++ b/headphones/importer.py @@ -173,18 +173,11 @@ def addArtisttoDB(artistid, extrasonly=False): try: release_dict = mb.getReleaseGroup(rgid) except Exception, e: - logger.info('Unable to get release information for %s - it may not be a valid release group \ - (or it might just not be tagged right in MusicBrainz)' % rg['title']) + logger.info('Unable to get release information for %s - it may not be a valid release group (or it might just not be tagged right in MusicBrainz)' % rg['title']) continue if not release_dict: continue - - release = mb.getRelease(release_dict['releaseid']) - - if not release: - logger.warn('Unable to get release information for %s. Skipping for now.' % rg['title']) - continue logger.info(u"Now adding/updating album: " + rg['title']) controlValueDict = {"AlbumID": rg['id']} @@ -206,7 +199,7 @@ def addArtisttoDB(artistid, extrasonly=False): "Type": rg['type'] } - if release['date'] > helpers.today(): + if release_dict['releasedate'] > helpers.today(): newValueDict['Status'] = "Wanted" else: newValueDict['Status'] = "Skipped" @@ -214,9 +207,12 @@ def addArtisttoDB(artistid, extrasonly=False): myDB.upsert("albums", newValueDict, controlValueDict) # I changed the albumid from releaseid -> rgid, so might need to delete albums that have a releaseid - myDB.action('DELETE from albums WHERE AlbumID=?', [release['id']]) - - for track in release['tracks']: + for release in release_dict['releaselist']: + myDB.action('DELETE from albums WHERE AlbumID=?', [release['releaseid']]) + myDB.action('DELETE from tracks WHERE AlbumID=?', [release['releaseid']]) + + myDB.action('DELETE from tracks WHERE AlbumID=?', [rg['id']]) + for track in release_dict['tracks']: controlValueDict = {"TrackID": track['id'], "AlbumID": rg['id']} @@ -226,6 +222,7 @@ def addArtisttoDB(artistid, extrasonly=False): "AlbumASIN": release_dict['asin'], "TrackTitle": track['title'], "TrackDuration": track['duration'], + "TrackNumber": track['number'] } myDB.upsert("tracks", newValueDict, controlValueDict) diff --git a/headphones/mb.py b/headphones/mb.py index 0af3174e..99abe7c7 100644 --- a/headphones/mb.py +++ b/headphones/mb.py @@ -162,7 +162,6 @@ def getReleaseGroup(rgid): with mb_lock: releaselist = [] - asinlist = [] inc = ws.ReleaseGroupIncludes(releases=True) releaseGroup = None @@ -237,24 +236,39 @@ def getReleaseGroup(rgid): release_dict = { 'hasasin': bool(releaseResult.asin), 'asin': releaseResult.asin, - 'tracks': len(releaseResult.getTracks()), + 'trackscount': len(releaseResult.getTracks()), 'releaseid': u.extractUuid(releaseResult.id), 'releasedate': releaseResult.getEarliestReleaseDate(), 'format': format, 'country': country } - - releaselist.append(release_dict) - if releaseResult.asin: - asinlist.append(releaseResult.asin) + tracks = [] + + i = 1 + for track in releaseResult.tracks: + + tracks.append({ + 'number': i, + 'title': track.title, + 'id': u.extractUuid(track.id), + 'url': track.id, + 'duration': track.duration + }) + i += 1 + + release_dict['tracks'] = tracks + + releaselist.append(release_dict) - a = multikeysort(releaselist, ['-hasasin', 'country', 'format', 'tracks']) + a = multikeysort(releaselist, ['-hasasin', 'country', 'format', 'trackscount']) release_dict = {'releaseid' :a[0]['releaseid'], 'releasedate' : releaselist[0]['releasedate'], - 'trackcount' : a[0]['tracks'], - 'asin' : a[0]['asin'] + 'trackcount' : a[0]['trackscount'], + 'tracks' : a[0]['tracks'], + 'asin' : a[0]['asin'], + 'releaselist' : releaselist } return release_dict diff --git a/headphones/postprocessor.py b/headphones/postprocessor.py index 1bb41132..87cc50ac 100644 --- a/headphones/postprocessor.py +++ b/headphones/postprocessor.py @@ -110,7 +110,7 @@ def doPostProcessing(albumid, albumpath, release, tracks, downloaded_track_list) def addAlbumArt(albumid, downloaded_track_list): - + logger.info('Adding album art') album_art_path = albumart.getAlbumArt(albumid) artwork = urllib.urlopen(album_art_path).read() @@ -125,14 +125,13 @@ def addAlbumArt(albumid, downloaded_track_list): f.save() def cleanupFiles(albumpath): - + logger.info('Cleaning up files') for r,d,f in os.walk(albumpath): for files in f: if not any(files.endswith(x) for x in (".mp3", ".flac", ".aac", ".ogg", ".ape", ".m4a")): os.remove(os.path.join(r, files)) def moveFiles(albumpath, release, tracks): - try: year = release['ReleaseDate'][:4] except TypeError: @@ -147,6 +146,7 @@ def moveFiles(albumpath, release, tracks): folder = helpers.replace_all(headphones.FOLDER_FORMAT, values) destination_path = os.path.join(headphones.DESTINATION_DIR, folder) + logger.info('Moving files from %s to %s' % (folder, destination_path)) try: os.makedirs(destination_path) @@ -159,7 +159,7 @@ def moveFiles(albumpath, release, tracks): shutil.move(os.path.join(r, files), destination_path) try: - os.rmdir(albumpath) + os.removedirs(albumpath) except Exception, e: logger.error('Could not remove directory: %s. %s' % (albumpath, e)) diff --git a/headphones/searcher.py b/headphones/searcher.py index 6a6d1a13..5ebe4790 100644 --- a/headphones/searcher.py +++ b/headphones/searcher.py @@ -154,7 +154,11 @@ def searchNZB(albumid=None, new=False): data = urllib.urlopen(searchURL).read() logger.info(u"Parsing results from "+searchURL) - d = minidom.parseString(data) + + try: + d = minidom.parseString(data) + except ExpatError: + logger.error('Unable to get the NZBs.org feed. Check that your settings are correct - post a bug if they are') node = d.documentElement items = d.getElementsByTagName("item") diff --git a/headphones/updater.py b/headphones/updater.py index ae626109..77fe5485 100644 --- a/headphones/updater.py +++ b/headphones/updater.py @@ -6,7 +6,7 @@ def dbUpdate(): myDB = db.DBConnection() - activeartists = myDB.select('SELECT ArtistID, ArtistName from artists WHERE Status="Active" or Status="Loading"') + activeartists = myDB.select('SELECT ArtistID, ArtistName from artists WHERE Status="Active" or Status="Loading" order by ArtistSortName collate nocase') logger.info('Starting update for %i active artists' % len(activeartists))