Merge pull request #1839 from rembo10/develop

Merge develop

headphones/__init__.py

@@ -376,36 +376,14 @@ def initialize():
             return False
 
         # Make sure all the config sections exist
-        CheckSection('General')
+        for section in ('General', 'SABnzbd', 'NZBget', 'Transmission',
-        CheckSection('SABnzbd')
+                        'uTorrent', 'Headphones', 'Newznab', 'NZBsorg',
-        CheckSection('NZBget')
+                        'omgwtfnzbs', 'Piratebay', 'Kat', 'Mininova', 'Waffles',
-        CheckSection('Transmission')
+                        'Rutracker', 'What.cd', 'Growl', 'Prowl', 'Pushover',
-        CheckSection('uTorrent')
+                        'PushBullet', 'XBMC', 'LMS', 'Plex', 'NMA', 'Pushalot',
-        CheckSection('Headphones')
+                        'Synoindex', 'Twitter', 'OSX_Notify', 'Boxcar',
-        CheckSection('Newznab')
+                        'Songkick', 'Advanced'):
-        CheckSection('NZBsorg')
+            CheckSection(section)
-        CheckSection('omgwtfnzbs')
-        CheckSection('Piratebay')
-        CheckSection('Kat')
-        CheckSection('Mininova')
-        CheckSection('Waffles')
-        CheckSection('Rutracker')
-        CheckSection('What.cd')
-        CheckSection('Growl')
-        CheckSection('Prowl')
-        CheckSection('Pushover')
-        CheckSection('PushBullet')
-        CheckSection('XBMC')
-        CheckSection('LMS')
-        CheckSection('Plex')
-        CheckSection('NMA')
-        CheckSection('Pushalot')
-        CheckSection('Synoindex')
-        CheckSection('Twitter')
-        CheckSection('OSX_Notify')
-        CheckSection('Boxcar')
-        CheckSection('Songkick')
-        CheckSection('Advanced')
 
         # Set global variables based on config file or use defaults
         CONFIG_VERSION = check_setting_str(CFG, 'General', 'config_version', '0')

headphones/importer.py

@@ -193,14 +193,17 @@ def addArtisttoDB(artistid, extrasonly=False, forcefull=False):
     except IndexError:
         includeExtras = False
 
-    #Clean all references to release group in dB that are no longer referenced in musicbrainz
+    #Clean all references to release group in dB that are no longer referenced from the musicbrainz refresh
     group_list = []
     force_repackage = 0
     #Don't nuke the database if there's a MusicBrainz error
-    if len(artist['releasegroups']) != 0 and not extrasonly:
+    if len(artist['releasegroups']) != 0:
         for groups in artist['releasegroups']:
             group_list.append(groups['id'])
-        remove_missing_groups_from_albums = myDB.action("SELECT AlbumID FROM albums WHERE ArtistID=?", [artistid])
+        if not extrasonly:
+            remove_missing_groups_from_albums = myDB.select("SELECT AlbumID FROM albums WHERE ArtistID=?", [artistid])
+        else:
+            remove_missing_groups_from_albums = myDB.select('SELECT AlbumID FROM albums WHERE ArtistID=? AND Status="Skipped" AND Type!="Album"', [artistid])
         for items in remove_missing_groups_from_albums:
             if items['AlbumID'] not in group_list:
                 # Remove all from albums/tracks that aren't in release groups
@@ -209,13 +212,12 @@ def addArtisttoDB(artistid, extrasonly=False, forcefull=False):
                 myDB.action("DELETE FROM tracks WHERE AlbumID=?", [items['AlbumID']])
                 myDB.action("DELETE FROM alltracks WHERE AlbumID=?", [items['AlbumID']])
                 myDB.action('DELETE from releases WHERE ReleaseGroupID=?', [items['AlbumID']])
-                logger.info("[%s] Removing all references to release group %s to reflect MusicBrainz" % (artist['artist_name'], items['AlbumID']))
+                logger.info("[%s] Removing all references to release group %s to reflect MusicBrainz refresh" % (artist['artist_name'], items['AlbumID']))
-                force_repackage = 1
+                if not extrasonly:
-    elif extrasonly:
+                    force_repackage = 1
-        # Not really sure what we're doing here but don't want to log the message below if we're fetching extras only
-        pass
     else:
-        logger.info("[%s] There was either an error pulling data from MusicBrainz or there might not be any releases for this category" % artist['artist_name'])
+        if not extrasonly:
+            logger.info("[%s] There was either an error pulling data from MusicBrainz or there might not be any releases for this category" % artist['artist_name'])
 
     # Then search for releases within releasegroups, if releases don't exist, then remove from allalbums/alltracks
     album_searches = []
@@ -488,6 +490,7 @@ def addArtisttoDB(artistid, extrasonly=False, forcefull=False):
             if skip_log == 0:
                 logger.info(u"[%s] No new releases, so no changes made to %s" % (artist['artist_name'], rg['title']))
 
+    time.sleep(3)
     finalize_update(artistid, artist['artist_name'], errors)
 
     logger.info(u"Seeing if we need album art for: %s" % artist['artist_name'])

headphones/librarysync.py

@@ -50,7 +50,7 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, cron=Fal
 
     if not append:
         # Clean up bad filepaths
-        tracks = myDB.select('SELECT Location, TrackID from alltracks WHERE Location IS NOT NULL')
+        tracks = myDB.select('SELECT Location from alltracks WHERE Location IS NOT NULL UNION SELECT Location from tracks WHERE Location IS NOT NULL')
 
         for track in tracks:
             encoded_track_string = track['Location'].encode(headphones.SYS_ENCODING)
@@ -216,6 +216,7 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, cron=Fal
         if song['ArtistName'] and song['AlbumTitle'] and song['TrackTitle']:
 
             track = myDB.action('SELECT ArtistName, AlbumTitle, TrackTitle, AlbumID from tracks WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [song['ArtistName'], song['AlbumTitle'], song['TrackTitle']]).fetchone()
+            have_updated = False
             if track:
                 controlValueDict = { 'ArtistName' : track['ArtistName'],
                                      'AlbumTitle' : track['AlbumTitle'],
@@ -228,6 +229,7 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, cron=Fal
                 controlValueDict2 = { 'Location' : song['Location']}
                 newValueDict2 = { 'Matched' : track['AlbumID']}
                 myDB.upsert("have", newValueDict2, controlValueDict2)
+                have_updated = True
             else:
                 track = myDB.action('SELECT CleanName, AlbumID from tracks WHERE CleanName LIKE ?', [song['CleanName']]).fetchone()
                 if track:
@@ -240,11 +242,12 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, cron=Fal
                     controlValueDict2 = { 'Location' : song['Location']}
                     newValueDict2 = { 'Matched' : track['AlbumID']}
                     myDB.upsert("have", newValueDict2, controlValueDict2)
+                    have_updated = True
                 else:
                     controlValueDict2 = { 'Location' : song['Location']}
                     newValueDict2 = { 'Matched' : "Failed"}
                     myDB.upsert("have", newValueDict2, controlValueDict2)
-
+                    have_updated = True
 
             alltrack = myDB.action('SELECT ArtistName, AlbumTitle, TrackTitle, AlbumID from alltracks WHERE ArtistName LIKE ? AND AlbumTitle LIKE ? AND TrackTitle LIKE ?', [song['ArtistName'], song['AlbumTitle'], song['TrackTitle']]).fetchone()
             if alltrack:
@@ -272,9 +275,12 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None, cron=Fal
                     newValueDict2 = { 'Matched' : alltrack['AlbumID']}
                     myDB.upsert("have", newValueDict2, controlValueDict2)
                 else:
-                    controlValueDict2 = { 'Location' : song['Location']}
+                    # alltracks may not exist if adding album manually, have should only be set to failed if not already updated in tracks
-                    newValueDict2 = { 'Matched' : "Failed"}
+                    if not have_updated:
-                    myDB.upsert("have", newValueDict2, controlValueDict2)
+                        controlValueDict2 = { 'Location' : song['Location']}
+                        newValueDict2 = { 'Matched' : "Failed"}
+                        myDB.upsert("have", newValueDict2, controlValueDict2)
+
         else:
             controlValueDict2 = { 'Location' : song['Location']}
             newValueDict2 = { 'Matched' : "Failed"}

headphones/mb.py

@@ -174,6 +174,14 @@ def findRelease(name, limit=1, artist=None):
                         formats += str(count) + 'x'
                     formats += format
 
+            rg_type = ''
+            if 'type' in result['release-group']:
+                rg_type = result['release-group']['type']
+                if rg_type == 'Album' and 'secondary-type-list' in result['release-group']:
+                    secondary_type = result['release-group']['secondary-type-list'][0]
+                    if secondary_type != rg_type:
+                        rg_type = secondary_type
+
             releaselist.append({
                         'uniquename':        unicode(result['artist-credit'][0]['artist']['name']),
                         'title':             unicode(title),
@@ -187,7 +195,7 @@ def findRelease(name, limit=1, artist=None):
                         'formats':           unicode(formats),
                         'tracks':            unicode(tracks),
                         'rgid':              unicode(result['release-group']['id']),
-                        'rgtype':            unicode(result['release-group']['type']) if 'type' in result['release-group'] else ''
+                        'rgtype':            unicode(rg_type)
                         })
         return releaselist
 
@@ -263,14 +271,18 @@ def getArtist(artistid, extrasonly=False):
 
         if includeExtras:
 
-            # Need to convert extras string from something like '2,5.6' to ['ep','live','remix']
+            # Need to convert extras string from something like '2,5.6' to ['ep','live','remix'] (append new extras to end)
-            extras = db_artist['Extras']
+            if db_artist['Extras']:
-            extras_list = ["single", "ep", "compilation", "soundtrack", "live", "remix", "dj-mix", "mixtape/street", "spokenword", "audiobook", "broadcast", "interview", "other"]
+                extras = map(int, db_artist['Extras'].split(','))
+            else:
+                extras = []
+            extras_list = ["single", "ep", "compilation", "soundtrack", "live", "remix", "spokenword", "audiobook", "other", "dj-mix", "mixtape/street", "broadcast", "interview"]
+
             includes = []
 
             i = 1
             for extra in extras_list:
-                if str(i) in extras:
+                if i in extras:
                     includes.append(extra)
                 i += 1
 
@@ -289,11 +301,18 @@ def getArtist(artistid, extrasonly=False):
                     time.sleep(5)
 
                 for rg in mb_extras_list:
+
+                    rg_type = rg['type']
+                    if rg_type == 'Album' and 'secondary-type-list' in rg:
+                        secondary_type = rg['secondary-type-list'][0]
+                        if secondary_type != rg_type:
+                            rg_type = secondary_type
+
                     releasegroups.append({
                             'title':        unicode(rg['title']),
                             'id':           unicode(rg['id']),
                             'url':          u"http://musicbrainz.org/release-group/" + rg['id'],
-                            'type':         unicode(rg['type'])
+                            'type':         unicode(rg_type)
                         })
 
         artist_dict['releasegroups'] = releasegroups
@@ -364,8 +383,15 @@ def getRelease(releaseid, include_artist_info=True):
                 release['rg_title'] = unicode(results['release-group']['title'])
                 try:
                     release['rg_type'] = unicode(results['release-group']['type'])
+
+                    if release['rg_type'] == 'Album' and 'secondary-type-list' in results['release-group']:
+                        secondary_type = unicode(results['release-group']['secondary-type-list'][0])
+                        if secondary_type != release['rg_type']:
+                            release['rg_type'] = secondary_type
+
                 except KeyError:
                     release['rg_type'] = u'Unknown'
+
             else:
                 logger.warn("Release " + releaseid + "had no ReleaseGroup associated")
 
@@ -445,6 +471,10 @@ def get_new_releases(rgid,includeExtras=False,forcefull=False):
                 raise Exception('No release group associated with release id ' + releasedata['id'] + ' album id' + rgid)
             release['Type'] = unicode(releasedata['release-group']['type'])
 
+            if release['Type'] == 'Album' and 'secondary-type-list' in releasedata['release-group']:
+                secondary_type = unicode(releasedata['release-group']['secondary-type-list'][0])
+                if secondary_type != release['Type']:
+                    release['Type'] = secondary_type
 
             #making the assumption that the most important artist will be first in the list
             if 'artist-credit' in releasedata:

headphones/postprocessor.py

@@ -30,6 +30,49 @@ from headphones import logger, helpers, request, mb, music_encoder
 
 postprocessor_lock = threading.Lock()
 
+
+def find_in_path(albumpath, extra_formats=None, use_MF=True):
+    """ Takes a path and optionally extra formats.
+        Finds files matching the MEDIA_FORMATS and returns them as a list.
+        If use_MF is disabled MEDIA_FORMATS will be ignored.
+    """
+    if not os.path.isdir(albumpath):
+        return False
+    found_tracks = []
+    if extra_formats:
+        if type(extra_formats) is not list:
+            extra_formats = [extra_formats]
+        if not use_MF:
+            mf = extra_formats
+        else:
+            mf = headphones.MEDIA_FORMATS + extra_formats
+    else:
+        if not use_MF:
+            # if not using MF then extra_formats must be set
+            return False
+        mf = headphones.MEDIA_FORMATS
+    for r, d, f in os.walk(albumpath):
+        for _file in f:
+            if any(_file.lower().endswith('.' + x.lower()) for x in mf):
+                found_tracks.append(os.path.join(r, _file))
+    return found_tracks
+
+
+def count_matches(track_list, match, inverted=False):
+    """ Takes a list of tracks and a pattern to match.
+        Returns the number of matched items and a list of the matches.
+        If inverted is set to True the returned list will be inverted and only
+        contain the NON-matches.
+    """
+    if not track_list or not match:
+        return False, False
+    new_list = [x for x in track_list if x.lower().endswith(match)]
+    nr_of_matches = len(new_list)
+    if inverted:
+        new_list = [x for x in track_list if not x.lower().endswith(match)]
+    return nr_of_matches, new_list
+
+
 def checkFolder():
 
     with postprocessor_lock:
@@ -152,20 +195,19 @@ def verify(albumid, albumpath, Kind=None, forced=False):
         release = myDB.action('SELECT * from albums WHERE AlbumID=?', [albumid]).fetchone()
         tracks = myDB.select('SELECT * from tracks WHERE AlbumID=?', [albumid])
 
-    downloaded_track_list = []
+    all_matched = find_in_path(albumpath, ["cue", "part", "upart"])
-    downloaded_cuecount = 0
+    incomplete_matches, remaining_matches = count_matches(all_matched,
-
+                                                          (".part", ".upart"),
-    for r,d,f in os.walk(albumpath):
+                                                          inverted = True)
-        for files in f:
+    if incomplete_matches and not forced:
-            if any(files.lower().endswith('.' + x.lower()) for x in headphones.MEDIA_FORMATS):
+        _a = os.path.basename(albumpath).decode(headphones.SYS_ENCODING,
-                downloaded_track_list.append(os.path.join(r, files))
+                                                'replace')
-            elif files.lower().endswith('.cue'):
+        logger.info("Looks like " + _a + " isn't complete yet. Will try again on the next run")
-                downloaded_cuecount += 1
+        return
-            # if any of the files end in *.part, we know the torrent isn't done yet. Process if forced, though
-            elif files.lower().endswith(('.part', '.utpart')) and not forced:
-                logger.info("Looks like " + os.path.basename(albumpath).decode(headphones.SYS_ENCODING, 'replace') + " isn't complete yet. Will try again on the next run")
-                return
 
+    downloaded_cuecount, downloaded_track_list = count_matches(remaining_matches,
+                                                               ".cue",
+                                                               inverted = True)
 
     # use xld to split cue
 
@@ -207,11 +249,7 @@ def verify(albumid, albumpath, Kind=None, forced=False):
 
             # count files, should now be more than original if xld successfully split
 
-            new_downloaded_track_list_count = 0
+            new_downloaded_track_list_count = len(find_in_path(albumpath))
-            for r,d,f in os.walk(albumpath):
-                for file in f:
-                    if any(file.lower().endswith('.' + x.lower()) for x in headphones.MEDIA_FORMATS):
-                        new_downloaded_track_list_count += 1
 
             if new_downloaded_track_list_count > len(downloaded_track_list):
 
@@ -220,12 +258,8 @@ def verify(albumid, albumpath, Kind=None, forced=False):
                     os.rename(downloaded_track, downloaded_track + '.original')
 
                 #reload
+                downloaded_track_list = find_in_path(albumpath)
 
-                downloaded_track_list = []
-                for r,d,f in os.walk(albumpath):
-                    for file in f:
-                        if any(file.lower().endswith('.' + x.lower()) for x in headphones.MEDIA_FORMATS):
-                            downloaded_track_list.append(os.path.join(r, file))
 
     # test #1: metadata - usually works
     logger.debug('Verifying metadata...')
@@ -330,15 +364,12 @@ def doPostProcessing(albumid, albumpath, release, tracks, downloaded_track_list,
         # Need to update the downloaded track list with the new location.
         # Could probably just throw in the "headphones-modified" folder,
         # but this is good to make sure we're not counting files that may have failed to move
-        downloaded_track_list = []
-        downloaded_cuecount = 0
 
-        for r,d,f in os.walk(albumpath):
+        all_matches = find_in_path(albumpath, "cue")
-            for files in f:
+        downloaded_cuecount, downloaded_track_list = count_matches(all_matches,
-                if any(files.lower().endswith('.' + x.lower()) for x in headphones.MEDIA_FORMATS):
+                                                                   ".cue",
-                    downloaded_track_list.append(os.path.join(r, files))
+                                                                   True)
-                elif files.lower().endswith('.cue'):
+
-                    downloaded_cuecount += 1
 
     # Check if files are valid media files and are writeable, before the steps
     # below are executed. This simplifies errors and prevents unfinished steps.
@@ -391,7 +422,7 @@ def doPostProcessing(albumid, albumpath, release, tracks, downloaded_track_list,
 
     if headphones.KEEP_NFO:
         renameNFO(albumpath)
-        
+
     if headphones.ADD_ALBUM_ART and artwork:
         addAlbumArt(artwork, albumpath, release)
 
@@ -577,26 +608,30 @@ def addAlbumArt(artwork, albumpath, release):
 
 def cleanupFiles(albumpath):
     logger.info('Cleaning up files')
-    for r,d,f in os.walk(albumpath):
+    if headphones.KEEP_NFO:
-        for files in f:
+        files = find_in_path(albumpath, extra_formats="nfo", inverted=True)
-            if not any(files.lower().endswith('.' + x.lower()) for x in headphones.MEDIA_FORMATS):
+    else:
-                if not (headphones.KEEP_NFO and files.lower().endswith('.nfo')):
+        files = find_in_path(albumpath, inverted=True)
-                    logger.debug('Removing: %s' % files)
+    for _f in files:
-                    try:
+        logger.debug('Removing: %s' % _f)
-                        os.remove(os.path.join(r, files))
+        try:
-                    except Exception, e:
+            os.remove(_f)
-                        logger.error(u'Could not remove file: %s. Error: %s' % (files.decode(headphones.SYS_ENCODING, 'replace'), e))
+        except Exception, e:
+            _f_decoded = _f.decode(headphones.SYS_ENCODING, 'replace')
+            logger.error(u'Could not remove file: %s. Error: %s' % (_f_decoded,
+                                                                    e))
 
 def renameNFO(albumpath):
-    for r,d,f in os.walk(albumpath):
+    files = find_in_path(albumpath, extra_formats="nfo", use_MF=False)
-        for file in f:
+    for _f in files:
-            if file.lower().endswith('.nfo'):
+        _f_decoded = _f.decode(headphones.SYS_ENCODING, 'replace')
-                logger.debug('Renaming: "%s" to "%s"' % (file.decode(headphones.SYS_ENCODING, 'replace'), file.decode(headphones.SYS_ENCODING, 'replace') + '-orig'))
+        logger.debug('Renaming: "%s" to "%s"' % (_f_decoded, _f_decoded + '-orig'))
-                try:          
+        try:
-                    new_file_name = os.path.join(r, file)[:-3] + 'orig.nfo'
+            new_file_name = _f[:-3] + 'orig.nfo'
-                    os.rename(os.path.join(r, file), new_file_name)
+            os.rename(_f, new_file_name)
-                except Exception, e:
+        except Exception, e:
-                    logger.error(u'Could not rename file: %s. Error: %s' % (os.path.join(r, file).decode(headphones.SYS_ENCODING, 'replace'), e))
+            logger.error(u'Could not rename file: %s. Error: %s' % (_f_decoded,
+                                                                    e))
 
 def moveFiles(albumpath, release, tracks):
 
@@ -622,12 +657,12 @@ def moveFiles(albumpath, release, tracks):
         firstchar = '0-9'
     else:
         firstchar = sortname[0]
-    
+
     for r,d,f in os.walk(albumpath):
             try:
                 origfolder = os.path.basename(os.path.normpath(r))
             except:
-                origfolder = ''   
+                origfolder = ''
 
     values = {  '$Artist':  artist,
                 '$SortArtist': sortname,
@@ -662,13 +697,14 @@ def moveFiles(albumpath, release, tracks):
     lossy_media = False
     lossless_media = False
 
-    for r,d,f in os.walk(albumpath):
+    if find_in_path(albumpath,
-        for files in f:
+                    extra_formats=headphones.LOSSY_MEDIA_FORMATS,
-            files_to_move.append(os.path.join(r, files))
+                    use_MF=False):
-            if any(files.lower().endswith('.' + x.lower()) for x in headphones.LOSSY_MEDIA_FORMATS):
+        lossy_media = True
-                lossy_media = True
+    if find_in_path(albumpath,
-            if any(files.lower().endswith('.' + x.lower()) for x in headphones.LOSSLESS_MEDIA_FORMATS):
+                    extra_formats=headphones.LOSSLESS_MEDIA_FORMATS,
-                lossless_media = True
+                    use_MF=False):
+        lossless_media = True
 
     # Do some sanity checking to see what directories we need to create:
     make_lossy_folder = False

headphones/sab.py

@@ -62,7 +62,7 @@ def sendNZB(nzb):
         # Sanitize the file a bit, since we can only use ascii chars with MultiPartPostHandler
         nzbdata = helpers.latinToAscii(nzb.extraInfo[0])
         params['mode'] = 'addfile'
-        multiPartParams = {"nzbfile": (nzb.name+".nzb", nzbdata)}
+        multiPartParams = {"nzbfile": (helpers.latinToAscii(nzb.name)+".nzb", nzbdata)}
 
     if not headphones.SAB_HOST.startswith('http'):
         headphones.SAB_HOST = 'http://' + headphones.SAB_HOST

headphones/torrentfinished.py

@@ -15,7 +15,7 @@
 
 import threading
 import headphones
-from headphones import db, utorrent, transmission, logger
+from headphones import db, utorrent, transmission
 
 postprocessor_lock = threading.Lock()
 

headphones/utorrent.py

@@ -180,7 +180,7 @@ def setSeedRatio(hash, ratio):
         uTorrentClient.setprops(hash,'seed_ratio', ratio * 10)
     else:
         # TODO passing -1 should be unlimited
-        uTorrentClient.setprops(hash,'seed_ratio', -1.00)
+        uTorrentClient.setprops(hash,'seed_ratio', -10)
 
 def dirTorrent(hash, cacheid=None, return_name=None):
 
@@ -214,8 +214,8 @@ def addTorrent(link, hash):
     # Get Active Directory from settings
     active_dir, completed_dir = getSettingsDirectories()
 
-    if not active_dir or not completed_dir:
+    if not active_dir:
-        logger.error('Could not get "Put new downloads in:" or "Move completed downloads to:" directories from uTorrent settings, please ensure they are set')
+        logger.error('Could not get "Put new downloads in:" directory from uTorrent settings, please ensure it is set')
         return None
 
     uTorrentClient.add_url(link)

headphones/webserve.py

@@ -34,6 +34,7 @@ from headphones import logger, searcher, db, importer, mb, lastfm, librarysync,
 from headphones.helpers import checked, radio,today, cleanName
 
 import lib.simplejson as simplejson
+from lib.simplejson import OrderedDict
 
 import sys
 
@@ -84,18 +85,18 @@ class WebInterface(object):
         if not artist:
             raise cherrypy.HTTPRedirect("home")
 
-        # Serve the extras up as a dict to make things easier for new templates
+        # Serve the extras up as a dict to make things easier for new templates (append new extras to the end)
-        extras_list = ["single", "ep", "compilation", "soundtrack", "live", "remix", "djmix", "mixtape_street", "spokenword", "audiobook", "broadcast", "interview", "other"]
+        extras_list = ["single", "ep", "compilation", "soundtrack", "live", "remix", "spokenword", "audiobook", "other", "djmix", "mixtape_street", "broadcast", "interview"]
-        extras_dict = {}
+        if artist['Extras']:
-
+            artist_extras = map(int, artist['Extras'].split(','))
-        if not artist['Extras']:
-            artist_extras = ""
         else:
-            artist_extras = artist['Extras']
+            artist_extras = []
+
+        extras_dict = OrderedDict()
 
         i = 1
         for extra in extras_list:
-            if str(i) in artist_extras:
+            if i in artist_extras:
                 extras_dict[extra] = "checked"
             else:
                 extras_dict[extra] = ""
@@ -161,7 +162,7 @@ class WebInterface(object):
             temp_extras_list = []
             # TODO: Put these extras as a global variable
             i = 1
-            for extra in ["single", "ep", "compilation", "soundtrack", "live", "remix", "djmix", "mixtape_street", "spokenword", "audiobook", "broadcast", "interview", "other"]:
+            for extra in ["single", "ep", "compilation", "soundtrack", "live", "remix", "spokenword", "audiobook", "other", "djmix", "mixtape_street", "broadcast", "interview"]:
                 if extra in kwargs:
                     temp_extras_list.append(i)
                 i += 1
@@ -299,8 +300,8 @@ class WebInterface(object):
         if action == "ignore":
             myDB = db.DBConnection()
             for artist in args:
-                myDB.action('DELETE FROM newartists WHERE ArtistName=?', [artist])
+                myDB.action('DELETE FROM newartists WHERE ArtistName=?', [artist.decode(headphones.SYS_ENCODING, 'replace')])
-                myDB.action('UPDATE have SET Matched="Ignored" WHERE ArtistName=?', [artist])
+                myDB.action('UPDATE have SET Matched="Ignored" WHERE ArtistName=?', [artist.decode(headphones.SYS_ENCODING, 'replace')])
                 logger.info("Artist %s removed from new artist list and set to ignored" % artist)
         raise cherrypy.HTTPRedirect("home")
     addArtists.exposed = True
@@ -378,6 +379,12 @@ class WebInterface(object):
     def deleteAlbum(self, AlbumID, ArtistID=None):
         logger.info(u"Deleting all traces of album: " + AlbumID)
         myDB = db.DBConnection()
+
+        myDB.action('DELETE from have WHERE Matched=?', [AlbumID])
+        album = myDB.action('SELECT ArtistName, AlbumTitle from albums where AlbumID=?', [AlbumID]).fetchone()
+        if album:
+            myDB.action('DELETE from have WHERE ArtistName=? AND AlbumTitle=?', [album['ArtistName'], album['AlbumTitle']])
+
         myDB.action('DELETE from albums WHERE AlbumID=?', [AlbumID])
         myDB.action('DELETE from tracks WHERE AlbumID=?', [AlbumID])
         myDB.action('DELETE from allalbums WHERE AlbumID=?', [AlbumID])
@@ -460,8 +467,9 @@ class WebInterface(object):
                     have_album_dictionary.append(have_dict)
         headphones_albums = myDB.select('SELECT ArtistName, AlbumTitle from albums ORDER BY ArtistName')
         for albums in headphones_albums:
-            headphones_dict = { 'ArtistName' : albums['ArtistName'], 'AlbumTitle' : albums['AlbumTitle'] }
+            if albums['ArtistName'] and albums['AlbumTitle']:
-            headphones_album_dictionary.append(headphones_dict)
+                headphones_dict = { 'ArtistName' : albums['ArtistName'], 'AlbumTitle' : albums['AlbumTitle'] }
+                headphones_album_dictionary.append(headphones_dict)
         #unmatchedalbums = [f for f in have_album_dictionary if f not in [x for x in headphones_album_dictionary]]
 
         check = set([(cleanName(d['ArtistName']).lower(), cleanName(d['AlbumTitle']).lower()) for d in headphones_album_dictionary])
@@ -1143,13 +1151,18 @@ class WebInterface(object):
                     "mpc_enabled": checked(headphones.MPC_ENABLED)
                 }
 
-        # Need to convert EXTRAS to a dictionary we can pass to the config: it'll come in as a string like 2,5,6,8
+        # Need to convert EXTRAS to a dictionary we can pass to the config: it'll come in as a string like 2,5,6,8 (append new extras to the end)
-        extras_list = ["single", "ep", "compilation", "soundtrack", "live", "remix", "dj-mix", "mixtape/street", "spokenword", "audiobook", "broadcast", "interview", "other"]
+        extras_list = ["single", "ep", "compilation", "soundtrack", "live", "remix", "spokenword", "audiobook", "other", "djmix", "mixtape_street", "broadcast", "interview"]
-        extras_dict = {}
+        if headphones.EXTRAS:
+            extras = map(int, headphones.EXTRAS.split(','))
+        else:
+            extras = []
+
+        extras_dict = OrderedDict()
 
         i = 1
         for extra in extras_list:
-            if str(i) in headphones.EXTRAS:
+            if i in extras:
                 extras_dict[extra] = "checked"
             else:
                 extras_dict[extra] = ""
@@ -1168,8 +1181,8 @@ class WebInterface(object):
         preferred_words=None, required_words=None, ignored_words=None, preferred_quality=0, preferred_bitrate=None, detect_bitrate=0, move_files=0, torrentblackhole_dir=None, download_torrent_dir=None,
         numberofseeders=None, use_piratebay=0, piratebay_proxy_url=None, piratebay_ratio=None, use_kat=0, kat_proxy_url=None, kat_ratio=None, use_mininova=0, mininova_ratio=None, waffles=0, waffles_uid=None, waffles_passkey=None, waffles_ratio=None, whatcd=0, whatcd_username=None, whatcd_password=None, whatcd_ratio=None,
         rutracker=0, rutracker_user=None, rutracker_password=None, rutracker_ratio=None, rename_files=0, correct_metadata=0, cleanup_files=0, keep_nfo=0, add_album_art=0, album_art_format=None, embed_album_art=0, embed_lyrics=0, replace_existing_folders=False,
-        destination_dir=None, lossless_destination_dir=None, folder_format=None, file_format=None, file_underscores=0, include_extras=0, single=0, ep=0, compilation=0, soundtrack=0, live=0,
+        destination_dir=None, lossless_destination_dir=None, folder_format=None, file_format=None, file_underscores=0, include_extras=0, single=0, ep=0, compilation=0, soundtrack=0, live=0, remix=0, spokenword=0, audiobook=0, other=0, djmix=0, mixtape_street=0, broadcast=0, interview=0,
-        remix=0, djmix=0, mixtape_street=0, broadcast=0, interview=0, spokenword=0, audiobook=0, other=0, autowant_upcoming=False, autowant_all=False, keep_torrent_files=False, prefer_torrents=0, open_magnet_links=0, interface=None, log_dir=None, cache_dir=None, music_encoder=0, encoder=None, xldprofile=None,
+        autowant_upcoming=False, autowant_all=False, keep_torrent_files=False, prefer_torrents=0, open_magnet_links=0, interface=None, log_dir=None, cache_dir=None, music_encoder=0, encoder=None, xldprofile=None,
         bitrate=None, samplingfrequency=None, encoderfolder=None, advancedencoder=None, encoderoutputformat=None, encodervbrcbr=None, encoderquality=None, encoderlossless=0,
         delete_lossless_files=0, growl_enabled=0, growl_onsnatch=0, growl_host=None, growl_password=None, prowl_enabled=0, prowl_onsnatch=0, prowl_keys=None, prowl_priority=0, xbmc_enabled=0, xbmc_host=None, xbmc_username=None, xbmc_password=None,
         xbmc_update=0, xbmc_notify=0, nma_enabled=False, nma_apikey=None, nma_priority=0, nma_onsnatch=0, pushalot_enabled=False, pushalot_apikey=None, pushalot_onsnatch=0, synoindex_enabled=False, lms_enabled=0, lms_host=None,
@@ -1382,9 +1395,9 @@ class WebInterface(object):
 
                 headphones.EXTRA_NEWZNABS.append((newznab_host, newznab_api, newznab_enabled))
 
-        # Convert the extras to list then string. Coming in as 0 or 1
+        # Convert the extras to list then string. Coming in as 0 or 1 (append new extras to the end)
         temp_extras_list = []
-        extras_list = [single, ep, compilation, soundtrack, live, remix, djmix, mixtape_street, spokenword, audiobook, broadcast, interview, other]
+        extras_list = [single, ep, compilation, soundtrack, live, remix, spokenword, audiobook, other, djmix, mixtape_street, broadcast, interview]
 
         i = 1
         for extra in extras_list: