Post Processing! Very beta, but improvements coming soon. Also: specify log dir, automatically add extras for artists, release group selection process improvements

headphones/__init__.py

@@ -11,7 +11,7 @@ from lib.configobj import ConfigObj
 
 import cherrypy
 
-from headphones import updater, searcher, importer, versioncheck, logger
+from headphones import updater, searcher, importer, versioncheck, logger, postprocessor
 
 FULL_PATH = None
 PROG_DIR = None
@@ -52,13 +52,14 @@ LATEST_VERSION = None
 COMMITS_BEHIND = None
 
 MUSIC_DIR = None
+DESTINATION_DIR = None
 FOLDER_FORMAT = None
 FILE_FORMAT = None
 PATH_TO_XML = None
 PREFERRED_QUALITY = None
 PREFERRED_BITRATE = None
 DETECT_BITRATE = False
-FLAC_TO_MP3 = False
+CORRECT_METADATA = False
 MOVE_FILES = False
 RENAME_FILES = False
 CLEANUP_FILES = False
@@ -67,6 +68,7 @@ DOWNLOAD_DIR = None
 BLACKHOLE = None
 BLACKHOLE_DIR = None
 USENET_RETENTION = None
+INCLUDE_EXTRAS = False
 
 NZB_SEARCH_INTERVAL = 360
 LIBRARYSCAN_INTERVAL = 60
@@ -141,8 +143,8 @@ def initialize():
 	
 		global __INITIALIZED__, FULL_PATH, PROG_DIR, QUIET, DAEMON, DATA_DIR, CONFIG_FILE, CFG, LOG_DIR, CACHE_DIR, \
 				HTTP_PORT, HTTP_HOST, HTTP_USERNAME, HTTP_PASSWORD, HTTP_ROOT, LAUNCH_BROWSER, GIT_PATH, \
-				CURRENT_VERSION, LATEST_VERSION, MUSIC_DIR, PREFERRED_QUALITY, PREFERRED_BITRATE, DETECT_BITRATE, \
+				CURRENT_VERSION, LATEST_VERSION, MUSIC_DIR, DESTINATION_DIR, PREFERRED_QUALITY, PREFERRED_BITRATE, DETECT_BITRATE, \
-				FLAC_TO_MP3, MOVE_FILES, RENAME_FILES, FOLDER_FORMAT, FILE_FORMAT, CLEANUP_FILES, \
+				CORRECT_METADATA, MOVE_FILES, RENAME_FILES, FOLDER_FORMAT, FILE_FORMAT, CLEANUP_FILES, INCLUDE_EXTRAS, \
 				ADD_ALBUM_ART, DOWNLOAD_DIR, BLACKHOLE, BLACKHOLE_DIR, USENET_RETENTION, NZB_SEARCH_INTERVAL, \
 				LIBRARYSCAN_INTERVAL, SAB_HOST, SAB_USERNAME, SAB_PASSWORD, SAB_APIKEY, SAB_CATEGORY, \
 				NZBMATRIX, NZBMATRIX_USERNAME, NZBMATRIX_APIKEY, NEWZNAB, NEWZNAB_HOST, NEWZNAB_APIKEY, \
@@ -173,22 +175,25 @@ def initialize():
 		HTTP_ROOT = check_setting_str(CFG, 'General', 'http_root', '/')
 		LAUNCH_BROWSER = bool(check_setting_int(CFG, 'General', 'launch_browser', 1))
 		GIT_PATH = check_setting_str(CFG, 'General', 'git_path', '')
+		LOG_DIR = check_setting_str(CFG, 'General', 'log_dir', '')
 		
 		MUSIC_DIR = check_setting_str(CFG, 'General', 'music_dir', '')
+		DESTINATION_DIR = check_setting_str(CFG, 'General', 'destination_dir', '')
 		PREFERRED_QUALITY = check_setting_int(CFG, 'General', 'preferred_quality', 0)
 		PREFERRED_BITRATE = check_setting_int(CFG, 'General', 'preferred_bitrate', '')
 		DETECT_BITRATE = bool(check_setting_int(CFG, 'General', 'detect_bitrate', 0))
-		FLAC_TO_MP3 = bool(check_setting_int(CFG, 'General', 'flac_to_mp3', 0))
+		CORRECT_METADATA = bool(check_setting_int(CFG, 'General', 'correct_metadata', 0))
 		MOVE_FILES = bool(check_setting_int(CFG, 'General', 'move_files', 0))
 		RENAME_FILES = bool(check_setting_int(CFG, 'General', 'rename_files', 0))
-		FOLDER_FORMAT = check_setting_str(CFG, 'General', 'folder_format', '%artist/%album/%track')
+		FOLDER_FORMAT = check_setting_str(CFG, 'General', 'folder_format', 'artist/album [year]')
-		FILE_FORMAT = check_setting_str(CFG, 'General', 'file_format', '%tracknumber %artist - %album - %title')
+		FILE_FORMAT = check_setting_str(CFG, 'General', 'file_format', 'tracknumber artist - album [year]- title')
 		CLEANUP_FILES = bool(check_setting_int(CFG, 'General', 'cleanup_files', 0))
 		ADD_ALBUM_ART = bool(check_setting_int(CFG, 'General', 'add_album_art', 0))
 		DOWNLOAD_DIR = check_setting_str(CFG, 'General', 'download_dir', '')
 		BLACKHOLE = bool(check_setting_int(CFG, 'General', 'blackhole', 0))
 		BLACKHOLE_DIR = check_setting_str(CFG, 'General', 'blackhole_dir', '')
 		USENET_RETENTION = check_setting_int(CFG, 'General', 'usenet_retention', '')
+		INCLUDE_EXTRAS = bool(check_setting_int(CFG, 'General', 'include_extras', 0))
 		
 		NZB_SEARCH_INTERVAL = check_setting_int(CFG, 'General', 'nzb_search_interval', 360)
 		LIBRARYSCAN_INTERVAL = check_setting_int(CFG, 'General', 'libraryscan_interval', 180)
@@ -211,8 +216,9 @@ def initialize():
 		NZBSORG_UID = check_setting_str(CFG, 'NZBsorg', 'nzbsorg_uid', '')
 		NZBSORG_HASH = check_setting_str(CFG, 'NZBsorg', 'nzbsorg_hash', '')
 	
-		# Put the log dir in the data dir for now
+		if not LOG_DIR:
-		LOG_DIR = os.path.join(DATA_DIR, 'logs')
+			LOG_DIR = os.path.join(DATA_DIR, 'logs')
+		
 		if not os.path.exists(LOG_DIR):
 			try:
 				os.makedirs(LOG_DIR)
@@ -223,6 +229,11 @@ def initialize():
 		# Start the logger, silence console logging if we need to
 		logger.headphones_log.initLogger(quiet=QUIET)
 		
+		# Update some old config code:
+		if FOLDER_FORMAT == '%artist/%album/%track':
+			FOLDER_FORMAT = 'artist/album [year]'
+		if FILE_FORMAT == '%tracknumber %artist - %album - %title':
+			FILE_FORMAT = 'tracknumber artist - album - title'
 		
 		# Put the cache dir in the data dir for now
 		CACHE_DIR = os.path.join(DATA_DIR, 'cache')
@@ -232,8 +243,6 @@ def initialize():
 			except OSError:
 				logger.error('Could not create cache dir. Check permissions of datadir: ' + DATA_DIR)
 		
-		
-		
 		# Initialize the database
 		logger.info('Checking to see if the database has all tables....')
 		try:
@@ -318,13 +327,15 @@ def config_write():
 	new_config['General']['http_password'] = HTTP_PASSWORD
 	new_config['General']['http_root'] = HTTP_ROOT
 	new_config['General']['launch_browser'] = int(LAUNCH_BROWSER)
+	new_config['General']['log_dir'] = LOG_DIR
 	new_config['General']['git_path'] = GIT_PATH
 
 	new_config['General']['music_dir'] = MUSIC_DIR
+	new_config['General']['destination_dir'] = DESTINATION_DIR
 	new_config['General']['preferred_quality'] = PREFERRED_QUALITY
 	new_config['General']['preferred_bitrate'] = PREFERRED_BITRATE
 	new_config['General']['detect_bitrate'] = int(DETECT_BITRATE)
-	new_config['General']['flac_to_mp3'] = int(FLAC_TO_MP3)
+	new_config['General']['correct_metadata'] = int(CORRECT_METADATA)
 	new_config['General']['move_files'] = int(MOVE_FILES)
 	new_config['General']['rename_files'] = int(RENAME_FILES)
 	new_config['General']['folder_format'] = FOLDER_FORMAT
@@ -335,6 +346,7 @@ def config_write():
 	new_config['General']['blackhole'] = int(BLACKHOLE)
 	new_config['General']['blackhole_dir'] = BLACKHOLE_DIR
 	new_config['General']['usenet_retention'] = USENET_RETENTION
+	new_config['General']['include_extras'] = int(INCLUDE_EXTRAS)
 	
 	new_config['General']['nzb_search_interval'] = NZB_SEARCH_INTERVAL
 	new_config['General']['libraryscan_interval'] = LIBRARYSCAN_INTERVAL
@@ -376,6 +388,8 @@ def start():
 		SCHED.add_interval_job(searcher.searchNZB, minutes=NZB_SEARCH_INTERVAL)
 		SCHED.add_interval_job(importer.scanMusic, minutes=LIBRARYSCAN_INTERVAL)
 		SCHED.add_interval_job(versioncheck.checkGithub, minutes=300)
+		SCHED.add_interval_job(postprocessor.checkFolder, minutes=5)
+		postprocessor.checkFolder()
 
 		SCHED.start()
 		
@@ -388,7 +402,7 @@ def dbcheck():
 	c.execute('CREATE TABLE IF NOT EXISTS artists (ArtistID TEXT UNIQUE, ArtistName TEXT, ArtistSortName TEXT, DateAdded TEXT, Status TEXT, IncludeExtras INTEGER, LatestAlbum TEXT, ReleaseDate TEXT, AlbumID TEXT, HaveTracks INTEGER, TotalTracks INTEGER)')
 	c.execute('CREATE TABLE IF NOT EXISTS albums (ArtistID TEXT, ArtistName TEXT, AlbumTitle TEXT, AlbumASIN TEXT, ReleaseDate TEXT, DateAdded TEXT, AlbumID TEXT UNIQUE, Status TEXT, Type TEXT)')
 	c.execute('CREATE TABLE IF NOT EXISTS tracks (ArtistID TEXT, ArtistName TEXT, AlbumTitle TEXT, AlbumASIN TEXT, AlbumID TEXT, TrackTitle TEXT, TrackDuration, TrackID TEXT)')
-	c.execute('CREATE TABLE IF NOT EXISTS snatched (AlbumID TEXT, Title TEXT, Size INTEGER, URL TEXT, DateAdded TEXT, Status TEXT)')
+	c.execute('CREATE TABLE IF NOT EXISTS snatched (AlbumID TEXT, Title TEXT, Size INTEGER, URL TEXT, DateAdded TEXT, Status TEXT, FolderName TEXT)')
 	c.execute('CREATE TABLE IF NOT EXISTS have (ArtistName TEXT, AlbumTitle TEXT, TrackNumber TEXT, TrackTitle TEXT, TrackLength TEXT, BitRate TEXT, Genre TEXT, Date TEXT, TrackID TEXT)')
 	
 	try:
@@ -425,6 +439,11 @@ def dbcheck():
 		c.execute('SELECT Type from albums')
 	except sqlite3.OperationalError:
 		c.execute('ALTER TABLE albums ADD COLUMN Type TEXT DEFAULT "Album"')
+		
+	try:
+		c.execute('SELECT FolderName from snatched')
+	except sqlite3.OperationalError:
+		c.execute('ALTER TABLE snatched ADD COLUMN FolderName TEXT')	
 	
 	conn.commit()
 	c.close()

headphones/albumart.py

@@ -0,0 +1,10 @@
+from headphones import db
+
+def getAlbumArt(albumid):
+
+	myDB = db.DBConnection()
+	asin = myDB.action('SELECT AlbumASIN from albums WHERE AlbumID=?', [albumid]).fetchone()[0]
+	
+	url = 'http://ec1.images-amazon.com/images/P/%s.01.LZZZZZZZ.jpg' % asin
+	
+	return url

headphones/importer.py

@@ -4,7 +4,7 @@ import os
 from lib.beets.mediafile import MediaFile
 
 import headphones
-from headphones import logger, helpers, db, mb
+from headphones import logger, helpers, db, mb, albumart
 
 various_artists_mbid = '89ad4ac3-39f7-470e-963a-56509c546377'
 
@@ -158,6 +158,9 @@ def addArtisttoDB(artistid, extrasonly=False):
 					"DateAdded": 		helpers.today(),
 					"Status": 			"Loading"}
 	
+	if headphones.INCLUDE_EXTRAS:
+		newValueDict['IncludeExtras'] = 1
+	
 	myDB.upsert("artists", newValueDict, controlValueDict)
 
 	for rg in artist['releasegroups']:
@@ -170,7 +173,8 @@ def addArtisttoDB(artistid, extrasonly=False):
 		try:	
 			release_dict = mb.getReleaseGroup(rgid)
 		except Exception, e:
-			logger.info('Unable to get release information for %s - it may not be a valid release group' % rg['title'])
+			logger.info('Unable to get release information for %s - it may not be a valid release group \
+				(or it might just not be tagged right in MusicBrainz)' % rg['title'])
 			continue
 			
 		if not release_dict:
@@ -187,7 +191,7 @@ def addArtisttoDB(artistid, extrasonly=False):
 		
 		if len(rg_exists):
 		
-			newValueDict = {"AlbumASIN":		release['asin'],
+			newValueDict = {"AlbumASIN":		release_dict['asin'],
 							"ReleaseDate":		release_dict['releasedate'],
 							}
 		
@@ -196,7 +200,7 @@ def addArtisttoDB(artistid, extrasonly=False):
 			newValueDict = {"ArtistID":			artistid,
 							"ArtistName": 		artist['artist_name'],
 							"AlbumTitle":		rg['title'],
-							"AlbumASIN":		release['asin'],
+							"AlbumASIN":		release_dict['asin'],
 							"ReleaseDate":		release_dict['releasedate'],
 							"DateAdded":		helpers.today(),
 							"Type":				rg['type']
@@ -219,7 +223,7 @@ def addArtisttoDB(artistid, extrasonly=False):
 			newValueDict = {"ArtistID":		artistid,
 						"ArtistName": 		artist['artist_name'],
 						"AlbumTitle":		rg['title'],
-						"AlbumASIN":		release['asin'],
+						"AlbumASIN":		release_dict['asin'],
 						"TrackTitle":		track['title'],
 						"TrackDuration":	track['duration'],
 						}

headphones/mb.py

@@ -9,8 +9,9 @@ import lib.musicbrainz2.utils as u
 
 from lib.musicbrainz2.webservice import WebServiceError
 
+import headphones
 from headphones import logger, db
-from headphones.helpers import multikeysort
+from headphones.helpers import multikeysort, replace_all
 
 q = ws.Query()
 mb_lock = threading.Lock()
@@ -83,7 +84,7 @@ def getArtist(artistid, extrasonly=False):
 				artist = q.getArtistById(artistid, inc)
 				break
 			except WebServiceError, e:
-				logger.warn('Attempt to retrieve artist information from MusicBrainz failed for artistid: %s. Sleeping 10 seconds' % artistid)
+				logger.warn('Attempt to retrieve artist information from MusicBrainz failed for artistid: %s. Sleeping 5 seconds' % artistid)
 				attempt += 1
 				time.sleep(5)
 				
@@ -120,7 +121,7 @@ def getArtist(artistid, extrasonly=False):
 		except IndexError:
 			includeExtras = False
 		
-		if includeExtras:
+		if includeExtras or headphones.INCLUDE_EXTRAS:
 			includes = [m.Release.TYPE_COMPILATION, m.Release.TYPE_REMIX, m.Release.TYPE_SINGLE, m.Release.TYPE_LIVE, m.Release.TYPE_EP]
 			for include in includes:
 				inc = ws.ArtistIncludes(releases=(m.Release.TYPE_OFFICIAL, include), releaseGroups=True)
@@ -134,7 +135,7 @@ def getArtist(artistid, extrasonly=False):
 						artist = q.getArtistById(artistid, inc)
 						break
 					except WebServiceError, e:
-						logger.warn('Attempt to retrieve artist information from MusicBrainz failed for artistid: %s. Sleeping 10 seconds' % artistid)
+						logger.warn('Attempt to retrieve artist information from MusicBrainz failed for artistid: %s. Sleeping 5 seconds' % artistid)
 						attempt += 1
 						time.sleep(5)
 						
@@ -161,6 +162,7 @@ def getReleaseGroup(rgid):
 	with mb_lock:
 	
 		releaselist = []
+		asinlist = []
 		
 		inc = ws.ReleaseGroupIncludes(releases=True)
 		releaseGroup = None
@@ -172,7 +174,7 @@ def getReleaseGroup(rgid):
 				releaseGroup = q.getReleaseGroupById(rgid, inc)
 				break
 			except WebServiceError, e:
-				logger.warn('Attempt to retrieve information from MusicBrainz for release group "%s" failed. Sleeping 10 seconds' % rgid)
+				logger.warn('Attempt to retrieve information from MusicBrainz for release group "%s" failed. Sleeping 5 seconds' % rgid)
 				attempt += 1
 				time.sleep(5)
 	
@@ -194,28 +196,66 @@ def getReleaseGroup(rgid):
 					releaseResult = q.getReleaseById(release.id, inc)
 					break
 				except WebServiceError, e:
-					logger.warn('Attempt to retrieve release information for %s from MusicBrainz failed: %s. Sleeping 10 seconds' % (releaseResult.title, e))
+					logger.warn('Attempt to retrieve release information for %s from MusicBrainz failed: %s. Sleeping 5 seconds' % (releaseResult.title, e))
 					attempt += 1
 					time.sleep(5)		
 			
 			if not releaseResult:
 				continue
 				
+			if releaseResult.title.lower() != releaseGroup.title.lower():
+				continue
+				
 			time.sleep(1)
 			
+			formats = {
+				'2xVinyl':			'2',
+				'Vinyl':			'2',
+				'CD':				'0',
+				'Cassette':			'3',			
+				'2xCD':				'1',
+				'Digital Media':	'0'
+				}
+				
+			country = {
+				'US':	'0',
+				'GB':	'1',
+				'JP':	'1',
+				}
+
+			
+			try:
+				format = int(replace_all(u.extractFragment(releaseResult.releaseEvents[0].format), formats))
+			except:
+				format = 3
+				
+			try:
+				country = int(replace_all(releaseResult.releaseEvents[0].country, country))
+			except:
+				country = 2
+			
 			release_dict = {
-				'asin':			bool(releaseResult.asin),
+				'hasasin':		bool(releaseResult.asin),
+				'asin':			releaseResult.asin,
 				'tracks':		len(releaseResult.getTracks()),
 				'releaseid':	u.extractUuid(releaseResult.id),
-				'releasedate':	releaseResult.getEarliestReleaseDate()
+				'releasedate':	releaseResult.getEarliestReleaseDate(),
+				'format':		format,
+				'country':		country
 				}
-			
+				
 			releaselist.append(release_dict)
+			
+			if releaseResult.asin:
+				asinlist.append(releaseResult.asin)
 	
-		a = multikeysort(releaselist, ['-asin', '-tracks'])
+		a = multikeysort(releaselist, ['-hasasin', 'country', 'format', 'tracks'])
 		
 		release_dict = {'releaseid' :a[0]['releaseid'],
-						'releasedate'	: releaselist[0]['releasedate']}
+						'releasedate'	: releaselist[0]['releasedate'],
+						'trackcount'	: a[0]['tracks'],
+						'asin'	:	a[0]['asin']
+						}
 		
 		return release_dict
 	
@@ -237,7 +277,7 @@ def getRelease(releaseid):
 				results = q.getReleaseById(releaseid, inc)
 				break
 			except WebServiceError, e:
-				logger.warn('Attempt to retrieve information from MusicBrainz for release "%s" failed: %s. SLeeping 10 seconds' % (releaseid, e))
+				logger.warn('Attempt to retrieve information from MusicBrainz for release "%s" failed: %s. SLeeping 5 seconds' % (releaseid, e))
 				attempt += 1
 				time.sleep(5)	
 		
@@ -286,7 +326,7 @@ def findArtistbyAlbum(name):
 			results = q.getReleaseGroups(f)
 			break
 		except WebServiceError, e:
-			logger.warn('Attempt to query MusicBrainz for %s failed: %s. Sleeping 10 seconds.' % (name, e))
+			logger.warn('Attempt to query MusicBrainz for %s failed: %s. Sleeping 5 seconds.' % (name, e))
 			attempt += 1
 			time.sleep(5)	
 	

headphones/postprocessor.py

@@ -0,0 +1,190 @@
+import os
+import time
+
+import urllib, shutil
+
+from lib.beets.mediafile import MediaFile
+import lib.musicbrainz2.webservice as ws
+
+import headphones
+from headphones import db, albumart, logger, helpers
+
+def checkFolder():
+
+	myDB = db.DBConnection()
+	snatched = myDB.select('SELECT * from snatched WHERE Status="Snatched" or Status="Unprocessed"')
+
+	for album in snatched:
+		
+		if album['FolderName']:
+		
+			album_path = os.path.join(headphones.DOWNLOAD_DIR, album['FolderName'])
+
+			if os.path.exists(album_path):
+				verify(album['AlbumID'], album_path)
+
+def verify(albumid, albumpath):
+
+	myDB = db.DBConnection()
+	release = myDB.action('SELECT * from albums WHERE AlbumID=?', [albumid]).fetchone()
+	tracks = myDB.select('SELECT * from tracks WHERE AlbumID=?', [albumid])
+	
+	downloaded_track_list = []
+	for r,d,f in os.walk(albumpath):
+		for files in f:
+			if any(files.endswith(x) for x in (".mp3", ".flac", ".aac", ".ogg", ".ape")):
+				downloaded_track_list.append(os.path.join(r, files))
+
+	# test #1: filenames
+	
+	
+	# test #2: metadata. 
+	for downloaded_track in downloaded_track_list:
+		try:
+			f = MediaFile(downloaded_track)
+		except:
+			continue
+		if helpers.latinToAscii(f.artist.lower()).encode('UTF-8') == helpers.latinToAscii(release['ArtistName'].lower()).encode('UTF-8') and helpers.latinToAscii(f.album.lower()).encode('UTF-8') == helpers.latinToAscii(release['AlbumTitle'].lower()).encode('UTF-8'):
+			doPostProcessing(albumid, albumpath, release, tracks, downloaded_track_list)
+			return
+			
+	# test #3: number of songs and duration
+	db_track_duration = 0
+	downloaded_track_duration = 0
+	
+	for track in tracks:
+		try:
+			db_track_duration += track['TrackDuration']/1000
+		except:
+			downloaded_track_duration = False
+			break
+			
+	for downloaded_track in downloaded_track_list:
+		try:
+			f = MediaFile(downloaded_track)
+			downloaded_track_duration += f.length
+		except:
+			downloaded_track_duration = False
+			break
+		
+	if downloaded_track_duration and db_track_duration:
+		delta = abs(downloaded_track_duration - db_track_duration)
+		if len(tracks) == len(downloaded_track_list) and delta < 100:
+			doPostProcessing(albumid, albumpath, release, tracks, downloaded_track_list)
+			return
+			
+	logger.warn('Could not identify album: %s. It may not be the intended album.' % albumpath)
+	myDB.action('UPDATE snatched SET status = "Unprocessed" WHERE AlbumID=?', [albumid])
+			
+def doPostProcessing(albumid, albumpath, release, tracks, downloaded_track_list):
+
+	logger.info('Starting post-processing for: %s - %s' % (release['ArtistName'], release['AlbumTitle']))
+
+	if headphones.ADD_ALBUM_ART:
+		addAlbumArt(albumid, downloaded_track_list)
+	
+	if headphones.CLEANUP_FILES:
+		cleanupFiles(albumpath)
+		
+	if headphones.CORRECT_METADATA:
+		correctMetadata(albumid, release, downloaded_track_list)
+		
+	if headphones.RENAME_FILES:
+		renameFiles(albumpath, downloaded_track_list, release)
+	
+	if headphones.MOVE_FILES and headphones.DESTINATION_DIR:
+		moveFiles(albumpath, release, tracks)
+		
+	myDB = db.DBConnection()
+	myDB.action('UPDATE albums SET status = "Downloaded" WHERE AlbumID=?', [albumid])
+	myDB.action('UPDATE snatched SET status = "Processed" WHERE AlbumID=?', [albumid])
+	logger.info('Post-processing for %s - %s complete' % (release['ArtistName'], release['AlbumTitle']))
+	
+		
+def addAlbumArt(albumid, downloaded_track_list):
+
+	album_art_path = albumart.getAlbumArt(albumid)
+	
+	artwork = urllib.urlopen(album_art_path).read()
+	
+	for downloaded_track in downloaded_track_list:
+		try:
+			f = MediaFile(downloaded_track)
+		except:
+			continue
+			
+		f.art = artwork
+		f.save()
+	
+def cleanupFiles(albumpath):
+
+	for r,d,f in os.walk(albumpath):
+		for files in f:
+			if not any(files.endswith(x) for x in (".mp3", ".flac", ".aac", ".ogg", ".ape", ".m4a")):
+				os.remove(os.path.join(r, files))
+				
+def moveFiles(albumpath, release, tracks):
+
+	try:
+		year = release['ReleaseDate'][:4]
+	except TypeError:
+		year = ''
+
+	values = {	'artist':	release['ArtistName'],
+				'album':	release['AlbumTitle'],
+				'year':		year
+			}
+			
+	
+	folder = helpers.replace_all(headphones.FOLDER_FORMAT, values)
+	
+	destination_path = os.path.join(headphones.DESTINATION_DIR, folder)
+	
+	try:
+		os.makedirs(destination_path)
+	except Exception, e:
+		logger.error('Could not create folder for %s. Not moving' % release['AlbumName'])
+		return
+		
+	for r,d,f in os.walk(albumpath):
+		for files in f:
+			shutil.move(os.path.join(r, files), destination_path)
+			
+	try:
+		os.rmdir(albumpath)
+	except Exception, e:
+		logger.error('Could not remove directory: %s. %s' % (albumpath, e))
+		
+def correctMetadata(albumid, release, downloaded_track_list):
+	
+	pass
+
+def renameFiles(albumpath, downloaded_track_list, release):
+
+	try:
+		year = release['ReleaseDate'][:4]
+	except TypeError:
+		year = ''
+	# Until tagging works better I'm going to rely on the already provided metadata
+	for downloaded_track in downloaded_track_list:
+		try:
+			f = MediaFile(downloaded_track)
+		except:
+			continue
+			
+		tracknumber = '%02d' % f.track
+			
+		values = {	'tracknumber':	tracknumber,
+					'title':		f.title,
+					'artist':		release['ArtistName'],
+					'album':		release['AlbumTitle'],
+					'year':			year
+					}
+					
+		ext = os.path.splitext(downloaded_track)[1]
+		
+		new_file_name = helpers.replace_all(headphones.FILE_FORMAT, values).replace('/','_') + ext
+
+		new_file = os.path.join(albumpath, new_file_name)
+		
+		shutil.move(downloaded_track, new_file)

headphones/searcher.py

@@ -244,7 +244,9 @@ def searchNZB(albumid=None, new=False):
 			
 			
 			logger.info(u"Found best result: %s (%s) - %s" % (bestqual[0], bestqual[2], helpers.bytes_to_mb(bestqual[1])))
+			
 			downloadurl = bestqual[2]
+			nzb_folder_name = '%s - %s [%s]' % (helpers.latinToAscii(albums[0]).encode('UTF-8'), helpers.latinToAscii(albums[1]).encode('UTF-8'), year)
 
 			if headphones.SAB_HOST and not headphones.BLACKHOLE:
 				linkparams = {}
@@ -262,7 +264,7 @@ def searchNZB(albumid=None, new=False):
 								
 				linkparams["name"] = downloadurl
 
-				linkparams["nzbname"] = ('%s - %s [%s]' % (helpers.latinToAscii(albums[0]).encode('UTF-8'), helpers.latinToAscii(albums[1]).encode('UTF-8'), year))
+				linkparams["nzbname"] = nzb_folder_name
 					
 				saburl = 'http://' + headphones.SAB_HOST + '/sabnzbd/api?' + urllib.urlencode(linkparams)
 				logger.info(u"Sending link to SABNZBD: " + saburl)
@@ -275,12 +277,12 @@ def searchNZB(albumid=None, new=False):
 					break
 					
 				myDB.action('UPDATE albums SET status = "Snatched" WHERE AlbumID=?', [albums[2]])
-				myDB.action('INSERT INTO snatched VALUES( ?, ?, ?, ?, DATETIME("NOW", "localtime"), ?)', [albums[2], bestqual[0], bestqual[1], bestqual[2], "Snatched"])
+				myDB.action('INSERT INTO snatched VALUES( ?, ?, ?, ?, DATETIME("NOW", "localtime"), ?, ?)', [albums[2], bestqual[0], bestqual[1], bestqual[2], "Snatched", nzb_folder_name])
 
 			
 			elif headphones.BLACKHOLE:
 			
-				nzb_name = ('%s - %s [%s].nzb' % (albums[0], albums[1], year))
+				nzb_name = nzb_folder_name + '.nzb'
 				download_path = os.path.join(headphones.BLACKHOLE_DIR, nzb_name)
 				
 				try:
@@ -290,4 +292,4 @@ def searchNZB(albumid=None, new=False):
 					break
 					
 				myDB.action('UPDATE albums SET status = "Snatched" WHERE AlbumID=?', [albums[2]])
-				myDB.action('INSERT INTO snatched VALUES( ?, ?, ?, ?, DATETIME("NOW", "localtime"), ?)', [albums[2], bestqual[0], bestqual[1], bestqual[2], "Snatched"])
+				myDB.action('INSERT INTO snatched VALUES( ?, ?, ?, ?, DATETIME("NOW", "localtime"), ?, ?)', [albums[2], bestqual[0], bestqual[1], bestqual[2], "Snatched", nzb_folder_name])

headphones/templates.py

@@ -62,7 +62,8 @@ configform = form = '''
 		  	<a href="#web_interface" >Web Interface</a> |
 		  	<a href="#download" class="smalltext">Download Settings</a> |
 		  	<a href="#providers" class="smalltext">Search Providers</a> |
-		  	<a href="#post_processing" class="smalltext">Quality &amp; Post Processing</a>
+		  	<a href="#post_processing" class="smalltext">Quality &amp; Post Processing</a> |
+		  	<a href="#advanced_settings" class="smalltext">Advanced Settings</a>
 		  </div>
   	</center>
   <div class="table">
@@ -152,7 +153,7 @@ configform = form = '''
                     <p>Music Download Directory:</p><input type="text" name="download_dir" value="%s" size="60"><br>
 
                     <i class="smalltext">Full path to the directory where SAB downloads your music<br>
-                    i.e. Downloads/music or /Users/name/Downloads/music</i>
+                    i.e. /Users/name/Downloads/music</i>
                 </td>
             </tr>
             
@@ -277,10 +278,10 @@ configform = form = '''
                 <td>
                       <b>Post-Processing:</b>
                       <p>
-                      	<input type="checkbox" name="move_files" value="1" %s />Move downloads to Music Folder<br />
+                      	<input type="checkbox" name="move_files" value="1" %s />Move downloads to Destination Folder<br />
-                      	<input type="checkbox" name="flac_to_mp3" value="1" %s />Convert lossless to mp3<br>
+                      	<input type="checkbox" name="rename_files" value="1" %s />Rename files<br>
-                        <input type="checkbox" name="rename_files" value="1" %s />Rename &amp; add metadata<br>
+                        <input type="checkbox" name="correct_metadata" value="1" %s />Correct metadata<br>
-                        <input type="checkbox" name="cleanup_files" value="1" %s />Delete leftover files<br>
+                        <input type="checkbox" name="cleanup_files" value="1" %s />Delete leftover files (.m3u, .nfo, .sfv, .nzb, etc.)<br>
                       	<input type="checkbox" name="add_album_art" value="1" %s>Add album art
                       </p>
                 </td>
@@ -290,12 +291,42 @@ configform = form = '''
                 <td>
                     <br>
 
-                    <p><b>Path to Music folder</b>:<br><input type="text" name="music_dir" value="%s" size="60" maxlength="200">
+                    <p><b>Path to Destination folder</b>:<br><input type="text" name="destination_dir" value="%s" size="60">
                       <br>
                       <i class="smalltext">i.e. /Users/name/Music/iTunes or /Volumes/share/music</i>
                     </p>
                 </td>
             </tr>
+		</table>
+		
+        <a name="advanced_settings"><h1><u>Advanced Settings</u></h1></a>
+        
+        <table class="configtable" summary="Advanced Settings">
+            <tr>
+                <td>
+                	<b>Renaming Options:</b>
+                    <br>
+
+                    <p><b>Folder Format</b>:<br><input type="text" name="folder_format" value="%s" size="60">
+                      <br>
+                      <i class="smalltext">Use: artist, album and year</i>
+                    </p>
+                    
+                    <p><b>File Format</b>:<br><input type="text" name="file_format" value="%s" size="60">
+                      <br>
+                      <i class="smalltext">Use: tracknumber, title, artist, album and year</i>
+                    </p>
+                </td>
+                <td>
+                      <b>Miscellaneous:</b>
+                      <p>
+                      	<input type="checkbox" name="include_extras" value="1" %s />Automatically Include Extras When Adding an Artist<br />
+                      <i class="smalltext">Extras includes: EPs, Compilations, Live Albums, Remix Albums and Singles</i>
+                      </p>
+                    <p><b>Log Directory</b>:<br><input type="text" name="log_dir" value="%s" size="60">
+                    </p>
+                </td>
+            </tr>
 
         </table>
 
@@ -346,7 +377,7 @@ def displayAlbums(ArtistID, Type=None):
 		elif results[i][3] == 'Wanted':
 			newStatus = '''<b>%s</b>[<A class="external" href="unqueueAlbum?AlbumID=%s&ArtistID=%s">skip</a>]''' % (results[i][3], results[i][2], ArtistID)				
 		elif results[i][3] == 'Downloaded':
-			newStatus = '''<b>%s</b>[<A class="external" href="queueAlbum?AlbumID=%s&ArtistID=%s">retry</a>]''' % (results[i][3], results[i][2], ArtistID)
+			newStatus = '''<b>%s</b>[<A class="external" href="queueAlbum?AlbumID=%s&ArtistID=%s">retry</a>][<A class="external" href="queueAlbum?AlbumID=%s&ArtistID=%s&new=True">new</a>]''' % (results[i][3], results[i][2], ArtistID, results[i][2], ArtistID)
 		elif results[i][3] == 'Snatched':
 			newStatus = '''<b>%s</b>[<A class="external" href="queueAlbum?AlbumID=%s&ArtistID=%s">retry</a>][<A class="external" href="queueAlbum?AlbumID=%s&ArtistID=%s&new=True">new</a>]''' % (results[i][3], results[i][2], ArtistID, results[i][2], ArtistID)
 		else:

headphones/webserve.py

@@ -550,11 +550,15 @@ class WebInterface(object):
 		headphones.PREFERRED_BITRATE,
 		checked(headphones.DETECT_BITRATE),
 		checked(headphones.MOVE_FILES),
-		checked(headphones.FLAC_TO_MP3),
 		checked(headphones.RENAME_FILES),
+		checked(headphones.CORRECT_METADATA),
 		checked(headphones.CLEANUP_FILES),
 		checked(headphones.ADD_ALBUM_ART),
-		headphones.MUSIC_DIR
+		headphones.DESTINATION_DIR,
+		headphones.FOLDER_FORMAT,
+		headphones.FILE_FORMAT,
+		checked(headphones.INCLUDE_EXTRAS),
+		headphones.LOG_DIR
 		))
 		page.append(templates._footer % headphones.CURRENT_VERSION)
 		return page
@@ -565,7 +569,8 @@ class WebInterface(object):
 	def configUpdate(self, http_host='0.0.0.0', http_username=None, http_port=8181, http_password=None, launch_browser=0,
 		sab_host=None, sab_username=None, sab_apikey=None, sab_password=None, sab_category=None, download_dir=None, blackhole=0, blackhole_dir=None,
 		usenet_retention=None, nzbmatrix=0, nzbmatrix_username=None, nzbmatrix_apikey=None, newznab=0, newznab_host=None, newznab_apikey=None,
-		nzbsorg=0, nzbsorg_uid=None, nzbsorg_hash=None, preferred_quality=0, preferred_bitrate=None, detect_bitrate=0, flac_to_mp3=0, move_files=0, music_dir=None, rename_files=0, cleanup_files=0, add_album_art=0):
+		nzbsorg=0, nzbsorg_uid=None, nzbsorg_hash=None, preferred_quality=0, preferred_bitrate=None, detect_bitrate=0, move_files=0, 
+		rename_files=0, correct_metadata=0, cleanup_files=0, add_album_art=0, destination_dir=None, folder_format=None, file_format=None, include_extras=0, log_dir=None):
 		
 		headphones.HTTP_HOST = http_host
 		headphones.HTTP_PORT = http_port
@@ -593,12 +598,16 @@ class WebInterface(object):
 		headphones.PREFERRED_QUALITY = int(preferred_quality)
 		headphones.PREFERRED_BITRATE = preferred_bitrate
 		headphones.DETECT_BITRATE = detect_bitrate
-		headphones.FLAC_TO_MP3 = flac_to_mp3
 		headphones.MOVE_FILES = move_files
-		headphones.MUSIC_DIR = music_dir
+		headphones.CORRECT_METADATA = correct_metadata
 		headphones.RENAME_FILES = rename_files
 		headphones.CLEANUP_FILES = cleanup_files
 		headphones.ADD_ALBUM_ART = add_album_art
+		headphones.DESTINATION_DIR = destination_dir
+		headphones.FOLDER_FORMAT = folder_format
+		headphones.FILE_FORMAT = file_format
+		headphones.INCLUDE_EXTRAS = include_extras
+		headphones.LOG_DIR = log_dir
 		
 		headphones.config_write()