From b9e17c96efdf21aa885c9d61c9f98fa02cf16ec0 Mon Sep 17 00:00:00 2001 From: rembo10 Date: Mon, 10 Dec 2012 06:49:17 -0500 Subject: [PATCH 1/5] Possible fix for encoder commands not working (on windows only?) when there are spaces in the path to encoder --- headphones/music_encoder.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/headphones/music_encoder.py b/headphones/music_encoder.py index 1f03aca7..916c0335 100644 --- a/headphones/music_encoder.py +++ b/headphones/music_encoder.py @@ -156,7 +156,7 @@ def command(encoder,musicSource,musicDest,albumPath): if XLD: xldDestDir = os.path.split(musicDest)[0] - cmd = encoder + cmd = '"' + encoder + '"' cmd = cmd + ' "' + musicSource + '"' cmd = cmd + ' --profile' cmd = cmd + ' "' + xldProfile + '"' @@ -165,7 +165,7 @@ def command(encoder,musicSource,musicDest,albumPath): elif headphones.ENCODER == 'lame': if headphones.ADVANCEDENCODER =='': - cmd=encoder + ' -h' + cmd='"' + encoder + '"' + ' -h' if headphones.ENCODERVBRCBR=='cbr': cmd=cmd+ ' --resample ' + str(headphones.SAMPLINGFREQUENCY) + ' -b ' + str(headphones.BITRATE) elif headphones.ENCODERVBRCBR=='vbr': @@ -177,7 +177,7 @@ def command(encoder,musicSource,musicDest,albumPath): cmd=cmd+ ' "' + musicDest +'"' elif headphones.ENCODER == 'ffmpeg': - cmd=encoder+ ' -i' + cmd='"' + encoder + '"' + ' -i' cmd=cmd+ ' "' + musicSource + '"' if headphones.ADVANCEDENCODER =='': if headphones.ENCODEROUTPUTFORMAT=='ogg': From a060fc4b94ae0683496b2263b9afce88d64f90e8 Mon Sep 17 00:00:00 2001 From: rembo10 Date: Fri, 14 Dec 2012 07:12:37 -0500 Subject: [PATCH 2/5] Initial setup for nzbsrus, removed nzbmatrix and newzbin from config :-( --- data/interfaces/default/config.html | 40 +++++++++-------------------- headphones/__init__.py | 16 +++++++++++- headphones/webserve.py | 36 +++++++++++++++----------- 3 files changed, 48 insertions(+), 44 deletions(-) diff --git a/data/interfaces/default/config.html b/data/interfaces/default/config.html index 58baf946..29978480 100644 --- a/data/interfaces/default/config.html +++ b/data/interfaces/default/config.html @@ -184,23 +184,7 @@ - - + +
-
- NZBMatrix -
- -
-
-
- - -
-
- - -
-
-
- + NZBs
Newznab
@@ -254,9 +238,7 @@
- 		+
NZBs.org
@@ -270,21 +252,24 @@
- Newsbin + NZBsRus
- +
- - + +
- - + +
+
Torrents
@@ -1038,9 +1023,8 @@ $( "#tabs" ).tabs(); }); initActions(); - initConfigCheckbox("#usenzbmatrix"); initConfigCheckbox("#usenewznab"); - initConfigCheckbox("#usenewzbin"); + initConfigCheckbox("#usenzbsrus"); initConfigCheckbox("#usenzbsorg"); initConfigCheckbox("#usewaffles"); initConfigCheckbox("#userutracker"); diff --git a/headphones/__init__.py b/headphones/__init__.py index 9286835f..efbfe7e5 100644 --- a/headphones/__init__.py +++ b/headphones/__init__.py @@ -140,6 +140,10 @@ NEWZBIN = False NEWZBIN_UID = None NEWZBIN_PASSWORD = None +NZBSRUS = False +NZBSRUS_UID = None +NZBSRUS_APIKEY = None + LASTFM_USERNAME = None LOSSY_MEDIA_FORMATS = ["mp3", "aac", "ogg", "ape", "m4a"] @@ -268,7 +272,7 @@ def initialize(): RUTRACKER, RUTRACKER_USER, RUTRACKER_PASSWORD, WHATCD, WHATCD_USERNAME, WHATCD_PASSWORD, DOWNLOAD_TORRENT_DIR, \ LIBRARYSCAN, LIBRARYSCAN_INTERVAL, DOWNLOAD_SCAN_INTERVAL, SAB_HOST, SAB_USERNAME, SAB_PASSWORD, SAB_APIKEY, SAB_CATEGORY, \ NZBMATRIX, NZBMATRIX_USERNAME, NZBMATRIX_APIKEY, NEWZNAB, NEWZNAB_HOST, NEWZNAB_APIKEY, NEWZNAB_ENABLED, EXTRA_NEWZNABS,\ - NZBSORG, NZBSORG_UID, NZBSORG_HASH, NEWZBIN, NEWZBIN_UID, NEWZBIN_PASSWORD, LASTFM_USERNAME, INTERFACE, FOLDER_PERMISSIONS, \ + NZBSORG, NZBSORG_UID, NZBSORG_HASH, NEWZBIN, NEWZBIN_UID, NEWZBIN_PASSWORD, NZBSRUS, NZBSRUS_UID, NZBSRUS_APIKEY, LASTFM_USERNAME, INTERFACE, FOLDER_PERMISSIONS, \ ENCODERFOLDER, ENCODER_PATH, ENCODER, XLDPROFILE, BITRATE, SAMPLINGFREQUENCY, MUSIC_ENCODER, ADVANCEDENCODER, ENCODEROUTPUTFORMAT, ENCODERQUALITY, \ ENCODERVBRCBR, ENCODERLOSSLESS, DELETE_LOSSLESS_FILES, PROWL_ENABLED, PROWL_PRIORITY, PROWL_KEYS, PROWL_ONSNATCH, \ PUSHOVER_ENABLED, PUSHOVER_PRIORITY, PUSHOVER_KEYS, PUSHOVER_ONSNATCH, MIRRORLIST, \ @@ -285,6 +289,7 @@ def initialize(): CheckSection('NZBMatrix') CheckSection('Newznab') CheckSection('NZBsorg') + CheckSection('NZBsRus') CheckSection('Newzbin') CheckSection('Waffles') CheckSection('Rutracker') @@ -400,6 +405,10 @@ def initialize(): NEWZBIN = bool(check_setting_int(CFG, 'Newzbin', 'newzbin', 0)) NEWZBIN_UID = check_setting_str(CFG, 'Newzbin', 'newzbin_uid', '') NEWZBIN_PASSWORD = check_setting_str(CFG, 'Newzbin', 'newzbin_password', '') + + NZBSRUS = bool(check_setting_int(CFG, 'NZBsRus', 'nzbsrus', 0)) + NZBSRUS_UID = check_setting_str(CFG, 'NZBsRus', 'nzbsrus_uid', '') + NZBSRUS_APIKEY = check_setting_str(CFG, 'NZBsRus', 'nzbsrus_apikey', '') LASTFM_USERNAME = check_setting_str(CFG, 'General', 'lastfm_username', '') @@ -725,6 +734,11 @@ def config_write(): new_config['Newzbin']['newzbin_uid'] = NEWZBIN_UID new_config['Newzbin']['newzbin_password'] = NEWZBIN_PASSWORD + new_config['NZBsRus'] = {} + new_config['NZBsRus']['nzbsrus'] = int(NZBSRUS) + new_config['NZBsRus']['nzbsrus_uid'] = NZBSRUS_UID + new_config['NZBsRus']['nzbsrus_apikey'] = NZBSRUS_APIKEY + new_config['Prowl'] = {} new_config['Prowl']['prowl_enabled'] = int(PROWL_ENABLED) new_config['Prowl']['prowl_keys'] = PROWL_KEYS diff --git a/headphones/webserve.py b/headphones/webserve.py index f7b52f01..926e379a 100644 --- a/headphones/webserve.py +++ b/headphones/webserve.py @@ -568,9 +568,9 @@ class WebInterface(object): "use_blackhole" : checked(headphones.BLACKHOLE), "blackhole_dir" : headphones.BLACKHOLE_DIR, "usenet_retention" : headphones.USENET_RETENTION, - "use_nzbmatrix" : checked(headphones.NZBMATRIX), - "nzbmatrix_user" : headphones.NZBMATRIX_USERNAME, - "nzbmatrix_api" : headphones.NZBMATRIX_APIKEY, +# "use_nzbmatrix" : checked(headphones.NZBMATRIX), +# "nzbmatrix_user" : headphones.NZBMATRIX_USERNAME, +# "nzbmatrix_api" : headphones.NZBMATRIX_APIKEY, "use_newznab" : checked(headphones.NEWZNAB), "newznab_host" : headphones.NEWZNAB_HOST, "newznab_api" : headphones.NEWZNAB_APIKEY, @@ -579,9 +579,12 @@ class WebInterface(object): "use_nzbsorg" : checked(headphones.NZBSORG), "nzbsorg_uid" : headphones.NZBSORG_UID, "nzbsorg_hash" : headphones.NZBSORG_HASH, - "use_newzbin" : checked(headphones.NEWZBIN), - "newzbin_uid" : headphones.NEWZBIN_UID, - "newzbin_pass" : headphones.NEWZBIN_PASSWORD, +# "use_newzbin" : checked(headphones.NEWZBIN), +# "newzbin_uid" : headphones.NEWZBIN_UID, +# "newzbin_pass" : headphones.NEWZBIN_PASSWORD, + "use_nzbsrus" : checked(headphones.NZBSRUS), + "nzbsrus_uid" : headphones.NZBSRUS_UID, + "nzbsrus_apikey" : headphones.NZBSRUS_APIKEY, "torrentblackhole_dir" : headphones.TORRENTBLACKHOLE_DIR, "download_torrent_dir" : headphones.DOWNLOAD_TORRENT_DIR, "numberofseeders" : headphones.NUMBEROFSEEDERS, @@ -683,9 +686,9 @@ class WebInterface(object): def configUpdate(self, http_host='0.0.0.0', http_username=None, http_port=8181, http_password=None, launch_browser=0, api_enabled=0, api_key=None, download_scan_interval=None, nzb_search_interval=None, libraryscan_interval=None, sab_host=None, sab_username=None, sab_apikey=None, sab_password=None, - sab_category=None, download_dir=None, blackhole=0, blackhole_dir=None, usenet_retention=None, nzbmatrix=0, nzbmatrix_username=None, nzbmatrix_apikey=None, - newznab=0, newznab_host=None, newznab_apikey=None, newznab_enabled=0, nzbsorg=0, nzbsorg_uid=None, nzbsorg_hash=None, newzbin=0, newzbin_uid=None, - newzbin_password=None, preferred_quality=0, preferred_bitrate=None, detect_bitrate=0, move_files=0, torrentblackhole_dir=None, download_torrent_dir=None, + sab_category=None, download_dir=None, blackhole=0, blackhole_dir=None, usenet_retention=None, newznab=0, newznab_host=None, newznab_apikey=None, + newznab_enabled=0, nzbsorg=0, nzbsorg_uid=None, nzbsorg_hash=None, nzbsrus=0, nzbsrus_uid=None, nzbsrus_apikey=None, preferred_quality=0, preferred_bitrate=None, + detect_bitrate=0, move_files=0, torrentblackhole_dir=None, download_torrent_dir=None, numberofseeders=10, use_isohunt=0, use_kat=0, use_mininova=0, waffles=0, waffles_uid=None, waffles_passkey=None, whatcd=0, whatcd_username=None, whatcd_password=None, rutracker=0, rutracker_user=None, rutracker_password=None, rename_files=0, correct_metadata=0, cleanup_files=0, add_album_art=0, embed_album_art=0, embed_lyrics=0, destination_dir=None, lossless_destination_dir=None, folder_format=None, file_format=None, include_extras=0, single=0, ep=0, compilation=0, soundtrack=0, live=0, @@ -715,9 +718,9 @@ class WebInterface(object): headphones.BLACKHOLE = blackhole headphones.BLACKHOLE_DIR = blackhole_dir headphones.USENET_RETENTION = usenet_retention - headphones.NZBMATRIX = nzbmatrix - headphones.NZBMATRIX_USERNAME = nzbmatrix_username - headphones.NZBMATRIX_APIKEY = nzbmatrix_apikey +# headphones.NZBMATRIX = nzbmatrix +# headphones.NZBMATRIX_USERNAME = nzbmatrix_username +# headphones.NZBMATRIX_APIKEY = nzbmatrix_apikey headphones.NEWZNAB = newznab headphones.NEWZNAB_HOST = newznab_host headphones.NEWZNAB_APIKEY = newznab_apikey @@ -725,9 +728,12 @@ class WebInterface(object): headphones.NZBSORG = nzbsorg headphones.NZBSORG_UID = nzbsorg_uid headphones.NZBSORG_HASH = nzbsorg_hash - headphones.NEWZBIN = newzbin - headphones.NEWZBIN_UID = newzbin_uid - headphones.NEWZBIN_PASSWORD = newzbin_password +# headphones.NEWZBIN = newzbin +# headphones.NEWZBIN_UID = newzbin_uid +# headphones.NEWZBIN_PASSWORD = newzbin_password + headphones.NZBSRUS = nzbsrus + headphones.NZBSRUS_UID = nzbsrus_uid + headphones.NZBSRUS_APIKEY = nzbsrus_apikey headphones.TORRENTBLACKHOLE_DIR = torrentblackhole_dir headphones.NUMBEROFSEEDERS = numberofseeders headphones.DOWNLOAD_TORRENT_DIR = download_torrent_dir From 8479cd277f897c88bbd65a901dba88618cb68930 Mon Sep 17 00:00:00 2001 From: rembo10 Date: Fri, 14 Dec 2012 07:48:15 -0500 Subject: [PATCH 3/5] Commented out nzbmatrix and newzbin code, added in nzbsrus api - but service appears to be offline so can't test it right now - might need to switch to rss feed --- headphones/searcher.py | 334 ++++++++++++++++++++++++----------------- 1 file changed, 197 insertions(+), 137 deletions(-) diff --git a/headphones/searcher.py b/headphones/searcher.py index 99ef1d63..0b46f55b 100644 --- a/headphones/searcher.py +++ b/headphones/searcher.py @@ -173,65 +173,65 @@ def searchNZB(albumid=None, new=False, losslessOnly=False): resultlist = [] - if headphones.NZBMATRIX: - provider = "nzbmatrix" - if headphones.PREFERRED_QUALITY == 3 or losslessOnly: - categories = "23" - elif headphones.PREFERRED_QUALITY: - categories = "23,22" - else: - categories = "22" - - # Search Audiobooks/Singles/etc - if albums['Type'] == "Other": - categories = "49" - logger.info("Album type is audiobook/spokenword. Using audiobook category") - if albums['Type'] == "Single": - categories = "47" - logger.info("Album type is 'Single'. Using singles category") - - # For some reason NZBMatrix is erroring out/timing out when the term starts with a "The" right now - # so we'll strip it out for the time being. This may get fixed on their end, it may not, but - # hopefully this will fix it for now. If you notice anything else it gets stuck on, please post it - # on Github so it can be added - if term.lower().startswith("the "): - term = term[4:] - - - params = { "page": "download", - "username": headphones.NZBMATRIX_USERNAME, - "apikey": headphones.NZBMATRIX_APIKEY, - "subcat": categories, - "maxage": headphones.USENET_RETENTION, - "english": 1, - "ssl": 1, - "scenename": 1, - "term": term - } - - searchURL = "https://rss.nzbmatrix.com/rss.php?" + urllib.urlencode(params) - logger.info(u'Parsing results from NZBMatrix' % searchURL) - try: - data = urllib2.urlopen(searchURL, timeout=20).read() - except urllib2.URLError, e: - logger.warn('Error fetching data from NZBMatrix: %s' % e) - data = False - - if data: - - d = feedparser.parse(data) - - for item in d.entries: - try: - url = item.link - title = item.title - size = int(item.links[1]['length']) - - resultlist.append((title, size, url, provider)) - logger.info('Found %s. Size: %s' % (title, helpers.bytes_to_mb(size))) - - except AttributeError, e: - logger.info(u"No results found from NZBMatrix for %s" % term) +# if headphones.NZBMATRIX: +# provider = "nzbmatrix" +# if headphones.PREFERRED_QUALITY == 3 or losslessOnly: +# categories = "23" +# elif headphones.PREFERRED_QUALITY: +# categories = "23,22" +# else: +# categories = "22" +# +# # Search Audiobooks/Singles/etc +# if albums['Type'] == "Other": +# categories = "49" +# logger.info("Album type is audiobook/spokenword. Using audiobook category") +# if albums['Type'] == "Single": +# categories = "47" +# logger.info("Album type is 'Single'. Using singles category") +# +# # For some reason NZBMatrix is erroring out/timing out when the term starts with a "The" right now +# # so we'll strip it out for the time being. This may get fixed on their end, it may not, but +# # hopefully this will fix it for now. If you notice anything else it gets stuck on, please post it +# # on Github so it can be added +# if term.lower().startswith("the "): +# term = term[4:] +# +# +# params = { "page": "download", +# "username": headphones.NZBMATRIX_USERNAME, +# "apikey": headphones.NZBMATRIX_APIKEY, +# "subcat": categories, +# "maxage": headphones.USENET_RETENTION, +# "english": 1, +# "ssl": 1, +# "scenename": 1, +# "term": term +# } +# +# searchURL = "https://rss.nzbmatrix.com/rss.php?" + urllib.urlencode(params) +# logger.info(u'Parsing results from NZBMatrix' % searchURL) +# try: +# data = urllib2.urlopen(searchURL, timeout=20).read() +# except urllib2.URLError, e: +# logger.warn('Error fetching data from NZBMatrix: %s' % e) +# data = False +# +# if data: +# +# d = feedparser.parse(data) +# +# for item in d.entries: +# try: +# url = item.link +# title = item.title +# size = int(item.links[1]['length']) +# +# resultlist.append((title, size, url, provider)) +# logger.info('Found %s. Size: %s' % (title, helpers.bytes_to_mb(size))) +# +# except AttributeError, e: +# logger.info(u"No results found from NZBMatrix for %s" % term) if headphones.NEWZNAB: @@ -359,89 +359,149 @@ def searchNZB(albumid=None, new=False, losslessOnly=False): except Exception, e: logger.error(u"An unknown error occurred trying to parse the feed: %s" % e) - - if headphones.NEWZBIN: - provider = "newzbin" - providerurl = "https://www.newzbin2.es/" + + if headphones.NZBSRUS: + + provider = "nzbsrus" + categories = "54" + if headphones.PREFERRED_QUALITY == 3 or losslessOnly: - categories = "7" #music - format = "2" #flac + sub = "16" elif headphones.PREFERRED_QUALITY: - categories = "7" #music - format = "10" #mp3+flac + sub = "" else: - categories = "7" #music - format = "8" #mp3 - + sub = "15" + if albums['Type'] == 'Other': - categories = "13" - format = "16" - logger.info("Album type is audiobook/spokenword. Using audiobook category") - - params = { - "fpn": "p", - 'u_nfo_posts_only': 0, - 'u_url_posts_only': 0, - 'u_comment_posts_only': 0, - 'u_show_passworded': 0, - "searchaction": "Search", - #"dl": 1, - "category": categories, - "retention": headphones.USENET_RETENTION, - "ps_rb_audio_format": format, - "feed": "rss", - "u_post_results_amt": 50, #this can default to a high number per user - "hauth": 1, - "q": term - } - searchURL = providerurl + "search/?%s" % urllib.urlencode(params) - try: - data = getNewzbinURL(searchURL) - except exceptions.NewzbinAPIThrottled: - #try again if we were throttled - data = getNewzbinURL(searchURL) - if data: - logger.info(u'Parsing results from %s' % (searchURL, providerurl)) - - try: - d = minidom.parseString(data) - node = d.documentElement - items = d.getElementsByTagName("item") - except ExpatError: - logger.info('Unable to get the NEWZBIN feed. Check that your settings are correct - post a bug if they are') - items = [] - - if len(items): - - for item in items: - - sizenode = item.getElementsByTagName("report:size")[0].childNodes - titlenode = item.getElementsByTagName("title")[0].childNodes - linknode = item.getElementsByTagName("link")[0].childNodes - - for node in sizenode: - size = int(node.data) - for node in titlenode: - title = node.data - for node in linknode: - url = node.data - - #exract the reportid from the link nodes - id_regex = re.escape(providerurl) + 'browse/post/(\d+)/' - id_match = re.match(id_regex, url) - if not id_match: - logger.info("Didn't find a valid Newzbin reportid in linknode") - else: - url = id_match.group(1) #we have to make a post request later, need the id - if url: - resultlist.append((title, size, url, provider)) - logger.info('Found %s. Size: %s' % (title, helpers.bytes_to_mb(size))) - else: - logger.info('No url link found in nzb. Skipping.') - - else: - logger.info('No results found from NEWZBIN for %s' % term) + sub = "" + logger.info("Album type is audiobook/spokenword. Searching all music categories") + params = { "uid": headphones.NZBSRUS_UID, + "key": headphones.NZBSRUS_APIKEY, + "cat": categories, + "sub": sub, + "age": headphones.USENET_RETENTION, + "searchtext": term + } + + searchURL = 'https://www.nzbsrus.com/api.php?' + urllib.urlencode(params) + + # Add a user-agent + request = urllib2.Request(searchURL) + request.add_header('User-Agent', 'headphones/0.0 +https://github.com/rembo10/headphones') + opener = urllib2.build_opener() + + logger.info(u'Parsing results from NZBsRus' % searchURL) + + try: + data = opener.open(request).read() + except Exception, e: + logger.warn('Error fetching data from NZBsRus: %s' % e) + data = False + + if data: + + d = feedparser.parse(data) + + if not len(d.entries): + logger.info(u"No results found from NZBsRus for %s" % term) + pass + + else: + for item in d.entries: + try: + url = item.link + title = item.title + size = int(item.links[1]['length']) + + resultlist.append((title, size, url, provider)) + logger.info('Found %s. Size: %s' % (title, helpers.bytes_to_mb(size))) + + except Exception, e: + logger.error(u"An unknown error occurred trying to parse the feed: %s" % e) + +# if headphones.NEWZBIN: +# provider = "newzbin" +# providerurl = "https://www.newzbin2.es/" +# if headphones.PREFERRED_QUALITY == 3 or losslessOnly: +# categories = "7" #music +# format = "2" #flac +# elif headphones.PREFERRED_QUALITY: +# categories = "7" #music +# format = "10" #mp3+flac +# else: +# categories = "7" #music +# format = "8" #mp3 +# +# if albums['Type'] == 'Other': +# categories = "13" +# format = "16" +# logger.info("Album type is audiobook/spokenword. Using audiobook category") +# +# params = { +# "fpn": "p", +# 'u_nfo_posts_only': 0, +# 'u_url_posts_only': 0, +# 'u_comment_posts_only': 0, +# 'u_show_passworded': 0, +# "searchaction": "Search", +# #"dl": 1, +# "category": categories, +# "retention": headphones.USENET_RETENTION, +# "ps_rb_audio_format": format, +# "feed": "rss", +# "u_post_results_amt": 50, #this can default to a high number per user +# "hauth": 1, +# "q": term +# } +# searchURL = providerurl + "search/?%s" % urllib.urlencode(params) +# try: +# data = getNewzbinURL(searchURL) +# except exceptions.NewzbinAPIThrottled: +# #try again if we were throttled +# data = getNewzbinURL(searchURL) +# if data: +# logger.info(u'Parsing results from %s' % (searchURL, providerurl)) +# +# try: +# d = minidom.parseString(data) +# node = d.documentElement +# items = d.getElementsByTagName("item") +# except ExpatError: +# logger.info('Unable to get the NEWZBIN feed. Check that your settings are correct - post a bug if they are') +# items = [] +# +# if len(items): +# +# for item in items: +# +# sizenode = item.getElementsByTagName("report:size")[0].childNodes +# titlenode = item.getElementsByTagName("title")[0].childNodes +# linknode = item.getElementsByTagName("link")[0].childNodes +# +# for node in sizenode: +# size = int(node.data) +# for node in titlenode: +# title = node.data +# for node in linknode: +# url = node.data +# +# #exract the reportid from the link nodes +# id_regex = re.escape(providerurl) + 'browse/post/(\d+)/' +# id_match = re.match(id_regex, url) +# if not id_match: +# logger.info("Didn't find a valid Newzbin reportid in linknode") +# else: +# url = id_match.group(1) #we have to make a post request later, need the id +# if url: +# resultlist.append((title, size, url, provider)) +# logger.info('Found %s. Size: %s' % (title, helpers.bytes_to_mb(size))) +# else: +# logger.info('No url link found in nzb. Skipping.') +# +# else: +# logger.info('No results found from NEWZBIN for %s' % term) +# #attempt to verify that this isn't a substring result #when looking for "Foo - Foo" we don't want "Foobar" #this should be less of an issue when it isn't a self-titled album so we'll only check vs artist From 02fb83592ad2c836d6960f85345eb3f6ae1311f9 Mon Sep 17 00:00:00 2001 From: rembo10 Date: Tue, 18 Dec 2012 05:18:55 -0500 Subject: [PATCH 4/5] Added option to preserve torrent dir for seeding after post processing --- data/interfaces/default/config.html | 13 ++++++++----- headphones/__init__.py | 12 ++++++++++-- headphones/postprocessor.py | 27 ++++++++++++++++++--------- headphones/searcher.py | 4 ++-- headphones/webserve.py | 4 +++- 5 files changed, 41 insertions(+), 19 deletions(-) diff --git a/data/interfaces/default/config.html b/data/interfaces/default/config.html index 29978480..565464f1 100644 --- a/data/interfaces/default/config.html +++ b/data/interfaces/default/config.html @@ -156,8 +156,12 @@ Full path where your torrent client downloads your music e.g. /Users/name/Downloads/music -
-
+ +
+ + +
+
Usenet
@@ -355,9 +359,8 @@
- - - + + diff --git a/headphones/__init__.py b/headphones/__init__.py index efbfe7e5..8ea2597d 100644 --- a/headphones/__init__.py +++ b/headphones/__init__.py @@ -110,6 +110,7 @@ INCLUDE_EXTRAS = False EXTRAS = None AUTOWANT_UPCOMING = False AUTOWANT_ALL = False +KEEP_TORRENT_FILES = False SEARCH_INTERVAL = 360 LIBRARYSCAN = False @@ -266,7 +267,7 @@ def initialize(): HTTP_PORT, HTTP_HOST, HTTP_USERNAME, HTTP_PASSWORD, HTTP_ROOT, HTTP_PROXY, LAUNCH_BROWSER, API_ENABLED, API_KEY, GIT_PATH, \ CURRENT_VERSION, LATEST_VERSION, CHECK_GITHUB, CHECK_GITHUB_ON_STARTUP, CHECK_GITHUB_INTERVAL, MUSIC_DIR, DESTINATION_DIR, \ LOSSLESS_DESTINATION_DIR, PREFERRED_QUALITY, PREFERRED_BITRATE, DETECT_BITRATE, ADD_ARTISTS, CORRECT_METADATA, MOVE_FILES, \ - RENAME_FILES, FOLDER_FORMAT, FILE_FORMAT, CLEANUP_FILES, INCLUDE_EXTRAS, EXTRAS, AUTOWANT_UPCOMING, AUTOWANT_ALL, \ + RENAME_FILES, FOLDER_FORMAT, FILE_FORMAT, CLEANUP_FILES, INCLUDE_EXTRAS, EXTRAS, AUTOWANT_UPCOMING, AUTOWANT_ALL, KEEP_TORRENT_FILES, \ ADD_ALBUM_ART, EMBED_ALBUM_ART, EMBED_LYRICS, DOWNLOAD_DIR, BLACKHOLE, BLACKHOLE_DIR, USENET_RETENTION, SEARCH_INTERVAL, \ TORRENTBLACKHOLE_DIR, NUMBEROFSEEDERS, ISOHUNT, KAT, MININOVA, WAFFLES, WAFFLES_UID, WAFFLES_PASSKEY, \ RUTRACKER, RUTRACKER_USER, RUTRACKER_PASSWORD, WHATCD, WHATCD_USERNAME, WHATCD_PASSWORD, DOWNLOAD_TORRENT_DIR, \ @@ -354,6 +355,7 @@ def initialize(): EXTRAS = check_setting_str(CFG, 'General', 'extras', '') AUTOWANT_UPCOMING = bool(check_setting_int(CFG, 'General', 'autowant_upcoming', 1)) AUTOWANT_ALL = bool(check_setting_int(CFG, 'General', 'autowant_all', 0)) + KEEP_TORRENT_FILES = bool(check_setting_int(CFG, 'General', 'keep_torrent_files', 0)) SEARCH_INTERVAL = check_setting_int(CFG, 'General', 'search_interval', 1440) LIBRARYSCAN = bool(check_setting_int(CFG, 'General', 'libraryscan', 1)) @@ -671,6 +673,7 @@ def config_write(): new_config['General']['extras'] = EXTRAS new_config['General']['autowant_upcoming'] = int(AUTOWANT_UPCOMING) new_config['General']['autowant_all'] = int(AUTOWANT_ALL) + new_config['General']['keep_torrent_files'] = int(KEEP_TORRENT_FILES) new_config['General']['numberofseeders'] = NUMBEROFSEEDERS new_config['General']['torrentblackhole_dir'] = TORRENTBLACKHOLE_DIR @@ -830,7 +833,7 @@ def dbcheck(): c.execute('CREATE TABLE IF NOT EXISTS tracks (ArtistID TEXT, ArtistName TEXT, AlbumTitle TEXT, AlbumASIN TEXT, AlbumID TEXT, TrackTitle TEXT, TrackDuration, TrackID TEXT, TrackNumber INTEGER, Location TEXT, BitRate INTEGER, CleanName TEXT, Format TEXT, ReleaseID TEXT)') # Format here means mp3, flac, etc. c.execute('CREATE TABLE IF NOT EXISTS allalbums (ArtistID TEXT, ArtistName TEXT, AlbumTitle TEXT, AlbumASIN TEXT, ReleaseDate TEXT, AlbumID TEXT, Type TEXT, ReleaseID TEXT, ReleaseCountry TEXT, ReleaseFormat TEXT)') c.execute('CREATE TABLE IF NOT EXISTS alltracks (ArtistID TEXT, ArtistName TEXT, AlbumTitle TEXT, AlbumASIN TEXT, AlbumID TEXT, TrackTitle TEXT, TrackDuration, TrackID TEXT, TrackNumber INTEGER, Location TEXT, BitRate INTEGER, CleanName TEXT, Format TEXT, ReleaseID TEXT)') - c.execute('CREATE TABLE IF NOT EXISTS snatched (AlbumID TEXT, Title TEXT, Size INTEGER, URL TEXT, DateAdded TEXT, Status TEXT, FolderName TEXT)') + c.execute('CREATE TABLE IF NOT EXISTS snatched (AlbumID TEXT, Title TEXT, Size INTEGER, URL TEXT, DateAdded TEXT, Status TEXT, FolderName TEXT, Kind TEXT)') c.execute('CREATE TABLE IF NOT EXISTS have (ArtistName TEXT, AlbumTitle TEXT, TrackNumber TEXT, TrackTitle TEXT, TrackLength TEXT, BitRate TEXT, Genre TEXT, Date TEXT, TrackID TEXT, Location TEXT, CleanName TEXT, Format TEXT, Matched TEXT)') # Matched is a temporary value used to see if there was a match found in alltracks c.execute('CREATE TABLE IF NOT EXISTS lastfmcloud (ArtistName TEXT, ArtistID TEXT, Count INTEGER)') c.execute('CREATE TABLE IF NOT EXISTS descriptions (ArtistID TEXT, ReleaseGroupID TEXT, ReleaseID TEXT, Summary TEXT, Content TEXT, LastUpdated TEXT)') @@ -993,6 +996,11 @@ def dbcheck(): for artist in artists: if artist[1]: c.execute('UPDATE artists SET Extras=? WHERE ArtistID=?', ("1,2,3,4,5,6,7,8", artist[0])) + + try: + c.execute('SELECT Kind from snatched') + except sqlite3.OperationalError: + c.execute('ALTER TABLE snatched ADD COLUMN Kind TEXT DEFAULT NULL') conn.commit() c.close() diff --git a/headphones/postprocessor.py b/headphones/postprocessor.py index cb37c5c9..719628d2 100644 --- a/headphones/postprocessor.py +++ b/headphones/postprocessor.py @@ -60,13 +60,13 @@ def checkFolder(): if os.path.exists(nzb_album_path): logger.debug('Found %s in NZB download folder. Verifying....' % album['FolderName']) - verify(album['AlbumID'], nzb_album_path) + verify(album['AlbumID'], nzb_album_path, album['Kind']) if os.path.exists(torrent_album_path): logger.debug('Found %s in torrent download folder. Verifying....' % album['FolderName']) - verify(album['AlbumID'], torrent_album_path) + verify(album['AlbumID'], torrent_album_path, album['Kind']) -def verify(albumid, albumpath): +def verify(albumid, albumpath, Kind=None): myDB = db.DBConnection() release = myDB.action('SELECT * from albums WHERE AlbumID=?', [albumid]).fetchone() @@ -254,7 +254,7 @@ def verify(albumid, albumpath): logger.debug('Matching metadata album: %s with album name: %s' % (metaalbum, dbalbum)) if metaartist == dbartist and metaalbum == dbalbum: - doPostProcessing(albumid, albumpath, release, tracks, downloaded_track_list) + doPostProcessing(albumid, albumpath, release, tracks, downloaded_track_list, Kind) return # test #2: filenames @@ -272,7 +272,7 @@ def verify(albumid, albumpath): logger.debug('Checking if track title: %s is in file name: %s' % (dbtrack, filetrack)) if dbtrack in filetrack: - doPostProcessing(albumid, albumpath, release, tracks, downloaded_track_list) + doPostProcessing(albumid, albumpath, release, tracks, downloaded_track_list, Kind) return # test #3: number of songs and duration @@ -304,7 +304,7 @@ def verify(albumid, albumpath): logger.debug('Database track duration: %i' % db_track_duration) delta = abs(downloaded_track_duration - db_track_duration) if delta < 240: - doPostProcessing(albumid, albumpath, release, tracks, downloaded_track_list) + doPostProcessing(albumid, albumpath, release, tracks, downloaded_track_list, Kind) return logger.warn(u'Could not identify album: %s. It may not be the intended album.' % albumpath.decode(headphones.SYS_ENCODING, 'replace')) @@ -315,9 +315,18 @@ def verify(albumid, albumpath): else: logger.info(u"Already marked as unprocessed: " + albumpath.decode(headphones.SYS_ENCODING, 'replace')) -def doPostProcessing(albumid, albumpath, release, tracks, downloaded_track_list): +def doPostProcessing(albumid, albumpath, release, tracks, downloaded_track_list, Kind=None): logger.info('Starting post-processing for: %s - %s' % (release['ArtistName'], release['AlbumTitle'])) + # Check to see if we're preserving the torrent dir + if headphones.KEEP_TORRENT_FILES and Kind="torrent": + new_folder = os.path.join(os.path.dirname(albumpath), ('temp' + release['AlbumTitle'][:5]).encode(headphones.SYS_ENCODING, 'replace')) + try: + shutil.copytree(albumpath, new_folder) + albumpath = new_folder + except Exception, e: + logger.warn("Cannot copy/move files to temp folder: " + new_folder.decode(headphones.SYS_ENCODING, 'replace') + ". Not continuing. Error: " + str(e)) + return #start encoding if headphones.MUSIC_ENCODER: downloaded_track_list=music_encoder.encode(albumpath) @@ -830,10 +839,10 @@ def forcePostProcess(): # First try to see if there's a match in the snatched table, then we'll try to parse the foldername # TODO: Iterate through underscores -> spaces, spaces -> dots, underscores -> dots (this might be hit or miss since it assumes # all spaces/underscores came from sab replacing values - snatched = myDB.action('SELECT AlbumID, Title from snatched WHERE FolderName LIKE ?', [folder_basename]).fetchone() + snatched = myDB.action('SELECT AlbumID, Title, Kind from snatched WHERE FolderName LIKE ?', [folder_basename]).fetchone() if snatched: logger.info('Found a match in the database: %s. Verifying to make sure it is the correct album' % snatched['Title']) - verify(snatched['AlbumID'], folder) + verify(snatched['AlbumID'], folder, kind) continue # Try to parse the folder name into a valid format diff --git a/headphones/searcher.py b/headphones/searcher.py index 0b46f55b..01ee5de7 100644 --- a/headphones/searcher.py +++ b/headphones/searcher.py @@ -621,7 +621,7 @@ def searchNZB(albumid=None, new=False, losslessOnly=False): break myDB.action('UPDATE albums SET status = "Snatched" WHERE AlbumID=?', [albums[2]]) - myDB.action('INSERT INTO snatched VALUES( ?, ?, ?, ?, DATETIME("NOW", "localtime"), ?, ?)', [albums[2], bestqual[0], bestqual[1], bestqual[2], "Snatched", nzb_folder_name]) + myDB.action('INSERT INTO snatched VALUES( ?, ?, ?, ?, DATETIME("NOW", "localtime"), ?, ?, ?)', [albums[2], bestqual[0], bestqual[1], bestqual[2], "Snatched", nzb_folder_name, "nzb"]) return "found" else: return "none" @@ -1291,7 +1291,7 @@ def searchTorrent(albumid=None, new=False, losslessOnly=False): break myDB.action('UPDATE albums SET status = "Snatched" WHERE AlbumID=?', [albums[2]]) - myDB.action('INSERT INTO snatched VALUES( ?, ?, ?, ?, DATETIME("NOW", "localtime"), ?, ?)', [albums[2], bestqual[0], bestqual[1], bestqual[2], "Snatched", torrent_folder_name]) + myDB.action('INSERT INTO snatched VALUES( ?, ?, ?, ?, DATETIME("NOW", "localtime"), ?, ?, ?)', [albums[2], bestqual[0], bestqual[1], bestqual[2], "Snatched", torrent_folder_name, "torrent"]) def preprocesstorrent(resultlist, pre_sorted_list=False): selresult = "" diff --git a/headphones/webserve.py b/headphones/webserve.py index 926e379a..f3af617d 100644 --- a/headphones/webserve.py +++ b/headphones/webserve.py @@ -622,6 +622,7 @@ class WebInterface(object): "include_extras" : checked(headphones.INCLUDE_EXTRAS), "autowant_upcoming" : checked(headphones.AUTOWANT_UPCOMING), "autowant_all" : checked(headphones.AUTOWANT_ALL), + "keep_torrent_files" : checked(headphones.KEEP_TORRENT_FILES), "log_dir" : headphones.LOG_DIR, "cache_dir" : headphones.CACHE_DIR, "interface_list" : interface_list, @@ -692,7 +693,7 @@ class WebInterface(object): numberofseeders=10, use_isohunt=0, use_kat=0, use_mininova=0, waffles=0, waffles_uid=None, waffles_passkey=None, whatcd=0, whatcd_username=None, whatcd_password=None, rutracker=0, rutracker_user=None, rutracker_password=None, rename_files=0, correct_metadata=0, cleanup_files=0, add_album_art=0, embed_album_art=0, embed_lyrics=0, destination_dir=None, lossless_destination_dir=None, folder_format=None, file_format=None, include_extras=0, single=0, ep=0, compilation=0, soundtrack=0, live=0, - remix=0, spokenword=0, audiobook=0, autowant_upcoming=False, autowant_all=False, interface=None, log_dir=None, cache_dir=None, music_encoder=0, encoder=None, xldprofile=None, + remix=0, spokenword=0, audiobook=0, autowant_upcoming=False, autowant_all=False, keep_torrent_files=False, interface=None, log_dir=None, cache_dir=None, music_encoder=0, encoder=None, xldprofile=None, bitrate=None, samplingfrequency=None, encoderfolder=None, advancedencoder=None, encoderoutputformat=None, encodervbrcbr=None, encoderquality=None, encoderlossless=0, delete_lossless_files=0, prowl_enabled=0, prowl_onsnatch=0, prowl_keys=None, prowl_priority=0, xbmc_enabled=0, xbmc_host=None, xbmc_username=None, xbmc_password=None, xbmc_update=0, xbmc_notify=0, nma_enabled=False, nma_apikey=None, nma_priority=0, nma_onsnatch=0, synoindex_enabled=False, @@ -768,6 +769,7 @@ class WebInterface(object): headphones.INCLUDE_EXTRAS = include_extras headphones.AUTOWANT_UPCOMING = autowant_upcoming headphones.AUTOWANT_ALL = autowant_all + headphones.KEEP_TORRENT_FILES = keep_torrent_files headphones.INTERFACE = interface headphones.LOG_DIR = log_dir headphones.CACHE_DIR = cache_dir From 307a564ad56f92460a90d88ad582f264447f5196 Mon Sep 17 00:00:00 2001 From: rembo10 Date: Wed, 19 Dec 2012 10:53:57 -0500 Subject: [PATCH 5/5] Some bug fixes for preserving torrent files after post processing --- headphones/postprocessor.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/headphones/postprocessor.py b/headphones/postprocessor.py index 719628d2..853ed3fa 100644 --- a/headphones/postprocessor.py +++ b/headphones/postprocessor.py @@ -319,7 +319,7 @@ def doPostProcessing(albumid, albumpath, release, tracks, downloaded_track_list, logger.info('Starting post-processing for: %s - %s' % (release['ArtistName'], release['AlbumTitle'])) # Check to see if we're preserving the torrent dir - if headphones.KEEP_TORRENT_FILES and Kind="torrent": + if headphones.KEEP_TORRENT_FILES and Kind=="torrent": new_folder = os.path.join(os.path.dirname(albumpath), ('temp' + release['AlbumTitle'][:5]).encode(headphones.SYS_ENCODING, 'replace')) try: shutil.copytree(albumpath, new_folder) @@ -842,7 +842,7 @@ def forcePostProcess(): snatched = myDB.action('SELECT AlbumID, Title, Kind from snatched WHERE FolderName LIKE ?', [folder_basename]).fetchone() if snatched: logger.info('Found a match in the database: %s. Verifying to make sure it is the correct album' % snatched['Title']) - verify(snatched['AlbumID'], folder, kind) + verify(snatched['AlbumID'], folder, Snatched['Kind']) continue # Try to parse the folder name into a valid format