From a66aa5f41888e8d0102d5ea7f9a915fc5902ee57 Mon Sep 17 00:00:00 2001 From: Remy Date: Tue, 9 Aug 2011 05:08:45 +0800 Subject: [PATCH 1/9] Fixed wanted albums showing Retry Download subhead --- data/interfaces/default/album.html | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/data/interfaces/default/album.html b/data/interfaces/default/album.html index e8406f63..2308f32b 100644 --- a/data/interfaces/default/album.html +++ b/data/interfaces/default/album.html @@ -9,11 +9,11 @@ From bdd576f8b29c3d72f63c7aaa53b3c1c83bb12abb Mon Sep 17 00:00:00 2001 From: Remy Date: Tue, 9 Aug 2011 05:09:37 +0800 Subject: [PATCH 2/9] Changed artist already in db message to debug --- headphones/importer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/headphones/importer.py b/headphones/importer.py index 5ddbb20e..43726f59 100644 --- a/headphones/importer.py +++ b/headphones/importer.py @@ -96,7 +96,7 @@ def is_exists(artistid): artistlist = myDB.select('SELECT ArtistID, ArtistName from artists WHERE ArtistID=?', [artistid]) if any(artistid in x for x in artistlist): - logger.info(artistlist[0][1] + u" is already in the database. Updating 'have tracks', but not artist information") + logger.debug(artistlist[0][1] + u" is already in the database. Updating 'have tracks', but not artist information") return True else: return False From fd74af60ee9545e5f5407fb3f2ec6db33d0ff3b4 Mon Sep 17 00:00:00 2001 From: Remy Date: Tue, 9 Aug 2011 05:13:30 +0800 Subject: [PATCH 3/9] Fixed default logging to show results by most recent --- data/interfaces/default/logs.html | 1 + 1 file changed, 1 insertion(+) diff --git a/data/interfaces/default/logs.html b/data/interfaces/default/logs.html index 8434e3b6..cc6436f3 100644 --- a/data/interfaces/default/logs.html +++ b/data/interfaces/default/logs.html @@ -57,6 +57,7 @@ "bStateSave": true, "iDisplayLength": 100, "sPaginationType": "full_numbers", + "aaSorting": [] }); }); From e70f893ff3e94edf6aa38df23aa3e9dcff3fd5a1 Mon Sep 17 00:00:00 2001 From: Remy Date: Tue, 9 Aug 2011 16:22:51 +0800 Subject: [PATCH 4/9] Fixed album title exception in post processor --- headphones/postprocessor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/headphones/postprocessor.py b/headphones/postprocessor.py index 313f30af..e28033c3 100644 --- a/headphones/postprocessor.py +++ b/headphones/postprocessor.py @@ -306,7 +306,7 @@ def moveFiles(albumpath, release, tracks): os.chmod(temp_f, 0755) except Exception, e: - logger.error('Could not create folder for %s. Not moving' % release['AlbumName']) + logger.error('Could not create folder for %s. Not moving: %s' % (release['AlbumTitle'], e)) return albumpath for r,d,f in os.walk(albumpath): From 2dd5ab94eee8e2f78bfd5331fd18e20e25b29981 Mon Sep 17 00:00:00 2001 From: Remy Date: Wed, 10 Aug 2011 05:49:25 +0800 Subject: [PATCH 5/9] Fixed bug with log page not displaying if non-utf8 characters --- data/css/style.css | 2 +- data/interfaces/default/logs.html | 23 +++++++++-------------- headphones/__init__.py | 2 ++ headphones/helpers.py | 9 +++++++-- headphones/logger.py | 24 ++++++++++++++---------- headphones/webserve.py | 8 +------- 6 files changed, 34 insertions(+), 34 deletions(-) diff --git a/data/css/style.css b/data/css/style.css index 511c8ae7..c8b0eb22 100755 --- a/data/css/style.css +++ b/data/css/style.css @@ -190,7 +190,7 @@ table#log_table { background-color: white; } table#log_table th#timestamp { text-align: left; min-width: 165px; } table#log_table th#level { text-align: left; min-width: 75px; } -table#log_table th#message { text-align: left; min-width: 200px; } +table#log_table th#message { text-align: left; min-width: 500px; } table#upcoming_table th#albumart { text-align: center; min-width: 50px; } table#upcoming_table th#albumname { text-align: center; min-width: 200px; } diff --git a/data/interfaces/default/logs.html b/data/interfaces/default/logs.html index cc6436f3..98004034 100644 --- a/data/interfaces/default/logs.html +++ b/data/interfaces/default/logs.html @@ -15,23 +15,18 @@ %for line in lineList: <% - out_tuple = helpers.extract_logline(line) + timestamp, message, level, threadname = line + + if level == 'WARNING' or level == 'ERROR': + grade = 'X' + else: + grade = 'Z' %> - %if out_tuple: - <% - if out_tuple[1] == 'DEBUG': - continue - elif out_tuple[1] == 'WARNING' or out_tuple[1] == 'ERROR': - grade = 'X' - else: - grade = 'Z' - %> - ${out_tuple[0]} - ${out_tuple[1]} - ${out_tuple[3].decode('utf-8')} + ${timestamp} + ${level} + ${message} - %endif %endfor diff --git a/headphones/__init__.py b/headphones/__init__.py index 47e8ed29..4671177a 100644 --- a/headphones/__init__.py +++ b/headphones/__init__.py @@ -37,6 +37,8 @@ CFG = None DB_FILE = None LOG_DIR = None +LOG_LIST = [] + CACHE_DIR = None HTTP_PORT = None diff --git a/headphones/helpers.py b/headphones/helpers.py index 9dc3aa10..4ecc6941 100644 --- a/headphones/helpers.py +++ b/headphones/helpers.py @@ -5,8 +5,6 @@ import re import headphones -from headphones import logger - def multikeysort(items, columns): comparers = [ ((itemgetter(col[1:].strip()), -1) if col.startswith('-') else (itemgetter(col.strip()), 1)) for col in columns] @@ -91,6 +89,10 @@ def today(): yyyymmdd = datetime.date.isoformat(today) return yyyymmdd +def now(): + now = datetime.datetime.now() + return now.strftime("%Y-%m-%d %H:%M:%S") + def bytes_to_mb(bytes): mb = int(bytes)/1048576 @@ -103,6 +105,9 @@ def replace_all(text, dic): return text def extract_data(s): + + from headphones import logger + #headphones default format pattern = re.compile(r'(?P.*?)\s\-\s(?P.*?)\s\[(?P.*?)\]', re.VERBOSE) match = pattern.match(s) diff --git a/headphones/logger.py b/headphones/logger.py index f861a4c3..93c77c57 100644 --- a/headphones/logger.py +++ b/headphones/logger.py @@ -4,6 +4,7 @@ import logging from logging import handlers import headphones +from headphones import helpers MAX_SIZE = 1000000 # 1mb MAX_FILES = 5 @@ -45,33 +46,36 @@ class RotatingLogger(object): l.addHandler(consolehandler) def log(self, message, level): - + logger = logging.getLogger('headphones') threadname = threading.currentThread().getName() - message = threadname + ' : ' + message - if level == 'debug': + if level != 'DEBUG': + headphones.LOG_LIST.insert(0, (helpers.now(), message, level, threadname)) + + message = threadname + ' : ' + message + + if level == 'DEBUG': logger.debug(message) - elif level == 'info': + elif level == 'INFO': logger.info(message) - elif level == 'warn': + elif level == 'WARN': logger.warn(message) else: logger.error(message) - headphones_log = RotatingLogger('headphones.log', MAX_SIZE, MAX_FILES) def debug(message): - headphones_log.log(message, level='debug') + headphones_log.log(message, level='DEBUG') def info(message): - headphones_log.log(message, level='info') + headphones_log.log(message, level='INFO') def warn(message): - headphones_log.log(message, level='warn') + headphones_log.log(message, level='WARN') def error(message): - headphones_log.log(message, level='error') + headphones_log.log(message, level='ERROR') diff --git a/headphones/webserve.py b/headphones/webserve.py index fd406306..779f6da0 100644 --- a/headphones/webserve.py +++ b/headphones/webserve.py @@ -231,13 +231,7 @@ class WebInterface(object): history.exposed = True def logs(self): - log_file = os.path.join(headphones.LOG_DIR, 'headphones.log') - if os.path.isfile(log_file): - fileHandle = open(log_file) - lineList = fileHandle.readlines() - fileHandle.close() - lineList.reverse() - return serve_template(templatename="logs.html", title="Log", lineList=lineList[0:500]) + return serve_template(templatename="logs.html", title="Log", lineList=headphones.LOG_LIST) logs.exposed = True def clearhistory(self, type=None): From 24c6ea9e5bac6d0f2597d51dff773c8c067b824e Mon Sep 17 00:00:00 2001 From: Remy Date: Wed, 10 Aug 2011 05:58:56 +0800 Subject: [PATCH 6/9] Added download directory scan interval as a config option in config.ini --- headphones/__init__.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/headphones/__init__.py b/headphones/__init__.py index 4671177a..e5c8ba27 100644 --- a/headphones/__init__.py +++ b/headphones/__init__.py @@ -76,6 +76,7 @@ INCLUDE_EXTRAS = False NZB_SEARCH_INTERVAL = 360 LIBRARYSCAN_INTERVAL = 60 +DOWNLOAD_SCAN_INTERVAL = 5 SAB_HOST = None SAB_USERNAME = None @@ -158,7 +159,7 @@ def initialize(): CURRENT_VERSION, LATEST_VERSION, MUSIC_DIR, DESTINATION_DIR, PREFERRED_QUALITY, PREFERRED_BITRATE, DETECT_BITRATE, \ CORRECT_METADATA, MOVE_FILES, RENAME_FILES, FOLDER_FORMAT, FILE_FORMAT, CLEANUP_FILES, INCLUDE_EXTRAS, \ ADD_ALBUM_ART, EMBED_ALBUM_ART, DOWNLOAD_DIR, BLACKHOLE, BLACKHOLE_DIR, USENET_RETENTION, NZB_SEARCH_INTERVAL, \ - LIBRARYSCAN_INTERVAL, SAB_HOST, SAB_USERNAME, SAB_PASSWORD, SAB_APIKEY, SAB_CATEGORY, \ + LIBRARYSCAN_INTERVAL, DOWNLOAD_SCAN_INTERVAL, SAB_HOST, SAB_USERNAME, SAB_PASSWORD, SAB_APIKEY, SAB_CATEGORY, \ NZBMATRIX, NZBMATRIX_USERNAME, NZBMATRIX_APIKEY, NEWZNAB, NEWZNAB_HOST, NEWZNAB_APIKEY, \ NZBSORG, NZBSORG_UID, NZBSORG_HASH, NEWZBIN, NEWZBIN_UID, NEWZBIN_PASSWORD, LASTFM_USERNAME @@ -211,6 +212,7 @@ def initialize(): NZB_SEARCH_INTERVAL = check_setting_int(CFG, 'General', 'nzb_search_interval', 360) LIBRARYSCAN_INTERVAL = check_setting_int(CFG, 'General', 'libraryscan_interval', 180) + DOWNLOAD_SCAN_INTERVAL = check_setting_int(CFG, 'General', 'download_scan_interval', 5) SAB_HOST = check_setting_str(CFG, 'SABnzbd', 'sab_host', '') SAB_USERNAME = check_setting_str(CFG, 'SABnzbd', 'sab_username', '') @@ -371,6 +373,7 @@ def config_write(): new_config['General']['nzb_search_interval'] = NZB_SEARCH_INTERVAL new_config['General']['libraryscan_interval'] = LIBRARYSCAN_INTERVAL + new_config['General']['download_scan_interval'] = DOWNLOAD_SCAN_INTERVAL new_config['SABnzbd'] = {} new_config['SABnzbd']['sab_host'] = SAB_HOST @@ -416,7 +419,7 @@ def start(): SCHED.add_interval_job(searcher.searchNZB, minutes=NZB_SEARCH_INTERVAL) SCHED.add_interval_job(importer.scanMusic, minutes=LIBRARYSCAN_INTERVAL) SCHED.add_interval_job(versioncheck.checkGithub, minutes=300) - SCHED.add_interval_job(postprocessor.checkFolder, minutes=5) + SCHED.add_interval_job(postprocessor.checkFolder, minutes=DOWNLOAD_SCAN_INTERVAL) SCHED.start() From 957a7522dfadf1824159effae95427eae714e724 Mon Sep 17 00:00:00 2001 From: sbuser Date: Wed, 10 Aug 2011 10:12:11 -0500 Subject: [PATCH 7/9] Added exceptions.py to throw our own exceptions. Attempted to mimic sickbeard exception handling of newzbin throttling. --- headphones/exceptions.py | 26 ++++++++++++++++++++++++++ headphones/searcher.py | 14 +++++++++----- 2 files changed, 35 insertions(+), 5 deletions(-) create mode 100644 headphones/exceptions.py diff --git a/headphones/exceptions.py b/headphones/exceptions.py new file mode 100644 index 00000000..d591edff --- /dev/null +++ b/headphones/exceptions.py @@ -0,0 +1,26 @@ +def ex(e): + """ + Returns a string from the exception text if it exists. + """ + + # sanity check + if not e.args or not e.args[0]: + return "" + + e_message = e.args[0] + + # if fixStupidEncodings doesn't fix it then maybe it's not a string, in which case we'll try printing it anyway + if not e_message: + try: + e_message = str(e.args[0]) + except: + e_message = "" + + return e_message + + +class HeadphonesException(Exception): + "Generic Headphones Exception - should never be thrown, only subclassed" + +class NewzbinAPIThrottled(HeadphonesException): + "Newzbin has throttled us, deal with it" diff --git a/headphones/searcher.py b/headphones/searcher.py index 00cbfeb0..c2070ee9 100644 --- a/headphones/searcher.py +++ b/headphones/searcher.py @@ -4,7 +4,7 @@ from xml.dom import minidom from xml.parsers.expat import ExpatError import os, re, time -import headphones +import headphones, exceptions from headphones import logger, db, helpers, classes, sab class NewzbinDownloader(urllib.FancyURLopener): @@ -31,11 +31,9 @@ class NewzbinDownloader(urllib.FancyURLopener): #raise exceptions.AuthException("Newzbin account not premium status, can't download NZBs") logger.info("Newzbin error 402") - logger.info("Newzbin throttled our NZB downloading, pausing for " + result.group(1) + "seconds") - + logger.info("Newzbin throttled our NZB downloading, pausing for " + result.group(1) + " seconds") time.sleep(int(result.group(1))) - - #raise exceptions.NewzbinAPIThrottled() + raise exceptions.NewzbinAPIThrottled() #this should be in a class somewhere def getNewzbinURL(url): @@ -459,6 +457,12 @@ def getresultNZB(result): nzb = urllib.urlopen(url, data=params).read() except urllib2.URLError, e: logger.warn('Error fetching nzb from url: %s. Error: %s' % (url, e)) + except exceptions.NewzbinAPIThrottled: + #TODO: This has created a potentially infinite loop? As long as they keep throttling we keep trying. + logger.info("Done waiting for Newzbin API throttle limit, starting downloads again") + getresultNZB(result) + except AttributeError: + logger.warn("AttributeError in getresultNZB.") else: try: nzb = urllib2.urlopen(result[2], timeout=30).read() From fd5d937cc4625109673564cc0b9da37e4b99e079 Mon Sep 17 00:00:00 2001 From: sbuser Date: Wed, 10 Aug 2011 10:16:38 -0500 Subject: [PATCH 8/9] Fix for TypeError: object of type 'NoneType' has no len() in Newzbin results. --- headphones/searcher.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/headphones/searcher.py b/headphones/searcher.py index c2070ee9..3f7e77ee 100644 --- a/headphones/searcher.py +++ b/headphones/searcher.py @@ -304,7 +304,7 @@ def searchNZB(albumid=None, new=False): items = d.getElementsByTagName("item") except ExpatError: logger.info('Unable to get the NEWZBIN feed. Check that your settings are correct - post a bug if they are') - items = None + items = [] if len(items): From 5eac6d466ecfe945a43917c61af9b435288ac50b Mon Sep 17 00:00:00 2001 From: sbuser Date: Wed, 10 Aug 2011 10:25:31 -0500 Subject: [PATCH 9/9] More Newzbin throttling fixes. --- headphones/searcher.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/headphones/searcher.py b/headphones/searcher.py index 3f7e77ee..c08ed287 100644 --- a/headphones/searcher.py +++ b/headphones/searcher.py @@ -23,6 +23,10 @@ class NewzbinDownloader(urllib.FancyURLopener): rtext = str(headers.getheader('X-DNZB-RText')) result = re.search("wait (\d+) seconds", rtext) + logger.info("Newzbin throttled our NZB downloading, pausing for " + result.group(1) + " seconds") + time.sleep(int(result.group(1))) + raise exceptions.NewzbinAPIThrottled() + elif newzbinErrCode == 401: logger.info("Newzbin error 401") #raise exceptions.AuthException("Newzbin username or password incorrect") @@ -31,10 +35,6 @@ class NewzbinDownloader(urllib.FancyURLopener): #raise exceptions.AuthException("Newzbin account not premium status, can't download NZBs") logger.info("Newzbin error 402") - logger.info("Newzbin throttled our NZB downloading, pausing for " + result.group(1) + " seconds") - time.sleep(int(result.group(1))) - raise exceptions.NewzbinAPIThrottled() - #this should be in a class somewhere def getNewzbinURL(url): @@ -294,7 +294,11 @@ def searchNZB(albumid=None, new=False): "q": term } searchURL = providerurl + "search/?%s" % urllib.urlencode(params) - data = getNewzbinURL(searchURL) + try: + data = getNewzbinURL(searchURL) + except exceptions.NewzbinAPIThrottled: + #try again if we were throttled + data = getNewzbinURL(searchURL) if data: logger.info(u'Parsing results from %s' % (searchURL, providerurl))