From b84b9b997ed22282e1351081f6dd78543f54b568 Mon Sep 17 00:00:00 2001 From: sbuser Date: Sat, 6 Aug 2011 16:12:11 -0500 Subject: [PATCH 1/2] Added headphones.MEDIA_FORMATS and referenced it in various places. M4A had been left out in scanMusic. Should just have to change the literal in the future. --- headphones/__init__.py | 1 + headphones/importer.py | 2 +- headphones/postprocessor.py | 6 +++--- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/headphones/__init__.py b/headphones/__init__.py index 84023254..6b9c91a6 100644 --- a/headphones/__init__.py +++ b/headphones/__init__.py @@ -99,6 +99,7 @@ NEWZBIN_PASSWORD = None LASTFM_USERNAME = None +MEDIA_FORMATS = ["mp3", "flac", "aac", "ogg", "ape", "m4a"] def CheckSection(sec): """ Check if INI section exists, if not create it """ diff --git a/headphones/importer.py b/headphones/importer.py index a64e959e..aa4a544b 100644 --- a/headphones/importer.py +++ b/headphones/importer.py @@ -24,7 +24,7 @@ def scanMusic(dir=None): for r,d,f in os.walk(dir): for files in f: - if any(files.endswith(x) for x in (".mp3", ".flac", ".aac", ".ogg", ".ape", ".m4a")): + if any(files.endswith('.' + x) for x in headphones.MEDIA_FORMATS): results.append(os.path.join(r, files)) logger.info(u'%i music files found. Reading metadata....' % len(results)) diff --git a/headphones/postprocessor.py b/headphones/postprocessor.py index e97d46b1..13b753e3 100644 --- a/headphones/postprocessor.py +++ b/headphones/postprocessor.py @@ -113,7 +113,7 @@ def verify(albumid, albumpath): downloaded_track_list = [] for r,d,f in os.walk(albumpath): for files in f: - if any(files.endswith(x) for x in (".mp3", ".flac", ".aac", ".ogg", ".ape", ".m4a")): + if any(files.endswith('.' + x) for x in headphones.MEDIA_FORMATS): downloaded_track_list.append(os.path.join(r, files)) # test #1: metadata - usually works @@ -240,7 +240,7 @@ def cleanupFiles(albumpath): logger.info('Cleaning up files') for r,d,f in os.walk(albumpath): for files in f: - if not any(files.endswith(x) for x in (".mp3", ".flac", ".aac", ".ogg", ".ape", ".m4a")): + if not any(files.endswith('.' + x) for x in headphones.MEDIA_FORMATS): logger.debug('Removing: %s' % files) try: os.remove(os.path.join(r, files)) @@ -393,7 +393,7 @@ def updateHave(albumpath): for r,d,f in os.walk(albumpath): for files in f: - if any(files.endswith(x) for x in (".mp3", ".flac", ".aac", ".ogg", ".ape")): + if any(files.endswith('.' + x) for x in headphones.MEDIA_FORMATS): results.append(os.path.join(r, files)) if results: From 86861df6f87abaf81a33dec504025e33c6ad8934 Mon Sep 17 00:00:00 2001 From: sbuser Date: Sun, 7 Aug 2011 21:16:35 -0500 Subject: [PATCH 2/2] Searcher now downloads the nzb itself in order to pre-process the nzb and take action. Currently it just checks the date in the nzb to make sure it is within retention range. NZBs are sent directly to SAB or put into the blackhole. --- headphones/sab.py | 4 -- headphones/searcher.py | 111 ++++++++++++++++++++--------------------- 2 files changed, 54 insertions(+), 61 deletions(-) diff --git a/headphones/sab.py b/headphones/sab.py index bc9d8053..d6433ce7 100644 --- a/headphones/sab.py +++ b/headphones/sab.py @@ -67,10 +67,6 @@ def sendNZB(nzb): url = "http://" + headphones.SAB_HOST + "/" + "api?" + urllib.urlencode(params) - logger.info(u"Sending NZB to SABnzbd") - - logger.info(u"URL: " + url) - try: if nzb.resultType == "nzb": diff --git a/headphones/searcher.py b/headphones/searcher.py index d7bdb4c5..6645f47f 100644 --- a/headphones/searcher.py +++ b/headphones/searcher.py @@ -400,72 +400,31 @@ def searchNZB(albumid=None, new=False): logger.info(u"Found best result: %s (%s) - %s" % (bestqual[0], bestqual[2], helpers.bytes_to_mb(bestqual[1]))) - - if bestqual[3] == "newzbin": - #logger.info("Found a newzbin result") - reportid = bestqual[2] - params = urllib.urlencode({"username": headphones.NEWZBIN_UID, "password": headphones.NEWZBIN_PASSWORD, "reportid": reportid}) - url = providerurl + "/api/dnzb/" - urllib._urlopener = NewzbinDownloader() - data = urllib.urlopen(url, data=params).read() - nzb = classes.NZBDataSearchResult() - nzb.extraInfo.append(data) - nzb_folder_name = '%s - %s [%s]' % (helpers.latinToAscii(albums[0]).encode('UTF-8').replace('/', '_'), helpers.latinToAscii(albums[1]).encode('UTF-8').replace('/', '_'), year) - nzb.name = nzb_folder_name - logger.info(u"Sending FILE to SABNZBD: " + nzb.name) - sab.sendNZB(nzb) - - myDB.action('UPDATE albums SET status = "Snatched" WHERE AlbumID=?', [albums[2]]) - myDB.action('INSERT INTO snatched VALUES( ?, ?, ?, ?, DATETIME("NOW", "localtime"), ?, ?)', [albums[2], bestqual[0], bestqual[1], bestqual[2], "Snatched", nzb_folder_name]) - else: - downloadurl = bestqual[2] - nzb_folder_name = '%s - %s [%s]' % (helpers.latinToAscii(albums[0]).encode('UTF-8').replace('/', '_'), helpers.latinToAscii(albums[1]).encode('UTF-8').replace('/', '_'), year) - + logger.info(u"Pre-processing result") + (data, bestqual) = preprocess(nzblist) + if data and bestqual: + nzb_folder_name = '%s - %s [%s]' % (helpers.latinToAscii(albums[0]).encode('UTF-8').replace('/', '_'), helpers.latinToAscii(albums[1]).encode('UTF-8').replace('/', '_'), year) if headphones.SAB_HOST and not headphones.BLACKHOLE: - linkparams = {} - - linkparams["mode"] = "addurl" - - if headphones.SAB_APIKEY: - linkparams["apikey"] = headphones.SAB_APIKEY - if headphones.SAB_USERNAME: - linkparams["ma_username"] = headphones.SAB_USERNAME - if headphones.SAB_PASSWORD: - linkparams["ma_password"] = headphones.SAB_PASSWORD - if headphones.SAB_CATEGORY: - linkparams["cat"] = headphones.SAB_CATEGORY - - linkparams["name"] = downloadurl - - linkparams["nzbname"] = nzb_folder_name - - saburl = 'http://' + headphones.SAB_HOST + '/sabnzbd/api?' + urllib.urlencode(linkparams) - logger.info(u"Sending link to SABNZBD: " + saburl) - - try: - urllib.urlopen(saburl) - - except: - logger.error(u"Unable to send link. Are you sure the host address is correct?") - break - - myDB.action('UPDATE albums SET status = "Snatched" WHERE AlbumID=?', [albums[2]]) - myDB.action('INSERT INTO snatched VALUES( ?, ?, ?, ?, DATETIME("NOW", "localtime"), ?, ?)', [albums[2], bestqual[0], bestqual[1], bestqual[2], "Snatched", nzb_folder_name]) - - + + nzb = classes.NZBDataSearchResult() + nzb.extraInfo.append(data) + nzb.name = nzb_folder_name + sab.sendNZB(nzb) + elif headphones.BLACKHOLE: nzb_name = nzb_folder_name + '.nzb' download_path = os.path.join(headphones.BLACKHOLE_DIR, nzb_name) - try: - urllib.urlretrieve(downloadurl, download_path) + f = open(download_path) + f.write(data) + f.close() except Exception, e: - logger.error('Couldn\'t retrieve NZB: %s' % e) + logger.error('Couldn\'t write NZB file: %s' % e) break - myDB.action('UPDATE albums SET status = "Snatched" WHERE AlbumID=?', [albums[2]]) - myDB.action('INSERT INTO snatched VALUES( ?, ?, ?, ?, DATETIME("NOW", "localtime"), ?, ?)', [albums[2], bestqual[0], bestqual[1], bestqual[2], "Snatched", nzb_folder_name]) + myDB.action('UPDATE albums SET status = "Snatched" WHERE AlbumID=?', [albums[2]]) + myDB.action('INSERT INTO snatched VALUES( ?, ?, ?, ?, DATETIME("NOW", "localtime"), ?, ?)', [albums[2], bestqual[0], bestqual[1], bestqual[2], "Snatched", nzb_folder_name]) def verifyresult(title, term): @@ -482,3 +441,41 @@ def verifyresult(title, term): return False else: return True + +def getresultNZB(result): + if result[3] == 'Newzbin': + params = urllib.urlencode({"username": headphones.NEWZBIN_UID, "password": headphones.NEWZBIN_PASSWORD, "reportid": result[2]}) + url = "https://www.newzbin.com" + "/api/dnzb/" + urllib._urlopener = NewzbinDownloader() + try: + nzb = urllib.urlopen(url, data=params).read() + except urllib2.URLError, e: + logger.warn('Error fetching nzb from url: ' + url + ' %s' % e) + else: + try: + nzb = urllib2.urlopen(result[2], timeout=20).read() + except: + logger.warn('Error fetching nzb from url: ' + result[2] + ' %s' % e) + return nzb + +def preprocess(resultlist): + for result in resultlist: + nzb = getresultNZB(result) + if nzb: + try: + d = minidom.parseString(nzb) + node = d.documentElement + nzbfiles = d.getElementsByTagName("file") + for nzbfile in nzbfiles: + if nzbfile.getAttribute("date") < (time.time() - int(headphones.USENET_RETENTION) * 86400): + logger.error('NZB contains a file out of your retention. Skipping.') + continue + #TODO: Do we want rar checking in here to try to keep unknowns out? + #or at least the option to do so? + except ExpatError: + logger.error('Unable to parse the best result NZB. Skipping.') + continue + return nzb, result + else: + logger.error("Couldn't retrieve the best nzb. Skipping.") + return (False, False) \ No newline at end of file