From b84b9b997ed22282e1351081f6dd78543f54b568 Mon Sep 17 00:00:00 2001 From: sbuser Date: Sat, 6 Aug 2011 16:12:11 -0500 Subject: [PATCH 1/9] Added headphones.MEDIA_FORMATS and referenced it in various places. M4A had been left out in scanMusic. Should just have to change the literal in the future. --- headphones/__init__.py | 1 + headphones/importer.py | 2 +- headphones/postprocessor.py | 6 +++--- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/headphones/__init__.py b/headphones/__init__.py index 84023254..6b9c91a6 100644 --- a/headphones/__init__.py +++ b/headphones/__init__.py @@ -99,6 +99,7 @@ NEWZBIN_PASSWORD = None LASTFM_USERNAME = None +MEDIA_FORMATS = ["mp3", "flac", "aac", "ogg", "ape", "m4a"] def CheckSection(sec): """ Check if INI section exists, if not create it """ diff --git a/headphones/importer.py b/headphones/importer.py index a64e959e..aa4a544b 100644 --- a/headphones/importer.py +++ b/headphones/importer.py @@ -24,7 +24,7 @@ def scanMusic(dir=None): for r,d,f in os.walk(dir): for files in f: - if any(files.endswith(x) for x in (".mp3", ".flac", ".aac", ".ogg", ".ape", ".m4a")): + if any(files.endswith('.' + x) for x in headphones.MEDIA_FORMATS): results.append(os.path.join(r, files)) logger.info(u'%i music files found. Reading metadata....' % len(results)) diff --git a/headphones/postprocessor.py b/headphones/postprocessor.py index e97d46b1..13b753e3 100644 --- a/headphones/postprocessor.py +++ b/headphones/postprocessor.py @@ -113,7 +113,7 @@ def verify(albumid, albumpath): downloaded_track_list = [] for r,d,f in os.walk(albumpath): for files in f: - if any(files.endswith(x) for x in (".mp3", ".flac", ".aac", ".ogg", ".ape", ".m4a")): + if any(files.endswith('.' + x) for x in headphones.MEDIA_FORMATS): downloaded_track_list.append(os.path.join(r, files)) # test #1: metadata - usually works @@ -240,7 +240,7 @@ def cleanupFiles(albumpath): logger.info('Cleaning up files') for r,d,f in os.walk(albumpath): for files in f: - if not any(files.endswith(x) for x in (".mp3", ".flac", ".aac", ".ogg", ".ape", ".m4a")): + if not any(files.endswith('.' + x) for x in headphones.MEDIA_FORMATS): logger.debug('Removing: %s' % files) try: os.remove(os.path.join(r, files)) @@ -393,7 +393,7 @@ def updateHave(albumpath): for r,d,f in os.walk(albumpath): for files in f: - if any(files.endswith(x) for x in (".mp3", ".flac", ".aac", ".ogg", ".ape")): + if any(files.endswith('.' + x) for x in headphones.MEDIA_FORMATS): results.append(os.path.join(r, files)) if results: From 86861df6f87abaf81a33dec504025e33c6ad8934 Mon Sep 17 00:00:00 2001 From: sbuser Date: Sun, 7 Aug 2011 21:16:35 -0500 Subject: [PATCH 2/9] Searcher now downloads the nzb itself in order to pre-process the nzb and take action. Currently it just checks the date in the nzb to make sure it is within retention range. NZBs are sent directly to SAB or put into the blackhole. --- headphones/sab.py | 4 -- headphones/searcher.py | 111 ++++++++++++++++++++--------------------- 2 files changed, 54 insertions(+), 61 deletions(-) diff --git a/headphones/sab.py b/headphones/sab.py index bc9d8053..d6433ce7 100644 --- a/headphones/sab.py +++ b/headphones/sab.py @@ -67,10 +67,6 @@ def sendNZB(nzb): url = "http://" + headphones.SAB_HOST + "/" + "api?" + urllib.urlencode(params) - logger.info(u"Sending NZB to SABnzbd") - - logger.info(u"URL: " + url) - try: if nzb.resultType == "nzb": diff --git a/headphones/searcher.py b/headphones/searcher.py index d7bdb4c5..6645f47f 100644 --- a/headphones/searcher.py +++ b/headphones/searcher.py @@ -400,72 +400,31 @@ def searchNZB(albumid=None, new=False): logger.info(u"Found best result: %s (%s) - %s" % (bestqual[0], bestqual[2], helpers.bytes_to_mb(bestqual[1]))) - - if bestqual[3] == "newzbin": - #logger.info("Found a newzbin result") - reportid = bestqual[2] - params = urllib.urlencode({"username": headphones.NEWZBIN_UID, "password": headphones.NEWZBIN_PASSWORD, "reportid": reportid}) - url = providerurl + "/api/dnzb/" - urllib._urlopener = NewzbinDownloader() - data = urllib.urlopen(url, data=params).read() - nzb = classes.NZBDataSearchResult() - nzb.extraInfo.append(data) - nzb_folder_name = '%s - %s [%s]' % (helpers.latinToAscii(albums[0]).encode('UTF-8').replace('/', '_'), helpers.latinToAscii(albums[1]).encode('UTF-8').replace('/', '_'), year) - nzb.name = nzb_folder_name - logger.info(u"Sending FILE to SABNZBD: " + nzb.name) - sab.sendNZB(nzb) - - myDB.action('UPDATE albums SET status = "Snatched" WHERE AlbumID=?', [albums[2]]) - myDB.action('INSERT INTO snatched VALUES( ?, ?, ?, ?, DATETIME("NOW", "localtime"), ?, ?)', [albums[2], bestqual[0], bestqual[1], bestqual[2], "Snatched", nzb_folder_name]) - else: - downloadurl = bestqual[2] - nzb_folder_name = '%s - %s [%s]' % (helpers.latinToAscii(albums[0]).encode('UTF-8').replace('/', '_'), helpers.latinToAscii(albums[1]).encode('UTF-8').replace('/', '_'), year) - + logger.info(u"Pre-processing result") + (data, bestqual) = preprocess(nzblist) + if data and bestqual: + nzb_folder_name = '%s - %s [%s]' % (helpers.latinToAscii(albums[0]).encode('UTF-8').replace('/', '_'), helpers.latinToAscii(albums[1]).encode('UTF-8').replace('/', '_'), year) if headphones.SAB_HOST and not headphones.BLACKHOLE: - linkparams = {} - - linkparams["mode"] = "addurl" - - if headphones.SAB_APIKEY: - linkparams["apikey"] = headphones.SAB_APIKEY - if headphones.SAB_USERNAME: - linkparams["ma_username"] = headphones.SAB_USERNAME - if headphones.SAB_PASSWORD: - linkparams["ma_password"] = headphones.SAB_PASSWORD - if headphones.SAB_CATEGORY: - linkparams["cat"] = headphones.SAB_CATEGORY - - linkparams["name"] = downloadurl - - linkparams["nzbname"] = nzb_folder_name - - saburl = 'http://' + headphones.SAB_HOST + '/sabnzbd/api?' + urllib.urlencode(linkparams) - logger.info(u"Sending link to SABNZBD: " + saburl) - - try: - urllib.urlopen(saburl) - - except: - logger.error(u"Unable to send link. Are you sure the host address is correct?") - break - - myDB.action('UPDATE albums SET status = "Snatched" WHERE AlbumID=?', [albums[2]]) - myDB.action('INSERT INTO snatched VALUES( ?, ?, ?, ?, DATETIME("NOW", "localtime"), ?, ?)', [albums[2], bestqual[0], bestqual[1], bestqual[2], "Snatched", nzb_folder_name]) - - + + nzb = classes.NZBDataSearchResult() + nzb.extraInfo.append(data) + nzb.name = nzb_folder_name + sab.sendNZB(nzb) + elif headphones.BLACKHOLE: nzb_name = nzb_folder_name + '.nzb' download_path = os.path.join(headphones.BLACKHOLE_DIR, nzb_name) - try: - urllib.urlretrieve(downloadurl, download_path) + f = open(download_path) + f.write(data) + f.close() except Exception, e: - logger.error('Couldn\'t retrieve NZB: %s' % e) + logger.error('Couldn\'t write NZB file: %s' % e) break - myDB.action('UPDATE albums SET status = "Snatched" WHERE AlbumID=?', [albums[2]]) - myDB.action('INSERT INTO snatched VALUES( ?, ?, ?, ?, DATETIME("NOW", "localtime"), ?, ?)', [albums[2], bestqual[0], bestqual[1], bestqual[2], "Snatched", nzb_folder_name]) + myDB.action('UPDATE albums SET status = "Snatched" WHERE AlbumID=?', [albums[2]]) + myDB.action('INSERT INTO snatched VALUES( ?, ?, ?, ?, DATETIME("NOW", "localtime"), ?, ?)', [albums[2], bestqual[0], bestqual[1], bestqual[2], "Snatched", nzb_folder_name]) def verifyresult(title, term): @@ -482,3 +441,41 @@ def verifyresult(title, term): return False else: return True + +def getresultNZB(result): + if result[3] == 'Newzbin': + params = urllib.urlencode({"username": headphones.NEWZBIN_UID, "password": headphones.NEWZBIN_PASSWORD, "reportid": result[2]}) + url = "https://www.newzbin.com" + "/api/dnzb/" + urllib._urlopener = NewzbinDownloader() + try: + nzb = urllib.urlopen(url, data=params).read() + except urllib2.URLError, e: + logger.warn('Error fetching nzb from url: ' + url + ' %s' % e) + else: + try: + nzb = urllib2.urlopen(result[2], timeout=20).read() + except: + logger.warn('Error fetching nzb from url: ' + result[2] + ' %s' % e) + return nzb + +def preprocess(resultlist): + for result in resultlist: + nzb = getresultNZB(result) + if nzb: + try: + d = minidom.parseString(nzb) + node = d.documentElement + nzbfiles = d.getElementsByTagName("file") + for nzbfile in nzbfiles: + if nzbfile.getAttribute("date") < (time.time() - int(headphones.USENET_RETENTION) * 86400): + logger.error('NZB contains a file out of your retention. Skipping.') + continue + #TODO: Do we want rar checking in here to try to keep unknowns out? + #or at least the option to do so? + except ExpatError: + logger.error('Unable to parse the best result NZB. Skipping.') + continue + return nzb, result + else: + logger.error("Couldn't retrieve the best nzb. Skipping.") + return (False, False) \ No newline at end of file From 6159ca61edf2f881e6cc4db64907c0d62e39a3e7 Mon Sep 17 00:00:00 2001 From: Remy Date: Mon, 8 Aug 2011 13:18:10 -0700 Subject: [PATCH 3/9] Changed the way album descriptions are retrieved from lastfm --- headphones/importer.py | 2 +- headphones/lastfm.py | 47 +++++++++++++++++++++++++++++++++++++++++- headphones/mb.py | 2 +- 3 files changed, 48 insertions(+), 3 deletions(-) diff --git a/headphones/importer.py b/headphones/importer.py index 27e3bfbe..4f8dae3b 100644 --- a/headphones/importer.py +++ b/headphones/importer.py @@ -215,7 +215,7 @@ def addArtisttoDB(artistid, extrasonly=False): myDB.upsert("albums", newValueDict, controlValueDict) - lastfm.getAlbumDescription(rg['id'], release_dict['releaselist']) + lastfm.getAlbumDescription(rg['id'], artist['artist_name'], rg['title']) # I changed the albumid from releaseid -> rgid, so might need to delete albums that have a releaseid for release in release_dict['releaselist']: diff --git a/headphones/lastfm.py b/headphones/lastfm.py index 0e39a110..814a35e8 100644 --- a/headphones/lastfm.py +++ b/headphones/lastfm.py @@ -91,7 +91,52 @@ def getArtists(): for artistid in artistlist: importer.addArtisttoDB(artistid) -def getAlbumDescription(rgid, releaselist): +def getAlbumDescription(rgid, artist, album): + + myDB = db.DBConnection() + result = myDB.select('SELECT Summary from descriptions WHERE ReleaseGroupID=?', [rgid]) + + if result: + return + + params = { "method": 'album.getInfo', + "api_key": api_key, + "artist": artist.encode('utf-8'), + "album": album.encode('utf-8') + } + + searchURL = 'http://ws.audioscrobbler.com/2.0/?' + urllib.urlencode(params) + data = urllib.urlopen(searchURL).read() + + if data == 'Album not found': + return + + try: + d = minidom.parseString(data) + + albuminfo = d.getElementsByTagName("album") + + for item in albuminfo: + summarynode = item.getElementsByTagName("summary")[0].childNodes + contentnode = item.getElementsByTagName("content")[0].childNodes + for node in summarynode: + summary = node.data + for node in contentnode: + content = node.data + + controlValueDict = {'ReleaseGroupID': rgid} + newValueDict = {'Summary': summary, + 'Content': content} + myDB.upsert("descriptions", newValueDict, controlValueDict) + + except: + return + +def getAlbumDescriptionOld(rgid, releaselist): + """ + This was a dumb way to do it - going to just use artist & album name but keeping this here + because I may use it to fetch and cache album art + """ myDB = db.DBConnection() result = myDB.select('SELECT Summary from descriptions WHERE ReleaseGroupID=?', [rgid]) diff --git a/headphones/mb.py b/headphones/mb.py index 92723fda..1d6b20cd 100644 --- a/headphones/mb.py +++ b/headphones/mb.py @@ -306,7 +306,7 @@ def getReleaseGroup(rgid): releaselist.append(release_dict) - average_tracks = sum(x['trackscount'] for x in releaselist) / len(releaselist) + average_tracks = sum(x['trackscount'] for x in releaselist) / float(len(releaselist)) for item in releaselist: item['trackscount_delta'] = abs(average_tracks - item['trackscount']) From 6251df64a3de7b0177afe6e4176a3e7a23e81c50 Mon Sep 17 00:00:00 2001 From: Remy Date: Mon, 8 Aug 2011 14:08:45 -0700 Subject: [PATCH 4/9] Fixed wanted albums showing Retry Download subhead --- data/interfaces/default/album.html | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/data/interfaces/default/album.html b/data/interfaces/default/album.html index e8406f63..2308f32b 100644 --- a/data/interfaces/default/album.html +++ b/data/interfaces/default/album.html @@ -9,11 +9,11 @@ From 1d361892a463f7bfce2478fb6275cc7eb5dfdc10 Mon Sep 17 00:00:00 2001 From: Remy Date: Mon, 8 Aug 2011 14:09:37 -0700 Subject: [PATCH 5/9] Changed artist already in db message to debug --- headphones/importer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/headphones/importer.py b/headphones/importer.py index 4f8dae3b..2efc2a27 100644 --- a/headphones/importer.py +++ b/headphones/importer.py @@ -96,7 +96,7 @@ def is_exists(artistid): artistlist = myDB.select('SELECT ArtistID, ArtistName from artists WHERE ArtistID=?', [artistid]) if any(artistid in x for x in artistlist): - logger.info(artistlist[0][1] + u" is already in the database. Updating 'have tracks', but not artist information") + logger.debug(artistlist[0][1] + u" is already in the database. Updating 'have tracks', but not artist information") return True else: return False From ed95b27945bafa043dd1b175cec8c83867e3240b Mon Sep 17 00:00:00 2001 From: Remy Date: Mon, 8 Aug 2011 14:13:30 -0700 Subject: [PATCH 6/9] Fixed default logging to show results by most recent --- data/interfaces/default/logs.html | 1 + 1 file changed, 1 insertion(+) diff --git a/data/interfaces/default/logs.html b/data/interfaces/default/logs.html index 8434e3b6..cc6436f3 100644 --- a/data/interfaces/default/logs.html +++ b/data/interfaces/default/logs.html @@ -57,6 +57,7 @@ "bStateSave": true, "iDisplayLength": 100, "sPaginationType": "full_numbers", + "aaSorting": [] }); }); From 856ebded0d55bbb5e50a3e335819906960f63123 Mon Sep 17 00:00:00 2001 From: Remy Date: Mon, 8 Aug 2011 14:43:41 -0700 Subject: [PATCH 7/9] Fixed some formatting on the config page --- data/interfaces/default/config.html | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/data/interfaces/default/config.html b/data/interfaces/default/config.html index b6d36986..0bd12fc0 100644 --- a/data/interfaces/default/config.html +++ b/data/interfaces/default/config.html @@ -177,8 +177,6 @@ -
-

Newzbin Password:

@@ -242,7 +240,7 @@

Automatically Include Extras When Adding an Artist


Extras includes: EPs, Compilations, Live Albums, Remix Albums and Singles -


+

Log Directory:

From 5cc29413d245657ff88319018494b79e5cfa2c23 Mon Sep 17 00:00:00 2001 From: Remy Date: Mon, 8 Aug 2011 17:56:40 -0700 Subject: [PATCH 8/9] Changed some config layout --- data/css/style.css | 4 +-- data/interfaces/default/config.html | 49 +++++++++++++---------------- 2 files changed, 24 insertions(+), 29 deletions(-) diff --git a/data/css/style.css b/data/css/style.css index 511c8ae7..e3d9a282 100755 --- a/data/css/style.css +++ b/data/css/style.css @@ -138,8 +138,8 @@ div#main { margin: 0; padding: 80px 0 0 0; } .table_wrapper_right{ padding: 25px; background-color: #ffffff; width: 40%; min-height: 100px; margin-top: 25px; margin-left: auto; margin-right: 30px; -moz-border-radius: 20px; border-radius: 20px; } .configtable { font-size: 14px; line-height:18px; } -.configtable td { width: 350px; padding: 10px; vertical-align: middle; } -.configtable tr { vertical-align: text-top; } +.configtable td { width: 350px; padding: 10px; } +.configtable td#middle { vertical-align: middle; } table#artist_table { background-color: white; width: 100%; padding: 20px; } diff --git a/data/interfaces/default/config.html b/data/interfaces/default/config.html index 0bd12fc0..fe464f75 100644 --- a/data/interfaces/default/config.html +++ b/data/interfaces/default/config.html @@ -109,7 +109,7 @@

Search Providers

 - @@ -125,59 +125,55 @@ - - - @@ -190,21 +186,21 @@ @@ -251,4 +247,3 @@ (Web Interface changes require a restart to take effect) -<%inherit file="base.html" /> \ No newline at end of file From e3bc4a68efb2f525384094db2d78fdca3ad18efb Mon Sep 17 00:00:00 2001 From: Remy Date: Mon, 8 Aug 2011 19:15:49 -0700 Subject: [PATCH 9/9] Blackhole option not creating the nzb files - added write argument to f.open --- headphones/searcher.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/headphones/searcher.py b/headphones/searcher.py index 3eb293be..333b12eb 100644 --- a/headphones/searcher.py +++ b/headphones/searcher.py @@ -415,7 +415,7 @@ def searchNZB(albumid=None, new=False): nzb_name = nzb_folder_name + '.nzb' download_path = os.path.join(headphones.BLACKHOLE_DIR, nzb_name) try: - f = open(download_path) + f = open(download_path, 'w') f.write(data) f.close() except Exception, e:
+

NZBMatrix:

+

Newznab:

-

Newznab Host:

+

Newznab Host:


i.e. http://nzb.su 		- -

- Newznab API:

+

Newznab API:

-
- -

NZBs.org:

+ 		+

NZBs.org:

-

NZBs.org UID:

+

NZBs.org UID:

-

NZBs.org Hash:

+

NZBs.org Hash:

-

Newzbin:

+ 		+

Newzbin:

-

Newzbin UID:

+

Newzbin UID:

-

Newzbin Password:

+

Newzbin Password:

Album Quality:


- Highest Quality excluding Lossless
- Highest Quality including Lossless
- Lossless Only
- Preferred Bitrate: + Highest Quality excluding Lossless
+ Highest Quality including Lossless
+ Lossless Only
+ Preferred Bitrate: kbps
Auto-Detect Preferred Bitrate

Post-Processing:

- Move downloads to Destination Folder
- Rename files
- Correct metadata
- Delete leftover files (.m3u, .nfo, .sfv, .nzb, etc.)
- Add album art as 'folder.jpg' to album folder
- Embed album art in each file + Move downloads to Destination Folder
+ Rename files
+ Correct metadata
+ Delete leftover files (.m3u, .nfo, .sfv, .nzb, etc.)
+ Add album art as 'folder.jpg' to album folder
+ Embed album art in each file