From e88049dd960961942f561418f35349da36a5cc72 Mon Sep 17 00:00:00 2001 From: sbuser Date: Mon, 8 Aug 2011 13:46:52 -0500 Subject: [PATCH 1/7] bugfix --- headphones/searcher.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/headphones/searcher.py b/headphones/searcher.py index 824bb41d..1459481b 100644 --- a/headphones/searcher.py +++ b/headphones/searcher.py @@ -407,7 +407,7 @@ def searchNZB(albumid=None, new=False): if headphones.SAB_HOST and not headphones.BLACKHOLE: nzb = classes.NZBDataSearchResult() - nzb.extraInfo.append(data) + nzb.extraInfo.append(data) nzb.name = nzb_folder_name sab.sendNZB(nzb) From c07bef0db1e630d8a8203d78d69ce87cc9985af1 Mon Sep 17 00:00:00 2001 From: sbuser Date: Mon, 8 Aug 2011 18:26:10 -0500 Subject: [PATCH 2/7] Keep a local cache of releaseID->releaseGroupID relationships so we don't hammer MB unnecessarily when external programs addReleaseByID --- headphones/__init__.py | 1 + headphones/importer.py | 33 +++++++++++++++++++-------------- 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/headphones/__init__.py b/headphones/__init__.py index 6939f8b8..47e8ed29 100644 --- a/headphones/__init__.py +++ b/headphones/__init__.py @@ -431,6 +431,7 @@ def dbcheck(): c.execute('CREATE TABLE IF NOT EXISTS have (ArtistName TEXT, AlbumTitle TEXT, TrackNumber TEXT, TrackTitle TEXT, TrackLength TEXT, BitRate TEXT, Genre TEXT, Date TEXT, TrackID TEXT)') c.execute('CREATE TABLE IF NOT EXISTS lastfmcloud (ArtistName TEXT, ArtistID TEXT, Count INTEGER)') c.execute('CREATE TABLE IF NOT EXISTS descriptions (ReleaseGroupID TEXT, ReleaseID TEXT, Summary TEXT, Content TEXT)') + c.execute('CREATE TABLE IF NOT EXISTS releases (ReleaseID TEXT, ReleaseGroupID TEXT, UNIQUE(ReleaseID, ReleaseGroupID))') try: c.execute('SELECT IncludeExtras from artists') diff --git a/headphones/importer.py b/headphones/importer.py index 4f8dae3b..cfcd22ee 100644 --- a/headphones/importer.py +++ b/headphones/importer.py @@ -256,21 +256,26 @@ def addArtisttoDB(artistid, extrasonly=False): logger.info(u"Updating complete for: " + artist['artist_name']) def addReleaseById(rid): - + myDB = db.DBConnection() + + rgid = myDB.select("SELECT * from releases WHERE ReleaseID=?", [rid]) + if not rgid: + #we have to make a call to get the release no matter what so we can get the RGID + #need a way around this - a local cache maybe in the future maybe? + try: + release_dict = mb.getRelease(rid) + #keep a local cache of these so that external programs that are adding releasesByID don't hammer MB + myDB.action('INSERT INTO releases VALUES( ?, ?)', [rid, release_dict['rgid']]) + except Exception, e: + logger.info('Unable to get release information for Release: ' + str(rid) + " " + str(e)) + return + if not release_dict: + logger.info('Unable to get release information for Release: ' + str(rid) + " no dict") + return + + rgid = release_dict['rgid'] - #we have to make a call to get the release no matter what so we can get the RGID - #need a way around this - a local cache maybe in the future maybe? - try: - release_dict = mb.getRelease(rid) - except Exception, e: - logger.info('Unable to get release information for Release: ' + str(rid) + " " + str(e)) - return - if not release_dict: - logger.info('Unable to get release information for Release: ' + str(rid) + " no dict") - return - - rgid = release_dict['rgid'] #we don't want to make more calls to MB here unless we have to, could be happening quite a lot #TODO: why do I have to str() this here? I don't get it. @@ -316,7 +321,7 @@ def addReleaseById(rid): for track in release_dict['tracks']: controlValueDict = {"TrackID": track['id'], - "AlbumID": release_dict['rgid']} + "AlbumID": rgid} newValueDict = {"ArtistID": release_dict['artist_id'], "ArtistName": release_dict['artist_name'], "AlbumTitle": release_dict['rg_title'], From 12406b67571268ef35e1d66a0397db1b590a1030 Mon Sep 17 00:00:00 2001 From: sbuser Date: Mon, 8 Aug 2011 18:33:51 -0500 Subject: [PATCH 3/7] Bugfix for local ReleaseID->ReleaseGroupID cache --- headphones/importer.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/headphones/importer.py b/headphones/importer.py index cfcd22ee..b7231cc3 100644 --- a/headphones/importer.py +++ b/headphones/importer.py @@ -259,7 +259,9 @@ def addReleaseById(rid): myDB = db.DBConnection() - rgid = myDB.select("SELECT * from releases WHERE ReleaseID=?", [rid]) + results = myDB.select("SELECT ReleaseGroupID from releases WHERE ReleaseID=?", [rid]) + for result in results: + rgid = result['ReleaseGroupID'] if not rgid: #we have to make a call to get the release no matter what so we can get the RGID #need a way around this - a local cache maybe in the future maybe? From 089af04098dfdcabd87e2a02c8215da5016135ab Mon Sep 17 00:00:00 2001 From: sbuser Date: Mon, 8 Aug 2011 19:29:27 -0500 Subject: [PATCH 4/7] More bug fixes for ReleaseID->ReleaseGroupID cache. --- headphones/importer.py | 39 ++++++++++++++++++++++++++------------- 1 file changed, 26 insertions(+), 13 deletions(-) diff --git a/headphones/importer.py b/headphones/importer.py index b7231cc3..4b14e43c 100644 --- a/headphones/importer.py +++ b/headphones/importer.py @@ -259,16 +259,19 @@ def addReleaseById(rid): myDB = db.DBConnection() - results = myDB.select("SELECT ReleaseGroupID from releases WHERE ReleaseID=?", [rid]) + rgid = None + artistid = None + release_dict = None + results = myDB.select("SELECT albums.ArtistID, releases.ReleaseGroupID from releases, albums WHERE releases.ReleaseID=? and releases.ReleaseGroupID=albums.AlbumID LIMIT 1", [rid]) for result in results: rgid = result['ReleaseGroupID'] + artistid = result['ArtistID'] + logger.debug("Found a cached releaseid : releasegroupid relationship: " + rid + " : " + rgid) if not rgid: - #we have to make a call to get the release no matter what so we can get the RGID - #need a way around this - a local cache maybe in the future maybe? + #didn't find it in the cache, get the information from MB + logger.debug("Didn't find releaseID %s in the cache. Looking up its ReleaseGroupID", [rid]) try: release_dict = mb.getRelease(rid) - #keep a local cache of these so that external programs that are adding releasesByID don't hammer MB - myDB.action('INSERT INTO releases VALUES( ?, ?)', [rid, release_dict['rgid']]) except Exception, e: logger.info('Unable to get release information for Release: ' + str(rid) + " " + str(e)) return @@ -277,15 +280,15 @@ def addReleaseById(rid): return rgid = release_dict['rgid'] - + artistid = release_dict['artist_id'] #we don't want to make more calls to MB here unless we have to, could be happening quite a lot - #TODO: why do I have to str() this here? I don't get it. rg_exists = myDB.select("SELECT * from albums WHERE AlbumID=?", [rgid]) #make sure the artist exists since I don't know what happens later if it doesn't - artist_exists = myDB.select("SELECT * from artists WHERE ArtistID=?", [release_dict['artist_id']]) - if not artist_exists: + artist_exists = myDB.select("SELECT * from artists WHERE ArtistID=?", [artistid]) + + if not artist_exists and release_dict: if release_dict['artist_name'].startswith('The '): sortname = release_dict['artist_name'][4:] else: @@ -303,8 +306,13 @@ def addReleaseById(rid): newValueDict['IncludeExtras'] = 1 myDB.upsert("artists", newValueDict, controlValueDict) - - if not rg_exists: + + elif not artist_exists and not release_dict: + logger.error("Artist does not exist in the database and did not get a valid response from MB. Skipping release.") + return + + if not rg_exists and release_dict: #it should never be the case that we have an rg and not the artist + #but if it is this will fail logger.info(u"Now adding-by-id album (" + release_dict['title'] + ") from id: " + rgid) controlValueDict = {"AlbumID": rgid} @@ -319,6 +327,9 @@ def addReleaseById(rid): } myDB.upsert("albums", newValueDict, controlValueDict) + + #keep a local cache of these so that external programs that are adding releasesByID don't hammer MB + myDB.action('INSERT INTO releases VALUES( ?, ?)', [rid, release_dict['rgid']]) for track in release_dict['tracks']: @@ -334,10 +345,12 @@ def addReleaseById(rid): } myDB.upsert("tracks", newValueDict, controlValueDict) - - + #start a search for the album import searcher searcher.searchNZB(rgid, False) + elif not rg_exists and not release_dict: + logger.error("ReleaseGroup does not exist in the database and did not get a valid response from MB. Skipping release.") + return else: logger.info('Release ' + str(rid) + " already exists in the database!") \ No newline at end of file From 696647b9f093ba5706c34d40812d000bcbc3620d Mon Sep 17 00:00:00 2001 From: sbuser Date: Mon, 8 Aug 2011 20:19:53 -0500 Subject: [PATCH 5/7] Still bug fixing ReleaseID lookups. --- headphones/importer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/headphones/importer.py b/headphones/importer.py index 4b14e43c..5ddbb20e 100644 --- a/headphones/importer.py +++ b/headphones/importer.py @@ -269,7 +269,7 @@ def addReleaseById(rid): logger.debug("Found a cached releaseid : releasegroupid relationship: " + rid + " : " + rgid) if not rgid: #didn't find it in the cache, get the information from MB - logger.debug("Didn't find releaseID %s in the cache. Looking up its ReleaseGroupID", [rid]) + logger.debug("Didn't find releaseID " + rid + " in the cache. Looking up its ReleaseGroupID") try: release_dict = mb.getRelease(rid) except Exception, e: From c2092211f8e90cfc7348e59150ad0efb4dfc71a0 Mon Sep 17 00:00:00 2001 From: sbuser Date: Tue, 9 Aug 2011 18:27:35 -0500 Subject: [PATCH 6/7] Handle an error in searcher. --- headphones/searcher.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/headphones/searcher.py b/headphones/searcher.py index 28f717a7..35def60f 100644 --- a/headphones/searcher.py +++ b/headphones/searcher.py @@ -453,7 +453,7 @@ def getresultNZB(result): else: try: nzb = urllib2.urlopen(result[2], timeout=20).read() - except: + except urllib2.URLError, e: logger.warn('Error fetching nzb from url: ' + result[2] + ' %s' % e) return nzb From 63b48a06639b3279f1b762d668f38f4fa795095d Mon Sep 17 00:00:00 2001 From: sbuser Date: Tue, 9 Aug 2011 18:33:11 -0500 Subject: [PATCH 7/7] Fixed Newzbin downloads in new searcher with preprocessing. Upped timeout as my newznab provider lags. Make a config option for this? --- headphones/searcher.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/headphones/searcher.py b/headphones/searcher.py index 35def60f..66216222 100644 --- a/headphones/searcher.py +++ b/headphones/searcher.py @@ -442,7 +442,7 @@ def verifyresult(title, term): return True def getresultNZB(result): - if result[3] == 'Newzbin': + if result[3] == 'newzbin': params = urllib.urlencode({"username": headphones.NEWZBIN_UID, "password": headphones.NEWZBIN_PASSWORD, "reportid": result[2]}) url = "https://www.newzbin.com" + "/api/dnzb/" urllib._urlopener = NewzbinDownloader() @@ -452,7 +452,7 @@ def getresultNZB(result): logger.warn('Error fetching nzb from url: ' + url + ' %s' % e) else: try: - nzb = urllib2.urlopen(result[2], timeout=20).read() + nzb = urllib2.urlopen(result[2], timeout=30).read() except urllib2.URLError, e: logger.warn('Error fetching nzb from url: ' + result[2] + ' %s' % e) return nzb