From a717594950518493c1cb0fa42446ac48ff4df8ea Mon Sep 17 00:00:00 2001 From: Bas Stottelaar Date: Sun, 6 Apr 2014 16:15:11 +0200 Subject: [PATCH] Replaced urllib2 in albumart and cache --- headphones/albumart.py | 39 ++--- headphones/cache.py | 377 ++++++++++++++++++----------------------- 2 files changed, 177 insertions(+), 239 deletions(-) diff --git a/headphones/albumart.py b/headphones/albumart.py index 88406858..69150924 100644 --- a/headphones/albumart.py +++ b/headphones/albumart.py @@ -13,39 +13,30 @@ # You should have received a copy of the GNU General Public License # along with Headphones. If not, see . -import urllib2 -from headphones import db +from headphones import helpers, db def getAlbumArt(albumid): - myDB = db.DBConnection() asin = myDB.action('SELECT AlbumASIN from albums WHERE AlbumID=?', [albumid]).fetchone()[0] - - if not asin: - return None - - url = 'http://ec1.images-amazon.com/images/P/%s.01.LZZZZZZZ.jpg' % asin - - return url - + + if asin: + return 'http://ec1.images-amazon.com/images/P/%s.01.LZZZZZZZ.jpg' % asin + def getCachedArt(albumid): - from headphones import cache - + c = cache.Cache() - artwork_path = c.get_artwork_from_cache(AlbumID=albumid) - + if not artwork_path: - return None - + return + if artwork_path.startswith('http://'): - try: - artwork = urllib2.urlopen(artwork_path, timeout=20).read() - return artwork - except: + artwork = helpers.request_content(artwork_path, timeout=20) + + if not artwork: logger.warn("Unable to open url: %s", artwork_path) - return None + return else: - artwork = open(artwork_path, "r").read() - return artwork + with open(artwork_path, "r") as fp: + return fp.read() diff --git a/headphones/cache.py b/headphones/cache.py index 25acb19d..064c4166 100644 --- a/headphones/cache.py +++ b/headphones/cache.py @@ -14,7 +14,7 @@ # along with Headphones. If not, see . import os -import glob, urllib, urllib2 +import glob, urllib import lib.simplejson as simplejson @@ -25,42 +25,42 @@ lastfm_apikey = "690e1ed3bc00bc91804cd8f7fe5ed6d4" class Cache(object): """ - This class deals with getting, storing and serving up artwork (album + This class deals with getting, storing and serving up artwork (album art, artist images, etc) and info/descriptions (album info, artist descrptions) - to and from the cache folder. This can be called from within a web interface, + to and from the cache folder. This can be called from within a web interface, for example, using the helper functions getInfo(id) and getArtwork(id), to utilize the cached images rather than having to retrieve them every time the page is reloaded. - + So you can call cache.getArtwork(id) which will return an absolute path to the image file on the local machine, or if the cache directory doesn't exist, or can not be written to, it will return a url to the image. - + Call cache.getInfo(id) to grab the artist/album info; will return the text description - + The basic format for art in the cache is .. and for info it is ..txt """ - + path_to_art_cache = os.path.join(headphones.CACHE_DIR, 'artwork') - + id = None id_type = None # 'artist' or 'album' - set automatically depending on whether ArtistID or AlbumID is passed query_type = None # 'artwork','thumb' or 'info' - set automatically - + artwork_files = [] thumb_files = [] - + artwork_errors = False artwork_url = None - + thumb_errors = False thumb_url = None - + info_summary = None info_content = None - + def __init__(self): - + pass def _findfilesstartingwith(self,pattern,folder): @@ -70,7 +70,7 @@ class Cache(object): if fname.startswith(pattern): files.append(os.path.join(folder,fname)) return files - + def _exists(self, type): self.artwork_files = [] self.thumb_files = [] @@ -93,53 +93,53 @@ class Cache(object): # There's probably a better way to do this split_date = date.split('-') days_old = int(split_date[0])*365 + int(split_date[1])*30 + int(split_date[2]) - + return days_old - - + + def _is_current(self, filename=None, date=None): - + if filename: base_filename = os.path.basename(filename) date = base_filename.split('.')[1] - + # Calculate how old the cached file is based on todays date & file date stamp # helpers.today() returns todays date in yyyy-mm-dd format if self._get_age(helpers.today()) - self._get_age(date) < 30: return True else: return False - + def _get_thumb_url(self, data): - + thumb_url = None - + try: images = data[self.id_type]['image'] except KeyError: return None - + for image in images: if image['size'] == 'medium': thumb_url = image['#text'] break - + return thumb_url - + def get_artwork_from_cache(self, ArtistID=None, AlbumID=None): ''' Pass a musicbrainz id to this function (either ArtistID or AlbumID) ''' - + self.query_type = 'artwork' - + if ArtistID: self.id = ArtistID self.id_type = 'artist' else: self.id = AlbumID self.id_type = 'album' - + if self._exists('artwork') and self._is_current(filename=self.artwork_files[0]): return self.artwork_files[0] else: @@ -151,21 +151,21 @@ class Cache(object): return self.artwork_files[0] else: return None - + def get_thumb_from_cache(self, ArtistID=None, AlbumID=None): ''' Pass a musicbrainz id to this function (either ArtistID or AlbumID) ''' - + self.query_type = 'thumb' - + if ArtistID: self.id = ArtistID self.id_type = 'artist' else: self.id = AlbumID self.id_type = 'album' - + if self._exists('thumb') and self._is_current(filename=self.thumb_files[0]): return self.thumb_files[0] else: @@ -177,13 +177,13 @@ class Cache(object): return self.thumb_files[0] else: return None - + def get_info_from_cache(self, ArtistID=None, AlbumID=None): - + self.query_type = 'info' myDB = db.DBConnection() - - if ArtistID: + + if ArtistID: self.id = ArtistID self.id_type = 'artist' db_info = myDB.action('SELECT Summary, Content, LastUpdated FROM descriptions WHERE ArtistID=?', [self.id]).fetchone() @@ -193,7 +193,7 @@ class Cache(object): db_info = myDB.action('SELECT Summary, Content, LastUpdated FROM descriptions WHERE ReleaseGroupID=?', [self.id]).fetchone() if not db_info or not db_info['LastUpdated'] or not self._is_current(date=db_info['LastUpdated']): - + self._update_cache() info_dict = { 'Summary' : self.info_summary, 'Content' : self.info_content } return info_dict @@ -201,188 +201,146 @@ class Cache(object): else: info_dict = { 'Summary' : db_info['Summary'], 'Content' : db_info['Content'] } return info_dict - + def get_image_links(self, ArtistID=None, AlbumID=None): ''' Here we're just going to open up the last.fm url, grab the image links and return them Won't save any image urls, or save the artwork in the cache. Useful for search results, etc. ''' if ArtistID: - + self.id_type = 'artist' - + params = { "method": "artist.getInfo", "api_key": lastfm_apikey, "mbid": ArtistID, "format": "json" } - - url = "http://ws.audioscrobbler.com/2.0/?" + urllib.urlencode(params) - logger.debug('Retrieving artist information from: %s', url) - - try: - result = urllib2.urlopen(url, timeout=20).read() - except: - logger.warn('Could not open url: %s', url) - return - - if result: - - try: - data = simplejson.JSONDecoder().decode(result) - except: - logger.warn('Could not parse data from url: %s', url) - return - try: - image_url = data['artist']['image'][-1]['#text'] - except Exception: - logger.debug('No artist image found on url: %s', url) - image_url = None - - thumb_url = self._get_thumb_url(data) - if not thumb_url: - logger.debug('No artist thumbnail image found on url: %s', url) - + url = "http://ws.audioscrobbler.com/2.0/" + data = helpers.request_json(url, params=params, timeout=20) + + if not data: + return + + try: + image_url = data['artist']['image'][-1]['#text'] + except Exception: + logger.debug('No artist image found on url: %s', url) + image_url = None + + thumb_url = self._get_thumb_url(data) + if not thumb_url: + logger.debug('No artist thumbnail image found on url: %s', url) + else: - + self.id_type = 'album' - + params = { "method": "album.getInfo", "api_key": lastfm_apikey, "mbid": AlbumID, "format": "json" } - + url = "http://ws.audioscrobbler.com/2.0/?" + urllib.urlencode(params) - logger.debug('Retrieving album information from: %s', url) - - try: - result = urllib2.urlopen(url, timeout=20).read() - except: - logger.warn('Could not open url: %s', url) + data = helpers.request_json(url, params=params, timeout=20) + + if not data: return - - if result: - - try: - data = simplejson.JSONDecoder().decode(result) - except: - logger.warn('Could not parse data from url: %s', url) - return - - try: - image_url = data['artist']['image'][-1]['#text'] - except Exception: - logger.debug('No artist image found on url: %s', url) - image_url = None - - thumb_url = self._get_thumb_url(data) - - if not thumb_url: - logger.debug('No artist thumbnail image found on url: %s', url) - - image_dict = {'artwork' : image_url, 'thumbnail' : thumb_url } - return image_dict - + + try: + image_url = data['artist']['image'][-1]['#text'] + except Exception: + logger.debug('No artist image found on url: %s', url) + image_url = None + + thumb_url = self._get_thumb_url(data) + + if not thumb_url: + logger.debug('No artist thumbnail image found on url: %s', url) + + return {'artwork' : image_url, 'thumbnail' : thumb_url } + def _update_cache(self): ''' Since we call the same url for both info and artwork, we'll update both at the same time ''' myDB = db.DBConnection() - + # Since lastfm uses release ids rather than release group ids for albums, we have to do a artist + album search for albums if self.id_type == 'artist': - + params = { "method": "artist.getInfo", "api_key": lastfm_apikey, "mbid": self.id, "format": "json" } - - url = "http://ws.audioscrobbler.com/2.0/?" + urllib.urlencode(params) - logger.debug('Retrieving artist information from: %s', url) - - try: - result = urllib2.urlopen(url, timeout=20).read() - except: - logger.warn('Could not open url: %s', url) + + url = "http://ws.audioscrobbler.com/2.0/" + data = helpers.request_json(url, timeout=20, params=params) + + if not data: return - - if result: - - try: - data = simplejson.JSONDecoder().decode(result) - except: - logger.warn('Could not parse data from url: %s', url) - return - try: - self.info_summary = data['artist']['bio']['summary'] - except Exception: - logger.debug('No artist bio summary found on url: %s', url) - self.info_summary = None - try: - self.info_content = data['artist']['bio']['content'] - except Exception: - logger.debug('No artist bio found on url: %s', url) - self.info_content = None - try: - image_url = data['artist']['image'][-1]['#text'] - except Exception: - logger.debug('No artist image found on url: %s', url) - image_url = None - - thumb_url = self._get_thumb_url(data) - if not thumb_url: - logger.debug('No artist thumbnail image found on url: %s', url) - + + try: + self.info_summary = data['artist']['bio']['summary'] + except Exception: + logger.debug('No artist bio summary found on url: %s', url) + self.info_summary = None + try: + self.info_content = data['artist']['bio']['content'] + except Exception: + logger.debug('No artist bio found on url: %s', url) + self.info_content = None + try: + image_url = data['artist']['image'][-1]['#text'] + except Exception: + logger.debug('No artist image found on url: %s', url) + image_url = None + + thumb_url = self._get_thumb_url(data) + if not thumb_url: + logger.debug('No artist thumbnail image found on url: %s', url) + else: dbartist = myDB.action('SELECT ArtistName, AlbumTitle FROM albums WHERE AlbumID=?', [self.id]).fetchone() - + params = { "method": "album.getInfo", "api_key": lastfm_apikey, "artist": dbartist['ArtistName'].encode('utf-8'), "album": dbartist['AlbumTitle'].encode('utf-8'), "format": "json" } - - url = "http://ws.audioscrobbler.com/2.0/?" + urllib.urlencode(params) - - logger.debug('Retrieving artist information from: %s', url) - try: - result = urllib2.urlopen(url, timeout=20).read() - except: - logger.warn('Could not open url: %s', url) - return - - if result: - try: - data = simplejson.JSONDecoder().decode(result) - except: - logger.warn('Could not parse data from url: %s', url) - return - try: - self.info_summary = data['album']['wiki']['summary'] - except Exception: - logger.debug('No album summary found from: %s', url) - self.info_summary = None - try: - self.info_content = data['album']['wiki']['content'] - except Exception: - logger.debug('No album infomation found from: %s', url) - self.info_content = None - try: - image_url = data['album']['image'][-1]['#text'] - except Exception: - logger.debug('No album image link found on url: %s', url) - image_url = None - - thumb_url = self._get_thumb_url(data) - if not thumb_url: - logger.debug('No album thumbnail image found on url: %s', url) - + url = "http://ws.audioscrobbler.com/2.0/" + data = helpers.request_json(url, timeout=20, params=params) + + if not data: + return + + try: + self.info_summary = data['album']['wiki']['summary'] + except Exception: + logger.debug('No album summary found from: %s', url) + self.info_summary = None + try: + self.info_content = data['album']['wiki']['content'] + except Exception: + logger.debug('No album infomation found from: %s', url) + self.info_content = None + try: + image_url = data['album']['image'][-1]['#text'] + except Exception: + logger.debug('No album image link found on url: %s', url) + image_url = None + + thumb_url = self._get_thumb_url(data) + + if not thumb_url: + logger.debug('No album thumbnail image found on url: %s', url) + #Save the content & summary to the database no matter what if we've opened up the url if self.id_type == 'artist': controlValueDict = {"ArtistID": self.id} @@ -392,33 +350,28 @@ class Cache(object): newValueDict = {"Summary": self.info_summary, "Content": self.info_content, "LastUpdated": helpers.today()} - + myDB.upsert("descriptions", newValueDict, controlValueDict) - + # Save the image URL to the database if image_url: if self.id_type == 'artist': myDB.action('UPDATE artists SET ArtworkURL=? WHERE ArtistID=?', [image_url, self.id]) else: myDB.action('UPDATE albums SET ArtworkURL=? WHERE AlbumID=?', [image_url, self.id]) - + # Save the thumb URL to the database if thumb_url: if self.id_type == 'artist': myDB.action('UPDATE artists SET ThumbURL=? WHERE ArtistID=?', [thumb_url, self.id]) else: myDB.action('UPDATE albums SET ThumbURL=? WHERE AlbumID=?', [thumb_url, self.id]) - + # Should we grab the artwork here if we're just grabbing thumbs or info?? Probably not since the files can be quite big if image_url and self.query_type == 'artwork': - try: - artwork = urllib2.urlopen(image_url, timeout=20).read() - except Exception, e: - logger.error('Unable to open url "%s". Error: %s', image_url, e) - artwork = None - + artwork = helpers.request_content(image_url, timeout=20) + if artwork: - # Make sure the artwork dir exists: if not os.path.isdir(self.path_to_art_cache): try: @@ -427,16 +380,16 @@ class Cache(object): logger.error('Unable to create artwork cache dir. Error: %s', e) self.artwork_errors = True self.artwork_url = image_url - + #Delete the old stuff for artwork_file in self.artwork_files: try: os.remove(artwork_file) except: logger.error('Error deleting file from the cache: %s', artwork_file) - + ext = os.path.splitext(image_url)[1] - + artwork_path = os.path.join(self.path_to_art_cache, self.id + '.' + helpers.today() + ext) try: f = open(artwork_path, 'wb') @@ -446,18 +399,12 @@ class Cache(object): logger.error('Unable to write to the cache dir: %s', e) self.artwork_errors = True self.artwork_url = image_url - + # Grab the thumbnail as well if we're getting the full artwork (as long as it's missing/outdated if thumb_url and self.query_type in ['thumb','artwork'] and not (self.thumb_files and self._is_current(self.thumb_files[0])): - - try: - artwork = urllib2.urlopen(thumb_url, timeout=20).read() - except Exception, e: - logger.error('Unable to open url "%s". Error: %s', thumb_url, e) - artwork = None - + artwork = helpers.request_content(thumb_url, timeout=20) + if artwork: - # Make sure the artwork dir exists: if not os.path.isdir(self.path_to_art_cache): try: @@ -466,16 +413,16 @@ class Cache(object): logger.error('Unable to create artwork cache dir. Error: %s' + e) self.thumb_errors = True self.thumb_url = thumb_url - + #Delete the old stuff for thumb_file in self.thumb_files: try: os.remove(thumb_file) except: logger.error('Error deleting file from the cache: %s', thumb_file) - + ext = os.path.splitext(image_url)[1] - + thumb_path = os.path.join(self.path_to_art_cache, 'T_' + self.id + '.' + helpers.today() + ext) try: f = open(thumb_path, 'wb') @@ -487,44 +434,44 @@ class Cache(object): self.thumb_url = image_url def getArtwork(ArtistID=None, AlbumID=None): - + c = Cache() artwork_path = c.get_artwork_from_cache(ArtistID, AlbumID) - + if not artwork_path: return None - + if artwork_path.startswith('http://'): return artwork_path else: artwork_file = os.path.basename(artwork_path) return "cache/artwork/" + artwork_file - + def getThumb(ArtistID=None, AlbumID=None): - + c = Cache() artwork_path = c.get_thumb_from_cache(ArtistID, AlbumID) - + if not artwork_path: return None - + if artwork_path.startswith('http://'): return artwork_path else: thumbnail_file = os.path.basename(artwork_path) return "cache/artwork/" + thumbnail_file - + def getInfo(ArtistID=None, AlbumID=None): - + c = Cache() - + info_dict = c.get_info_from_cache(ArtistID, AlbumID) - + return info_dict - + def getImageLinks(ArtistID=None, AlbumID=None): - + c = Cache() image_links = c.get_image_links(ArtistID, AlbumID) - + return image_links