Replaced urllib2 in albumart and cache

This commit is contained in:
Bas Stottelaar
2014-04-06 16:15:11 +02:00
parent 084bf59b6a
commit a717594950
2 changed files with 177 additions and 239 deletions

View File

@@ -13,39 +13,30 @@
# You should have received a copy of the GNU General Public License
# along with Headphones. If not, see <http://www.gnu.org/licenses/>.
import urllib2
from headphones import db
from headphones import helpers, db
def getAlbumArt(albumid):
myDB = db.DBConnection()
asin = myDB.action('SELECT AlbumASIN from albums WHERE AlbumID=?', [albumid]).fetchone()[0]
if not asin:
return None
url = 'http://ec1.images-amazon.com/images/P/%s.01.LZZZZZZZ.jpg' % asin
return url
if asin:
return 'http://ec1.images-amazon.com/images/P/%s.01.LZZZZZZZ.jpg' % asin
def getCachedArt(albumid):
from headphones import cache
c = cache.Cache()
artwork_path = c.get_artwork_from_cache(AlbumID=albumid)
if not artwork_path:
return None
return
if artwork_path.startswith('http://'):
try:
artwork = urllib2.urlopen(artwork_path, timeout=20).read()
return artwork
except:
artwork = helpers.request_content(artwork_path, timeout=20)
if not artwork:
logger.warn("Unable to open url: %s", artwork_path)
return None
return
else:
artwork = open(artwork_path, "r").read()
return artwork
with open(artwork_path, "r") as fp:
return fp.read()

View File

@@ -14,7 +14,7 @@
# along with Headphones. If not, see <http://www.gnu.org/licenses/>.
import os
import glob, urllib, urllib2
import glob, urllib
import lib.simplejson as simplejson
@@ -25,42 +25,42 @@ lastfm_apikey = "690e1ed3bc00bc91804cd8f7fe5ed6d4"
class Cache(object):
"""
This class deals with getting, storing and serving up artwork (album
This class deals with getting, storing and serving up artwork (album
art, artist images, etc) and info/descriptions (album info, artist descrptions)
to and from the cache folder. This can be called from within a web interface,
to and from the cache folder. This can be called from within a web interface,
for example, using the helper functions getInfo(id) and getArtwork(id), to utilize the cached
images rather than having to retrieve them every time the page is reloaded.
So you can call cache.getArtwork(id) which will return an absolute path
to the image file on the local machine, or if the cache directory
doesn't exist, or can not be written to, it will return a url to the image.
Call cache.getInfo(id) to grab the artist/album info; will return the text description
The basic format for art in the cache is <musicbrainzid>.<date>.<ext>
and for info it is <musicbrainzid>.<date>.txt
"""
path_to_art_cache = os.path.join(headphones.CACHE_DIR, 'artwork')
id = None
id_type = None # 'artist' or 'album' - set automatically depending on whether ArtistID or AlbumID is passed
query_type = None # 'artwork','thumb' or 'info' - set automatically
artwork_files = []
thumb_files = []
artwork_errors = False
artwork_url = None
thumb_errors = False
thumb_url = None
info_summary = None
info_content = None
def __init__(self):
pass
def _findfilesstartingwith(self,pattern,folder):
@@ -70,7 +70,7 @@ class Cache(object):
if fname.startswith(pattern):
files.append(os.path.join(folder,fname))
return files
def _exists(self, type):
self.artwork_files = []
self.thumb_files = []
@@ -93,53 +93,53 @@ class Cache(object):
# There's probably a better way to do this
split_date = date.split('-')
days_old = int(split_date[0])*365 + int(split_date[1])*30 + int(split_date[2])
return days_old
def _is_current(self, filename=None, date=None):
if filename:
base_filename = os.path.basename(filename)
date = base_filename.split('.')[1]
# Calculate how old the cached file is based on todays date & file date stamp
# helpers.today() returns todays date in yyyy-mm-dd format
if self._get_age(helpers.today()) - self._get_age(date) < 30:
return True
else:
return False
def _get_thumb_url(self, data):
thumb_url = None
try:
images = data[self.id_type]['image']
except KeyError:
return None
for image in images:
if image['size'] == 'medium':
thumb_url = image['#text']
break
return thumb_url
def get_artwork_from_cache(self, ArtistID=None, AlbumID=None):
'''
Pass a musicbrainz id to this function (either ArtistID or AlbumID)
'''
self.query_type = 'artwork'
if ArtistID:
self.id = ArtistID
self.id_type = 'artist'
else:
self.id = AlbumID
self.id_type = 'album'
if self._exists('artwork') and self._is_current(filename=self.artwork_files[0]):
return self.artwork_files[0]
else:
@@ -151,21 +151,21 @@ class Cache(object):
return self.artwork_files[0]
else:
return None
def get_thumb_from_cache(self, ArtistID=None, AlbumID=None):
'''
Pass a musicbrainz id to this function (either ArtistID or AlbumID)
'''
self.query_type = 'thumb'
if ArtistID:
self.id = ArtistID
self.id_type = 'artist'
else:
self.id = AlbumID
self.id_type = 'album'
if self._exists('thumb') and self._is_current(filename=self.thumb_files[0]):
return self.thumb_files[0]
else:
@@ -177,13 +177,13 @@ class Cache(object):
return self.thumb_files[0]
else:
return None
def get_info_from_cache(self, ArtistID=None, AlbumID=None):
self.query_type = 'info'
myDB = db.DBConnection()
if ArtistID:
if ArtistID:
self.id = ArtistID
self.id_type = 'artist'
db_info = myDB.action('SELECT Summary, Content, LastUpdated FROM descriptions WHERE ArtistID=?', [self.id]).fetchone()
@@ -193,7 +193,7 @@ class Cache(object):
db_info = myDB.action('SELECT Summary, Content, LastUpdated FROM descriptions WHERE ReleaseGroupID=?', [self.id]).fetchone()
if not db_info or not db_info['LastUpdated'] or not self._is_current(date=db_info['LastUpdated']):
self._update_cache()
info_dict = { 'Summary' : self.info_summary, 'Content' : self.info_content }
return info_dict
@@ -201,188 +201,146 @@ class Cache(object):
else:
info_dict = { 'Summary' : db_info['Summary'], 'Content' : db_info['Content'] }
return info_dict
def get_image_links(self, ArtistID=None, AlbumID=None):
'''
Here we're just going to open up the last.fm url, grab the image links and return them
Won't save any image urls, or save the artwork in the cache. Useful for search results, etc.
'''
if ArtistID:
self.id_type = 'artist'
params = { "method": "artist.getInfo",
"api_key": lastfm_apikey,
"mbid": ArtistID,
"format": "json"
}
url = "http://ws.audioscrobbler.com/2.0/?" + urllib.urlencode(params)
logger.debug('Retrieving artist information from: %s', url)
try:
result = urllib2.urlopen(url, timeout=20).read()
except:
logger.warn('Could not open url: %s', url)
return
if result:
try:
data = simplejson.JSONDecoder().decode(result)
except:
logger.warn('Could not parse data from url: %s', url)
return
try:
image_url = data['artist']['image'][-1]['#text']
except Exception:
logger.debug('No artist image found on url: %s', url)
image_url = None
thumb_url = self._get_thumb_url(data)
if not thumb_url:
logger.debug('No artist thumbnail image found on url: %s', url)
url = "http://ws.audioscrobbler.com/2.0/"
data = helpers.request_json(url, params=params, timeout=20)
if not data:
return
try:
image_url = data['artist']['image'][-1]['#text']
except Exception:
logger.debug('No artist image found on url: %s', url)
image_url = None
thumb_url = self._get_thumb_url(data)
if not thumb_url:
logger.debug('No artist thumbnail image found on url: %s', url)
else:
self.id_type = 'album'
params = { "method": "album.getInfo",
"api_key": lastfm_apikey,
"mbid": AlbumID,
"format": "json"
}
url = "http://ws.audioscrobbler.com/2.0/?" + urllib.urlencode(params)
logger.debug('Retrieving album information from: %s', url)
try:
result = urllib2.urlopen(url, timeout=20).read()
except:
logger.warn('Could not open url: %s', url)
data = helpers.request_json(url, params=params, timeout=20)
if not data:
return
if result:
try:
data = simplejson.JSONDecoder().decode(result)
except:
logger.warn('Could not parse data from url: %s', url)
return
try:
image_url = data['artist']['image'][-1]['#text']
except Exception:
logger.debug('No artist image found on url: %s', url)
image_url = None
thumb_url = self._get_thumb_url(data)
if not thumb_url:
logger.debug('No artist thumbnail image found on url: %s', url)
image_dict = {'artwork' : image_url, 'thumbnail' : thumb_url }
return image_dict
try:
image_url = data['artist']['image'][-1]['#text']
except Exception:
logger.debug('No artist image found on url: %s', url)
image_url = None
thumb_url = self._get_thumb_url(data)
if not thumb_url:
logger.debug('No artist thumbnail image found on url: %s', url)
return {'artwork' : image_url, 'thumbnail' : thumb_url }
def _update_cache(self):
'''
Since we call the same url for both info and artwork, we'll update both at the same time
'''
myDB = db.DBConnection()
# Since lastfm uses release ids rather than release group ids for albums, we have to do a artist + album search for albums
if self.id_type == 'artist':
params = { "method": "artist.getInfo",
"api_key": lastfm_apikey,
"mbid": self.id,
"format": "json"
}
url = "http://ws.audioscrobbler.com/2.0/?" + urllib.urlencode(params)
logger.debug('Retrieving artist information from: %s', url)
try:
result = urllib2.urlopen(url, timeout=20).read()
except:
logger.warn('Could not open url: %s', url)
url = "http://ws.audioscrobbler.com/2.0/"
data = helpers.request_json(url, timeout=20, params=params)
if not data:
return
if result:
try:
data = simplejson.JSONDecoder().decode(result)
except:
logger.warn('Could not parse data from url: %s', url)
return
try:
self.info_summary = data['artist']['bio']['summary']
except Exception:
logger.debug('No artist bio summary found on url: %s', url)
self.info_summary = None
try:
self.info_content = data['artist']['bio']['content']
except Exception:
logger.debug('No artist bio found on url: %s', url)
self.info_content = None
try:
image_url = data['artist']['image'][-1]['#text']
except Exception:
logger.debug('No artist image found on url: %s', url)
image_url = None
thumb_url = self._get_thumb_url(data)
if not thumb_url:
logger.debug('No artist thumbnail image found on url: %s', url)
try:
self.info_summary = data['artist']['bio']['summary']
except Exception:
logger.debug('No artist bio summary found on url: %s', url)
self.info_summary = None
try:
self.info_content = data['artist']['bio']['content']
except Exception:
logger.debug('No artist bio found on url: %s', url)
self.info_content = None
try:
image_url = data['artist']['image'][-1]['#text']
except Exception:
logger.debug('No artist image found on url: %s', url)
image_url = None
thumb_url = self._get_thumb_url(data)
if not thumb_url:
logger.debug('No artist thumbnail image found on url: %s', url)
else:
dbartist = myDB.action('SELECT ArtistName, AlbumTitle FROM albums WHERE AlbumID=?', [self.id]).fetchone()
params = { "method": "album.getInfo",
"api_key": lastfm_apikey,
"artist": dbartist['ArtistName'].encode('utf-8'),
"album": dbartist['AlbumTitle'].encode('utf-8'),
"format": "json"
}
url = "http://ws.audioscrobbler.com/2.0/?" + urllib.urlencode(params)
logger.debug('Retrieving artist information from: %s', url)
try:
result = urllib2.urlopen(url, timeout=20).read()
except:
logger.warn('Could not open url: %s', url)
return
if result:
try:
data = simplejson.JSONDecoder().decode(result)
except:
logger.warn('Could not parse data from url: %s', url)
return
try:
self.info_summary = data['album']['wiki']['summary']
except Exception:
logger.debug('No album summary found from: %s', url)
self.info_summary = None
try:
self.info_content = data['album']['wiki']['content']
except Exception:
logger.debug('No album infomation found from: %s', url)
self.info_content = None
try:
image_url = data['album']['image'][-1]['#text']
except Exception:
logger.debug('No album image link found on url: %s', url)
image_url = None
thumb_url = self._get_thumb_url(data)
if not thumb_url:
logger.debug('No album thumbnail image found on url: %s', url)
url = "http://ws.audioscrobbler.com/2.0/"
data = helpers.request_json(url, timeout=20, params=params)
if not data:
return
try:
self.info_summary = data['album']['wiki']['summary']
except Exception:
logger.debug('No album summary found from: %s', url)
self.info_summary = None
try:
self.info_content = data['album']['wiki']['content']
except Exception:
logger.debug('No album infomation found from: %s', url)
self.info_content = None
try:
image_url = data['album']['image'][-1]['#text']
except Exception:
logger.debug('No album image link found on url: %s', url)
image_url = None
thumb_url = self._get_thumb_url(data)
if not thumb_url:
logger.debug('No album thumbnail image found on url: %s', url)
#Save the content & summary to the database no matter what if we've opened up the url
if self.id_type == 'artist':
controlValueDict = {"ArtistID": self.id}
@@ -392,33 +350,28 @@ class Cache(object):
newValueDict = {"Summary": self.info_summary,
"Content": self.info_content,
"LastUpdated": helpers.today()}
myDB.upsert("descriptions", newValueDict, controlValueDict)
# Save the image URL to the database
if image_url:
if self.id_type == 'artist':
myDB.action('UPDATE artists SET ArtworkURL=? WHERE ArtistID=?', [image_url, self.id])
else:
myDB.action('UPDATE albums SET ArtworkURL=? WHERE AlbumID=?', [image_url, self.id])
# Save the thumb URL to the database
if thumb_url:
if self.id_type == 'artist':
myDB.action('UPDATE artists SET ThumbURL=? WHERE ArtistID=?', [thumb_url, self.id])
else:
myDB.action('UPDATE albums SET ThumbURL=? WHERE AlbumID=?', [thumb_url, self.id])
# Should we grab the artwork here if we're just grabbing thumbs or info?? Probably not since the files can be quite big
if image_url and self.query_type == 'artwork':
try:
artwork = urllib2.urlopen(image_url, timeout=20).read()
except Exception, e:
logger.error('Unable to open url "%s". Error: %s', image_url, e)
artwork = None
artwork = helpers.request_content(image_url, timeout=20)
if artwork:
# Make sure the artwork dir exists:
if not os.path.isdir(self.path_to_art_cache):
try:
@@ -427,16 +380,16 @@ class Cache(object):
logger.error('Unable to create artwork cache dir. Error: %s', e)
self.artwork_errors = True
self.artwork_url = image_url
#Delete the old stuff
for artwork_file in self.artwork_files:
try:
os.remove(artwork_file)
except:
logger.error('Error deleting file from the cache: %s', artwork_file)
ext = os.path.splitext(image_url)[1]
artwork_path = os.path.join(self.path_to_art_cache, self.id + '.' + helpers.today() + ext)
try:
f = open(artwork_path, 'wb')
@@ -446,18 +399,12 @@ class Cache(object):
logger.error('Unable to write to the cache dir: %s', e)
self.artwork_errors = True
self.artwork_url = image_url
# Grab the thumbnail as well if we're getting the full artwork (as long as it's missing/outdated
if thumb_url and self.query_type in ['thumb','artwork'] and not (self.thumb_files and self._is_current(self.thumb_files[0])):
try:
artwork = urllib2.urlopen(thumb_url, timeout=20).read()
except Exception, e:
logger.error('Unable to open url "%s". Error: %s', thumb_url, e)
artwork = None
artwork = helpers.request_content(thumb_url, timeout=20)
if artwork:
# Make sure the artwork dir exists:
if not os.path.isdir(self.path_to_art_cache):
try:
@@ -466,16 +413,16 @@ class Cache(object):
logger.error('Unable to create artwork cache dir. Error: %s' + e)
self.thumb_errors = True
self.thumb_url = thumb_url
#Delete the old stuff
for thumb_file in self.thumb_files:
try:
os.remove(thumb_file)
except:
logger.error('Error deleting file from the cache: %s', thumb_file)
ext = os.path.splitext(image_url)[1]
thumb_path = os.path.join(self.path_to_art_cache, 'T_' + self.id + '.' + helpers.today() + ext)
try:
f = open(thumb_path, 'wb')
@@ -487,44 +434,44 @@ class Cache(object):
self.thumb_url = image_url
def getArtwork(ArtistID=None, AlbumID=None):
c = Cache()
artwork_path = c.get_artwork_from_cache(ArtistID, AlbumID)
if not artwork_path:
return None
if artwork_path.startswith('http://'):
return artwork_path
else:
artwork_file = os.path.basename(artwork_path)
return "cache/artwork/" + artwork_file
def getThumb(ArtistID=None, AlbumID=None):
c = Cache()
artwork_path = c.get_thumb_from_cache(ArtistID, AlbumID)
if not artwork_path:
return None
if artwork_path.startswith('http://'):
return artwork_path
else:
thumbnail_file = os.path.basename(artwork_path)
return "cache/artwork/" + thumbnail_file
def getInfo(ArtistID=None, AlbumID=None):
c = Cache()
info_dict = c.get_info_from_cache(ArtistID, AlbumID)
return info_dict
def getImageLinks(ArtistID=None, AlbumID=None):
c = Cache()
image_links = c.get_image_links(ArtistID, AlbumID)
return image_links