Merge branch 'develop'

data/interfaces/default/index.html

@@ -36,7 +36,7 @@
                         "aTargets": [0],
                         "mData":"ArtistID",
                         "mRender": function ( data, type, full ) {
-                            return '<div id="artistImg"><img class="albumArt" alt="" id="'+ data + '" data-src="artwork/thumbs/artist/' + data + '"/></div>';
+                            return '<div id="artistImg"><img class="albumArt" height="50" width="50" alt="" id="'+ data + '" data-src="artwork/thumbs/artist/' + data + '"/></div>';
                         }
                     },
                     {

headphones/cache.py

@@ -14,11 +14,17 @@
 #  along with Headphones.  If not, see <http://www.gnu.org/licenses/>.
 
 import os
+from six.moves.urllib.parse import urlencode
 
 import headphones
 from headphones import db, helpers, logger, lastfm, request, mb
 
-LASTFM_API_KEY = "690e1ed3bc00bc91804cd8f7fe5ed6d4"
+# LASTFM_API_KEY = "690e1ed3bc00bc91804cd8f7fe5ed6d4"
+LASTFM_API_KEY = "a1c959f36a912a165d73c4dc5b9520f5"
+
+FANART_URL = 'https://webservice.fanart.tv/v3/music/'
+FANART_PROJECT_KEY = '22b73c9603eba09d0c855f2d2bdba31c'
+FANART_CLIENT_KEY = '919b389a18a3f0b2c916090022ab3c7a'
 
 
 class Cache(object):
@@ -96,6 +102,11 @@ class Cache(object):
     def _is_current(self, filename=None, date=None):
 
         if filename:
+
+            # 2019 last.fm no longer allows access to artist images, for now we'll keep the cached artist image if it exists and get new from fanart.tv
+            if self.id_type == 'artist' and 'fanart' not in filename:
+                return True
+
             base_filename = os.path.basename(filename)
             date = base_filename.split('.')[1]
 
@@ -211,20 +222,34 @@ class Cache(object):
         if ArtistID:
 
             self.id_type = 'artist'
-            data = lastfm.request_lastfm("artist.getinfo", mbid=ArtistID, api_key=LASTFM_API_KEY)
+
+            # 2019 last.fm no longer allows access to artist images, try fanart.tv instead
+            image_url = None
+            thumb_url = None
+            data = request.request_json(FANART_URL + ArtistID, whitelist_status_code=404,
+                                        headers={'api-key': FANART_PROJECT_KEY, 'client-key': FANART_CLIENT_KEY})
 
             if not data:
                 return
 
-            try:
-                image_url = data['artist']['image'][-1]['#text']
-            except (KeyError, IndexError):
-                logger.debug('No artist image found')
-                image_url = None
+            if data.get('artistthumb'):
+                image_url = data['artistthumb'][0]['url']
+            elif data.get('artistbackground'):
+                image_url = data['artistbackground'][0]['url']
+            # elif data.get('hdmusiclogo'):
+            #    image_url = data['hdmusiclogo'][0]['url']
 
-            thumb_url = self._get_thumb_url(data)
-            if not thumb_url:
-                logger.debug('No artist thumbnail image found')
+            # fallback to 1st album cover if none of the above
+            elif 'albums' in data:
+                for mbid, art in data.get('albums', dict()).items():
+                    if 'albumcover' in art:
+                        image_url = art['albumcover'][0]['url']
+                        break
+
+            if image_url:
+                thumb_url = image_url
+            else:
+                logger.debug('No artist image found on fanart.tv for Artist Id: %s', self.id)
 
         else:
 
@@ -283,6 +308,7 @@ class Cache(object):
         """
 
         myDB = db.DBConnection()
+        fanart = False
 
         # Since lastfm uses release ids rather than release group ids for albums, we have to do a artist + album search for albums
         # Exception is when adding albums manually, then we should use release id
@@ -311,15 +337,42 @@ class Cache(object):
             except KeyError:
                 logger.debug('No artist bio found')
                 self.info_content = None
-            try:
-                image_url = data['artist']['image'][-1]['#text']
-            except KeyError:
-                logger.debug('No artist image found')
-                image_url = None
 
-            thumb_url = self._get_thumb_url(data)
-            if not thumb_url:
-                logger.debug('No artist thumbnail image found')
+            # 2019 last.fm no longer allows access to artist images, try fanart.tv instead
+            image_url = None
+            thumb_url = None
+            data = request.request_json(FANART_URL + self.id, whitelist_status_code=404,
+                                        headers={'api-key': FANART_PROJECT_KEY, 'client-key': FANART_CLIENT_KEY})
+
+            if data.get('artistthumb'):
+                image_url = data['artistthumb'][0]['url']
+            elif data.get('artistbackground'):
+                image_url = data['artistbackground'][0]['url']
+            # elif data.get('hdmusiclogo'):
+            #    image_url = data['hdmusiclogo'][0]['url']
+
+            # fallback to 1st album cover if none of the above
+            elif 'albums' in data:
+                for mbid, art in data.get('albums', dict()).items():
+                    if 'albumcover' in art:
+                        image_url = art['albumcover'][0]['url']
+                        break
+
+            # finally, use 1st album cover from last.fm
+            if image_url:
+                fanart = True
+                thumb_url = image_url
+            else:
+                dbalbum = myDB.action(
+                    'SELECT ArtworkURL, ThumbURL FROM albums WHERE ArtworkURL IS NOT NULL AND ArtistID=?',
+                    [self.id]).fetchone()
+                if dbalbum:
+                    fanart = True
+                    image_url = dbalbum['ArtworkURL']
+                    thumb_url = dbalbum['ThumbURL']
+
+            if not image_url:
+                logger.debug('No artist image found on fanart.tv for Artist Id: %s', self.id)
 
         else:
             dbalbum = myDB.action(
@@ -427,7 +480,11 @@ class Cache(object):
 
                 ext = os.path.splitext(image_url)[1]
 
-                artwork_path = os.path.join(self.path_to_art_cache,
+                if fanart:
+                    artwork_path = os.path.join(self.path_to_art_cache,
+                                                self.id + '_fanart_' + '.' + helpers.today() + ext)
+                else:
+                    artwork_path = os.path.join(self.path_to_art_cache,
                                             self.id + '.' + helpers.today() + ext)
                 try:
                     with open(artwork_path, 'wb') as f:
@@ -443,7 +500,9 @@ class Cache(object):
         # as it's missing/outdated.
         if thumb_url and self.query_type in ['thumb', 'artwork'] and not (
                 self.thumb_files and self._is_current(self.thumb_files[0])):
-            artwork = request.request_content(thumb_url, timeout=20)
+
+            if not (self.query_type == 'artwork' and 'fanart' in thumb_url and artwork):
+                artwork = request.request_content(thumb_url, timeout=20)
 
             if artwork:
                 # Make sure the artwork dir exists:
@@ -466,11 +525,34 @@ class Cache(object):
 
                 ext = os.path.splitext(image_url)[1]
 
-                thumb_path = os.path.join(self.path_to_art_cache,
-                                          'T_' + self.id + '.' + helpers.today() + ext)
+                if fanart:
+                    thumb_path = os.path.join(self.path_to_art_cache,
+                                              'T_' + self.id + '_fanart_' + '.' + helpers.today() + ext)
+                else:
+                    thumb_path = os.path.join(self.path_to_art_cache,
+                                              'T_' + self.id + '.' + helpers.today() + ext)
                 try:
-                    with open(thumb_path, 'wb') as f:
-                        f.write(artwork)
+                    if self.id_type != 'artist':
+                        with open(thumb_path, 'wb') as f:
+                            f.write(artwork)
+                    else:
+
+                        # 2019 last.fm no longer allows access to artist images, use the fanart.tv image to create a thumb
+                        artwork_thumb = None
+                        if 'fanart' in thumb_url:
+                            # Create thumb using image resizing service
+                            artwork_path = '{0}?{1}'.format('http://images.weserv.nl/', urlencode({
+                                'url': thumb_url.replace('http://', ''),
+                                'w': 300,
+                            }))
+                            artwork_thumb = request.request_content(artwork_path, timeout=20, whitelist_status_code=404)
+
+                        if artwork_thumb:
+                            with open(thumb_path, 'wb') as f:
+                                f.write(artwork_thumb)
+                        else:
+                            with open(thumb_path, 'wb') as f:
+                                f.write(artwork)
 
                     os.chmod(thumb_path, int(headphones.CONFIG.FILE_PERMISSIONS, 8))
                 except (OSError, IOError) as e:
@@ -486,7 +568,7 @@ def getArtwork(ArtistID=None, AlbumID=None):
     if not artwork_path:
         return None
 
-    if artwork_path.startswith('http://'):
+    if artwork_path.startswith(('http://', 'https://')):
         return artwork_path
     else:
         artwork_file = os.path.basename(artwork_path)
@@ -500,7 +582,7 @@ def getThumb(ArtistID=None, AlbumID=None):
     if not artwork_path:
         return None
 
-    if artwork_path.startswith('http://'):
+    if artwork_path.startswith(('http://', 'https://')):
         return artwork_path
     else:
         thumbnail_file = os.path.basename(artwork_path)

headphones/librarysync.py

@@ -364,6 +364,11 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None,
 
         for artist in unique_artists:
 
+            if not artist:
+                continue
+
+            logger.info('Processing artist: %s' % artist)
+
             # check if artist is already in the db
             artist_lookup = "\"" + artist.replace("\"", "\"\"") + "\""
 
@@ -399,20 +404,23 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None,
                 #             [artist]))
                 # )
 
-                havetracks = (
-                    len(myDB.select(
-                        'SELECT ArtistID From tracks WHERE ArtistID = ? AND Location IS NOT NULL',
-                        [artistid])) + len(myDB.select(
-                            'SELECT ArtistName FROM have WHERE ArtistName LIKE ' + artist_lookup + ' AND Matched = "Failed"'))
-                )
+                try:
+                    havetracks = (
+                        len(myDB.select(
+                            'SELECT ArtistID From tracks WHERE ArtistID = ? AND Location IS NOT NULL',
+                            [artistid])) + len(myDB.select(
+                                'SELECT ArtistName FROM have WHERE ArtistName LIKE ' + artist_lookup + ' AND Matched = "Failed"'))
+                    )
+                except Exception as e:
+                    logger.warn('Error updating counts for artist: %s: %s' % (artist, e))
 
                 # Note: some people complain about having "artist have tracks" > # of tracks total in artist official releases
                 # (can fix by getting rid of second len statement)
 
-                myDB.action('UPDATE artists SET HaveTracks = ? WHERE ArtistID = ?', [havetracks, artistid])
-
-                # Update albums to downloaded
                 if havetracks:
+                    myDB.action('UPDATE artists SET HaveTracks = ? WHERE ArtistID = ?', [havetracks, artistid])
+
+                    # Update albums to downloaded
                     update_album_status(ArtistID=artistid)
 
         logger.info('Found %i new artists' % len(new_artist_list))
@@ -436,11 +444,16 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None,
         logger.info('Updating artist track counts')
 
         artist_lookup = "\"" + ArtistName.replace("\"", "\"\"") + "\""
-        havetracks = len(
-            myDB.select('SELECT ArtistID FROM tracks WHERE ArtistID = ? AND Location IS NOT NULL',
-                        [ArtistID])) + len(myDB.select(
-                            'SELECT ArtistName FROM have WHERE ArtistName LIKE ' + artist_lookup + ' AND Matched = "Failed"'))
-        myDB.action('UPDATE artists SET HaveTracks=? WHERE ArtistID=?', [havetracks, ArtistID])
+        try:
+            havetracks = len(
+                myDB.select('SELECT ArtistID FROM tracks WHERE ArtistID = ? AND Location IS NOT NULL',
+                            [ArtistID])) + len(myDB.select(
+                                'SELECT ArtistName FROM have WHERE ArtistName LIKE ' + artist_lookup + ' AND Matched = "Failed"'))
+        except Exception as e:
+            logger.warn('Error updating counts for artist: %s: %s' % (ArtistName, e))
+
+        if havetracks:
+            myDB.action('UPDATE artists SET HaveTracks=? WHERE ArtistID=?', [havetracks, ArtistID])
 
     # Moved above to call for each artist
     # if not append:

headphones/mb.py

@@ -793,3 +793,27 @@ def getArtistForReleaseGroup(rgid):
         return False
     else:
         return releaseGroup['artist-credit'][0]['artist']['name']
+
+
+def getArtistRelationships(artistid):
+    """
+    Returns list of relationship urls. e.g. Discogs, Wikipedia etc.
+    """
+    urls = []
+    artist = None
+    try:
+        with mb_lock:
+            info = musicbrainzngs.get_artist_by_id(artistid, includes='url-rels')
+    except musicbrainzngs.WebServiceError as e:
+        logger.warn(
+            'Attempt to query MusicBrainz for %s failed "%s"' % (artistid, str(e)))
+        mb_lock.snooze(5)
+    if 'artist' in info:
+        artist = info['artist']
+        if 'url-relation-list' in artist:
+            for l in artist['url-relation-list']:
+                urls.append({
+                    'type': l['type'],
+                    'url': l['target']
+                })
+    return urls

headphones/notifiers.py

@@ -1017,7 +1017,7 @@ class Email(object):
 
 
 class TELEGRAM(object):
-    def notify(self, message, status):
+    def notify(self, message, status, rgid=None, image=None):
         if not headphones.CONFIG.TELEGRAM_ENABLED:
             return
 
@@ -1030,22 +1030,34 @@ class TELEGRAM(object):
         userid = headphones.CONFIG.TELEGRAM_USERID
 
         # Construct message
-        payload = {'chat_id': userid, 'text': status + ': ' + message}
+        message = '\n\n' + message
 
-        # Send message to user using Telegram's Bot API
-        try:
-            response = requests.post(TELEGRAM_API % (token, "sendMessage"),
-                                     data=payload)
-        except Exception, e:
-            logger.info(u'Telegram notify failed: ' + str(e))
+        # MusicBrainz link
+        if rgid:
+            message += '\n\n <a href="http://musicbrainz.org/' \
+                      'release-group/%s">MusicBrainz</a>' % rgid
+
+        # Send image
+        response = None
+        if image:
+            image_file = {'photo': (image, open(image, "rb"))}
+            payload = {'chat_id': userid, 'parse_mode': "HTML", 'caption': status + message}
+            try:
+                response = requests.post(TELEGRAM_API % (token, "sendPhoto"), data=payload, files=image_file)
+            except Exception, e:
+                logger.info(u'Telegram notify failed: ' + str(e))
+        # Sent text
+        else:
+            payload = {'chat_id': userid, 'parse_mode': "HTML", 'text': status + message}
+            try:
+                response = requests.post(TELEGRAM_API % (token, "sendMessage"), data=payload)
+            except Exception, e:
+                logger.info(u'Telegram notify failed: ' + str(e))
 
         # Error logging
         sent_successfuly = True
-        if not response.status_code == 200:
-            logger.info(
-                u'Could not send notification to TelegramBot '
-                u'(token=%s). Response: [%s]',
-                (token, response.text))
+        if response and not response.status_code == 200:
+            logger.info("Could not send notification to TelegramBot (token=%s). Response: [%s]", token, response.text)
             sent_successfuly = False
 
         logger.info(u"Telegram notifications sent.")

headphones/postprocessor.py

@@ -586,7 +586,7 @@ def doPostProcessing(albumid, albumpath, release, tracks, downloaded_track_list,
     if headphones.CONFIG.TELEGRAM_ENABLED:
         logger.info(u"Telegram request")
         telegram = notifiers.TELEGRAM()
-        telegram.notify(pushmessage, statusmessage)
+        telegram.notify(statusmessage, pushmessage)
 
     if headphones.CONFIG.TWITTER_ENABLED:
         logger.info(u"Sending Twitter notification")

headphones/searcher.py

@@ -1075,8 +1075,12 @@ def send_to_downloader(data, bestqual, album):
         slack.notify(name, "Download started")
     if headphones.CONFIG.TELEGRAM_ENABLED and headphones.CONFIG.TELEGRAM_ONSNATCH:
         logger.info(u"Sending Telegram notification")
+        from headphones import cache
+        c = cache.Cache()
+        album_art = c.get_artwork_from_cache(None, rgid)
         telegram = notifiers.TELEGRAM()
-        telegram.notify(name, "Download started")
+        message = 'Snatched from ' + provider + '. ' + name
+        telegram.notify(message, "Snatched: " + title, rgid, image=album_art)
     if headphones.CONFIG.TWITTER_ENABLED and headphones.CONFIG.TWITTER_ONSNATCH:
         logger.info(u"Sending Twitter notification")
         twitter = notifiers.TwitterNotifier()