Upgraded beets to 1.3.10, including patches

lib/beetsplug/embedart.py

@@ -16,6 +16,9 @@
 import os.path
 import logging
 import imghdr
+import subprocess
+import platform
+from tempfile import NamedTemporaryFile
 
 from beets.plugins import BeetsPlugin
 from beets import mediafile
@@ -25,6 +28,7 @@ from beets.util import syspath, normpath, displayable_path
 from beets.util.artresizer import ArtResizer
 from beets import config
 
+
 log = logging.getLogger('beets')
 
 
@@ -36,12 +40,19 @@ class EmbedCoverArtPlugin(BeetsPlugin):
         self.config.add({
             'maxwidth': 0,
             'auto': True,
+            'compare_threshold': 0,
+            'ifempty': False,
         })
-        if self.config['maxwidth'].get(int) and \
-                not ArtResizer.shared.local:
+
+        if self.config['maxwidth'].get(int) and not ArtResizer.shared.local:
             self.config['maxwidth'] = 0
             log.warn(u"embedart: ImageMagick or PIL not found; "
                      u"'maxwidth' option ignored")
+        if self.config['compare_threshold'].get(int) and not \
+                ArtResizer.shared.can_compare:
+            self.config['compare_threshold'] = 0
+            log.warn(u"embedart: ImageMagick 6.8.7 or higher not installed; "
+                     u"'compare_threshold' option ignored")
 
     def commands(self):
         # Embed command.
@@ -52,12 +63,15 @@ class EmbedCoverArtPlugin(BeetsPlugin):
             '-f', '--file', metavar='PATH', help='the image file to embed'
         )
         maxwidth = config['embedart']['maxwidth'].get(int)
+        compare_threshold = config['embedart']['compare_threshold'].get(int)
+        ifempty = config['embedart']['ifempty'].get(bool)
 
         def embed_func(lib, opts, args):
             if opts.file:
                 imagepath = normpath(opts.file)
                 for item in lib.items(decargs(args)):
-                    embed_item(item, imagepath, maxwidth)
+                    embed_item(item, imagepath, maxwidth, None,
+                               compare_threshold, ifempty)
             else:
                 for album in lib.albums(decargs(args)):
                     embed_album(album, maxwidth)
@@ -72,7 +86,8 @@ class EmbedCoverArtPlugin(BeetsPlugin):
 
         def extract_func(lib, opts, args):
             outpath = normpath(opts.outpath or 'cover')
-            extract(lib, outpath, decargs(args))
+            item = lib.items(decargs(args)).get()
+            extract(outpath, item)
         extract_cmd.func = extract_func
 
         # Clear command.
@@ -91,23 +106,43 @@ def album_imported(lib, album):
     """Automatically embed art into imported albums.
     """
     if album.artpath and config['embedart']['auto']:
-        embed_album(album, config['embedart']['maxwidth'].get(int))
+        embed_album(album, config['embedart']['maxwidth'].get(int), True)
 
 
-def embed_item(item, imagepath, maxwidth=None, itempath=None):
+def embed_item(item, imagepath, maxwidth=None, itempath=None,
+               compare_threshold=0, ifempty=False, as_album=False):
     """Embed an image into the item's media file.
     """
+    if compare_threshold:
+        if not check_art_similarity(item, imagepath, compare_threshold):
+            log.warn(u'Image not similar; skipping.')
+            return
+    if ifempty:
+        art = get_art(item)
+        if not art:
+            pass
+        else:
+            log.debug(u'embedart: media file contained art already {0}'.format(
+                displayable_path(imagepath)
+            ))
+            return
+    if maxwidth and not as_album:
+        imagepath = resize_image(imagepath, maxwidth)
+
     try:
+        log.debug(u'embedart: embedding {0}'.format(
+            displayable_path(imagepath)
+        ))
         item['images'] = [_mediafile_image(imagepath, maxwidth)]
-        item.try_write(itempath)
     except IOError as exc:
         log.error(u'embedart: could not read image file: {0}'.format(exc))
-    finally:
-        # We don't want to store the image in the database
+    else:
+        # We don't want to store the image in the database.
+        item.try_write(itempath)
         del item['images']
 
 
-def embed_album(album, maxwidth=None):
+def embed_album(album, maxwidth=None, quiet=False):
     """Embed album art into all of the album's items.
     """
     imagepath = album.artpath
@@ -115,39 +150,78 @@ def embed_album(album, maxwidth=None):
         log.info(u'No album art present: {0} - {1}'.
                  format(album.albumartist, album.album))
         return
-    if not os.path.isfile(imagepath):
+    if not os.path.isfile(syspath(imagepath)):
         log.error(u'Album art not found at {0}'
-                  .format(imagepath))
+                  .format(displayable_path(imagepath)))
         return
+    if maxwidth:
+        imagepath = resize_image(imagepath, maxwidth)
 
-    log.info(u'Embedding album art into {0.albumartist} - {0.album}.'
-             .format(album))
+    log.log(
+        logging.DEBUG if quiet else logging.INFO,
+        u'Embedding album art into {0.albumartist} - {0.album}.'.format(album),
+    )
 
     for item in album.items():
-        embed_item(item, imagepath, maxwidth)
+        embed_item(item, imagepath, maxwidth, None,
+                   config['embedart']['compare_threshold'].get(int),
+                   config['embedart']['ifempty'].get(bool), as_album=True)
+
+
+def resize_image(imagepath, maxwidth):
+    """Returns path to an image resized to maxwidth.
+    """
+    log.info(u'Resizing album art to {0} pixels wide'
+             .format(maxwidth))
+    imagepath = ArtResizer.shared.resize(maxwidth, syspath(imagepath))
+    return imagepath
+
+
+def check_art_similarity(item, imagepath, compare_threshold):
+    """A boolean indicating if an image is similar to embedded item art.
+    """
+    with NamedTemporaryFile(delete=True) as f:
+        art = extract(f.name, item)
+
+        if art:
+            # Converting images to grayscale tends to minimize the weight
+            # of colors in the diff score
+            cmd = 'convert {0} {1} -colorspace gray MIFF:- | ' \
+                  'compare -metric PHASH - null:'.format(syspath(imagepath),
+                                                         syspath(art))
+
+            proc = subprocess.Popen(cmd, stdout=subprocess.PIPE,
+                                    stderr=subprocess.PIPE,
+                                    close_fds=platform.system() != 'Windows',
+                                    shell=True)
+            stdout, stderr = proc.communicate()
+            if proc.returncode:
+                if proc.returncode != 1:
+                    log.warn(u'embedart: IM phashes compare failed for {0}, \
+                   {1}'.format(displayable_path(imagepath),
+                               displayable_path(art)))
+                    return
+                phashDiff = float(stderr)
+            else:
+                phashDiff = float(stdout)
+
+            log.info(u'embedart: compare PHASH score is {0}'.format(phashDiff))
+            if phashDiff > compare_threshold:
+                return False
+
+    return True
 
 
 def _mediafile_image(image_path, maxwidth=None):
     """Return a `mediafile.Image` object for the path.
-
-    If maxwidth is set the image is resized if necessary.
     """
-    if maxwidth:
-        image_path = ArtResizer.shared.resize(maxwidth, syspath(image_path))
 
     with open(syspath(image_path), 'rb') as f:
         data = f.read()
     return mediafile.Image(data, type=mediafile.ImageType.front)
 
 
-# 'extractart' command.
-
-def extract(lib, outpath, query):
-    item = lib.items(query).get()
-    if not item:
-        log.error(u'No item matches query.')
-        return
-
+def get_art(item):
     # Extract the art.
     try:
         mf = mediafile.MediaFile(syspath(item.path))
@@ -157,7 +231,18 @@ def extract(lib, outpath, query):
         ))
         return
 
-    art = mf.art
+    return mf.art
+
+# 'extractart' command.
+
+
+def extract(outpath, item):
+    if not item:
+        log.error(u'No item matches query.')
+        return
+
+    art = get_art(item)
+
     if not art:
         log.error(u'No album art present in {0} - {1}.'
                   .format(item.artist, item.title))
@@ -170,10 +255,11 @@ def extract(lib, outpath, query):
         return
     outpath += '.' + ext
 
-    log.info(u'Extracting album art from: {0.artist} - {0.title}\n'
-             u'To: {1}'.format(item, displayable_path(outpath)))
+    log.info(u'Extracting album art from: {0.artist} - {0.title} '
+             u'to: {1}'.format(item, displayable_path(outpath)))
     with open(syspath(outpath), 'wb') as f:
         f.write(art)
+    return outpath
 
 
 # 'clearart' command.
@@ -190,5 +276,5 @@ def clear(lib, query):
                 displayable_path(item.path), exc
             ))
             continue
-        mf.art = None
+        del mf.art
         mf.save()

lib/beetsplug/fetchart.py

@@ -1,5 +1,5 @@
 # This file is part of beets.
-# Copyright 2013, Adrian Sampson.
+# Copyright 2014, Adrian Sampson.
 #
 # Permission is hereby granted, free of charge, to any person obtaining
 # a copy of this software and associated documentation files (the
@@ -22,12 +22,18 @@ from tempfile import NamedTemporaryFile
 
 import requests
 
-from beets.plugins import BeetsPlugin
-from beets.util.artresizer import ArtResizer
+from beets import plugins
 from beets import importer
 from beets import ui
 from beets import util
 from beets import config
+from beets.util.artresizer import ArtResizer
+
+try:
+    import itunes
+    HAVE_ITUNES = True
+except ImportError:
+    HAVE_ITUNES = False
 
 IMAGE_EXTENSIONS = ['png', 'jpg', 'jpeg']
 CONTENT_TYPES = ('image/jpeg',)
@@ -73,17 +79,14 @@ CAA_URL = 'http://coverartarchive.org/release/{mbid}/front-500.jpg'
 CAA_GROUP_URL = 'http://coverartarchive.org/release-group/{mbid}/front-500.jpg'
 
 
-def caa_art(release_id):
-    """Return the Cover Art Archive URL given a MusicBrainz release ID.
+def caa_art(album):
+    """Return the Cover Art Archive and Cover Art Archive release group URLs
+    using album MusicBrainz release ID and release group ID.
     """
-    return CAA_URL.format(mbid=release_id)
-
-
-def caa_group_art(release_group_id):
-    """Return the Cover Art Archive release group URL given a MusicBrainz
-    release group ID.
-    """
-    return CAA_GROUP_URL.format(mbid=release_group_id)
+    if album.mb_albumid:
+        yield CAA_URL.format(mbid=album.mb_albumid)
+    if album.mb_releasegroupid:
+        yield CAA_GROUP_URL.format(mbid=album.mb_releasegroupid)
 
 
 # Art from Amazon.
@@ -92,10 +95,12 @@ AMAZON_URL = 'http://images.amazon.com/images/P/%s.%02i.LZZZZZZZ.jpg'
 AMAZON_INDICES = (1, 2)
 
 
-def art_for_asin(asin):
-    """Generate URLs for an Amazon ID (ASIN) string."""
-    for index in AMAZON_INDICES:
-        yield AMAZON_URL % (asin, index)
+def art_for_asin(album):
+    """Generate URLs using Amazon ID (ASIN) string.
+    """
+    if album.asin:
+        for index in AMAZON_INDICES:
+            yield AMAZON_URL % (album.asin, index)
 
 
 # AlbumArt.org scraper.
@@ -104,11 +109,14 @@ AAO_URL = 'http://www.albumart.org/index_detail.php'
 AAO_PAT = r'href\s*=\s*"([^>"]*)"[^>]*title\s*=\s*"View larger image"'
 
 
-def aao_art(asin):
-    """Return art URL from AlbumArt.org given an ASIN."""
+def aao_art(album):
+    """Return art URL from AlbumArt.org using album ASIN.
+    """
+    if not album.asin:
+        return
     # Get the page from albumart.org.
     try:
-        resp = requests_session.get(AAO_URL, params={'asin': asin})
+        resp = requests_session.get(AAO_URL, params={'asin': album.asin})
         log.debug(u'fetchart: scraped art URL: {0}'.format(resp.url))
     except requests.RequestException:
         log.debug(u'fetchart: error scraping art page')
@@ -118,7 +126,7 @@ def aao_art(asin):
     m = re.search(AAO_PAT, resp.text)
     if m:
         image_url = m.group(1)
-        return image_url
+        yield image_url
     else:
         log.debug(u'fetchart: no image found on page')
 
@@ -132,6 +140,8 @@ def google_art(album):
     """Return art URL from google.org given an album title and
     interpreter.
     """
+    if not (album.albumartist and album.album):
+        return
     search_string = (album.albumartist + ',' + album.album).encode('utf-8')
     response = requests_session.get(GOOGLE_URL, params={
         'v': '1.0',
@@ -145,14 +155,39 @@ def google_art(album):
         data = results['responseData']
         dataInfo = data['results']
         for myUrl in dataInfo:
-            return myUrl['unescapedUrl']
+            yield myUrl['unescapedUrl']
     except:
         log.debug(u'fetchart: error scraping art page')
         return
 
 
+# Art from the iTunes Store.
+
+def itunes_art(album):
+    """Return art URL from iTunes Store given an album title.
+    """
+    search_string = (album.albumartist + ' ' + album.album).encode('utf-8')
+    try:
+        # Isolate bugs in the iTunes library while searching.
+        try:
+            itunes_album = itunes.search_album(search_string)[0]
+        except Exception as exc:
+            log.debug('fetchart: iTunes search failed: {0}'.format(exc))
+            return
+
+        if itunes_album.get_artwork()['100']:
+            small_url = itunes_album.get_artwork()['100']
+            big_url = small_url.replace('100x100', '1200x1200')
+            yield big_url
+        else:
+            log.debug(u'fetchart: album has no artwork in iTunes Store')
+    except IndexError:
+        log.debug(u'fetchart: album not found in iTunes Store')
+
+
 # Art from the filesystem.
 
+
 def filename_priority(filename, cover_names):
     """Sort order for image names.
 
@@ -164,7 +199,8 @@ def filename_priority(filename, cover_names):
 
 
 def art_in_path(path, cover_names, cautious):
-    """Look for album art files in a specified directory."""
+    """Look for album art files in a specified directory.
+    """
     if not os.path.isdir(path):
         return
 
@@ -195,31 +231,27 @@ def art_in_path(path, cover_names, cautious):
 
 # Try each source in turn.
 
+SOURCES_ALL = [u'coverart', u'itunes', u'amazon', u'albumart', u'google']
 
-def _source_urls(album):
+ART_FUNCS = {
+    u'coverart': caa_art,
+    u'itunes': itunes_art,
+    u'albumart': aao_art,
+    u'amazon': art_for_asin,
+    u'google': google_art,
+}
+
+
+def _source_urls(album, sources=SOURCES_ALL):
     """Generate possible source URLs for an album's art. The URLs are
     not guaranteed to work so they each need to be attempted in turn.
     This allows the main `art_for_album` function to abort iteration
     through this sequence early to avoid the cost of scraping when not
     necessary.
     """
-    # Cover Art Archive.
-    if album.mb_albumid:
-        yield caa_art(album.mb_albumid)
-    if album.mb_releasegroupid:
-        yield caa_group_art(album.mb_releasegroupid)
-
-    # Amazon and AlbumArt.org.
-    if album.asin:
-        for url in art_for_asin(album.asin):
-            yield url
-        url = aao_art(album.asin)
-        if url:
-            yield url
-
-    if config['fetchart']['google_search']:
-        url = google_art(album)
-        if url:
+    for s in sources:
+        urls = ART_FUNCS[s](album)
+        for url in urls:
             yield url
 
 
@@ -245,7 +277,8 @@ def art_for_album(album, paths, maxwidth=None, local_only=False):
     # Web art sources.
     remote_priority = config['fetchart']['remote_priority'].get(bool)
     if not local_only and (remote_priority or not out):
-        for url in _source_urls(album):
+        for url in _source_urls(album,
+                                config['fetchart']['sources'].as_str_seq()):
             if maxwidth:
                 url = ArtResizer.shared.proxy_url(maxwidth, url)
             candidate = _fetch_image(url)
@@ -286,7 +319,7 @@ def batch_fetch_art(lib, albums, force, maxwidth=None):
                                           message))
 
 
-class FetchArtPlugin(BeetsPlugin):
+class FetchArtPlugin(plugins.BeetsPlugin):
     def __init__(self):
         super(FetchArtPlugin, self).__init__()
 
@@ -297,6 +330,7 @@ class FetchArtPlugin(BeetsPlugin):
             'cautious': False,
             'google_search': False,
             'cover_names': ['cover', 'front', 'art', 'album', 'folder'],
+            'sources': SOURCES_ALL,
         })
 
         # Holds paths to downloaded images between fetching them and
@@ -309,6 +343,12 @@ class FetchArtPlugin(BeetsPlugin):
             self.import_stages = [self.fetch_art]
             self.register_listener('import_task_files', self.assign_art)
 
+        available_sources = list(SOURCES_ALL)
+        if not HAVE_ITUNES and u'itunes' in available_sources:
+            available_sources.remove(u'itunes')
+        self.config['sources'] = plugins.sanitize_choices(
+            self.config['sources'].as_str_seq(), available_sources)
+
     # Asynchronous; after music is added to the library.
     def fetch_art(self, session, task):
         """Find art for the album being imported."""

lib/beetsplug/lyrics.py

@@ -18,25 +18,26 @@ from __future__ import print_function
 
 import re
 import logging
-import urllib
+import requests
 import json
 import unicodedata
+import urllib
 import difflib
 import itertools
+from HTMLParser import HTMLParseError
 
-from beets.plugins import BeetsPlugin
-from beets import ui
-from beets import config
+from beets import plugins
+from beets import config, ui
 
 
 # Global logger.
 
 log = logging.getLogger('beets')
 
-DIV_RE = re.compile(r'<(/?)div>?')
+DIV_RE = re.compile(r'<(/?)div>?', re.I)
 COMMENT_RE = re.compile(r'<!--.*-->', re.S)
 TAG_RE = re.compile(r'<[^>]*>')
-BREAK_RE = re.compile(r'<br\s*/?>')
+BREAK_RE = re.compile(r'\n?\s*<br([\s|/][^>]*)*>\s*\n?', re.I)
 URL_CHARACTERS = {
     u'\u2018': u"'",
     u'\u2019': u"'",
@@ -60,10 +61,14 @@ def fetch_url(url):
     is unreachable.
     """
     try:
-        return urllib.urlopen(url).read()
-    except IOError as exc:
-        log.debug(u'failed to fetch: {0} ({1})'.format(url, unicode(exc)))
-        return None
+        r = requests.get(url, verify=False)
+    except requests.RequestException as exc:
+        log.debug(u'lyrics request failed: {0}'.format(exc))
+        return
+    if r.status_code == requests.codes.ok:
+        return r.text
+    else:
+        log.debug(u'failed to fetch: {0} ({1})'.format(url, r.status_code))
 
 
 def unescape(text):
@@ -79,10 +84,20 @@ def unescape(text):
     return out
 
 
-def extract_text(html, starttag):
+def extract_text_between(html, start_marker, end_marker):
+    try:
+        _, html = html.split(start_marker, 1)
+        html, _ = html.split(end_marker, 1)
+    except ValueError:
+        return u''
+    return html
+
+
+def extract_text_in(html, starttag):
     """Extract the text from a <DIV> tag in the HTML starting with
     ``starttag``. Returns None if parsing fails.
     """
+
     # Strip off the leading text before opening tag.
     try:
         _, html = html.split(starttag, 1)
@@ -101,7 +116,6 @@ def extract_text(html, starttag):
         else:  # Opening tag.
             if level == 0:
                 parts.append(html[pos:match.start()])
-
             level += 1
 
         if level == -1:
@@ -110,26 +124,7 @@ def extract_text(html, starttag):
     else:
         print('no closing tag found!')
         return
-    lyrics = ''.join(parts)
-    return strip_cruft(lyrics)
-
-
-def strip_cruft(lyrics, wscollapse=True):
-    """Clean up HTML from an extracted lyrics string. For example, <BR>
-    tags are replaced with newlines.
-    """
-    lyrics = COMMENT_RE.sub('', lyrics)
-    lyrics = unescape(lyrics)
-    if wscollapse:
-        lyrics = re.sub(r'\s+', ' ', lyrics)  # Whitespace collapse.
-    lyrics = re.sub(r'<(script).*?</\1>(?s)', '', lyrics)  # Strip script tags.
-    lyrics = BREAK_RE.sub('\n', lyrics)  # <BR> newlines.
-    lyrics = re.sub(r'\n +', '\n', lyrics)
-    lyrics = re.sub(r' +\n', '\n', lyrics)
-    lyrics = TAG_RE.sub('', lyrics)  # Strip remaining HTML tags.
-    lyrics = lyrics.replace('\r', '\n')
-    lyrics = lyrics.strip()
-    return lyrics
+    return u''.join(parts)
 
 
 def search_pairs(item):
@@ -140,7 +135,7 @@ def search_pairs(item):
 
     In addition to the artist and title obtained from the `item` the
     method tries to strip extra information like paranthesized suffixes
-    and featured artists from the strings and add them as caniddates.
+    and featured artists from the strings and add them as candidates.
     The method also tries to split multiple titles separated with `/`.
     """
 
@@ -149,7 +144,7 @@ def search_pairs(item):
     artists = [artist]
 
     # Remove any featuring artists from the artists name
-    pattern = r"(.*?) (&|\b(and|ft|feat(uring)?\b))"
+    pattern = r"(.*?) {0}".format(plugins.feat_tokens())
     match = re.search(pattern, artist, re.IGNORECASE)
     if match:
         artists.append(match.group(1))
@@ -162,8 +157,8 @@ def search_pairs(item):
         titles.append(match.group(1))
 
     # Remove any featuring artists from the title
-    pattern = r"(.*?) \b(ft|feat(uring)?)\b"
-    for title in titles:
+    pattern = r"(.*?) {0}".format(plugins.feat_tokens(for_artist=False))
+    for title in titles[:]:
         match = re.search(pattern, title, re.IGNORECASE)
         if match:
             titles.append(match.group(1))
@@ -189,6 +184,19 @@ def _encode(s):
         s = s.encode('utf8', 'ignore')
     return urllib.quote(s)
 
+# Musixmatch
+
+MUSIXMATCH_URL_PATTERN = 'https://www.musixmatch.com/lyrics/%s/%s'
+
+
+def fetch_musixmatch(artist, title):
+    url = MUSIXMATCH_URL_PATTERN % (_lw_encode(artist.title()),
+                                    _lw_encode(title.title()))
+    html = fetch_url(url)
+    if not html:
+        return
+    lyrics = extract_text_between(html, '"lyrics_body":', '"lyrics_language":')
+    return lyrics.strip(',"').replace('\\n', '\n')
 
 # LyricsWiki.
 
@@ -212,7 +220,7 @@ def fetch_lyricswiki(artist, title):
     if not html:
         return
 
-    lyrics = extract_text(html, "<div class='lyricbox'>")
+    lyrics = extract_text_in(html, u"<div class='lyricbox'>")
     if lyrics and 'Unfortunately, we are not licensed' not in lyrics:
         return lyrics
 
@@ -238,8 +246,8 @@ def fetch_lyricscom(artist, title):
     html = fetch_url(url)
     if not html:
         return
-
-    lyrics = extract_text(html, '<div id="lyric_space">')
+    lyrics = extract_text_between(html, '<div id="lyrics" class="SCREENONLY" '
+                                  'itemprop="description">', '</div>')
     if not lyrics:
         return
     for not_found_str in LYRICSCOM_NOT_FOUND:
@@ -280,7 +288,6 @@ def is_page_candidate(urlLink, urlTitle, title, artist):
     artist = slugify(artist.lower())
     sitename = re.search(u"//([^/]+)/.*", slugify(urlLink.lower())).group(1)
     urlTitle = slugify(urlTitle.lower())
-
     # Check if URL title contains song title (exact match)
     if urlTitle.find(title) != -1:
         return True
@@ -289,41 +296,11 @@ def is_page_candidate(urlLink, urlTitle, title, artist):
     tokens = [by + '_' + artist for by in BY_TRANS] + \
              [artist, sitename, sitename.replace('www.', '')] + LYRICS_TRANS
     songTitle = re.sub(u'(%s)' % u'|'.join(tokens), u'', urlTitle)
-
-    typoRatio = .8
+    songTitle = songTitle.strip('_|')
+    typoRatio = .9
     return difflib.SequenceMatcher(None, songTitle, title).ratio() >= typoRatio
 
 
-def insert_line_feeds(text):
-    """Insert newlines before upper-case characters.
-    """
-    tokensStr = re.split("([a-z][A-Z])", text)
-    for idx in range(1, len(tokensStr), 2):
-        ltoken = list(tokensStr[idx])
-        tokensStr[idx] = ltoken[0] + '\n' + ltoken[1]
-    return ''.join(tokensStr)
-
-
-def sanitize_lyrics(text):
-    """Clean text, returning raw lyrics as output or None if it happens
-    that input text is actually not lyrics content.  Clean (x)html tags
-    in text, correct layout and syntax...
-    """
-    text = strip_cruft(text, False)
-
-    # Restore \n in input text
-    if '\n' not in text:
-        text = insert_line_feeds(text)
-
-    while text.count('\n\n') > text.count('\n') // 4:
-        # Remove first occurrence of \n for each sequence of \n
-        text = re.sub(r'\n(\n+)', '\g<1>', text)
-
-    text = re.sub(r'\n\n+', '\n\n', text)   # keep at most two \n in a row
-
-    return text
-
-
 def remove_credits(text):
     """Remove first/last line of text if it contains the word 'lyrics'
     eg 'Lyrics by songsdatabase.com'
@@ -342,13 +319,12 @@ def is_lyrics(text, artist=None):
     """Determine whether the text seems to be valid lyrics.
     """
     if not text:
-        return
-
+        return False
     badTriggersOcc = []
     nbLines = text.count('\n')
     if nbLines <= 1:
         log.debug(u"Ignoring too short lyrics '{0}'".format(text))
-        return 0
+        return False
     elif nbLines < 5:
         badTriggersOcc.append('too_short')
     else:
@@ -356,7 +332,7 @@ def is_lyrics(text, artist=None):
         # down
         text = remove_credits(text)
 
-    badTriggers = ['lyrics', 'copyright', 'property']
+    badTriggers = ['lyrics', 'copyright', 'property', 'links']
     if artist:
         badTriggersOcc += [artist]
 
@@ -366,62 +342,58 @@ def is_lyrics(text, artist=None):
 
     if badTriggersOcc:
         log.debug(u'Bad triggers detected: {0}'.format(badTriggersOcc))
-
     return len(badTriggersOcc) < 2
 
 
-def scrape_lyrics_from_url(url):
+def _scrape_strip_cruft(html, plain_text_out=False):
+    """Clean up HTML
+    """
+    html = unescape(html)
+
+    html = html.replace('\r', '\n')  # Normalize EOL.
+    html = re.sub(r' +', ' ', html)  # Whitespaces collapse.
+    html = BREAK_RE.sub('\n', html)  # <br> eats up surrounding '\n'.
+    html = re.sub(r'<(script).*?</\1>(?s)', '', html)  # Strip script tags.
+
+    if plain_text_out:  # Strip remaining HTML tags
+        html = COMMENT_RE.sub('', html)
+        html = TAG_RE.sub('', html)
+
+    html = '\n'.join([x.strip() for x in html.strip().split('\n')])
+    html = re.sub(r'\n{3,}', r'\n\n', html)
+    return html
+
+
+def _scrape_merge_paragraphs(html):
+    html = re.sub(r'</p>\s*<p(\s*[^>]*)>', '\n', html)
+    return re.sub(r'<div .*>\s*</div>', '\n', html)
+
+
+def scrape_lyrics_from_html(html):
     """Scrape lyrics from a URL. If no lyrics can be found, return None
     instead.
     """
-    from bs4 import BeautifulSoup, Comment
-    html = fetch_url(url)
+    from bs4 import SoupStrainer, BeautifulSoup
+
     if not html:
         return None
 
-    soup = BeautifulSoup(html)
-
-    for tag in soup.findAll('br'):
-        tag.replaceWith('\n')
-
-    # Remove non relevant html parts
-    [s.extract() for s in soup(['head', 'script'])]
-    comments = soup.findAll(text=lambda text: isinstance(text, Comment))
-    [s.extract() for s in comments]
+    def is_text_notcode(text):
+        length = len(text)
+        return (length > 20 and
+                text.count(' ') > length / 25 and
+                (text.find('{') == -1 or text.find(';') == -1))
+    html = _scrape_strip_cruft(html)
+    html = _scrape_merge_paragraphs(html)
 
+    # extract all long text blocks that are not code
     try:
-        for tag in soup.findAll(True):
-            tag.name = 'p'          # keep tag contents
-
-    except Exception, e:
-        log.debug(u'Error {0} when replacing containing marker by p marker'
-                  .format(e, exc_info=True))
-
-    # Make better soup from current soup! The previous unclosed <p> sections
-    # are now closed.  Use str() rather than prettify() as it's more
-    # conservative concerning EOL
-    soup = BeautifulSoup(str(soup))
-
-    # In case lyrics are nested in no markup but <body>
-    # Insert the whole body in a <p>
-    bodyTag = soup.find('body')
-    if bodyTag:
-        pTag = soup.new_tag("p")
-        bodyTag.parent.insert(0, pTag)
-        pTag.insert(0, bodyTag)
-
-    tagTokens = []
-
-    for tag in soup.findAll('p'):
-        soup2 = BeautifulSoup(str(tag))
-        # Extract all text of <p> section.
-        tagTokens += soup2.findAll(text=True)
-
-    if tagTokens:
-        # Lyrics are expected to be the longest paragraph
-        tagTokens = sorted(tagTokens, key=len, reverse=True)
-        soup = BeautifulSoup(tagTokens[0])
-        return unescape(tagTokens[0].strip("\n\r: "))
+        soup = BeautifulSoup(html, "html.parser",
+                             parse_only=SoupStrainer(text=is_text_notcode))
+    except HTMLParseError:
+        return None
+    soup = sorted(soup.stripped_strings, key=len)[-1]
+    return soup
 
 
 def fetch_google(artist, title):
@@ -443,15 +415,14 @@ def fetch_google(artist, title):
     if 'items' in data.keys():
         for item in data['items']:
             urlLink = item['link']
-            urlTitle = item['title']
+            urlTitle = item.get('title', u'')
             if not is_page_candidate(urlLink, urlTitle, title, artist):
                 continue
-            lyrics = scrape_lyrics_from_url(urlLink)
+            html = fetch_url(urlLink)
+            lyrics = scrape_lyrics_from_html(html)
             if not lyrics:
                 continue
 
-            lyrics = sanitize_lyrics(lyrics)
-
             if is_lyrics(lyrics, artist):
                 log.debug(u'got lyrics from {0}'.format(item['displayLink']))
                 return lyrics
@@ -459,8 +430,16 @@ def fetch_google(artist, title):
 
 # Plugin logic.
 
+SOURCES = ['google', 'lyricwiki', 'lyrics.com', 'musixmatch']
+SOURCE_BACKENDS = {
+    'google': fetch_google,
+    'lyricwiki': fetch_lyricswiki,
+    'lyrics.com': fetch_lyricscom,
+    'musixmatch': fetch_musixmatch,
+}
 
-class LyricsPlugin(BeetsPlugin):
+
+class LyricsPlugin(plugins.BeetsPlugin):
     def __init__(self):
         super(LyricsPlugin, self).__init__()
         self.import_stages = [self.imported]
@@ -469,12 +448,19 @@ class LyricsPlugin(BeetsPlugin):
             'google_API_key': None,
             'google_engine_ID': u'009217259823014548361:lndtuqkycfu',
             'fallback': None,
+            'force': False,
+            'sources': SOURCES,
         })
 
-        self.backends = [fetch_lyricswiki, fetch_lyricscom]
-
-        if self.config['google_API_key'].get():
-            self.backends.insert(0, fetch_google)
+        available_sources = list(SOURCES)
+        if not self.config['google_API_key'].get() and \
+                'google' in SOURCES:
+            available_sources.remove('google')
+        self.config['sources'] = plugins.sanitize_choices(
+            self.config['sources'].as_str_seq(), available_sources)
+        self.backends = []
+        for key in self.config['sources'].as_str_seq():
+            self.backends.append(SOURCE_BACKENDS[key])
 
     def commands(self):
         cmd = ui.Subcommand('lyrics', help='fetch song lyrics')
@@ -490,8 +476,10 @@ class LyricsPlugin(BeetsPlugin):
             # import_write config value.
             write = config['import']['write'].get(bool)
             for item in lib.items(ui.decargs(args)):
-                self.fetch_item_lyrics(lib, logging.INFO, item, write,
-                                       opts.force_refetch)
+                self.fetch_item_lyrics(
+                    lib, logging.INFO, item, write,
+                    opts.force_refetch or self.config['force'],
+                )
                 if opts.printlyr and item.lyrics:
                     ui.print_(item.lyrics)
 
@@ -504,7 +492,7 @@ class LyricsPlugin(BeetsPlugin):
         if self.config['auto']:
             for item in task.imported_items():
                 self.fetch_item_lyrics(session.lib, logging.DEBUG, item,
-                                       False, False)
+                                       False, self.config['force'])
 
     def fetch_item_lyrics(self, lib, loglevel, item, write, force):
         """Fetch and store lyrics for a single item. If ``write``, then the
@@ -551,8 +539,6 @@ class LyricsPlugin(BeetsPlugin):
         for backend in self.backends:
             lyrics = backend(artist, title)
             if lyrics:
-                if isinstance(lyrics, str):
-                    lyrics = lyrics.decode('utf8', 'ignore')
                 log.debug(u'got lyrics from backend: {0}'
                           .format(backend.__name__))
-                return lyrics.strip()
+                return _scrape_strip_cruft(lyrics, True)