gronod/headphones
1
0
Fork 0
mirror of https://github.com/rembo10/headphones.git synced 2026-07-20 16:03:59 +01:00
Code Issues Packages Projects Releases Wiki Activity

Upgraded beets to 1.3.10, including patches

Browse Source
This commit is contained in:
Bas Stottelaar
2015-01-27 22:26:35 +01:00
parent ea842a95ca
commit cf6a6a876f
31 changed files with 4792 additions and 3002 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/beetsplug/lyrics.py
+122 -136
View File
@@ -18,25 +18,26 @@ from __future__ import print_function
import re
import logging
import urllib
import requests
import json
import unicodedata
import urllib
import difflib
import itertools
from HTMLParser import HTMLParseError
from beets.plugins import BeetsPlugin
from beets import ui
from beets import config
from beets import plugins
from beets import config, ui
# Global logger.
log = logging.getLogger('beets')
DIV_RE = re.compile(r'<(/?)div>?')
DIV_RE = re.compile(r'<(/?)div>?', re.I)
COMMENT_RE = re.compile(r'<!--.*-->', re.S)
TAG_RE = re.compile(r'<[^>]*>')
BREAK_RE = re.compile(r'<br\s*/?>')
BREAK_RE = re.compile(r'\n?\s*<br([\s|/][^>]*)*>\s*\n?', re.I)
URL_CHARACTERS = {
u'\u2018': u"'",
u'\u2019': u"'",
@@ -60,10 +61,14 @@ def fetch_url(url):
is unreachable.
"""
try:
return urllib.urlopen(url).read()
except IOError as exc:
log.debug(u'failed to fetch: {0} ({1})'.format(url, unicode(exc)))
return None
r = requests.get(url, verify=False)
except requests.RequestException as exc:
log.debug(u'lyrics request failed: {0}'.format(exc))
return
if r.status_code == requests.codes.ok:
return r.text
else:
log.debug(u'failed to fetch: {0} ({1})'.format(url, r.status_code))
def unescape(text):
@@ -79,10 +84,20 @@ def unescape(text):
return out
def extract_text(html, starttag):
def extract_text_between(html, start_marker, end_marker):
try:
_, html = html.split(start_marker, 1)
html, _ = html.split(end_marker, 1)
except ValueError:
return u''
return html
def extract_text_in(html, starttag):
"""Extract the text from a <DIV> tag in the HTML starting with
``starttag``. Returns None if parsing fails.
"""
# Strip off the leading text before opening tag.
try:
_, html = html.split(starttag, 1)
@@ -101,7 +116,6 @@ def extract_text(html, starttag):
else: # Opening tag.
if level == 0:
parts.append(html[pos:match.start()])
level += 1
if level == -1:
@@ -110,26 +124,7 @@ def extract_text(html, starttag):
else:
print('no closing tag found!')
return
lyrics = ''.join(parts)
return strip_cruft(lyrics)
def strip_cruft(lyrics, wscollapse=True):
"""Clean up HTML from an extracted lyrics string. For example, <BR>
tags are replaced with newlines.
"""
lyrics = COMMENT_RE.sub('', lyrics)
lyrics = unescape(lyrics)
if wscollapse:
lyrics = re.sub(r'\s+', ' ', lyrics) # Whitespace collapse.
lyrics = re.sub(r'<(script).*?</\1>(?s)', '', lyrics) # Strip script tags.
lyrics = BREAK_RE.sub('\n', lyrics) # <BR> newlines.
lyrics = re.sub(r'\n +', '\n', lyrics)
lyrics = re.sub(r' +\n', '\n', lyrics)
lyrics = TAG_RE.sub('', lyrics) # Strip remaining HTML tags.
lyrics = lyrics.replace('\r', '\n')
lyrics = lyrics.strip()
return lyrics
return u''.join(parts)
def search_pairs(item):
@@ -140,7 +135,7 @@ def search_pairs(item):
In addition to the artist and title obtained from the `item` the
method tries to strip extra information like paranthesized suffixes
and featured artists from the strings and add them as caniddates.
and featured artists from the strings and add them as candidates.
The method also tries to split multiple titles separated with `/`.
"""
@@ -149,7 +144,7 @@ def search_pairs(item):
artists = [artist]
# Remove any featuring artists from the artists name
pattern = r"(.*?) (&|\b(and|ft|feat(uring)?\b))"
pattern = r"(.*?) {0}".format(plugins.feat_tokens())
match = re.search(pattern, artist, re.IGNORECASE)
if match:
artists.append(match.group(1))
@@ -162,8 +157,8 @@ def search_pairs(item):
titles.append(match.group(1))
# Remove any featuring artists from the title
pattern = r"(.*?) \b(ft|feat(uring)?)\b"
for title in titles:
pattern = r"(.*?) {0}".format(plugins.feat_tokens(for_artist=False))
for title in titles[:]:
match = re.search(pattern, title, re.IGNORECASE)
if match:
titles.append(match.group(1))
@@ -189,6 +184,19 @@ def _encode(s):
s = s.encode('utf8', 'ignore')
return urllib.quote(s)
# Musixmatch
MUSIXMATCH_URL_PATTERN = 'https://www.musixmatch.com/lyrics/%s/%s'
def fetch_musixmatch(artist, title):
url = MUSIXMATCH_URL_PATTERN % (_lw_encode(artist.title()),
_lw_encode(title.title()))
html = fetch_url(url)
if not html:
return
lyrics = extract_text_between(html, '"lyrics_body":', '"lyrics_language":')
return lyrics.strip(',"').replace('\\n', '\n')
# LyricsWiki.
@@ -212,7 +220,7 @@ def fetch_lyricswiki(artist, title):
if not html:
return
lyrics = extract_text(html, "<div class='lyricbox'>")
lyrics = extract_text_in(html, u"<div class='lyricbox'>")
if lyrics and 'Unfortunately, we are not licensed' not in lyrics:
return lyrics
@@ -238,8 +246,8 @@ def fetch_lyricscom(artist, title):
html = fetch_url(url)
if not html:
return
lyrics = extract_text(html, '<div id="lyric_space">')
lyrics = extract_text_between(html, '<div id="lyrics" class="SCREENONLY" '
'itemprop="description">', '</div>')
if not lyrics:
return
for not_found_str in LYRICSCOM_NOT_FOUND:
@@ -280,7 +288,6 @@ def is_page_candidate(urlLink, urlTitle, title, artist):
artist = slugify(artist.lower())
sitename = re.search(u"//([^/]+)/.*", slugify(urlLink.lower())).group(1)
urlTitle = slugify(urlTitle.lower())
# Check if URL title contains song title (exact match)
if urlTitle.find(title) != -1:
return True
@@ -289,41 +296,11 @@ def is_page_candidate(urlLink, urlTitle, title, artist):
tokens = [by + '_' + artist for by in BY_TRANS] + \
[artist, sitename, sitename.replace('www.', '')] + LYRICS_TRANS
songTitle = re.sub(u'(%s)' % u'|'.join(tokens), u'', urlTitle)
typoRatio = .8
songTitle = songTitle.strip('_|')
typoRatio = .9
return difflib.SequenceMatcher(None, songTitle, title).ratio() >= typoRatio
def insert_line_feeds(text):
"""Insert newlines before upper-case characters.
"""
tokensStr = re.split("([a-z][A-Z])", text)
for idx in range(1, len(tokensStr), 2):
ltoken = list(tokensStr[idx])
tokensStr[idx] = ltoken[0] + '\n' + ltoken[1]
return ''.join(tokensStr)
def sanitize_lyrics(text):
"""Clean text, returning raw lyrics as output or None if it happens
that input text is actually not lyrics content. Clean (x)html tags
in text, correct layout and syntax...
"""
text = strip_cruft(text, False)
# Restore \n in input text
if '\n' not in text:
text = insert_line_feeds(text)
while text.count('\n\n') > text.count('\n') // 4:
# Remove first occurrence of \n for each sequence of \n
text = re.sub(r'\n(\n+)', '\g<1>', text)
text = re.sub(r'\n\n+', '\n\n', text) # keep at most two \n in a row
return text
def remove_credits(text):
"""Remove first/last line of text if it contains the word 'lyrics'
eg 'Lyrics by songsdatabase.com'
@@ -342,13 +319,12 @@ def is_lyrics(text, artist=None):
"""Determine whether the text seems to be valid lyrics.
"""
if not text:
return
return False
badTriggersOcc = []
nbLines = text.count('\n')
if nbLines <= 1:
log.debug(u"Ignoring too short lyrics '{0}'".format(text))
return 0
return False
elif nbLines < 5:
badTriggersOcc.append('too_short')
else:
@@ -356,7 +332,7 @@ def is_lyrics(text, artist=None):
# down
text = remove_credits(text)
badTriggers = ['lyrics', 'copyright', 'property']
badTriggers = ['lyrics', 'copyright', 'property', 'links']
if artist:
badTriggersOcc += [artist]
@@ -366,62 +342,58 @@ def is_lyrics(text, artist=None):
if badTriggersOcc:
log.debug(u'Bad triggers detected: {0}'.format(badTriggersOcc))
return len(badTriggersOcc) < 2
def scrape_lyrics_from_url(url):
def _scrape_strip_cruft(html, plain_text_out=False):
"""Clean up HTML
"""
html = unescape(html)
html = html.replace('\r', '\n') # Normalize EOL.
html = re.sub(r' +', ' ', html) # Whitespaces collapse.
html = BREAK_RE.sub('\n', html) # <br> eats up surrounding '\n'.
html = re.sub(r'<(script).*?</\1>(?s)', '', html) # Strip script tags.
if plain_text_out: # Strip remaining HTML tags
html = COMMENT_RE.sub('', html)
html = TAG_RE.sub('', html)
html = '\n'.join([x.strip() for x in html.strip().split('\n')])
html = re.sub(r'\n{3,}', r'\n\n', html)
return html
def _scrape_merge_paragraphs(html):
html = re.sub(r'</p>\s*<p(\s*[^>]*)>', '\n', html)
return re.sub(r'<div .*>\s*</div>', '\n', html)
def scrape_lyrics_from_html(html):
"""Scrape lyrics from a URL. If no lyrics can be found, return None
instead.
"""
from bs4 import BeautifulSoup, Comment
html = fetch_url(url)
from bs4 import SoupStrainer, BeautifulSoup
if not html:
return None
soup = BeautifulSoup(html)
for tag in soup.findAll('br'):
tag.replaceWith('\n')
# Remove non relevant html parts
[s.extract() for s in soup(['head', 'script'])]
comments = soup.findAll(text=lambda text: isinstance(text, Comment))
[s.extract() for s in comments]
def is_text_notcode(text):
length = len(text)
return (length > 20 and
text.count(' ') > length / 25 and
(text.find('{') == -1 or text.find(';') == -1))
html = _scrape_strip_cruft(html)
html = _scrape_merge_paragraphs(html)
# extract all long text blocks that are not code
try:
for tag in soup.findAll(True):
tag.name = 'p' # keep tag contents
except Exception, e:
log.debug(u'Error {0} when replacing containing marker by p marker'
.format(e, exc_info=True))
# Make better soup from current soup! The previous unclosed <p> sections
# are now closed. Use str() rather than prettify() as it's more
# conservative concerning EOL
soup = BeautifulSoup(str(soup))
# In case lyrics are nested in no markup but <body>
# Insert the whole body in a <p>
bodyTag = soup.find('body')
if bodyTag:
pTag = soup.new_tag("p")
bodyTag.parent.insert(0, pTag)
pTag.insert(0, bodyTag)
tagTokens = []
for tag in soup.findAll('p'):
soup2 = BeautifulSoup(str(tag))
# Extract all text of <p> section.
tagTokens += soup2.findAll(text=True)
if tagTokens:
# Lyrics are expected to be the longest paragraph
tagTokens = sorted(tagTokens, key=len, reverse=True)
soup = BeautifulSoup(tagTokens[0])
return unescape(tagTokens[0].strip("\n\r: "))
soup = BeautifulSoup(html, "html.parser",
parse_only=SoupStrainer(text=is_text_notcode))
except HTMLParseError:
return None
soup = sorted(soup.stripped_strings, key=len)[-1]
return soup
def fetch_google(artist, title):
@@ -443,15 +415,14 @@ def fetch_google(artist, title):
if 'items' in data.keys():
for item in data['items']:
urlLink = item['link']
urlTitle = item['title']
urlTitle = item.get('title', u'')
if not is_page_candidate(urlLink, urlTitle, title, artist):
continue
lyrics = scrape_lyrics_from_url(urlLink)
html = fetch_url(urlLink)
lyrics = scrape_lyrics_from_html(html)
if not lyrics:
continue
lyrics = sanitize_lyrics(lyrics)
if is_lyrics(lyrics, artist):
log.debug(u'got lyrics from {0}'.format(item['displayLink']))
return lyrics
@@ -459,8 +430,16 @@ def fetch_google(artist, title):
# Plugin logic.
SOURCES = ['google', 'lyricwiki', 'lyrics.com', 'musixmatch']
SOURCE_BACKENDS = {
'google': fetch_google,
'lyricwiki': fetch_lyricswiki,
'lyrics.com': fetch_lyricscom,
'musixmatch': fetch_musixmatch,
}
class LyricsPlugin(BeetsPlugin):
class LyricsPlugin(plugins.BeetsPlugin):
def __init__(self):
super(LyricsPlugin, self).__init__()
self.import_stages = [self.imported]
@@ -469,12 +448,19 @@ class LyricsPlugin(BeetsPlugin):
'google_API_key': None,
'google_engine_ID': u'009217259823014548361:lndtuqkycfu',
'fallback': None,
'force': False,
'sources': SOURCES,
})
self.backends = [fetch_lyricswiki, fetch_lyricscom]
if self.config['google_API_key'].get():
self.backends.insert(0, fetch_google)
available_sources = list(SOURCES)
if not self.config['google_API_key'].get() and \
'google' in SOURCES:
available_sources.remove('google')
self.config['sources'] = plugins.sanitize_choices(
self.config['sources'].as_str_seq(), available_sources)
self.backends = []
for key in self.config['sources'].as_str_seq():
self.backends.append(SOURCE_BACKENDS[key])
def commands(self):
cmd = ui.Subcommand('lyrics', help='fetch song lyrics')
@@ -490,8 +476,10 @@ class LyricsPlugin(BeetsPlugin):
# import_write config value.
write = config['import']['write'].get(bool)
for item in lib.items(ui.decargs(args)):
self.fetch_item_lyrics(lib, logging.INFO, item, write,
opts.force_refetch)
self.fetch_item_lyrics(
lib, logging.INFO, item, write,
opts.force_refetch or self.config['force'],
)
if opts.printlyr and item.lyrics:
ui.print_(item.lyrics)
@@ -504,7 +492,7 @@ class LyricsPlugin(BeetsPlugin):
if self.config['auto']:
for item in task.imported_items():
self.fetch_item_lyrics(session.lib, logging.DEBUG, item,
False, False)
False, self.config['force'])
def fetch_item_lyrics(self, lib, loglevel, item, write, force):
"""Fetch and store lyrics for a single item. If ``write``, then the
@@ -551,8 +539,6 @@ class LyricsPlugin(BeetsPlugin):
for backend in self.backends:
lyrics = backend(artist, title)
if lyrics:
if isinstance(lyrics, str):
lyrics = lyrics.decode('utf8', 'ignore')
log.debug(u'got lyrics from backend: {0}'
.format(backend.__name__))
return lyrics.strip()
return _scrape_strip_cruft(lyrics, True)