gronod/headphones
1
0
Fork 0
mirror of https://github.com/rembo10/headphones.git synced 2026-07-21 08:24:00 +01:00
Code Issues Packages Projects Releases Wiki Activity

update Beets

Browse Source
This commit is contained in:
AdeHub
2024-08-24 16:44:41 +12:00
parent a63098a919
commit 046d4d82b4
116 changed files with 17353 additions and 9964 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/beets/__init__.py
+8 -7
View File
@@ -13,28 +13,29 @@
# included in all copies or substantial portions of the Software.
import confuse
from sys import stderr
__version__ = '1.6.0'
__author__ = 'Adrian Sampson <adrian@radbox.org>'
import confuse
__version__ = "2.0.0"
__author__ = "Adrian Sampson <adrian@radbox.org>"
class IncludeLazyConfig(confuse.LazyConfig):
"""A version of Confuse's LazyConfig that also merges in data from
YAML files specified in an `include` setting.
"""
def read(self, user=True, defaults=True):
super().read(user, defaults)
try:
for view in self['include']:
for view in self["include"]:
self.set_file(view.as_filename())
except confuse.NotFoundError:
pass
except confuse.ConfigReadError as err:
stderr.write("configuration `import` failed: {}"
.format(err.reason))
stderr.write("configuration `import` failed: {}".format(err.reason))
config = IncludeLazyConfig('beets', __name__)
config = IncludeLazyConfig("beets", __name__)
lib/beets/__main__.py
+1
View File
@@ -18,6 +18,7 @@
import sys
from .ui import main
if __name__ == "__main__":
lib/beets/art.py
+108 -112
View File
@@ -17,21 +17,19 @@ music and items' embedded album art.
"""
import subprocess
import platform
from tempfile import NamedTemporaryFile
import os
from tempfile import NamedTemporaryFile
from beets.util import displayable_path, syspath, bytestring_path
from beets.util.artresizer import ArtResizer
import mediafile
from beets.util import bytestring_path, displayable_path, syspath
from beets.util.artresizer import ArtResizer
def mediafile_image(image_path, maxwidth=None):
"""Return a `mediafile.Image` object for the path.
"""
"""Return a `mediafile.Image` object for the path."""
with open(syspath(image_path), 'rb') as f:
with open(syspath(image_path), "rb") as f:
data = f.read()
return mediafile.Image(data, type=mediafile.ImageType.front)
@@ -41,170 +39,168 @@ def get_art(log, item):
try:
mf = mediafile.MediaFile(syspath(item.path))
except mediafile.UnreadableFileError as exc:
log.warning('Could not extract art from {0}: {1}',
displayable_path(item.path), exc)
log.warning(
"Could not extract art from {0}: {1}",
displayable_path(item.path),
exc,
)
return
return mf.art
def embed_item(log, item, imagepath, maxwidth=None, itempath=None,
compare_threshold=0, ifempty=False, as_album=False, id3v23=None,
quality=0):
"""Embed an image into the item's media file.
"""
# Conditions and filters.
def embed_item(
log,
item,
imagepath,
maxwidth=None,
itempath=None,
compare_threshold=0,
ifempty=False,
as_album=False,
id3v23=None,
quality=0,
):
"""Embed an image into the item's media file."""
# Conditions.
if compare_threshold:
if not check_art_similarity(log, item, imagepath, compare_threshold):
log.info('Image not similar; skipping.')
is_similar = check_art_similarity(
log, item, imagepath, compare_threshold
)
if is_similar is None:
log.warning("Error while checking art similarity; skipping.")
return
elif not is_similar:
log.info("Image not similar; skipping.")
return
if ifempty and get_art(log, item):
log.info('media file already contained art')
log.info("media file already contained art")
return
# Filters.
if maxwidth and not as_album:
imagepath = resize_image(log, imagepath, maxwidth, quality)
# Get the `Image` object from the file.
try:
log.debug('embedding {0}', displayable_path(imagepath))
log.debug("embedding {0}", displayable_path(imagepath))
image = mediafile_image(imagepath, maxwidth)
except OSError as exc:
log.warning('could not read image file: {0}', exc)
log.warning("could not read image file: {0}", exc)
return
# Make sure the image kind is safe (some formats only support PNG
# and JPEG).
if image.mime_type not in ('image/jpeg', 'image/png'):
log.info('not embedding image of unsupported type: {}',
image.mime_type)
if image.mime_type not in ("image/jpeg", "image/png"):
log.info("not embedding image of unsupported type: {}", image.mime_type)
return
item.try_write(path=itempath, tags={'images': [image]}, id3v23=id3v23)
item.try_write(path=itempath, tags={"images": [image]}, id3v23=id3v23)
def embed_album(log, album, maxwidth=None, quiet=False, compare_threshold=0,
ifempty=False, quality=0):
"""Embed album art into all of the album's items.
"""
def embed_album(
log,
album,
maxwidth=None,
quiet=False,
compare_threshold=0,
ifempty=False,
quality=0,
):
"""Embed album art into all of the album's items."""
imagepath = album.artpath
if not imagepath:
log.info('No album art present for {0}', album)
log.info("No album art present for {0}", album)
return
if not os.path.isfile(syspath(imagepath)):
log.info('Album art not found at {0} for {1}',
displayable_path(imagepath), album)
log.info(
"Album art not found at {0} for {1}",
displayable_path(imagepath),
album,
)
return
if maxwidth:
imagepath = resize_image(log, imagepath, maxwidth, quality)
log.info('Embedding album art into {0}', album)
log.info("Embedding album art into {0}", album)
for item in album.items():
embed_item(log, item, imagepath, maxwidth, None, compare_threshold,
ifempty, as_album=True, quality=quality)
embed_item(
log,
item,
imagepath,
maxwidth,
None,
compare_threshold,
ifempty,
as_album=True,
quality=quality,
)
def resize_image(log, imagepath, maxwidth, quality):
"""Returns path to an image resized to maxwidth and encoded with the
specified quality level.
"""
log.debug('Resizing album art to {0} pixels wide and encoding at quality \
level {1}', maxwidth, quality)
imagepath = ArtResizer.shared.resize(maxwidth, syspath(imagepath),
quality=quality)
log.debug(
"Resizing album art to {0} pixels wide and encoding at quality \
level {1}",
maxwidth,
quality,
)
imagepath = ArtResizer.shared.resize(
maxwidth, syspath(imagepath), quality=quality
)
return imagepath
def check_art_similarity(log, item, imagepath, compare_threshold):
def check_art_similarity(
log,
item,
imagepath,
compare_threshold,
artresizer=None,
):
"""A boolean indicating if an image is similar to embedded item art.
If no embedded art exists, always return `True`. If the comparison fails
for some reason, the return value is `None`.
This must only be called if `ArtResizer.shared.can_compare` is `True`.
"""
with NamedTemporaryFile(delete=True) as f:
art = extract(log, f.name, item)
if art:
is_windows = platform.system() == "Windows"
if not art:
return True
# Converting images to grayscale tends to minimize the weight
# of colors in the diff score. So we first convert both images
# to grayscale and then pipe them into the `compare` command.
# On Windows, ImageMagick doesn't support the magic \\?\ prefix
# on paths, so we pass `prefix=False` to `syspath`.
convert_cmd = ['convert', syspath(imagepath, prefix=False),
syspath(art, prefix=False),
'-colorspace', 'gray', 'MIFF:-']
compare_cmd = ['compare', '-metric', 'PHASH', '-', 'null:']
log.debug('comparing images with pipeline {} | {}',
convert_cmd, compare_cmd)
convert_proc = subprocess.Popen(
convert_cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
close_fds=not is_windows,
)
compare_proc = subprocess.Popen(
compare_cmd,
stdin=convert_proc.stdout,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
close_fds=not is_windows,
)
if artresizer is None:
artresizer = ArtResizer.shared
# Check the convert output. We're not interested in the
# standard output; that gets piped to the next stage.
convert_proc.stdout.close()
convert_stderr = convert_proc.stderr.read()
convert_proc.stderr.close()
convert_proc.wait()
if convert_proc.returncode:
log.debug(
'ImageMagick convert failed with status {}: {!r}',
convert_proc.returncode,
convert_stderr,
)
return
# Check the compare output.
stdout, stderr = compare_proc.communicate()
if compare_proc.returncode:
if compare_proc.returncode != 1:
log.debug('ImageMagick compare failed: {0}, {1}',
displayable_path(imagepath),
displayable_path(art))
return
out_str = stderr
else:
out_str = stdout
try:
phash_diff = float(out_str)
except ValueError:
log.debug('IM output is not a number: {0!r}', out_str)
return
log.debug('ImageMagick compare score: {0}', phash_diff)
return phash_diff <= compare_threshold
return True
return artresizer.compare(art, imagepath, compare_threshold)
def extract(log, outpath, item):
art = get_art(log, item)
outpath = bytestring_path(outpath)
if not art:
log.info('No album art present in {0}, skipping.', item)
log.info("No album art present in {0}, skipping.", item)
return
# Add an extension to the filename.
ext = mediafile.image_extension(art)
if not ext:
log.warning('Unknown image type in {0}.',
displayable_path(item.path))
log.warning("Unknown image type in {0}.", displayable_path(item.path))
return
outpath += bytestring_path('.' + ext)
outpath += bytestring_path("." + ext)
log.info('Extracting album art from: {0} to: {1}',
item, displayable_path(outpath))
with open(syspath(outpath), 'wb') as f:
log.info(
"Extracting album art from: {0} to: {1}",
item,
displayable_path(outpath),
)
with open(syspath(outpath), "wb") as f:
f.write(art)
return outpath
@@ -218,7 +214,7 @@ def extract_first(log, outpath, items):
def clear(log, lib, query):
items = lib.items(query)
log.info('Clearing album art from {0} items', len(items))
log.info("Clearing album art from {0} items", len(items))
for item in items:
log.debug('Clearing art for {0}', item)
item.try_write(tags={'images': None})
log.debug("Clearing art for {0}", item)
item.try_write(tags={"images": None})
lib/beets/autotag/__init__.py
+99 -62
View File
@@ -14,78 +14,91 @@
"""Facilities for automatically determining files' correct metadata.
"""
from typing import Mapping
from beets import logging
from beets import config
from beets import config, logging
from beets.library import Item
# Parts of external interface.
from .hooks import ( # noqa
AlbumInfo,
TrackInfo,
AlbumMatch,
TrackMatch,
Distance,
TrackInfo,
TrackMatch,
)
from .match import tag_item, tag_album, Proposal # noqa
from .match import Recommendation # noqa
from .match import Proposal, current_metadata, tag_album, tag_item # noqa
# Global logger.
log = logging.getLogger('beets')
log = logging.getLogger("beets")
# Metadata fields that are already hardcoded, or where the tag name changes.
SPECIAL_FIELDS = {
'album': (
'va',
'releasegroup_id',
'artist_id',
'album_id',
'mediums',
'tracks',
'year',
'month',
'day',
'artist',
'artist_credit',
'artist_sort',
'data_url'
"album": (
"va",
"releasegroup_id",
"artist_id",
"artists_ids",
"album_id",
"mediums",
"tracks",
"year",
"month",
"day",
"artist",
"artists",
"artist_credit",
"artists_credit",
"artist_sort",
"artists_sort",
"data_url",
),
"track": (
"track_alt",
"artist_id",
"artists_ids",
"release_track_id",
"medium",
"index",
"medium_index",
"title",
"artist_credit",
"artists_credit",
"artist_sort",
"artists_sort",
"artist",
"artists",
"track_id",
"medium_total",
"data_url",
"length",
),
'track': (
'track_alt',
'artist_id',
'release_track_id',
'medium',
'index',
'medium_index',
'title',
'artist_credit',
'artist_sort',
'artist',
'track_id',
'medium_total',
'data_url',
'length'
)
}
# Additional utilities for the main interface.
def apply_item_metadata(item, track_info):
"""Set an item's metadata from its matched TrackInfo object.
"""
def apply_item_metadata(item: Item, track_info: TrackInfo):
"""Set an item's metadata from its matched TrackInfo object."""
item.artist = track_info.artist
item.artists = track_info.artists
item.artist_sort = track_info.artist_sort
item.artists_sort = track_info.artists_sort
item.artist_credit = track_info.artist_credit
item.artists_credit = track_info.artists_credit
item.title = track_info.title
item.mb_trackid = track_info.track_id
item.mb_releasetrackid = track_info.release_track_id
if track_info.artist_id:
item.mb_artistid = track_info.artist_id
if track_info.artists_ids:
item.mb_artistids = track_info.artists_ids
for field, value in track_info.items():
# We only overwrite fields that are not already hardcoded.
if field in SPECIAL_FIELDS['track']:
if field in SPECIAL_FIELDS["track"]:
continue
if value is None:
continue
@@ -95,45 +108,62 @@ def apply_item_metadata(item, track_info):
# and track number). Perhaps these should be emptied?
def apply_metadata(album_info, mapping):
def apply_metadata(album_info: AlbumInfo, mapping: Mapping[Item, TrackInfo]):
"""Set the items' metadata to match an AlbumInfo object using a
mapping from Items to TrackInfo objects.
"""
for item, track_info in mapping.items():
# Artist or artist credit.
if config['artist_credit']:
item.artist = (track_info.artist_credit or
track_info.artist or
album_info.artist_credit or
album_info.artist)
item.albumartist = (album_info.artist_credit or
album_info.artist)
if config["artist_credit"]:
item.artist = (
track_info.artist_credit
or track_info.artist
or album_info.artist_credit
or album_info.artist
)
item.artists = (
track_info.artists_credit
or track_info.artists
or album_info.artists_credit
or album_info.artists
)
item.albumartist = album_info.artist_credit or album_info.artist
item.albumartists = album_info.artists_credit or album_info.artists
else:
item.artist = (track_info.artist or album_info.artist)
item.artist = track_info.artist or album_info.artist
item.artists = track_info.artists or album_info.artists
item.albumartist = album_info.artist
item.albumartists = album_info.artists
# Album.
item.album = album_info.album
# Artist sort and credit names.
item.artist_sort = track_info.artist_sort or album_info.artist_sort
item.artist_credit = (track_info.artist_credit or
album_info.artist_credit)
item.artists_sort = track_info.artists_sort or album_info.artists_sort
item.artist_credit = (
track_info.artist_credit or album_info.artist_credit
)
item.artists_credit = (
track_info.artists_credit or album_info.artists_credit
)
item.albumartist_sort = album_info.artist_sort
item.albumartists_sort = album_info.artists_sort
item.albumartist_credit = album_info.artist_credit
item.albumartists_credit = album_info.artists_credit
# Release date.
for prefix in '', 'original_':
if config['original_date'] and not prefix:
for prefix in "", "original_":
if config["original_date"] and not prefix:
# Ignore specific release date.
continue
for suffix in 'year', 'month', 'day':
for suffix in "year", "month", "day":
key = prefix + suffix
value = getattr(album_info, key) or 0
# If we don't even have a year, apply nothing.
if suffix == 'year' and not value:
if suffix == "year" and not value:
break
# Otherwise, set the fetched value (or 0 for the month
@@ -142,13 +172,13 @@ def apply_metadata(album_info, mapping):
# If we're using original release date for both fields,
# also set item.year = info.original_year, etc.
if config['original_date']:
if config["original_date"]:
item[suffix] = value
# Title.
item.title = track_info.title
if config['per_disc_numbering']:
if config["per_disc_numbering"]:
# We want to let the track number be zero, but if the medium index
# is not provided we need to fall back to the overall index.
if track_info.medium_index is not None:
@@ -172,7 +202,14 @@ def apply_metadata(album_info, mapping):
item.mb_artistid = track_info.artist_id
else:
item.mb_artistid = album_info.artist_id
if track_info.artists_ids:
item.mb_artistids = track_info.artists_ids
else:
item.mb_artistids = album_info.artists_ids
item.mb_albumartistid = album_info.artist_id
item.mb_albumartistids = album_info.artists_ids
item.mb_releasegroupid = album_info.releasegroup_id
# Compilation flag.
@@ -184,17 +221,17 @@ def apply_metadata(album_info, mapping):
# Don't overwrite fields with empty values unless the
# field is explicitly allowed to be overwritten
for field, value in album_info.items():
if field in SPECIAL_FIELDS['album']:
if field in SPECIAL_FIELDS["album"]:
continue
clobber = field in config['overwrite_null']['album'].as_str_seq()
clobber = field in config["overwrite_null"]["album"].as_str_seq()
if value is None and not clobber:
continue
item[field] = value
for field, value in track_info.items():
if field in SPECIAL_FIELDS['track']:
if field in SPECIAL_FIELDS["track"]:
continue
clobber = field in config['overwrite_null']['track'].as_str_seq()
clobber = field in config["overwrite_null"]["track"].as_str_seq()
value = getattr(track_info, field)
if value is None and not clobber:
continue
lib/beets/autotag/hooks.py
+242 -184
View File
@@ -14,40 +14,51 @@
"""Glue between metadata sources and the matching logic."""
from __future__ import annotations
import re
from collections import namedtuple
from functools import total_ordering
import re
from typing import (
Any,
Callable,
Dict,
Iterable,
Iterator,
List,
Optional,
Tuple,
TypeVar,
Union,
cast,
)
from beets import logging
from beets import plugins
from beets import config
from beets.util import as_string
from beets.autotag import mb
from jellyfish import levenshtein_distance
from unidecode import unidecode
log = logging.getLogger('beets')
from beets import config, logging, plugins
from beets.autotag import mb
from beets.library import Item
from beets.util import as_string, cached_classproperty
# The name of the type for patterns in re changed in Python 3.7.
try:
Pattern = re._pattern_type
except AttributeError:
Pattern = re.Pattern
log = logging.getLogger("beets")
V = TypeVar("V")
# Classes used to represent candidate options.
class AttrDict(dict):
class AttrDict(Dict[str, V]):
"""A dictionary that supports attribute ("dot") access, so `d.field`
is equivalent to `d['field']`.
"""
def __getattr__(self, attr):
def __getattr__(self, attr: str) -> V:
if attr in self:
return self.get(attr)
return self[attr]
else:
raise AttributeError
def __setattr__(self, key, value):
def __setattr__(self, key: str, value: V):
self.__setitem__(key, value)
def __hash__(self):
@@ -68,32 +79,73 @@ class AlbumInfo(AttrDict):
The others are optional and may be None.
"""
def __init__(self, tracks, album=None, album_id=None, artist=None,
artist_id=None, asin=None, albumtype=None, va=False,
year=None, month=None, day=None, label=None, mediums=None,
artist_sort=None, releasegroup_id=None, catalognum=None,
script=None, language=None, country=None, style=None,
genre=None, albumstatus=None, media=None, albumdisambig=None,
releasegroupdisambig=None, artist_credit=None,
original_year=None, original_month=None,
original_day=None, data_source=None, data_url=None,
discogs_albumid=None, discogs_labelid=None,
discogs_artistid=None, **kwargs):
# TYPING: are all of these correct? I've assumed optional strings
def __init__(
self,
tracks: List[TrackInfo],
album: Optional[str] = None,
album_id: Optional[str] = None,
artist: Optional[str] = None,
artist_id: Optional[str] = None,
artists: Optional[List[str]] = None,
artists_ids: Optional[List[str]] = None,
asin: Optional[str] = None,
albumtype: Optional[str] = None,
albumtypes: Optional[List[str]] = None,
va: bool = False,
year: Optional[int] = None,
month: Optional[int] = None,
day: Optional[int] = None,
label: Optional[str] = None,
barcode: Optional[str] = None,
mediums: Optional[int] = None,
artist_sort: Optional[str] = None,
artists_sort: Optional[List[str]] = None,
releasegroup_id: Optional[str] = None,
release_group_title: Optional[str] = None,
catalognum: Optional[str] = None,
script: Optional[str] = None,
language: Optional[str] = None,
country: Optional[str] = None,
style: Optional[str] = None,
genre: Optional[str] = None,
albumstatus: Optional[str] = None,
media: Optional[str] = None,
albumdisambig: Optional[str] = None,
releasegroupdisambig: Optional[str] = None,
artist_credit: Optional[str] = None,
artists_credit: Optional[List[str]] = None,
original_year: Optional[int] = None,
original_month: Optional[int] = None,
original_day: Optional[int] = None,
data_source: Optional[str] = None,
data_url: Optional[str] = None,
discogs_albumid: Optional[str] = None,
discogs_labelid: Optional[str] = None,
discogs_artistid: Optional[str] = None,
**kwargs,
):
self.album = album
self.album_id = album_id
self.artist = artist
self.artist_id = artist_id
self.artists = artists or []
self.artists_ids = artists_ids or []
self.tracks = tracks
self.asin = asin
self.albumtype = albumtype
self.albumtypes = albumtypes or []
self.va = va
self.year = year
self.month = month
self.day = day
self.label = label
self.barcode = barcode
self.mediums = mediums
self.artist_sort = artist_sort
self.artists_sort = artists_sort or []
self.releasegroup_id = releasegroup_id
self.release_group_title = release_group_title
self.catalognum = catalognum
self.script = script
self.language = language
@@ -105,6 +157,7 @@ class AlbumInfo(AttrDict):
self.albumdisambig = albumdisambig
self.releasegroupdisambig = releasegroupdisambig
self.artist_credit = artist_credit
self.artists_credit = artists_credit or []
self.original_year = original_year
self.original_month = original_month
self.original_day = original_day
@@ -115,27 +168,7 @@ class AlbumInfo(AttrDict):
self.discogs_artistid = discogs_artistid
self.update(kwargs)
# Work around a bug in python-musicbrainz-ngs that causes some
# strings to be bytes rather than Unicode.
# https://github.com/alastair/python-musicbrainz-ngs/issues/85
def decode(self, codec='utf-8'):
"""Ensure that all string attributes on this object, and the
constituent `TrackInfo` objects, are decoded to Unicode.
"""
for fld in ['album', 'artist', 'albumtype', 'label', 'artist_sort',
'catalognum', 'script', 'language', 'country', 'style',
'genre', 'albumstatus', 'albumdisambig',
'releasegroupdisambig', 'artist_credit',
'media', 'discogs_albumid', 'discogs_labelid',
'discogs_artistid']:
value = getattr(self, fld)
if isinstance(value, bytes):
setattr(self, fld, value.decode(codec, 'ignore'))
for track in self.tracks:
track.decode(codec)
def copy(self):
def copy(self) -> AlbumInfo:
dupe = AlbumInfo([])
dupe.update(self)
dupe.tracks = [track.copy() for track in self.tracks]
@@ -154,20 +187,50 @@ class TrackInfo(AttrDict):
are all 1-based.
"""
def __init__(self, title=None, track_id=None, release_track_id=None,
artist=None, artist_id=None, length=None, index=None,
medium=None, medium_index=None, medium_total=None,
artist_sort=None, disctitle=None, artist_credit=None,
data_source=None, data_url=None, media=None, lyricist=None,
composer=None, composer_sort=None, arranger=None,
track_alt=None, work=None, mb_workid=None,
work_disambig=None, bpm=None, initial_key=None, genre=None,
**kwargs):
# TYPING: are all of these correct? I've assumed optional strings
def __init__(
self,
title: Optional[str] = None,
track_id: Optional[str] = None,
release_track_id: Optional[str] = None,
artist: Optional[str] = None,
artist_id: Optional[str] = None,
artists: Optional[List[str]] = None,
artists_ids: Optional[List[str]] = None,
length: Optional[float] = None,
index: Optional[int] = None,
medium: Optional[int] = None,
medium_index: Optional[int] = None,
medium_total: Optional[int] = None,
artist_sort: Optional[str] = None,
artists_sort: Optional[List[str]] = None,
disctitle: Optional[str] = None,
artist_credit: Optional[str] = None,
artists_credit: Optional[List[str]] = None,
data_source: Optional[str] = None,
data_url: Optional[str] = None,
media: Optional[str] = None,
lyricist: Optional[str] = None,
composer: Optional[str] = None,
composer_sort: Optional[str] = None,
arranger: Optional[str] = None,
track_alt: Optional[str] = None,
work: Optional[str] = None,
mb_workid: Optional[str] = None,
work_disambig: Optional[str] = None,
bpm: Optional[str] = None,
initial_key: Optional[str] = None,
genre: Optional[str] = None,
album: Optional[str] = None,
**kwargs,
):
self.title = title
self.track_id = track_id
self.release_track_id = release_track_id
self.artist = artist
self.artist_id = artist_id
self.artists = artists or []
self.artists_ids = artists_ids or []
self.length = length
self.index = index
self.media = media
@@ -175,8 +238,10 @@ class TrackInfo(AttrDict):
self.medium_index = medium_index
self.medium_total = medium_total
self.artist_sort = artist_sort
self.artists_sort = artists_sort or []
self.disctitle = disctitle
self.artist_credit = artist_credit
self.artists_credit = artists_credit or []
self.data_source = data_source
self.data_url = data_url
self.lyricist = lyricist
@@ -190,20 +255,10 @@ class TrackInfo(AttrDict):
self.bpm = bpm
self.initial_key = initial_key
self.genre = genre
self.album = album
self.update(kwargs)
# As above, work around a bug in python-musicbrainz-ngs.
def decode(self, codec='utf-8'):
"""Ensure that all string attributes on this object are decoded
to Unicode.
"""
for fld in ['title', 'artist', 'medium', 'artist_sort', 'disctitle',
'artist_credit', 'media']:
value = getattr(self, fld)
if isinstance(value, bytes):
setattr(self, fld, value.decode(codec, 'ignore'))
def copy(self):
def copy(self) -> TrackInfo:
dupe = TrackInfo()
dupe.update(self)
return dupe
@@ -213,23 +268,23 @@ class TrackInfo(AttrDict):
# Parameters for string distance function.
# Words that can be moved to the end of a string using a comma.
SD_END_WORDS = ['the', 'a', 'an']
SD_END_WORDS = ["the", "a", "an"]
# Reduced weights for certain portions of the string.
SD_PATTERNS = [
(r'^the ', 0.1),
(r'[\[\(]?(ep|single)[\]\)]?', 0.0),
(r'[\[\(]?(featuring|feat|ft)[\. :].+', 0.1),
(r'\(.*?\)', 0.3),
(r'\[.*?\]', 0.3),
(r'(, )?(pt\.|part) .+', 0.2),
(r"^the ", 0.1),
(r"[\[\(]?(ep|single)[\]\)]?", 0.0),
(r"[\[\(]?(featuring|feat|ft)[\. :].+", 0.1),
(r"\(.*?\)", 0.3),
(r"\[.*?\]", 0.3),
(r"(, )?(pt\.|part) .+", 0.2),
]
# Replacements to use before testing distance.
SD_REPLACE = [
(r'&', 'and'),
(r"&", "and"),
]
def _string_dist_basic(str1, str2):
def _string_dist_basic(str1: str, str2: str) -> float:
"""Basic edit distance between two strings, ignoring
non-alphanumeric characters and case. Comparisons are based on a
transliteration/lowering to ASCII characters. Normalized by string
@@ -239,14 +294,14 @@ def _string_dist_basic(str1, str2):
assert isinstance(str2, str)
str1 = as_string(unidecode(str1))
str2 = as_string(unidecode(str2))
str1 = re.sub(r'[^a-z0-9]', '', str1.lower())
str2 = re.sub(r'[^a-z0-9]', '', str2.lower())
str1 = re.sub(r"[^a-z0-9]", "", str1.lower())
str2 = re.sub(r"[^a-z0-9]", "", str2.lower())
if not str1 and not str2:
return 0.0
return levenshtein_distance(str1, str2) / float(max(len(str1), len(str2)))
def string_dist(str1, str2):
def string_dist(str1: Optional[str], str2: Optional[str]) -> float:
"""Gives an "intuitive" edit distance between two strings. This is
an edit distance, normalized by the string length, with a number of
tweaks that reflect intuition about text.
@@ -263,10 +318,10 @@ def string_dist(str1, str2):
# example, "the something" should be considered equal to
# "something, the".
for word in SD_END_WORDS:
if str1.endswith(', %s' % word):
str1 = '{} {}'.format(word, str1[:-len(word) - 2])
if str2.endswith(', %s' % word):
str2 = '{} {}'.format(word, str2[:-len(word) - 2])
if str1.endswith(", %s" % word):
str1 = "{} {}".format(word, str1[: -len(word) - 2])
if str2.endswith(", %s" % word):
str2 = "{} {}".format(word, str2[: -len(word) - 2])
# Perform a couple of basic normalizing substitutions.
for pat, repl in SD_REPLACE:
@@ -281,8 +336,8 @@ def string_dist(str1, str2):
penalty = 0.0
for pat, weight in SD_PATTERNS:
# Get strings that drop the pattern.
case_str1 = re.sub(pat, '', str1)
case_str2 = re.sub(pat, '', str2)
case_str1 = re.sub(pat, "", str1)
case_str2 = re.sub(pat, "", str2)
if case_str1 != str1 or case_str2 != str2:
# If the pattern was present (i.e., it is deleted in the
@@ -304,23 +359,6 @@ def string_dist(str1, str2):
return base_dist + penalty
class LazyClassProperty:
"""A decorator implementing a read-only property that is *lazy* in
the sense that the getter is only invoked once. Subsequent accesses
through *any* instance use the cached result.
"""
def __init__(self, getter):
self.getter = getter
self.computed = False
def __get__(self, obj, owner):
if not self.computed:
self.value = self.getter(owner)
self.computed = True
return self.value
@total_ordering
class Distance:
"""Keeps track of multiple distance penalties. Provides a single
@@ -330,12 +368,12 @@ class Distance:
def __init__(self):
self._penalties = {}
self.tracks: Dict[TrackInfo, Distance] = {}
@LazyClassProperty
def _weights(cls): # noqa: N805
"""A dictionary from keys to floating-point weights.
"""
weights_view = config['match']['distance_weights']
@cached_classproperty
def _weights(cls) -> Dict[str, float]: # noqa: N805
"""A dictionary from keys to floating-point weights."""
weights_view = config["match"]["distance_weights"]
weights = {}
for key in weights_view.keys():
weights[key] = weights_view[key].as_number()
@@ -344,7 +382,7 @@ class Distance:
# Access the components and their aggregates.
@property
def distance(self):
def distance(self) -> float:
"""Return a weighted and normalized distance across all
penalties.
"""
@@ -354,24 +392,22 @@ class Distance:
return 0.0
@property
def max_distance(self):
"""Return the maximum distance penalty (normalization factor).
"""
def max_distance(self) -> float:
"""Return the maximum distance penalty (normalization factor)."""
dist_max = 0.0
for key, penalty in self._penalties.items():
dist_max += len(penalty) * self._weights[key]
return dist_max
@property
def raw_distance(self):
"""Return the raw (denormalized) distance.
"""
def raw_distance(self) -> float:
"""Return the raw (denormalized) distance."""
dist_raw = 0.0
for key, penalty in self._penalties.items():
dist_raw += sum(penalty) * self._weights[key]
return dist_raw
def items(self):
def items(self) -> List[Tuple[str, float]]:
"""Return a list of (key, dist) pairs, with `dist` being the
weighted distance, sorted from highest to lowest. Does not
include penalties with a zero value.
@@ -385,87 +421,88 @@ class Distance:
# ascending order (for keys, when the penalty is equal) and
# still get the items with the biggest distance first.
return sorted(
list_,
key=lambda key_and_dist: (-key_and_dist[1], key_and_dist[0])
list_, key=lambda key_and_dist: (-key_and_dist[1], key_and_dist[0])
)
def __hash__(self):
def __hash__(self) -> int:
return id(self)
def __eq__(self, other):
def __eq__(self, other) -> bool:
return self.distance == other
# Behave like a float.
def __lt__(self, other):
def __lt__(self, other) -> bool:
return self.distance < other
def __float__(self):
def __float__(self) -> float:
return self.distance
def __sub__(self, other):
def __sub__(self, other) -> float:
return self.distance - other
def __rsub__(self, other):
def __rsub__(self, other) -> float:
return other - self.distance
def __str__(self):
def __str__(self) -> str:
return f"{self.distance:.2f}"
# Behave like a dict.
def __getitem__(self, key):
"""Returns the weighted distance for a named penalty.
"""
def __getitem__(self, key) -> float:
"""Returns the weighted distance for a named penalty."""
dist = sum(self._penalties[key]) * self._weights[key]
dist_max = self.max_distance
if dist_max:
return dist / dist_max
return 0.0
def __iter__(self):
def __iter__(self) -> Iterator[Tuple[str, float]]:
return iter(self.items())
def __len__(self):
def __len__(self) -> int:
return len(self.items())
def keys(self):
def keys(self) -> List[str]:
return [key for key, _ in self.items()]
def update(self, dist):
"""Adds all the distance penalties from `dist`.
"""
def update(self, dist: "Distance"):
"""Adds all the distance penalties from `dist`."""
if not isinstance(dist, Distance):
raise ValueError(
'`dist` must be a Distance object, not {}'.format(type(dist))
"`dist` must be a Distance object, not {}".format(type(dist))
)
for key, penalties in dist._penalties.items():
self._penalties.setdefault(key, []).extend(penalties)
# Adding components.
def _eq(self, value1, value2):
def _eq(self, value1: Union[re.Pattern[str], Any], value2: Any) -> bool:
"""Returns True if `value1` is equal to `value2`. `value1` may
be a compiled regular expression, in which case it will be
matched against `value2`.
"""
if isinstance(value1, Pattern):
if isinstance(value1, re.Pattern):
value2 = cast(str, value2)
return bool(value1.match(value2))
return value1 == value2
def add(self, key, dist):
def add(self, key: str, dist: float):
"""Adds a distance penalty. `key` must correspond with a
configured weight setting. `dist` must be a float between 0.0
and 1.0, and will be added to any existing distance penalties
for the same key.
"""
if not 0.0 <= dist <= 1.0:
raise ValueError(
f'`dist` must be between 0.0 and 1.0, not {dist}'
)
raise ValueError(f"`dist` must be between 0.0 and 1.0, not {dist}")
self._penalties.setdefault(key, []).append(dist)
def add_equality(self, key, value, options):
def add_equality(
self,
key: str,
value: Any,
options: Union[List[Any], Tuple[Any, ...], Any],
):
"""Adds a distance penalty of 1.0 if `value` doesn't match any
of the values in `options`. If an option is a compiled regular
expression, it will be considered equal if it matches against
@@ -481,7 +518,7 @@ class Distance:
dist = 1.0
self.add(key, dist)
def add_expr(self, key, expr):
def add_expr(self, key: str, expr: bool):
"""Adds a distance penalty of 1.0 if `expr` evaluates to True,
or 0.0.
"""
@@ -490,7 +527,7 @@ class Distance:
else:
self.add(key, 0.0)
def add_number(self, key, number1, number2):
def add_number(self, key: str, number1: int, number2: int):
"""Adds a distance penalty of 1.0 for each number of difference
between `number1` and `number2`, or 0.0 when there is no
difference. Use this when there is no upper limit on the
@@ -503,7 +540,12 @@ class Distance:
else:
self.add(key, 0.0)
def add_priority(self, key, value, options):
def add_priority(
self,
key: str,
value: Any,
options: Union[List[Any], Tuple[Any, ...], Any],
):
"""Adds a distance penalty that corresponds to the position at
which `value` appears in `options`. A distance penalty of 0.0
for the first option, or 1.0 if there is no matching option. If
@@ -521,7 +563,12 @@ class Distance:
dist = 1.0
self.add(key, dist)
def add_ratio(self, key, number1, number2):
def add_ratio(
self,
key: str,
number1: Union[int, float],
number2: Union[int, float],
):
"""Adds a distance penalty for `number1` as a ratio of `number2`.
`number1` is bound at 0 and `number2`.
"""
@@ -532,7 +579,7 @@ class Distance:
dist = 0.0
self.add(key, dist)
def add_string(self, key, str1, str2):
def add_string(self, key: str, str1: Optional[str], str2: Optional[str]):
"""Adds a distance penalty based on the edit distance between
`str1` and `str2`.
"""
@@ -542,64 +589,82 @@ class Distance:
# Structures that compose all the information for a candidate match.
AlbumMatch = namedtuple('AlbumMatch', ['distance', 'info', 'mapping',
'extra_items', 'extra_tracks'])
AlbumMatch = namedtuple(
"AlbumMatch", ["distance", "info", "mapping", "extra_items", "extra_tracks"]
)
TrackMatch = namedtuple('TrackMatch', ['distance', 'info'])
TrackMatch = namedtuple("TrackMatch", ["distance", "info"])
# Aggregation of sources.
def album_for_mbid(release_id):
def album_for_mbid(release_id: str) -> Optional[AlbumInfo]:
"""Get an AlbumInfo object for a MusicBrainz release ID. Return None
if the ID is not found.
"""
try:
album = mb.album_for_id(release_id)
if album:
plugins.send('albuminfo_received', info=album)
plugins.send("albuminfo_received", info=album)
return album
except mb.MusicBrainzAPIError as exc:
exc.log(log)
return None
def track_for_mbid(recording_id):
def track_for_mbid(recording_id: str) -> Optional[TrackInfo]:
"""Get a TrackInfo object for a MusicBrainz recording ID. Return None
if the ID is not found.
"""
try:
track = mb.track_for_id(recording_id)
if track:
plugins.send('trackinfo_received', info=track)
plugins.send("trackinfo_received", info=track)
return track
except mb.MusicBrainzAPIError as exc:
exc.log(log)
return None
def albums_for_id(album_id):
def albums_for_id(album_id: str) -> Iterable[AlbumInfo]:
"""Get a list of albums for an ID."""
a = album_for_mbid(album_id)
if a:
yield a
for a in plugins.album_for_id(album_id):
if a:
plugins.send('albuminfo_received', info=a)
plugins.send("albuminfo_received", info=a)
yield a
def tracks_for_id(track_id):
def tracks_for_id(track_id: str) -> Iterable[TrackInfo]:
"""Get a list of tracks for an ID."""
t = track_for_mbid(track_id)
if t:
yield t
for t in plugins.track_for_id(track_id):
if t:
plugins.send('trackinfo_received', info=t)
plugins.send("trackinfo_received", info=t)
yield t
@plugins.notify_info_yielded('albuminfo_received')
def album_candidates(items, artist, album, va_likely, extra_tags):
def invoke_mb(call_func: Callable, *args):
try:
return call_func(*args)
except mb.MusicBrainzAPIError as exc:
exc.log(log)
return ()
@plugins.notify_info_yielded("albuminfo_received")
def album_candidates(
items: List[Item],
artist: str,
album: str,
va_likely: bool,
extra_tags: Dict,
) -> Iterable[Tuple]:
"""Search for album matches. ``items`` is a list of Item objects
that make up the album. ``artist`` and ``album`` are the respective
names (strings), which may be derived from the item list or may be
@@ -609,40 +674,33 @@ def album_candidates(items, artist, album, va_likely, extra_tags):
constrain the search.
"""
# Base candidates if we have album and artist to match.
if artist and album:
try:
yield from mb.match_album(artist, album, len(items),
extra_tags)
except mb.MusicBrainzAPIError as exc:
exc.log(log)
if config["musicbrainz"]["enabled"]:
# Base candidates if we have album and artist to match.
if artist and album:
yield from invoke_mb(
mb.match_album, artist, album, len(items), extra_tags
)
# Also add VA matches from MusicBrainz where appropriate.
if va_likely and album:
try:
yield from mb.match_album(None, album, len(items),
extra_tags)
except mb.MusicBrainzAPIError as exc:
exc.log(log)
# Also add VA matches from MusicBrainz where appropriate.
if va_likely and album:
yield from invoke_mb(
mb.match_album, None, album, len(items), extra_tags
)
# Candidates from plugins.
yield from plugins.candidates(items, artist, album, va_likely,
extra_tags)
yield from plugins.candidates(items, artist, album, va_likely, extra_tags)
@plugins.notify_info_yielded('trackinfo_received')
def item_candidates(item, artist, title):
@plugins.notify_info_yielded("trackinfo_received")
def item_candidates(item: Item, artist: str, title: str) -> Iterable[Tuple]:
"""Search for item matches. ``item`` is the Item to be matched.
``artist`` and ``title`` are strings and either reflect the item or
are specified by the user.
"""
# MusicBrainz candidates.
if artist and title:
try:
yield from mb.match_track(artist, title)
except mb.MusicBrainzAPIError as exc:
exc.log(log)
if config["musicbrainz"]["enabled"] and artist and title:
yield from invoke_mb(mb.match_track, artist, title)
# Plugin candidates.
yield from plugins.item_candidates(item, artist, title)
lib/beets/autotag/match.py
+245 -149
View File
@@ -19,32 +19,53 @@ releases and tracks.
import datetime
import re
from munkres import Munkres
from collections import namedtuple
from typing import (
Any,
Dict,
Iterable,
List,
Optional,
Sequence,
Tuple,
TypeVar,
Union,
cast,
)
from beets import logging
from beets import plugins
from beets import config
from munkres import Munkres
from beets import config, logging, plugins
from beets.autotag import (
AlbumInfo,
AlbumMatch,
Distance,
TrackInfo,
TrackMatch,
hooks,
)
from beets.library import Item
from beets.util import plurality
from beets.autotag import hooks
from beets.util.enumeration import OrderedEnum
# Artist signals that indicate "various artists". These are used at the
# album level to determine whether a given release is likely a VA
# release and also on the track level to to remove the penalty for
# differing artists.
VA_ARTISTS = ('', 'various artists', 'various', 'va', 'unknown')
VA_ARTISTS = ("", "various artists", "various", "va", "unknown")
# Global logger.
log = logging.getLogger('beets')
log = logging.getLogger("beets")
# Recommendation enumeration.
class Recommendation(OrderedEnum):
"""Indicates a qualitative suggestion to the user about what should
be done with a given match.
"""
none = 0
low = 1
medium = 2
@@ -55,12 +76,15 @@ class Recommendation(OrderedEnum):
# consists of a list of possible candidates (i.e., AlbumInfo or TrackInfo
# objects) and a recommendation value.
Proposal = namedtuple('Proposal', ('candidates', 'recommendation'))
Proposal = namedtuple("Proposal", ("candidates", "recommendation"))
# Primary matching functionality.
def current_metadata(items):
def current_metadata(
items: Iterable[Item],
) -> Tuple[Dict[str, Any], Dict[str, Any]]:
"""Extract the likely current metadata for an album given a list of its
items. Return two dictionaries:
- The most common value for each field.
@@ -70,22 +94,36 @@ def current_metadata(items):
likelies = {}
consensus = {}
fields = ['artist', 'album', 'albumartist', 'year', 'disctotal',
'mb_albumid', 'label', 'catalognum', 'country', 'media',
'albumdisambig']
fields = [
"artist",
"album",
"albumartist",
"year",
"disctotal",
"mb_albumid",
"label",
"barcode",
"catalognum",
"country",
"media",
"albumdisambig",
]
for field in fields:
values = [item[field] for item in items if item]
likelies[field], freq = plurality(values)
consensus[field] = (freq == len(values))
consensus[field] = freq == len(values)
# If there's an album artist consensus, use this for the artist.
if consensus['albumartist'] and likelies['albumartist']:
likelies['artist'] = likelies['albumartist']
if consensus["albumartist"] and likelies["albumartist"]:
likelies["artist"] = likelies["albumartist"]
return likelies, consensus
def assign_items(items, tracks):
def assign_items(
items: Sequence[Item],
tracks: Sequence[TrackInfo],
) -> Tuple[Dict[Item, TrackInfo], List[Item], List[TrackInfo]]:
"""Given a list of Items and a list of TrackInfo objects, find the
best mapping between them. Returns a mapping from Items to TrackInfo
objects, a set of extra Items, and a set of extra TrackInfo
@@ -93,17 +131,17 @@ def assign_items(items, tracks):
of objects of the two types.
"""
# Construct the cost matrix.
costs = []
costs: List[List[Distance]] = []
for item in items:
row = []
for i, track in enumerate(tracks):
for track in tracks:
row.append(track_distance(item, track))
costs.append(row)
# Find a minimum-cost bipartite matching.
log.debug('Computing track assignment...')
log.debug("Computing track assignment...")
matching = Munkres().compute(costs)
log.debug('...done.')
log.debug("...done.")
# Produce the output matching.
mapping = {items[i]: tracks[j] for (i, j) in matching}
@@ -114,14 +152,18 @@ def assign_items(items, tracks):
return mapping, extra_items, extra_tracks
def track_index_changed(item, track_info):
def track_index_changed(item: Item, track_info: TrackInfo) -> bool:
"""Returns True if the item and track info index is different. Tolerates
per disc and per release numbering.
"""
return item.track not in (track_info.medium_index, track_info.index)
def track_distance(item, track_info, incl_artist=False):
def track_distance(
item: Item,
track_info: TrackInfo,
incl_artist: bool = False,
) -> Distance:
"""Determines the significance of a track metadata change. Returns a
Distance object. `incl_artist` indicates that a distance component should
be included for the track artist (i.e., for various-artist releases).
@@ -130,26 +172,37 @@ def track_distance(item, track_info, incl_artist=False):
# Length.
if track_info.length:
diff = abs(item.length - track_info.length) - \
config['match']['track_length_grace'].as_number()
dist.add_ratio('track_length', diff,
config['match']['track_length_max'].as_number())
item_length = cast(float, item.length)
track_length_grace = cast(
Union[float, int],
config["match"]["track_length_grace"].as_number(),
)
track_length_max = cast(
Union[float, int],
config["match"]["track_length_max"].as_number(),
)
diff = abs(item_length - track_info.length) - track_length_grace
dist.add_ratio("track_length", diff, track_length_max)
# Title.
dist.add_string('track_title', item.title, track_info.title)
dist.add_string("track_title", item.title, track_info.title)
# Artist. Only check if there is actually an artist in the track data.
if incl_artist and track_info.artist and \
item.artist.lower() not in VA_ARTISTS:
dist.add_string('track_artist', item.artist, track_info.artist)
if (
incl_artist
and track_info.artist
and item.artist.lower() not in VA_ARTISTS
):
dist.add_string("track_artist", item.artist, track_info.artist)
# Track index.
if track_info.index and item.track:
dist.add_expr('track_index', track_index_changed(item, track_info))
dist.add_expr("track_index", track_index_changed(item, track_info))
# Track ID.
if item.mb_trackid:
dist.add_expr('track_id', item.mb_trackid != track_info.track_id)
dist.add_expr("track_id", item.mb_trackid != track_info.track_id)
# Plugins.
dist.update(plugins.track_distance(item, track_info))
@@ -157,7 +210,11 @@ def track_distance(item, track_info, incl_artist=False):
return dist
def distance(items, album_info, mapping):
def distance(
items: Sequence[Item],
album_info: AlbumInfo,
mapping: Dict[Item, TrackInfo],
) -> Distance:
"""Determines how "significant" an album metadata change would be.
Returns a Distance object. `album_info` is an AlbumInfo object
reflecting the album to be compared. `items` is a sequence of all
@@ -172,90 +229,96 @@ def distance(items, album_info, mapping):
# Artist, if not various.
if not album_info.va:
dist.add_string('artist', likelies['artist'], album_info.artist)
dist.add_string("artist", likelies["artist"], album_info.artist)
# Album.
dist.add_string('album', likelies['album'], album_info.album)
dist.add_string("album", likelies["album"], album_info.album)
# Current or preferred media.
if album_info.media:
# Preferred media options.
patterns = config['match']['preferred']['media'].as_str_seq()
options = [re.compile(r'(\d+x)?(%s)' % pat, re.I) for pat in patterns]
patterns = config["match"]["preferred"]["media"].as_str_seq()
patterns = cast(Sequence[str], patterns)
options = [re.compile(r"(\d+x)?(%s)" % pat, re.I) for pat in patterns]
if options:
dist.add_priority('media', album_info.media, options)
dist.add_priority("media", album_info.media, options)
# Current media.
elif likelies['media']:
dist.add_equality('media', album_info.media, likelies['media'])
elif likelies["media"]:
dist.add_equality("media", album_info.media, likelies["media"])
# Mediums.
if likelies['disctotal'] and album_info.mediums:
dist.add_number('mediums', likelies['disctotal'], album_info.mediums)
if likelies["disctotal"] and album_info.mediums:
dist.add_number("mediums", likelies["disctotal"], album_info.mediums)
# Prefer earliest release.
if album_info.year and config['match']['preferred']['original_year']:
if album_info.year and config["match"]["preferred"]["original_year"]:
# Assume 1889 (earliest first gramophone discs) if we don't know the
# original year.
original = album_info.original_year or 1889
diff = abs(album_info.year - original)
diff_max = abs(datetime.date.today().year - original)
dist.add_ratio('year', diff, diff_max)
dist.add_ratio("year", diff, diff_max)
# Year.
elif likelies['year'] and album_info.year:
if likelies['year'] in (album_info.year, album_info.original_year):
elif likelies["year"] and album_info.year:
if likelies["year"] in (album_info.year, album_info.original_year):
# No penalty for matching release or original year.
dist.add('year', 0.0)
dist.add("year", 0.0)
elif album_info.original_year:
# Prefer matchest closest to the release year.
diff = abs(likelies['year'] - album_info.year)
diff_max = abs(datetime.date.today().year -
album_info.original_year)
dist.add_ratio('year', diff, diff_max)
diff = abs(likelies["year"] - album_info.year)
diff_max = abs(
datetime.date.today().year - album_info.original_year
)
dist.add_ratio("year", diff, diff_max)
else:
# Full penalty when there is no original year.
dist.add('year', 1.0)
dist.add("year", 1.0)
# Preferred countries.
patterns = config['match']['preferred']['countries'].as_str_seq()
patterns = config["match"]["preferred"]["countries"].as_str_seq()
patterns = cast(Sequence[str], patterns)
options = [re.compile(pat, re.I) for pat in patterns]
if album_info.country and options:
dist.add_priority('country', album_info.country, options)
dist.add_priority("country", album_info.country, options)
# Country.
elif likelies['country'] and album_info.country:
dist.add_string('country', likelies['country'], album_info.country)
elif likelies["country"] and album_info.country:
dist.add_string("country", likelies["country"], album_info.country)
# Label.
if likelies['label'] and album_info.label:
dist.add_string('label', likelies['label'], album_info.label)
if likelies["label"] and album_info.label:
dist.add_string("label", likelies["label"], album_info.label)
# Catalog number.
if likelies['catalognum'] and album_info.catalognum:
dist.add_string('catalognum', likelies['catalognum'],
album_info.catalognum)
if likelies["catalognum"] and album_info.catalognum:
dist.add_string(
"catalognum", likelies["catalognum"], album_info.catalognum
)
# Disambiguation.
if likelies['albumdisambig'] and album_info.albumdisambig:
dist.add_string('albumdisambig', likelies['albumdisambig'],
album_info.albumdisambig)
if likelies["albumdisambig"] and album_info.albumdisambig:
dist.add_string(
"albumdisambig", likelies["albumdisambig"], album_info.albumdisambig
)
# Album ID.
if likelies['mb_albumid']:
dist.add_equality('album_id', likelies['mb_albumid'],
album_info.album_id)
if likelies["mb_albumid"]:
dist.add_equality(
"album_id", likelies["mb_albumid"], album_info.album_id
)
# Tracks.
dist.tracks = {}
for item, track in mapping.items():
dist.tracks[track] = track_distance(item, track, album_info.va)
dist.add('tracks', dist.tracks[track].distance)
dist.add("tracks", dist.tracks[track].distance)
# Missing tracks.
for i in range(len(album_info.tracks) - len(mapping)):
dist.add('missing_tracks', 1.0)
for _ in range(len(album_info.tracks) - len(mapping)):
dist.add("missing_tracks", 1.0)
# Unmatched tracks.
for i in range(len(items) - len(mapping)):
dist.add('unmatched_tracks', 1.0)
for _ in range(len(items) - len(mapping)):
dist.add("unmatched_tracks", 1.0)
# Plugins.
dist.update(plugins.album_distance(items, album_info, mapping))
@@ -263,7 +326,7 @@ def distance(items, album_info, mapping):
return dist
def match_by_id(items):
def match_by_id(items: Iterable[Item]):
"""If the items are tagged with a MusicBrainz album ID, returns an
AlbumInfo object for the corresponding album. Otherwise, returns
None.
@@ -274,20 +337,22 @@ def match_by_id(items):
try:
first = next(albumids)
except StopIteration:
log.debug('No album ID found.')
log.debug("No album ID found.")
return None
# Is there a consensus on the MB album ID?
for other in albumids:
if other != first:
log.debug('No album ID consensus.')
log.debug("No album ID consensus.")
return None
# If all album IDs are equal, look up the album.
log.debug('Searching for discovered album ID: {0}', first)
log.debug("Searching for discovered album ID: {0}", first)
return hooks.album_for_mbid(first)
def _recommendation(results):
def _recommendation(
results: Sequence[Union[AlbumMatch, TrackMatch]],
) -> Recommendation:
"""Given a sorted list of AlbumMatch or TrackMatch objects, return a
recommendation based on the results' distances.
@@ -301,17 +366,19 @@ def _recommendation(results):
# Basic distance thresholding.
min_dist = results[0].distance
if min_dist < config['match']['strong_rec_thresh'].as_number():
if min_dist < config["match"]["strong_rec_thresh"].as_number():
# Strong recommendation level.
rec = Recommendation.strong
elif min_dist <= config['match']['medium_rec_thresh'].as_number():
elif min_dist <= config["match"]["medium_rec_thresh"].as_number():
# Medium recommendation level.
rec = Recommendation.medium
elif len(results) == 1:
# Only a single candidate.
rec = Recommendation.low
elif results[1].distance - min_dist >= \
config['match']['rec_gap_thresh'].as_number():
elif (
results[1].distance - min_dist
>= config["match"]["rec_gap_thresh"].as_number()
):
# Gap between first two candidates is large.
rec = Recommendation.low
else:
@@ -324,48 +391,60 @@ def _recommendation(results):
if isinstance(results[0], hooks.AlbumMatch):
for track_dist in min_dist.tracks.values():
keys.update(list(track_dist.keys()))
max_rec_view = config['match']['max_rec']
max_rec_view = config["match"]["max_rec"]
for key in keys:
if key in list(max_rec_view.keys()):
max_rec = max_rec_view[key].as_choice({
'strong': Recommendation.strong,
'medium': Recommendation.medium,
'low': Recommendation.low,
'none': Recommendation.none,
})
max_rec = max_rec_view[key].as_choice(
{
"strong": Recommendation.strong,
"medium": Recommendation.medium,
"low": Recommendation.low,
"none": Recommendation.none,
}
)
rec = min(rec, max_rec)
return rec
def _sort_candidates(candidates):
AnyMatch = TypeVar("AnyMatch", TrackMatch, AlbumMatch)
def _sort_candidates(candidates: Iterable[AnyMatch]) -> Sequence[AnyMatch]:
"""Sort candidates by distance."""
return sorted(candidates, key=lambda match: match.distance)
def _add_candidate(items, results, info):
def _add_candidate(
items: Sequence[Item],
results: Dict[Any, AlbumMatch],
info: AlbumInfo,
):
"""Given a candidate AlbumInfo object, attempt to add the candidate
to the output dictionary of AlbumMatch objects. This involves
checking the track count, ordering the items, checking for
duplicates, and calculating the distance.
"""
log.debug('Candidate: {0} - {1} ({2})',
info.artist, info.album, info.album_id)
log.debug(
"Candidate: {0} - {1} ({2})", info.artist, info.album, info.album_id
)
# Discard albums with zero tracks.
if not info.tracks:
log.debug('No tracks.')
log.debug("No tracks.")
return
# Don't duplicate.
if info.album_id in results:
log.debug('Duplicate.')
# Prevent duplicates.
if info.album_id and info.album_id in results:
log.debug("Duplicate.")
return
# Discard matches without required tags.
for req_tag in config['match']['required'].as_str_seq():
for req_tag in cast(
Sequence[str], config["match"]["required"].as_str_seq()
):
if getattr(info, req_tag) is None:
log.debug('Ignored. Missing required tag: {0}', req_tag)
log.debug("Ignored. Missing required tag: {0}", req_tag)
return
# Find mapping between the items and the track info.
@@ -376,18 +455,24 @@ def _add_candidate(items, results, info):
# Skip matches with ignored penalties.
penalties = [key for key, _ in dist]
for penalty in config['match']['ignored'].as_str_seq():
ignored = cast(Sequence[str], config["match"]["ignored"].as_str_seq())
for penalty in ignored:
if penalty in penalties:
log.debug('Ignored. Penalty: {0}', penalty)
log.debug("Ignored. Penalty: {0}", penalty)
return
log.debug('Success. Distance: {0}', dist)
results[info.album_id] = hooks.AlbumMatch(dist, info, mapping,
extra_items, extra_tracks)
log.debug("Success. Distance: {0}", dist)
results[info.album_id] = hooks.AlbumMatch(
dist, info, mapping, extra_items, extra_tracks
)
def tag_album(items, search_artist=None, search_album=None,
search_ids=[]):
def tag_album(
items,
search_artist: Optional[str] = None,
search_album: Optional[str] = None,
search_ids: List[str] = [],
) -> Tuple[str, str, Proposal]:
"""Return a tuple of the current artist name, the current album
name, and a `Proposal` containing `AlbumMatch` candidates.
@@ -407,20 +492,19 @@ def tag_album(items, search_artist=None, search_album=None,
"""
# Get current metadata.
likelies, consensus = current_metadata(items)
cur_artist = likelies['artist']
cur_album = likelies['album']
log.debug('Tagging {0} - {1}', cur_artist, cur_album)
cur_artist = cast(str, likelies["artist"])
cur_album = cast(str, likelies["album"])
log.debug("Tagging {0} - {1}", cur_artist, cur_album)
# The output result (distance, AlbumInfo) tuples (keyed by MB album
# ID).
candidates = {}
# The output result, keys are the MB album ID.
candidates: Dict[Any, AlbumMatch] = {}
# Search by explicit ID.
if search_ids:
for search_id in search_ids:
log.debug('Searching for album ID: {0}', search_id)
for id_candidate in hooks.albums_for_id(search_id):
_add_candidate(items, candidates, id_candidate)
log.debug("Searching for album ID: {0}", search_id)
for album_info_for_id in hooks.albums_for_id(search_id):
_add_candidate(items, candidates, album_info_for_id)
# Use existing metadata or text search.
else:
@@ -429,51 +513,58 @@ def tag_album(items, search_artist=None, search_album=None,
if id_info:
_add_candidate(items, candidates, id_info)
rec = _recommendation(list(candidates.values()))
log.debug('Album ID match recommendation is {0}', rec)
if candidates and not config['import']['timid']:
log.debug("Album ID match recommendation is {0}", rec)
if candidates and not config["import"]["timid"]:
# If we have a very good MBID match, return immediately.
# Otherwise, this match will compete against metadata-based
# matches.
if rec == Recommendation.strong:
log.debug('ID match.')
return cur_artist, cur_album, \
Proposal(list(candidates.values()), rec)
log.debug("ID match.")
return (
cur_artist,
cur_album,
Proposal(list(candidates.values()), rec),
)
# Search terms.
if not (search_artist and search_album):
# No explicit search terms -- use current metadata.
search_artist, search_album = cur_artist, cur_album
log.debug('Search terms: {0} - {1}', search_artist, search_album)
log.debug("Search terms: {0} - {1}", search_artist, search_album)
extra_tags = None
if config['musicbrainz']['extra_tags']:
tag_list = config['musicbrainz']['extra_tags'].get()
if config["musicbrainz"]["extra_tags"]:
tag_list = config["musicbrainz"]["extra_tags"].get()
extra_tags = {k: v for (k, v) in likelies.items() if k in tag_list}
log.debug('Additional search terms: {0}', extra_tags)
log.debug("Additional search terms: {0}", extra_tags)
# Is this album likely to be a "various artist" release?
va_likely = ((not consensus['artist']) or
(search_artist.lower() in VA_ARTISTS) or
any(item.comp for item in items))
log.debug('Album might be VA: {0}', va_likely)
va_likely = (
(not consensus["artist"])
or (search_artist.lower() in VA_ARTISTS)
or any(item.comp for item in items)
)
log.debug("Album might be VA: {0}", va_likely)
# Get the results from the data sources.
for matched_candidate in hooks.album_candidates(items,
search_artist,
search_album,
va_likely,
extra_tags):
for matched_candidate in hooks.album_candidates(
items, search_artist, search_album, va_likely, extra_tags
):
_add_candidate(items, candidates, matched_candidate)
log.debug('Evaluating {0} candidates.', len(candidates))
log.debug("Evaluating {0} candidates.", len(candidates))
# Sort and get the recommendation.
candidates = _sort_candidates(candidates.values())
rec = _recommendation(candidates)
return cur_artist, cur_album, Proposal(candidates, rec)
candidates_sorted = _sort_candidates(candidates.values())
rec = _recommendation(candidates_sorted)
return cur_artist, cur_album, Proposal(candidates_sorted, rec)
def tag_item(item, search_artist=None, search_title=None,
search_ids=[]):
def tag_item(
item,
search_artist: Optional[str] = None,
search_title: Optional[str] = None,
search_ids: Optional[List[str]] = None,
) -> Proposal:
"""Find metadata for a single track. Return a `Proposal` consisting
of `TrackMatch` objects.
@@ -485,26 +576,31 @@ def tag_item(item, search_artist=None, search_title=None,
# Holds candidates found so far: keys are MBIDs; values are
# (distance, TrackInfo) pairs.
candidates = {}
rec: Optional[Recommendation] = None
# First, try matching by MusicBrainz ID.
trackids = search_ids or [t for t in [item.mb_trackid] if t]
if trackids:
for trackid in trackids:
log.debug('Searching for track ID: {0}', trackid)
log.debug("Searching for track ID: {0}", trackid)
for track_info in hooks.tracks_for_id(trackid):
dist = track_distance(item, track_info, incl_artist=True)
candidates[track_info.track_id] = \
hooks.TrackMatch(dist, track_info)
candidates[track_info.track_id] = hooks.TrackMatch(
dist, track_info
)
# If this is a good match, then don't keep searching.
rec = _recommendation(_sort_candidates(candidates.values()))
if rec == Recommendation.strong and \
not config['import']['timid']:
log.debug('Track ID match.')
if (
rec == Recommendation.strong
and not config["import"]["timid"]
):
log.debug("Track ID match.")
return Proposal(_sort_candidates(candidates.values()), rec)
# If we're searching by ID, don't proceed.
if search_ids:
if candidates:
assert rec is not None
return Proposal(_sort_candidates(candidates.values()), rec)
else:
return Proposal([], Recommendation.none)
@@ -512,7 +608,7 @@ def tag_item(item, search_artist=None, search_title=None,
# Search terms.
if not (search_artist and search_title):
search_artist, search_title = item.artist, item.title
log.debug('Item search terms: {0} - {1}', search_artist, search_title)
log.debug("Item search terms: {0} - {1}", search_artist, search_title)
# Get and evaluate candidate metadata.
for track_info in hooks.item_candidates(item, search_artist, search_title):
@@ -520,7 +616,7 @@ def tag_item(item, search_artist=None, search_title=None,
candidates[track_info.track_id] = hooks.TrackMatch(dist, track_info)
# Sort by distance and return with recommendation.
log.debug('Found {0} candidates.', len(candidates))
candidates = _sort_candidates(candidates.values())
rec = _recommendation(candidates)
return Proposal(candidates, rec)
log.debug("Found {0} candidates.", len(candidates))
candidates_sorted = _sort_candidates(candidates.values())
rec = _recommendation(candidates_sorted)
return Proposal(candidates_sorted, rec)
lib/beets/autotag/mb.py
+556 -251
View File
File diff suppressed because it is too large Load Diff
lib/beets/config_default.yaml
+114 -41
View File
@@ -1,10 +1,34 @@
# --------------- Main ---------------
library: library.db
directory: ~/Music
statefile: state.pickle
# --------------- Plugins ---------------
plugins: []
pluginpath: []
# --------------- Import ---------------
clutter: ["Thumbs.DB", ".DS_Store"]
ignore: [".*", "*~", "System Volume Information", "lost+found"]
ignore_hidden: yes
import:
# common options
write: yes
copy: yes
move: no
timid: no
quiet: no
log:
# other options
default_action: apply
languages: []
quiet_fallback: skip
none_rec_action: ask
# rare options
link: no
hardlink: no
reflink: no
@@ -13,76 +37,117 @@ import:
incremental: no
incremental_skip_later: no
from_scratch: no
quiet_fallback: skip
none_rec_action: ask
timid: no
log:
autotag: yes
quiet: no
singletons: no
default_action: apply
languages: []
detail: no
flat: no
group_albums: no
pretend: false
search_ids: []
duplicate_keys:
album: albumartist album
item: artist title
duplicate_action: ask
duplicate_verbose_prompt: no
bell: no
set_fields: {}
ignored_alias_types: []
singleton_album_disambig: yes
clutter: ["Thumbs.DB", ".DS_Store"]
ignore: [".*", "*~", "System Volume Information", "lost+found"]
ignore_hidden: yes
# --------------- Paths ---------------
replace:
'[\\/]': _
'^\.': _
'[\x00-\x1f]': _
'[<>:"\?\*\|]': _
'\.$': _
'\s+$': ''
'^\s+': ''
'^-': _
path_sep_replace: _
drive_sep_replace: _
asciify_paths: false
art_filename: cover
max_filename_length: 0
replace:
# Replace bad characters with _
# prohibited in many filesystem paths
'[<>:\?\*\|]': _
# double quotation mark "
'\"': _
# path separators: \ or /
'[\\/]': _
# starting and closing periods
'^\.': _
'\.$': _
# control characters
'[\x00-\x1f]': _
# dash at the start of a filename (causes command line ambiguity)
'^-': _
# Replace bad characters with nothing
# starting and closing whitespace
'\s+$': ''
'^\s+': ''
aunique:
keys: albumartist album
disambiguators: albumtype year label catalognum albumdisambig releasegroupdisambig
bracket: '[]'
overwrite_null:
album: []
track: []
sunique:
keys: artist title
disambiguators: year trackdisambig
bracket: '[]'
# --------------- Tagging ---------------
plugins: []
pluginpath: []
threaded: yes
timeout: 5.0
per_disc_numbering: no
verbose: 0
terminal_encoding:
original_date: no
artist_credit: no
id3v23: no
va_name: "Various Artists"
paths:
default: $albumartist/$album%aunique{}/$track $title
singleton: Non-Album/$artist/$title
comp: Compilations/$album%aunique{}/$track $title
# --------------- Performance ---------------
threaded: yes
timeout: 5.0
# --------------- UI ---------------
verbose: 0
terminal_encoding:
ui:
terminal_width: 80
length_diff_thresh: 10.0
color: yes
colors:
text_success: green
text_warning: yellow
text_error: red
text_highlight: red
text_highlight_minor: lightgray
action_default: turquoise
action: blue
text_success: ['bold', 'green']
text_warning: ['bold', 'yellow']
text_error: ['bold', 'red']
text_highlight: ['bold', 'red']
text_highlight_minor: ['white']
action_default: ['bold', 'cyan']
action: ['bold', 'cyan']
# New Colors
text: ['normal']
text_faint: ['faint']
import_path: ['bold', 'blue']
import_path_items: ['bold', 'blue']
added: ['green']
removed: ['red']
changed: ['yellow']
added_highlight: ['bold', 'green']
removed_highlight: ['bold', 'red']
changed_highlight: ['bold', 'yellow']
text_diff_added: ['bold', 'red']
text_diff_removed: ['bold', 'red']
text_diff_changed: ['bold', 'red']
action_description: ['white']
import:
indentation:
match_header: 2
match_details: 2
match_tracklist: 5
layout: column
# --------------- Search ---------------
format_item: $artist - $album - $title
format_album: $albumartist - $album
@@ -93,14 +158,13 @@ sort_album: albumartist+ album+
sort_item: artist+ album+ disc+ track+
sort_case_insensitive: yes
paths:
default: $albumartist/$album%aunique{}/$track $title
singleton: Non-Album/$artist/$title
comp: Compilations/$album%aunique{}/$track $title
statefile: state.pickle
# --------------- Autotagger ---------------
overwrite_null:
album: []
track: []
musicbrainz:
enabled: yes
host: musicbrainz.org
https: no
ratelimit: 1
@@ -108,6 +172,13 @@ musicbrainz:
searchlimit: 5
extra_tags: []
genres: no
external_ids:
discogs: no
bandcamp: no
spotify: no
deezer: no
beatport: no
tidal: no
match:
strong_rec_thresh: 0.04
@@ -147,3 +218,5 @@ match:
ignore_video_tracks: yes
track_length_grace: 10
track_length_max: 30
album_disambig_fields: data_source media year country label catalognum albumdisambig
singleton_disambig_fields: data_source index track_alt album
lib/beets/dbcore/__init__.py
+14 -6
View File
@@ -16,12 +16,20 @@
Library.
"""
from .db import Model, Database
from .query import Query, FieldQuery, MatchQuery, AndQuery, OrQuery
from .db import Database, Model, Results
from .query import (
AndQuery,
FieldQuery,
InvalidQueryError,
MatchQuery,
OrQuery,
Query,
)
from .queryparse import (
parse_sorted_query,
query_from_strings,
sort_from_strings,
)
from .types import Type
from .queryparse import query_from_strings
from .queryparse import sort_from_strings
from .queryparse import parse_sorted_query
from .query import InvalidQueryError
# flake8: noqa
lib/beets/dbcore/db.py
+432 -238
View File
File diff suppressed because it is too large Load Diff
lib/beets/dbcore/query.py
+431 -288
View File
File diff suppressed because it is too large Load Diff
lib/beets/dbcore/queryparse.py
+93 -63
View File
@@ -12,30 +12,33 @@
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
"""Parsing of strings into DBCore queries.
"""
"""Parsing of strings into DBCore queries."""
import re
import itertools
from . import query
import re
from typing import Collection, Dict, List, Optional, Sequence, Tuple, Type
from . import Model, query
from .query import Sort
PARSE_QUERY_PART_REGEX = re.compile(
# Non-capturing optional segment for the keyword.
r'(-|\^)?' # Negation prefixes.
r'(?:'
r'(\S+?)' # The field key.
r'(?<!\\):' # Unescaped :
r')?'
r'(.*)', # The term itself.
re.I # Case-insensitive.
r"(-|\^)?" # Negation prefixes.
r"(?:"
r"(\S+?)" # The field key.
r"(?<!\\):" # Unescaped :
r")?"
r"(.*)", # The term itself.
re.I, # Case-insensitive.
)
def parse_query_part(part, query_classes={}, prefixes={},
default_class=query.SubstringQuery):
def parse_query_part(
part: str,
query_classes: Dict[str, Type[query.FieldQuery]] = {},
prefixes: Dict = {},
default_class: Type[query.SubstringQuery] = query.SubstringQuery,
) -> Tuple[Optional[str], str, Type[query.FieldQuery], bool]:
"""Parse a single *query part*, which is a chunk of a complete query
string representing a single criterion.
@@ -86,13 +89,13 @@ def parse_query_part(part, query_classes={}, prefixes={},
assert match # Regex should always match
negate = bool(match.group(1))
key = match.group(2)
term = match.group(3).replace('\\:', ':')
term = match.group(3).replace("\\:", ":")
# Check whether there's a prefix in the query and use the
# corresponding query type.
for pre, query_class in prefixes.items():
if term.startswith(pre):
return key, term[len(pre):], query_class, negate
return key, term[len(pre) :], query_class, negate
# No matching prefix, so use either the query class determined by
# the field or the default as a fallback.
@@ -100,7 +103,11 @@ def parse_query_part(part, query_classes={}, prefixes={},
return key, term, query_class, negate
def construct_query_part(model_cls, prefixes, query_part):
def construct_query_part(
model_cls: Type[Model],
prefixes: Dict,
query_part: str,
) -> query.Query:
"""Parse a *query part* string and return a :class:`Query` object.
:param model_cls: The :class:`Model` class that this is a query for.
@@ -116,40 +123,44 @@ def construct_query_part(model_cls, prefixes, query_part):
if not query_part:
return query.TrueQuery()
out_query: query.Query
# Use `model_cls` to build up a map from field (or query) names to
# `Query` classes.
query_classes = {}
for k, t in itertools.chain(model_cls._fields.items(),
model_cls._types.items()):
query_classes: Dict[str, Type[query.FieldQuery]] = {}
for k, t in itertools.chain(
model_cls._fields.items(), model_cls._types.items()
):
query_classes[k] = t.query
query_classes.update(model_cls._queries) # Non-field queries.
# Parse the string.
key, pattern, query_class, negate = \
parse_query_part(query_part, query_classes, prefixes)
key, pattern, query_class, negate = parse_query_part(
query_part, query_classes, prefixes
)
# If there's no key (field name) specified, this is a "match
# anything" query.
if key is None:
if issubclass(query_class, query.FieldQuery):
# The query type matches a specific field, but none was
# specified. So we use a version of the query that matches
# any field.
out_query = query.AnyFieldQuery(pattern, model_cls._search_fields,
query_class)
else:
# Non-field query type.
out_query = query_class(pattern)
# The query type matches a specific field, but none was
# specified. So we use a version of the query that matches
# any field.
out_query = query.AnyFieldQuery(
pattern, model_cls._search_fields, query_class
)
# Field queries get constructed according to the name of the field
# they are querying.
elif issubclass(query_class, query.FieldQuery):
key = key.lower()
out_query = query_class(key.lower(), pattern, key in model_cls._fields)
# Non-field (named) query.
else:
out_query = query_class(pattern)
field = table = key.lower()
if field in model_cls.shared_db_fields:
# This field exists in both tables, so SQLite will encounter
# an OperationalError if we try to query it in a join.
# Using an explicit table name resolves this.
table = f"{model_cls._table}.{field}"
field_in_db = field in model_cls.all_db_fields
out_query = query_class(table, pattern, field_in_db)
# Apply negation.
if negate:
@@ -158,7 +169,13 @@ def construct_query_part(model_cls, prefixes, query_part):
return out_query
def query_from_strings(query_cls, model_cls, prefixes, query_parts):
# TYPING ERROR
def query_from_strings(
query_cls: Type[query.CollectionQuery],
model_cls: Type[Model],
prefixes: Dict,
query_parts: Collection[str],
) -> query.Query:
"""Creates a collection query of type `query_cls` from a list of
strings in the format used by parse_query_part. `model_cls`
determines how queries are constructed from strings.
@@ -171,7 +188,11 @@ def query_from_strings(query_cls, model_cls, prefixes, query_parts):
return query_cls(subqueries)
def construct_sort_part(model_cls, part, case_insensitive=True):
def construct_sort_part(
model_cls: Type[Model],
part: str,
case_insensitive: bool = True,
) -> Sort:
"""Create a `Sort` from a single string criterion.
`model_cls` is the `Model` being queried. `part` is a single string
@@ -183,12 +204,13 @@ def construct_sort_part(model_cls, part, case_insensitive=True):
field = part[:-1]
assert field, "field is missing"
direction = part[-1]
assert direction in ('+', '-'), "part must end with + or -"
is_ascending = direction == '+'
assert direction in ("+", "-"), "part must end with + or -"
is_ascending = direction == "+"
if field in model_cls._sorts:
sort = model_cls._sorts[field](model_cls, is_ascending,
case_insensitive)
sort = model_cls._sorts[field](
model_cls, is_ascending, case_insensitive
)
elif field in model_cls._fields:
sort = query.FixedFieldSort(field, is_ascending, case_insensitive)
else:
@@ -197,23 +219,31 @@ def construct_sort_part(model_cls, part, case_insensitive=True):
return sort
def sort_from_strings(model_cls, sort_parts, case_insensitive=True):
"""Create a `Sort` from a list of sort criteria (strings).
"""
def sort_from_strings(
model_cls: Type[Model],
sort_parts: Sequence[str],
case_insensitive: bool = True,
) -> Sort:
"""Create a `Sort` from a list of sort criteria (strings)."""
if not sort_parts:
sort = query.NullSort()
return query.NullSort()
elif len(sort_parts) == 1:
sort = construct_sort_part(model_cls, sort_parts[0], case_insensitive)
return construct_sort_part(model_cls, sort_parts[0], case_insensitive)
else:
sort = query.MultipleSort()
for part in sort_parts:
sort.add_sort(construct_sort_part(model_cls, part,
case_insensitive))
return sort
sort.add_sort(
construct_sort_part(model_cls, part, case_insensitive)
)
return sort
def parse_sorted_query(model_cls, parts, prefixes={},
case_insensitive=True):
def parse_sorted_query(
model_cls: Type[Model],
parts: List[str],
prefixes: Dict = {},
case_insensitive: bool = True,
) -> Tuple[query.Query, Sort]:
"""Given a list of strings, create the `Query` and `Sort` that they
represent.
"""
@@ -224,24 +254,24 @@ def parse_sorted_query(model_cls, parts, prefixes={},
# Split up query in to comma-separated subqueries, each representing
# an AndQuery, which need to be joined together in one OrQuery
subquery_parts = []
for part in parts + [',']:
if part.endswith(','):
for part in parts + [","]:
if part.endswith(","):
# Ensure we can catch "foo, bar" as well as "foo , bar"
last_subquery_part = part[:-1]
if last_subquery_part:
subquery_parts.append(last_subquery_part)
# Parse the subquery in to a single AndQuery
# TODO: Avoid needlessly wrapping AndQueries containing 1 subquery?
query_parts.append(query_from_strings(
query.AndQuery, model_cls, prefixes, subquery_parts
))
query_parts.append(
query_from_strings(
query.AndQuery, model_cls, prefixes, subquery_parts
)
)
del subquery_parts[:]
else:
# Sort parts (1) end in + or -, (2) don't have a field, and
# (3) consist of more than just the + or -.
if part.endswith(('+', '-')) \
and ':' not in part \
and len(part) > 1:
if part.endswith(("+", "-")) and ":" not in part and len(part) > 1:
sort_parts.append(part)
else:
subquery_parts.append(part)
lib/beets/dbcore/types.py
+162 -67
View File
@@ -14,28 +14,46 @@
"""Representation of type information for DBCore model fields.
"""
import typing
from abc import ABC
from typing import Any, Generic, List, TypeVar, Union, cast
from . import query
from beets.util import str2bool
from .query import BooleanQuery, FieldQuery, NumericQuery, SubstringQuery
# Abstract base.
class Type:
class ModelType(typing.Protocol):
"""Protocol that specifies the required constructor for model types,
i.e. a function that takes any argument and attempts to parse it to the
given type.
"""
def __init__(self, value: Any = None): ...
# Generic type variables, used for the value type T and null type N (if
# nullable, else T and N are set to the same type for the concrete subclasses
# of Type).
N = TypeVar("N")
T = TypeVar("T", bound=ModelType)
class Type(ABC, Generic[T, N]):
"""An object encapsulating the type of a model field. Includes
information about how to store, query, format, and parse a given
field.
"""
sql = 'TEXT'
sql: str = "TEXT"
"""The SQLite column type for the value.
"""
query = query.SubstringQuery
query: typing.Type[FieldQuery] = SubstringQuery
"""The `Query` subclass to be used when querying the field.
"""
model_type = str
model_type: typing.Type[T]
"""The Python type that is used to represent the value in the model.
The model is guaranteed to return a value of this type if the field
@@ -44,12 +62,14 @@ class Type:
"""
@property
def null(self):
"""The value to be exposed when the underlying value is None.
"""
return self.model_type()
def null(self) -> N:
"""The value to be exposed when the underlying value is None."""
# Note that this default implementation only makes sense for T = N.
# It would be better to implement `null()` only in subclasses, or
# have a field null_type similar to `model_type` and use that here.
return cast(N, self.model_type())
def format(self, value):
def format(self, value: Union[N, T]) -> str:
"""Given a value of this type, produce a Unicode string
representing the value. This is used in template evaluation.
"""
@@ -57,13 +77,13 @@ class Type:
value = self.null
# `self.null` might be `None`
if value is None:
value = ''
if isinstance(value, bytes):
value = value.decode('utf-8', 'ignore')
return ""
elif isinstance(value, bytes):
return value.decode("utf-8", "ignore")
else:
return str(value)
return str(value)
def parse(self, string):
def parse(self, string: str) -> Union[T, N]:
"""Parse a (possibly human-written) string and return the
indicated value of this type.
"""
@@ -72,19 +92,23 @@ class Type:
except ValueError:
return self.null
def normalize(self, value):
def normalize(self, value: Any) -> Union[T, N]:
"""Given a value that will be assigned into a field of this
type, normalize the value to have the appropriate type. This
base implementation only reinterprets `None`.
"""
# TYPING ERROR
if value is None:
return self.null
else:
# TODO This should eventually be replaced by
# `self.model_type(value)`
return value
return cast(T, value)
def from_sql(self, sql_value):
def from_sql(
self,
sql_value: Union[None, int, float, str, bytes],
) -> Union[T, N]:
"""Receives the value stored in the SQL backend and return the
value to be stored in the model.
@@ -99,13 +123,13 @@ class Type:
and the method must handle these in addition.
"""
if isinstance(sql_value, memoryview):
sql_value = bytes(sql_value).decode('utf-8', 'ignore')
sql_value = bytes(sql_value).decode("utf-8", "ignore")
if isinstance(sql_value, str):
return self.parse(sql_value)
else:
return self.normalize(sql_value)
def to_sql(self, model_value):
def to_sql(self, model_value: Any) -> Union[None, int, float, str, bytes]:
"""Convert a value as stored in the model object to a value used
by the database adapter.
"""
@@ -114,18 +138,23 @@ class Type:
# Reusable types.
class Default(Type):
null = None
class Default(Type[str, None]):
model_type = str
@property
def null(self):
return None
class Integer(Type):
"""A basic integer type.
"""
sql = 'INTEGER'
query = query.NumericQuery
class BaseInteger(Type[int, N]):
"""A basic integer type."""
sql = "INTEGER"
query = NumericQuery
model_type = int
def normalize(self, value):
def normalize(self, value: Any) -> Union[int, N]:
try:
return self.model_type(round(float(value)))
except ValueError:
@@ -134,91 +163,153 @@ class Integer(Type):
return self.null
class PaddedInt(Integer):
class Integer(BaseInteger[int]):
@property
def null(self) -> int:
return 0
class NullInteger(BaseInteger[None]):
@property
def null(self) -> None:
return None
class BasePaddedInt(BaseInteger[N]):
"""An integer field that is formatted with a given number of digits,
padded with zeroes.
"""
def __init__(self, digits):
def __init__(self, digits: int):
self.digits = digits
def format(self, value):
return '{0:0{1}d}'.format(value or 0, self.digits)
def format(self, value: Union[int, N]) -> str:
return "{0:0{1}d}".format(value or 0, self.digits)
class NullPaddedInt(PaddedInt):
"""Same as `PaddedInt`, but does not normalize `None` to `0.0`.
"""
null = None
class PaddedInt(BasePaddedInt[int]):
pass
class NullPaddedInt(BasePaddedInt[None]):
"""Same as `PaddedInt`, but does not normalize `None` to `0`."""
@property
def null(self) -> None:
return None
class ScaledInt(Integer):
"""An integer whose formatting operation scales the number by a
constant and adds a suffix. Good for units with large magnitudes.
"""
def __init__(self, unit, suffix=''):
def __init__(self, unit: int, suffix: str = ""):
self.unit = unit
self.suffix = suffix
def format(self, value):
return '{}{}'.format((value or 0) // self.unit, self.suffix)
def format(self, value: int) -> str:
return "{}{}".format((value or 0) // self.unit, self.suffix)
class Id(Integer):
class Id(NullInteger):
"""An integer used as the row id or a foreign key in a SQLite table.
This type is nullable: None values are not translated to zero.
"""
null = None
def __init__(self, primary=True):
@property
def null(self) -> None:
return None
def __init__(self, primary: bool = True):
if primary:
self.sql = 'INTEGER PRIMARY KEY'
self.sql = "INTEGER PRIMARY KEY"
class Float(Type):
class BaseFloat(Type[float, N]):
"""A basic floating-point type. The `digits` parameter specifies how
many decimal places to use in the human-readable representation.
"""
sql = 'REAL'
query = query.NumericQuery
sql = "REAL"
query: typing.Type[FieldQuery[Any]] = NumericQuery
model_type = float
def __init__(self, digits=1):
def __init__(self, digits: int = 1):
self.digits = digits
def format(self, value):
return '{0:.{1}f}'.format(value or 0, self.digits)
def format(self, value: Union[float, N]) -> str:
return "{0:.{1}f}".format(value or 0, self.digits)
class NullFloat(Float):
"""Same as `Float`, but does not normalize `None` to `0.0`.
"""
null = None
class Float(BaseFloat[float]):
"""Floating-point type that normalizes `None` to `0.0`."""
@property
def null(self) -> float:
return 0.0
class String(Type):
"""A Unicode string type.
"""
sql = 'TEXT'
query = query.SubstringQuery
class NullFloat(BaseFloat[None]):
"""Same as `Float`, but does not normalize `None` to `0.0`."""
def normalize(self, value):
@property
def null(self) -> None:
return None
class BaseString(Type[T, N]):
"""A Unicode string type."""
sql = "TEXT"
query = SubstringQuery
def normalize(self, value: Any) -> Union[T, N]:
if value is None:
return self.null
else:
return self.model_type(value)
class Boolean(Type):
"""A boolean type.
class String(BaseString[str, Any]):
"""A Unicode string type."""
model_type = str
class DelimitedString(BaseString[List[str], List[str]]):
"""A list of Unicode strings, represented in-database by a single string
containing delimiter-separated values.
"""
sql = 'INTEGER'
query = query.BooleanQuery
model_type = list
def __init__(self, delimiter: str):
self.delimiter = delimiter
def format(self, value: List[str]):
return self.delimiter.join(value)
def parse(self, string: str):
if not string:
return []
return string.split(self.delimiter)
def to_sql(self, model_value: List[str]):
return self.delimiter.join(model_value)
class Boolean(Type):
"""A boolean type."""
sql = "INTEGER"
query = BooleanQuery
model_type = bool
def format(self, value):
def format(self, value: bool) -> str:
return str(bool(value))
def parse(self, string):
def parse(self, string: str) -> bool:
return str2bool(string)
@@ -231,3 +322,7 @@ FLOAT = Float()
NULL_FLOAT = NullFloat()
STRING = String()
BOOLEAN = Boolean()
SEMICOLON_SPACE_DSV = DelimitedString(delimiter="; ")
# Will set the proper null char in mediafile
MULTI_VALUE_DSV = DelimitedString(delimiter="\\␀")
lib/beets/importer.py
+445 -320
View File
File diff suppressed because it is too large Load Diff
lib/beets/library.py
+826 -547
View File
File diff suppressed because it is too large Load Diff
lib/beets/logging.py
+59 -47
View File
@@ -12,63 +12,50 @@
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
"""A drop-in replacement for the standard-library `logging` module that
allows {}-style log formatting on Python 2 and 3.
"""A drop-in replacement for the standard-library `logging` module.
Provides everything the "logging" module does. The only difference is
that when getLogger(name) instantiates a logger that logger uses
{}-style formatting.
Provides everything the "logging" module does. In addition, beets' logger
(as obtained by `getLogger(name)`) supports thread-local levels, and messages
use {}-style formatting and can interpolate keywords arguments to the logging
calls (`debug`, `info`, etc).
"""
from copy import copy
from logging import * # noqa
import subprocess
import logging
import threading
from copy import copy
def logsafe(val):
"""Coerce a potentially "problematic" value so it can be formatted
in a Unicode log string.
"""Coerce `bytes` to `str` to avoid crashes solely due to logging.
This works around a number of pitfalls when logging objects in
Python 2:
- Logging path names, which must be byte strings, requires
conversion for output.
- Some objects, including some exceptions, will crash when you call
`unicode(v)` while `str(v)` works fine. CalledProcessError is an
example.
This is particularly relevant for bytestring paths. Much of our code
explicitly uses `displayable_path` for them, but better be safe and prevent
any crashes that are solely due to log formatting.
"""
# Already Unicode.
if isinstance(val, str):
return val
# Bytestring: needs decoding.
elif isinstance(val, bytes):
# Bytestring: Needs decoding to be safe for substitution in format strings.
if isinstance(val, bytes):
# Blindly convert with UTF-8. Eventually, it would be nice to
# (a) only do this for paths, if they can be given a distinct
# type, and (b) warn the developer if they do this for other
# bytestrings.
return val.decode('utf-8', 'replace')
# A "problem" object: needs a workaround.
elif isinstance(val, subprocess.CalledProcessError):
try:
return str(val)
except UnicodeDecodeError:
# An object with a broken __unicode__ formatter. Use __str__
# instead.
return str(val).decode('utf-8', 'replace')
return val.decode("utf-8", "replace")
# Other objects are used as-is so field access, etc., still works in
# the format string.
else:
return val
# the format string. Relies on a working __str__ implementation.
return val
class StrFormatLogger(Logger):
class StrFormatLogger(logging.Logger):
"""A version of `Logger` that uses `str.format`-style formatting
instead of %-style formatting.
instead of %-style formatting and supports keyword arguments.
We cannot easily get rid of this even in the Python 3 era: This custom
formatting supports substitution from `kwargs` into the message, which the
default `logging.Logger._log()` implementation does not.
Remark by @sampsyo: https://stackoverflow.com/a/24683360 might be a way to
achieve this with less code.
"""
class _LogMessage:
@@ -82,19 +69,39 @@ class StrFormatLogger(Logger):
kwargs = {k: logsafe(v) for (k, v) in self.kwargs.items()}
return self.msg.format(*args, **kwargs)
def _log(self, level, msg, args, exc_info=None, extra=None, **kwargs):
def _log(
self,
level,
msg,
args,
exc_info=None,
extra=None,
stack_info=False,
**kwargs,
):
"""Log msg.format(*args, **kwargs)"""
m = self._LogMessage(msg, args, kwargs)
return super()._log(level, m, (), exc_info, extra)
stacklevel = kwargs.pop("stacklevel", 1)
stacklevel = {"stacklevel": stacklevel}
return super()._log(
level,
m,
(),
exc_info=exc_info,
extra=extra,
stack_info=stack_info,
**stacklevel,
)
class ThreadLocalLevelLogger(Logger):
"""A version of `Logger` whose level is thread-local instead of shared.
"""
class ThreadLocalLevelLogger(logging.Logger):
"""A version of `Logger` whose level is thread-local instead of shared."""
def __init__(self, name, level=NOTSET):
def __init__(self, name, level=logging.NOTSET):
self._thread_level = threading.local()
self.default_level = NOTSET
self.default_level = logging.NOTSET
super().__init__(name, level)
@property
@@ -121,12 +128,17 @@ class BeetsLogger(ThreadLocalLevelLogger, StrFormatLogger):
pass
my_manager = copy(Logger.manager)
my_manager = copy(logging.Logger.manager)
my_manager.loggerClass = BeetsLogger
# Act like the stdlib logging module by re-exporting its namespace.
from logging import * # noqa
# Override the `getLogger` to use our machinery.
def getLogger(name=None): # noqa
if name:
return my_manager.getLogger(name)
else:
return Logger.root
return logging.Logger.root
lib/beets/mediafile.py
+11 -3
View File
@@ -13,14 +13,22 @@
# included in all copies or substantial portions of the Software.
import warnings
import mediafile
import warnings
warnings.warn("beets.mediafile is deprecated; use mediafile instead")
warnings.warn(
"beets.mediafile is deprecated; use mediafile instead",
# Show the location of the `import mediafile` statement as the warning's
# source, rather than this file, such that the offending module can be
# identified easily.
stacklevel=2,
)
# Import everything from the mediafile module into this module.
for key, value in mediafile.__dict__.items():
if key not in ['__name__']:
if key not in ["__name__"]:
globals()[key] = value
# Cleanup namespace.
del key, value, warnings, mediafile
lib/beets/plugins.py
+124 -84
View File
@@ -15,26 +15,25 @@
"""Support for beets plugins."""
import traceback
import re
import inspect
import abc
import inspect
import re
import traceback
from collections import defaultdict
from functools import wraps
import mediafile
import beets
from beets import logging
import mediafile
PLUGIN_NAMESPACE = 'beetsplug'
PLUGIN_NAMESPACE = "beetsplug"
# Plugins using the Last.fm API can share the same API key.
LASTFM_KEY = '2dc3914abf35f0d9c92d97d8f8e42b43'
LASTFM_KEY = "2dc3914abf35f0d9c92d97d8f8e42b43"
# Global logger.
log = logging.getLogger('beets')
log = logging.getLogger("beets")
class PluginConflictException(Exception):
@@ -51,11 +50,10 @@ class PluginLogFilter(logging.Filter):
"""
def __init__(self, plugin):
self.prefix = f'{plugin.name}: '
self.prefix = f"{plugin.name}: "
def filter(self, record):
if hasattr(record.msg, 'msg') and isinstance(record.msg.msg,
str):
if hasattr(record.msg, "msg") and isinstance(record.msg.msg, str):
# A _LogMessage from our hacked-up Logging replacement.
record.msg.msg = self.prefix + record.msg.msg
elif isinstance(record.msg, str):
@@ -65,6 +63,7 @@ class PluginLogFilter(logging.Filter):
# Managing the plugins themselves.
class BeetsPlugin:
"""The base class for all beets plugins. Plugins provide
functionality by defining a subclass of BeetsPlugin and overriding
@@ -72,9 +71,8 @@ class BeetsPlugin:
"""
def __init__(self, name=None):
"""Perform one-time plugin setup.
"""
self.name = name or self.__module__.split('.')[-1]
"""Perform one-time plugin setup."""
self.name = name or self.__module__.split(".")[-1]
self.config = beets.config[self.name]
if not self.template_funcs:
self.template_funcs = {}
@@ -97,10 +95,11 @@ class BeetsPlugin:
return ()
def _set_stage_log_level(self, stages):
"""Adjust all the stages in `stages` to WARNING logging level.
"""
return [self._set_log_level_and_params(logging.WARNING, stage)
for stage in stages]
"""Adjust all the stages in `stages` to WARNING logging level."""
return [
self._set_log_level_and_params(logging.WARNING, stage)
for stage in stages
]
def get_early_import_stages(self):
"""Return a list of functions that should be called as importer
@@ -134,12 +133,11 @@ class BeetsPlugin:
def wrapper(*args, **kwargs):
assert self._log.level == logging.NOTSET
verbosity = beets.config['verbose'].get(int)
verbosity = beets.config["verbose"].get(int)
log_level = max(logging.DEBUG, base_log_level - 10 * verbosity)
self._log.setLevel(log_level)
if argspec.varkw is None:
kwargs = {k: v for k, v in kwargs.items()
if k in argspec.args}
kwargs = {k: v for k, v in kwargs.items() if k in argspec.args}
try:
return func(*args, **kwargs)
@@ -149,8 +147,7 @@ class BeetsPlugin:
return wrapper
def queries(self):
"""Should return a dict mapping prefixes to Query subclasses.
"""
"""Return a dict mapping prefixes to Query subclasses."""
return {}
def track_distance(self, item, info):
@@ -201,6 +198,7 @@ class BeetsPlugin:
"""
# Defer import to prevent circular dependency
from beets import library
mediafile.MediaFile.add_field(name, descriptor)
library.Item._media_fields.add(name)
@@ -208,8 +206,7 @@ class BeetsPlugin:
listeners = None
def register_listener(self, event, func):
"""Add a function as a listener for the specified event.
"""
"""Add a function as a listener for the specified event."""
wrapped_func = self._set_log_level_and_params(logging.WARNING, func)
cls = self.__class__
@@ -230,11 +227,13 @@ class BeetsPlugin:
function will be invoked as ``%name{}`` from path format
strings.
"""
def helper(func):
if cls.template_funcs is None:
cls.template_funcs = {}
cls.template_funcs[name] = func
return func
return helper
@classmethod
@@ -244,11 +243,13 @@ class BeetsPlugin:
strings. The function must accept a single parameter, the Item
being formatted.
"""
def helper(func):
if cls.template_fields is None:
cls.template_fields = {}
cls.template_fields[name] = func
return func
return helper
@@ -262,25 +263,29 @@ def load_plugins(names=()):
BeetsPlugin subclasses desired.
"""
for name in names:
modname = f'{PLUGIN_NAMESPACE}.{name}'
modname = f"{PLUGIN_NAMESPACE}.{name}"
try:
try:
namespace = __import__(modname, None, None)
except ImportError as exc:
# Again, this is hacky:
if exc.args[0].endswith(' ' + name):
log.warning('** plugin {0} not found', name)
if exc.args[0].endswith(" " + name):
log.warning("** plugin {0} not found", name)
else:
raise
else:
for obj in getattr(namespace, name).__dict__.values():
if isinstance(obj, type) and issubclass(obj, BeetsPlugin) \
and obj != BeetsPlugin and obj not in _classes:
if (
isinstance(obj, type)
and issubclass(obj, BeetsPlugin)
and obj != BeetsPlugin
and obj not in _classes
):
_classes.add(obj)
except Exception:
log.warning(
'** error loading plugin {}:\n{}',
"** error loading plugin {}:\n{}",
name,
traceback.format_exc(),
)
@@ -311,9 +316,9 @@ def find_plugins():
# Communication with plugins.
def commands():
"""Returns a list of Subcommand objects from all loaded plugins.
"""
"""Returns a list of Subcommand objects from all loaded plugins."""
out = []
for plugin in find_plugins():
out += plugin.commands()
@@ -332,16 +337,16 @@ def queries():
def types(model_cls):
# Gives us `item_types` and `album_types`
attr_name = f'{model_cls.__name__.lower()}_types'
attr_name = f"{model_cls.__name__.lower()}_types"
types = {}
for plugin in find_plugins():
plugin_types = getattr(plugin, attr_name, {})
for field in plugin_types:
if field in types and plugin_types[field] != types[field]:
raise PluginConflictException(
'Plugin {} defines flexible field {} '
'which has already been defined with '
'another type.'.format(plugin.name, field)
"Plugin {} defines flexible field {} "
"which has already been defined with "
"another type.".format(plugin.name, field)
)
types.update(plugin_types)
return types
@@ -349,7 +354,7 @@ def types(model_cls):
def named_queries(model_cls):
# Gather `item_queries` and `album_queries` from the plugins.
attr_name = f'{model_cls.__name__.lower()}_queries'
attr_name = f"{model_cls.__name__.lower()}_queries"
queries = {}
for plugin in find_plugins():
plugin_queries = getattr(plugin, attr_name, {})
@@ -362,6 +367,7 @@ def track_distance(item, info):
Returns a Distance object.
"""
from beets.autotag.hooks import Distance
dist = Distance()
for plugin in find_plugins():
dist.update(plugin.track_distance(item, info))
@@ -371,6 +377,7 @@ def track_distance(item, info):
def album_distance(items, album_info, mapping):
"""Returns the album distance calculated by plugins."""
from beets.autotag.hooks import Distance
dist = Distance()
for plugin in find_plugins():
dist.update(plugin.album_distance(items, album_info, mapping))
@@ -378,23 +385,21 @@ def album_distance(items, album_info, mapping):
def candidates(items, artist, album, va_likely, extra_tags=None):
"""Gets MusicBrainz candidates for an album from each plugin.
"""
"""Gets MusicBrainz candidates for an album from each plugin."""
for plugin in find_plugins():
yield from plugin.candidates(items, artist, album, va_likely,
extra_tags)
yield from plugin.candidates(
items, artist, album, va_likely, extra_tags
)
def item_candidates(item, artist, title):
"""Gets MusicBrainz candidates for an item from the plugins.
"""
"""Gets MusicBrainz candidates for an item from the plugins."""
for plugin in find_plugins():
yield from plugin.item_candidates(item, artist, title)
def album_for_id(album_id):
"""Get AlbumInfo objects for a given ID string.
"""
"""Get AlbumInfo objects for a given ID string."""
for plugin in find_plugins():
album = plugin.album_for_id(album_id)
if album:
@@ -402,8 +407,7 @@ def album_for_id(album_id):
def track_for_id(track_id):
"""Get TrackInfo objects for a given ID string.
"""
"""Get TrackInfo objects for a given ID string."""
for plugin in find_plugins():
track = plugin.track_for_id(track_id)
if track:
@@ -439,29 +443,44 @@ def import_stages():
# New-style (lazy) plugin-provided fields.
def _check_conflicts_and_merge(plugin, plugin_funcs, funcs):
"""Check the provided template functions for conflicts and merge into funcs.
Raises a `PluginConflictException` if a plugin defines template functions
for fields that another plugin has already defined template functions for.
"""
if plugin_funcs:
if not plugin_funcs.keys().isdisjoint(funcs.keys()):
conflicted_fields = ", ".join(plugin_funcs.keys() & funcs.keys())
raise PluginConflictException(
f"Plugin {plugin.name} defines template functions for "
f"{conflicted_fields} that conflict with another plugin."
)
funcs.update(plugin_funcs)
def item_field_getters():
"""Get a dictionary mapping field names to unary functions that
compute the field's value.
"""
funcs = {}
for plugin in find_plugins():
if plugin.template_fields:
funcs.update(plugin.template_fields)
_check_conflicts_and_merge(plugin, plugin.template_fields, funcs)
return funcs
def album_field_getters():
"""As above, for album fields.
"""
"""As above, for album fields."""
funcs = {}
for plugin in find_plugins():
if plugin.album_template_fields:
funcs.update(plugin.album_template_fields)
_check_conflicts_and_merge(plugin, plugin.album_template_fields, funcs)
return funcs
# Event dispatch.
def event_handlers():
"""Find all event handlers from plugins as a dictionary mapping
event names to sequences of callables.
@@ -482,7 +501,7 @@ def send(event, **arguments):
Return a list of non-None values returned from the handlers.
"""
log.debug('Sending event: {0}', event)
log.debug("Sending event: {0}", event)
results = []
for handler in event_handlers()[event]:
result = handler(**arguments)
@@ -497,11 +516,11 @@ def feat_tokens(for_artist=True):
The `for_artist` option determines whether the regex should be
suitable for matching artist fields (the default) or title fields.
"""
feat_words = ['ft', 'featuring', 'feat', 'feat.', 'ft.']
feat_words = ["ft", "featuring", "feat", "feat.", "ft."]
if for_artist:
feat_words += ['with', 'vs', 'and', 'con', '&']
return r'(?<=\s)(?:{})(?=\s)'.format(
'|'.join(re.escape(x) for x in feat_words)
feat_words += ["with", "vs", "and", "con", "&"]
return r"(?<=\s)(?:{})(?=\s)".format(
"|".join(re.escape(x) for x in feat_words)
)
@@ -517,7 +536,7 @@ def sanitize_choices(choices, choices_all):
if s not in seen:
if s in list(choices_all):
res.append(s)
elif s == '*':
elif s == "*":
res.extend(others)
seen.add(s)
return res
@@ -550,11 +569,11 @@ def sanitize_pairs(pairs, pairs_all):
if x not in seen:
seen.add(x)
res.append(x)
elif k == '*':
elif k == "*":
new = [o for o in others if o not in seen]
seen.update(new)
res.extend(new)
elif v == '*':
elif v == "*":
new = [o for o in others if o not in seen and o[0] == k]
seen.update(new)
res.extend(new)
@@ -568,12 +587,15 @@ def notify_info_yielded(event):
Each yielded value is passed to plugins using the 'info' parameter of
'send'.
"""
def decorator(generator):
def decorated(*args, **kwargs):
for v in generator(*args, **kwargs):
send(event, info=v)
yield v
return decorated
return decorator
@@ -583,7 +605,7 @@ def get_distance(config, data_source, info):
"""
dist = beets.autotag.Distance()
if info.data_source == data_source:
dist.add('source', config['source_weight'].as_number())
dist.add("source", config["source_weight"].as_number())
return dist
@@ -620,7 +642,7 @@ def apply_item_changes(lib, item, move, pretend, write):
class MetadataSourcePlugin(metaclass=abc.ABCMeta):
def __init__(self):
super().__init__()
self.config.add({'source_weight': 0.5})
self.config.add({"source_weight": 0.5})
@abc.abstractproperty
def id_regex(self):
@@ -643,7 +665,7 @@ class MetadataSourcePlugin(metaclass=abc.ABCMeta):
raise NotImplementedError
@abc.abstractmethod
def _search_api(self, query_type, filters, keywords=''):
def _search_api(self, query_type, filters, keywords=""):
raise NotImplementedError
@abc.abstractmethod
@@ -655,7 +677,7 @@ class MetadataSourcePlugin(metaclass=abc.ABCMeta):
raise NotImplementedError
@staticmethod
def get_artist(artists, id_key='id', name_key='name'):
def get_artist(artists, id_key="id", name_key="name", join_key=None):
"""Returns an artist string (all artists) and an artist_id (the main
artist) for a list of artist object dicts.
@@ -663,6 +685,8 @@ class MetadataSourcePlugin(metaclass=abc.ABCMeta):
and 'the') to the front and strips trailing disambiguation numbers. It
returns a tuple containing the comma-separated string of all
normalized artists and the ``id`` of the main/first artist.
Alternatively a keyword can be used to combine artists together into a
single string by passing the join_key argument.
:param artists: Iterable of artist dicts or lists returned by API.
:type artists: list[dict] or list[list]
@@ -673,39 +697,55 @@ class MetadataSourcePlugin(metaclass=abc.ABCMeta):
to concatenate for the artist string (containing all artists).
Defaults to 'name'.
:type name_key: str or int
:param join_key: Key or index corresponding to a field containing a
keyword to use for combining artists into a single string, for
example "Feat.", "Vs.", "And" or similar. The default is None
which keeps the default behaviour (comma-separated).
:type join_key: str or int
:return: Normalized artist string.
:rtype: str
"""
artist_id = None
artist_names = []
for artist in artists:
artist_string = ""
artists = list(artists) # In case a generator was passed.
total = len(artists)
for idx, artist in enumerate(artists):
if not artist_id:
artist_id = artist[id_key]
name = artist[name_key]
# Strip disambiguation number.
name = re.sub(r' \(\d+\)$', '', name)
name = re.sub(r" \(\d+\)$", "", name)
# Move articles to the front.
name = re.sub(r'^(.*?), (a|an|the)$', r'\2 \1', name, flags=re.I)
artist_names.append(name)
artist = ', '.join(artist_names).replace(' ,', ',') or None
return artist, artist_id
name = re.sub(r"^(.*?), (a|an|the)$", r"\2 \1", name, flags=re.I)
# Use a join keyword if requested and available.
if idx < (total - 1): # Skip joining on last.
if join_key and artist.get(join_key, None):
name += f" {artist[join_key]} "
else:
name += ", "
artist_string += name
def _get_id(self, url_type, id_):
return artist_string, artist_id
@staticmethod
def _get_id(url_type, id_, id_regex):
"""Parse an ID from its URL if necessary.
:param url_type: Type of URL. Either 'album' or 'track'.
:type url_type: str
:param id_: Album/track ID or URL.
:type id_: str
:param id_regex: A dictionary containing a regular expression
extracting an ID from an URL (if it's not an ID already) in
'pattern' and the number of the match group in 'match_group'.
:type id_regex: dict
:return: Album/track ID.
:rtype: str
"""
self._log.debug(
"Searching {} for {} '{}'", self.data_source, url_type, id_
)
match = re.search(self.id_regex['pattern'].format(url_type), str(id_))
log.debug("Extracting {} ID from '{}'", url_type, id_)
match = re.search(id_regex["pattern"].format(url_type), str(id_))
if match:
id_ = match.group(self.id_regex['match_group'])
id_ = match.group(id_regex["match_group"])
if id_:
return id_
return None
@@ -726,11 +766,11 @@ class MetadataSourcePlugin(metaclass=abc.ABCMeta):
:return: Candidate AlbumInfo objects.
:rtype: list[beets.autotag.hooks.AlbumInfo]
"""
query_filters = {'album': album}
query_filters = {"album": album}
if not va_likely:
query_filters['artist'] = artist
results = self._search_api(query_type='album', filters=query_filters)
albums = [self.album_for_id(album_id=r['id']) for r in results]
query_filters["artist"] = artist
results = self._search_api(query_type="album", filters=query_filters)
albums = [self.album_for_id(album_id=r["id"]) for r in results]
return [a for a in albums if a is not None]
def item_candidates(self, item, artist, title):
@@ -747,7 +787,7 @@ class MetadataSourcePlugin(metaclass=abc.ABCMeta):
:rtype: list[beets.autotag.hooks.TrackInfo]
"""
tracks = self._search_api(
query_type='track', keywords=title, filters={'artist': artist}
query_type="track", keywords=title, filters={"artist": artist}
)
return [self.track_for_id(track_data=track) for track in tracks]
lib/beets/random.py
+7 -8
View File
@@ -12,24 +12,22 @@
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
"""Get a random song or album from the library.
"""
"""Get a random song or album from the library."""
import random
from operator import attrgetter
from itertools import groupby
from operator import attrgetter
def _length(obj, album):
"""Get the duration of an item or album.
"""
"""Get the duration of an item or album."""
if album:
return sum(i.length for i in obj.items())
else:
return obj.length
def _equal_chance_permutation(objs, field='albumartist', random_gen=None):
def _equal_chance_permutation(objs, field="albumartist", random_gen=None):
"""Generate (lazily) a permutation of the objects where every group
with equal values for `field` have an equal chance of appearing in
any given position.
@@ -86,8 +84,9 @@ def _take_time(iter, secs, album):
return out
def random_objs(objs, album, number=1, time=None, equal_chance=False,
random_gen=None):
def random_objs(
objs, album, number=1, time=None, equal_chance=False, random_gen=None
):
"""Get a random subset of the provided `objs`.
If `number` is provided, produce that many matches. Otherwise, if
lib/beets/util/confit.py → lib/beets/test/__init__.py
+6 -15
View File
@@ -1,5 +1,5 @@
# This file is part of beets.
# Copyright 2016-2019, Adrian Sampson.
# Copyright 2024, Lars Kruse
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
@@ -12,17 +12,8 @@
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
import confuse
import warnings
warnings.warn("beets.util.confit is deprecated; use confuse instead")
# Import everything from the confuse module into this module.
for key, value in confuse.__dict__.items():
if key not in ['__name__']:
globals()[key] = value
# Cleanup namespace.
del key, value, warnings, confuse
"""This module contains components of beets' test environment, which
may be of use for testing procedures of external libraries or programs.
For example the 'TestHelper' class may be useful for creating an
in-memory beets library filled with a few example items.
"""
lib/beets/test/_common.py
+328
View File
@@ -0,0 +1,328 @@
# This file is part of beets.
# Copyright 2016, Adrian Sampson.
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
"""Some common functionality for beets' test cases."""
import os
import sys
import tempfile
import unittest
from contextlib import contextmanager
import beets
import beets.library
# Make sure the development versions of the plugins are used
import beetsplug
from beets import importer, logging, util
from beets.ui import commands
from beets.util import syspath
beetsplug.__path__ = [
os.path.abspath(
os.path.join(
os.path.dirname(__file__),
os.path.pardir,
os.path.pardir,
"beetsplug",
)
)
]
# Test resources path.
RSRC = util.bytestring_path(
os.path.abspath(
os.path.join(
os.path.dirname(__file__),
os.path.pardir,
os.path.pardir,
"test",
"rsrc",
)
)
)
PLUGINPATH = os.path.join(RSRC.decode(), "beetsplug")
# Propagate to root logger so the test runner can capture it
log = logging.getLogger("beets")
log.propagate = True
log.setLevel(logging.DEBUG)
# Dummy item creation.
_item_ident = 0
# OS feature test.
HAVE_SYMLINK = sys.platform != "win32"
HAVE_HARDLINK = sys.platform != "win32"
try:
import reflink
HAVE_REFLINK = reflink.supported_at(tempfile.gettempdir())
except ImportError:
HAVE_REFLINK = False
def item(lib=None):
global _item_ident
_item_ident += 1
i = beets.library.Item(
title="the title",
artist="the artist",
albumartist="the album artist",
album="the album",
genre="the genre",
lyricist="the lyricist",
composer="the composer",
arranger="the arranger",
grouping="the grouping",
work="the work title",
mb_workid="the work musicbrainz id",
work_disambig="the work disambiguation",
year=1,
month=2,
day=3,
track=4,
tracktotal=5,
disc=6,
disctotal=7,
lyrics="the lyrics",
comments="the comments",
bpm=8,
comp=True,
path=f"somepath{_item_ident}",
length=60.0,
bitrate=128000,
format="FLAC",
mb_trackid="someID-1",
mb_albumid="someID-2",
mb_artistid="someID-3",
mb_albumartistid="someID-4",
mb_releasetrackid="someID-5",
album_id=None,
mtime=12345,
)
if lib:
lib.add(i)
return i
def album(lib=None):
global _item_ident
_item_ident += 1
i = beets.library.Album(
artpath=None,
albumartist="some album artist",
albumartist_sort="some sort album artist",
albumartist_credit="some album artist credit",
album="the album",
genre="the genre",
year=2014,
month=2,
day=5,
tracktotal=0,
disctotal=1,
comp=False,
mb_albumid="someID-1",
mb_albumartistid="someID-1",
)
if lib:
lib.add(i)
return i
# Dummy import session.
def import_session(lib=None, loghandler=None, paths=[], query=[], cli=False):
cls = commands.TerminalImportSession if cli else importer.ImportSession
return cls(lib, loghandler, paths, query)
class Assertions:
"""A mixin with additional unit test assertions."""
def assertExists(self, path): # noqa
assert os.path.exists(syspath(path)), f"file does not exist: {path!r}"
def assertNotExists(self, path): # noqa
assert not os.path.exists(syspath(path)), f"file exists: {path!r}"
def assertIsFile(self, path): # noqa
self.assertExists(path)
assert os.path.isfile(
syspath(path)
), "path exists, but is not a regular file: {!r}".format(path)
def assertIsDir(self, path): # noqa
self.assertExists(path)
assert os.path.isdir(
syspath(path)
), "path exists, but is not a directory: {!r}".format(path)
def assert_equal_path(self, a, b):
"""Check that two paths are equal."""
a_bytes, b_bytes = util.normpath(a), util.normpath(b)
assert a_bytes == b_bytes, f"{a_bytes=} != {b_bytes=}"
# Mock I/O.
class InputException(Exception):
def __init__(self, output=None):
self.output = output
def __str__(self):
msg = "Attempt to read with no input provided."
if self.output is not None:
msg += f" Output: {self.output!r}"
return msg
class DummyOut:
encoding = "utf-8"
def __init__(self):
self.buf = []
def write(self, s):
self.buf.append(s)
def get(self):
return "".join(self.buf)
def flush(self):
self.clear()
def clear(self):
self.buf = []
class DummyIn:
encoding = "utf-8"
def __init__(self, out=None):
self.buf = []
self.reads = 0
self.out = out
def add(self, s):
self.buf.append(s + "\n")
def close(self):
pass
def readline(self):
if not self.buf:
if self.out:
raise InputException(self.out.get())
else:
raise InputException()
self.reads += 1
return self.buf.pop(0)
class DummyIO:
"""Mocks input and output streams for testing UI code."""
def __init__(self):
self.stdout = DummyOut()
self.stdin = DummyIn(self.stdout)
def addinput(self, s):
self.stdin.add(s)
def getoutput(self):
res = self.stdout.get()
self.stdout.clear()
return res
def readcount(self):
return self.stdin.reads
def install(self):
sys.stdin = self.stdin
sys.stdout = self.stdout
def restore(self):
sys.stdin = sys.__stdin__
sys.stdout = sys.__stdout__
# Utility.
def touch(path):
open(syspath(path), "a").close()
class Bag:
"""An object that exposes a set of fields given as keyword
arguments. Any field not found in the dictionary appears to be None.
Used for mocking Album objects and the like.
"""
def __init__(self, **fields):
self.fields = fields
def __getattr__(self, key):
return self.fields.get(key)
# Platform mocking.
@contextmanager
def platform_windows():
import ntpath
old_path = os.path
try:
os.path = ntpath
yield
finally:
os.path = old_path
@contextmanager
def platform_posix():
import posixpath
old_path = os.path
try:
os.path = posixpath
yield
finally:
os.path = old_path
@contextmanager
def system_mock(name):
import platform
old_system = platform.system
platform.system = lambda: name
try:
yield
finally:
platform.system = old_system
def slow_test(unused=None):
def _id(obj):
return obj
if "SKIP_SLOW_TESTS" in os.environ:
return unittest.skip("test is slow")
return _id
lib/beets/test/helper.py
+1003
View File
File diff suppressed because it is too large Load Diff
lib/beets/ui/__init__.py
+884 -304
View File
File diff suppressed because it is too large Load Diff
lib/beets/ui/commands.py
+1309 -654
View File
File diff suppressed because it is too large Load Diff
lib/beets/ui/completion_base.sh
+28 -5
View File
@@ -31,7 +31,7 @@
# plugins dynamically
#
# Currently, only Bash 3.2 and newer is supported and the
# `bash-completion` package is requied.
# `bash-completion` package (v2.8 or newer) is required.
#
# TODO
# ----
@@ -46,7 +46,30 @@
# * Support long options with `=`, e.g. `--config=file`. Debian's bash
# completion package can handle this.
#
# Note that 'bash-completion' v2.8 is a part of Debian 10, which is part of
# LTS until 2024-06-30. After this date, the minimum version requirement can
# be changed, and newer features can be used unconditionally. See PR#5301.
#
if [[ ${BASH_COMPLETION_VERSINFO[0]} -ne 2 \
|| ${BASH_COMPLETION_VERSINFO[1]} -lt 8 ]]; then
echo "Incompatible version of 'bash-completion'!"
return 1
fi
# The later code relies on 'bash-completion' version 2.12, but older versions
# are still supported. Here, we provide implementations of the newer functions
# in terms of older ones, if 'bash-completion' is too old to have them.
if [[ ${BASH_COMPLETION_VERSINFO[1]} -lt 12 ]]; then
_comp_get_words() {
_get_comp_words_by_ref "$@"
}
_comp_compgen_filedir() {
_filedir "$@"
}
fi
# Determines the beets subcommand and dispatches the completion
# accordingly.
@@ -54,7 +77,7 @@ _beet_dispatch() {
local cur prev cmd=
COMPREPLY=()
_get_comp_words_by_ref -n : cur prev
_comp_get_words -n : cur prev
# Look for the beets subcommand
local arg
@@ -99,7 +122,7 @@ _beet_complete() {
completions="${flags___common} ${opts} ${flags}"
COMPREPLY+=( $(compgen -W "$completions" -- $cur) )
else
_filedir
_comp_compgen_filedir
fi
}
@@ -114,12 +137,12 @@ _beet_complete_global() {
;;
-l|--library|-c|--config)
# Filename completion
_filedir
_comp_compgen_filedir
return
;;
-d|--directory)
# Directory completion
_filedir -d
_comp_compgen_filedir -d
return
;;
esac
lib/beets/util/__init__.py
+365 -351
View File
File diff suppressed because it is too large Load Diff
lib/beets/util/artresizer.py
+522 -315
View File
@@ -16,23 +16,20 @@
public resizing proxy if neither is available.
"""
import subprocess
import os
import os.path
import platform
import re
from tempfile import NamedTemporaryFile
import subprocess
from itertools import chain
from urllib.parse import urlencode
from beets import logging
from beets import util
# Resizing methods
PIL = 1
IMAGEMAGICK = 2
WEBPROXY = 3
from beets import logging, util
from beets.util import displayable_path, get_temp_filename, syspath
PROXY_URL = 'https://images.weserv.nl/'
PROXY_URL = "https://images.weserv.nl/"
log = logging.getLogger('beets')
log = logging.getLogger("beets")
def resize_url(url, maxwidth, quality=0):
@@ -40,265 +37,473 @@ def resize_url(url, maxwidth, quality=0):
maxwidth (preserving aspect ratio).
"""
params = {
'url': url.replace('http://', ''),
'w': maxwidth,
"url": url.replace("http://", ""),
"w": maxwidth,
}
if quality > 0:
params['q'] = quality
params["q"] = quality
return '{}?{}'.format(PROXY_URL, urlencode(params))
return "{}?{}".format(PROXY_URL, urlencode(params))
def temp_file_for(path):
"""Return an unused filename with the same extension as the
specified path.
"""
ext = os.path.splitext(path)[1]
with NamedTemporaryFile(suffix=util.py3_path(ext), delete=False) as f:
return util.bytestring_path(f.name)
class LocalBackendNotAvailableError(Exception):
pass
def pil_resize(maxwidth, path_in, path_out=None, quality=0, max_filesize=0):
"""Resize using Python Imaging Library (PIL). Return the output path
of resized image.
"""
path_out = path_out or temp_file_for(path_in)
from PIL import Image
_NOT_AVAILABLE = object()
log.debug('artresizer: PIL resizing {0} to {1}',
util.displayable_path(path_in), util.displayable_path(path_out))
try:
im = Image.open(util.syspath(path_in))
size = maxwidth, maxwidth
im.thumbnail(size, Image.ANTIALIAS)
class LocalBackend:
@classmethod
def available(cls):
try:
cls.version()
return True
except LocalBackendNotAvailableError:
return False
if quality == 0:
# Use PIL's default quality.
quality = -1
# progressive=False only affects JPEGs and is the default,
# but we include it here for explicitness.
im.save(util.py3_path(path_out), quality=quality, progressive=False)
class IMBackend(LocalBackend):
NAME = "ImageMagick"
if max_filesize > 0:
# If maximum filesize is set, we attempt to lower the quality of
# jpeg conversion by a proportional amount, up to 3 attempts
# First, set the maximum quality to either provided, or 95
if quality > 0:
lower_qual = quality
else:
lower_qual = 95
for i in range(5):
# 5 attempts is an abitrary choice
filesize = os.stat(util.syspath(path_out)).st_size
log.debug("PIL Pass {0} : Output size: {1}B", i, filesize)
if filesize <= max_filesize:
return path_out
# The relationship between filesize & quality will be
# image dependent.
lower_qual -= 10
# Restrict quality dropping below 10
if lower_qual < 10:
lower_qual = 10
# Use optimize flag to improve filesize decrease
im.save(util.py3_path(path_out), quality=lower_qual,
optimize=True, progressive=False)
log.warning("PIL Failed to resize file to below {0}B",
max_filesize)
return path_out
# These fields are used as a cache for `version()`. `_legacy` indicates
# whether the modern `magick` binary is available or whether to fall back
# to the old-style `convert`, `identify`, etc. commands.
_version = None
_legacy = None
@classmethod
def version(cls):
"""Obtain and cache ImageMagick version.
Raises `LocalBackendNotAvailableError` if not available.
"""
if cls._version is None:
for cmd_name, legacy in (("magick", False), ("convert", True)):
try:
out = util.command_output([cmd_name, "--version"]).stdout
except (subprocess.CalledProcessError, OSError) as exc:
log.debug("ImageMagick version check failed: {}", exc)
cls._version = _NOT_AVAILABLE
else:
if b"imagemagick" in out.lower():
pattern = rb".+ (\d+)\.(\d+)\.(\d+).*"
match = re.search(pattern, out)
if match:
cls._version = (
int(match.group(1)),
int(match.group(2)),
int(match.group(3)),
)
cls._legacy = legacy
if cls._version is _NOT_AVAILABLE:
raise LocalBackendNotAvailableError()
else:
return path_out
except OSError:
log.error("PIL cannot create thumbnail for '{0}'",
util.displayable_path(path_in))
return path_in
return cls._version
def __init__(self):
"""Initialize a wrapper around ImageMagick for local image operations.
def im_resize(maxwidth, path_in, path_out=None, quality=0, max_filesize=0):
"""Resize using ImageMagick.
Stores the ImageMagick version and legacy flag. If ImageMagick is not
available, raise an Exception.
"""
self.version()
Use the ``magick`` program or ``convert`` on older versions. Return
the output path of resized image.
"""
path_out = path_out or temp_file_for(path_in)
log.debug('artresizer: ImageMagick resizing {0} to {1}',
util.displayable_path(path_in), util.displayable_path(path_out))
# Use ImageMagick's magick binary when it's available.
# If it's not, fall back to the older, separate convert
# and identify commands.
if self._legacy:
self.convert_cmd = ["convert"]
self.identify_cmd = ["identify"]
self.compare_cmd = ["compare"]
else:
self.convert_cmd = ["magick"]
self.identify_cmd = ["magick", "identify"]
self.compare_cmd = ["magick", "compare"]
# "-resize WIDTHx>" shrinks images with the width larger
# than the given width while maintaining the aspect ratio
# with regards to the height.
# ImageMagick already seems to default to no interlace, but we include it
# here for the sake of explicitness.
cmd = ArtResizer.shared.im_convert_cmd + [
util.syspath(path_in, prefix=False),
'-resize', f'{maxwidth}x>',
'-interlace', 'none',
]
def resize(
self, maxwidth, path_in, path_out=None, quality=0, max_filesize=0
):
"""Resize using ImageMagick.
if quality > 0:
cmd += ['-quality', f'{quality}']
Use the ``magick`` program or ``convert`` on older versions. Return
the output path of resized image.
"""
if not path_out:
path_out = get_temp_filename(__name__, "resize_IM_", path_in)
# "-define jpeg:extent=SIZEb" sets the target filesize for imagemagick to
# SIZE in bytes.
if max_filesize > 0:
cmd += ['-define', f'jpeg:extent={max_filesize}b']
cmd.append(util.syspath(path_out, prefix=False))
try:
util.command_output(cmd)
except subprocess.CalledProcessError:
log.warning('artresizer: IM convert failed for {0}',
util.displayable_path(path_in))
return path_in
return path_out
BACKEND_FUNCS = {
PIL: pil_resize,
IMAGEMAGICK: im_resize,
}
def pil_getsize(path_in):
from PIL import Image
try:
im = Image.open(util.syspath(path_in))
return im.size
except OSError as exc:
log.error("PIL could not read file {}: {}",
util.displayable_path(path_in), exc)
def im_getsize(path_in):
cmd = ArtResizer.shared.im_identify_cmd + \
['-format', '%w %h', util.syspath(path_in, prefix=False)]
try:
out = util.command_output(cmd).stdout
except subprocess.CalledProcessError as exc:
log.warning('ImageMagick size query failed')
log.debug(
'`convert` exited with (status {}) when '
'getting size with command {}:\n{}',
exc.returncode, cmd, exc.output.strip()
"artresizer: ImageMagick resizing {0} to {1}",
displayable_path(path_in),
displayable_path(path_out),
)
return
try:
return tuple(map(int, out.split(b' ')))
except IndexError:
log.warning('Could not understand IM output: {0!r}', out)
# "-resize WIDTHx>" shrinks images with the width larger
# than the given width while maintaining the aspect ratio
# with regards to the height.
# ImageMagick already seems to default to no interlace, but we include
# it here for the sake of explicitness.
cmd = self.convert_cmd + [
syspath(path_in, prefix=False),
"-resize",
f"{maxwidth}x>",
"-interlace",
"none",
]
BACKEND_GET_SIZE = {
PIL: pil_getsize,
IMAGEMAGICK: im_getsize,
}
if quality > 0:
cmd += ["-quality", f"{quality}"]
# "-define jpeg:extent=SIZEb" sets the target filesize for imagemagick
# to SIZE in bytes.
if max_filesize > 0:
cmd += ["-define", f"jpeg:extent={max_filesize}b"]
def pil_deinterlace(path_in, path_out=None):
path_out = path_out or temp_file_for(path_in)
from PIL import Image
cmd.append(syspath(path_out, prefix=False))
try:
util.command_output(cmd)
except subprocess.CalledProcessError:
log.warning(
"artresizer: IM convert failed for {0}",
displayable_path(path_in),
)
return path_in
try:
im = Image.open(util.syspath(path_in))
im.save(util.py3_path(path_out), progressive=False)
return path_out
except IOError:
return path_in
def get_size(self, path_in):
cmd = self.identify_cmd + [
"-format",
"%w %h",
syspath(path_in, prefix=False),
]
def im_deinterlace(path_in, path_out=None):
path_out = path_out or temp_file_for(path_in)
try:
out = util.command_output(cmd).stdout
except subprocess.CalledProcessError as exc:
log.warning("ImageMagick size query failed")
log.debug(
"`convert` exited with (status {}) when "
"getting size with command {}:\n{}",
exc.returncode,
cmd,
exc.output.strip(),
)
return None
try:
return tuple(map(int, out.split(b" ")))
except IndexError:
log.warning("Could not understand IM output: {0!r}", out)
return None
cmd = ArtResizer.shared.im_convert_cmd + [
util.syspath(path_in, prefix=False),
'-interlace', 'none',
util.syspath(path_out, prefix=False),
]
def deinterlace(self, path_in, path_out=None):
if not path_out:
path_out = get_temp_filename(__name__, "deinterlace_IM_", path_in)
try:
util.command_output(cmd)
return path_out
except subprocess.CalledProcessError:
return path_in
cmd = self.convert_cmd + [
syspath(path_in, prefix=False),
"-interlace",
"none",
syspath(path_out, prefix=False),
]
try:
util.command_output(cmd)
return path_out
except subprocess.CalledProcessError:
# FIXME: Should probably issue a warning?
return path_in
DEINTERLACE_FUNCS = {
PIL: pil_deinterlace,
IMAGEMAGICK: im_deinterlace,
}
def get_format(self, filepath):
cmd = self.identify_cmd + ["-format", "%[magick]", syspath(filepath)]
try:
return util.command_output(cmd).stdout
except subprocess.CalledProcessError:
# FIXME: Should probably issue a warning?
return None
def im_get_format(filepath):
cmd = ArtResizer.shared.im_identify_cmd + [
'-format', '%[magick]',
util.syspath(filepath)
]
def convert_format(self, source, target, deinterlaced):
cmd = self.convert_cmd + [
syspath(source),
*(["-interlace", "none"] if deinterlaced else []),
syspath(target),
]
try:
return util.command_output(cmd).stdout
except subprocess.CalledProcessError:
return None
def pil_get_format(filepath):
from PIL import Image, UnidentifiedImageError
try:
with Image.open(util.syspath(filepath)) as im:
return im.format
except (ValueError, TypeError, UnidentifiedImageError, FileNotFoundError):
log.exception("failed to detect image format for {}", filepath)
return None
BACKEND_GET_FORMAT = {
PIL: pil_get_format,
IMAGEMAGICK: im_get_format,
}
def im_convert_format(source, target, deinterlaced):
cmd = ArtResizer.shared.im_convert_cmd + [
util.syspath(source),
*(["-interlace", "none"] if deinterlaced else []),
util.syspath(target),
]
try:
subprocess.check_call(
cmd,
stderr=subprocess.DEVNULL,
stdout=subprocess.DEVNULL
)
return target
except subprocess.CalledProcessError:
return source
def pil_convert_format(source, target, deinterlaced):
from PIL import Image, UnidentifiedImageError
try:
with Image.open(util.syspath(source)) as im:
im.save(util.py3_path(target), progressive=not deinterlaced)
try:
subprocess.check_call(
cmd, stderr=subprocess.DEVNULL, stdout=subprocess.DEVNULL
)
return target
except (ValueError, TypeError, UnidentifiedImageError, FileNotFoundError,
OSError):
log.exception("failed to convert image {} -> {}", source, target)
return source
except subprocess.CalledProcessError:
# FIXME: Should probably issue a warning?
return source
@property
def can_compare(self):
return self.version() > (6, 8, 7)
def compare(self, im1, im2, compare_threshold):
is_windows = platform.system() == "Windows"
# Converting images to grayscale tends to minimize the weight
# of colors in the diff score. So we first convert both images
# to grayscale and then pipe them into the `compare` command.
# On Windows, ImageMagick doesn't support the magic \\?\ prefix
# on paths, so we pass `prefix=False` to `syspath`.
convert_cmd = self.convert_cmd + [
syspath(im2, prefix=False),
syspath(im1, prefix=False),
"-colorspace",
"gray",
"MIFF:-",
]
compare_cmd = self.compare_cmd + [
"-define",
"phash:colorspaces=sRGB,HCLp",
"-metric",
"PHASH",
"-",
"null:",
]
log.debug(
"comparing images with pipeline {} | {}", convert_cmd, compare_cmd
)
convert_proc = subprocess.Popen(
convert_cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
close_fds=not is_windows,
)
compare_proc = subprocess.Popen(
compare_cmd,
stdin=convert_proc.stdout,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
close_fds=not is_windows,
)
# Check the convert output. We're not interested in the
# standard output; that gets piped to the next stage.
convert_proc.stdout.close()
convert_stderr = convert_proc.stderr.read()
convert_proc.stderr.close()
convert_proc.wait()
if convert_proc.returncode:
log.debug(
"ImageMagick convert failed with status {}: {!r}",
convert_proc.returncode,
convert_stderr,
)
return None
# Check the compare output.
stdout, stderr = compare_proc.communicate()
if compare_proc.returncode:
if compare_proc.returncode != 1:
log.debug(
"ImageMagick compare failed: {0}, {1}",
displayable_path(im2),
displayable_path(im1),
)
return None
out_str = stderr
else:
out_str = stdout
try:
phash_diff = float(out_str)
except ValueError:
log.debug("IM output is not a number: {0!r}", out_str)
return None
log.debug("ImageMagick compare score: {0}", phash_diff)
return phash_diff <= compare_threshold
@property
def can_write_metadata(self):
return True
def write_metadata(self, file, metadata):
assignments = list(
chain.from_iterable(("-set", k, v) for k, v in metadata.items())
)
command = self.convert_cmd + [file, *assignments, file]
util.command_output(command)
BACKEND_CONVERT_IMAGE_FORMAT = {
PIL: pil_convert_format,
IMAGEMAGICK: im_convert_format,
}
class PILBackend(LocalBackend):
NAME = "PIL"
@classmethod
def version(cls):
try:
__import__("PIL", fromlist=["Image"])
except ImportError:
raise LocalBackendNotAvailableError()
def __init__(self):
"""Initialize a wrapper around PIL for local image operations.
If PIL is not available, raise an Exception.
"""
self.version()
def resize(
self, maxwidth, path_in, path_out=None, quality=0, max_filesize=0
):
"""Resize using Python Imaging Library (PIL). Return the output path
of resized image.
"""
if not path_out:
path_out = get_temp_filename(__name__, "resize_PIL_", path_in)
from PIL import Image
log.debug(
"artresizer: PIL resizing {0} to {1}",
displayable_path(path_in),
displayable_path(path_out),
)
try:
im = Image.open(syspath(path_in))
size = maxwidth, maxwidth
im.thumbnail(size, Image.Resampling.LANCZOS)
if quality == 0:
# Use PIL's default quality.
quality = -1
# progressive=False only affects JPEGs and is the default,
# but we include it here for explicitness.
im.save(os.fsdecode(path_out), quality=quality, progressive=False)
if max_filesize > 0:
# If maximum filesize is set, we attempt to lower the quality
# of jpeg conversion by a proportional amount, up to 3 attempts
# First, set the maximum quality to either provided, or 95
if quality > 0:
lower_qual = quality
else:
lower_qual = 95
for i in range(5):
# 5 attempts is an arbitrary choice
filesize = os.stat(syspath(path_out)).st_size
log.debug("PIL Pass {0} : Output size: {1}B", i, filesize)
if filesize <= max_filesize:
return path_out
# The relationship between filesize & quality will be
# image dependent.
lower_qual -= 10
# Restrict quality dropping below 10
if lower_qual < 10:
lower_qual = 10
# Use optimize flag to improve filesize decrease
im.save(
os.fsdecode(path_out),
quality=lower_qual,
optimize=True,
progressive=False,
)
log.warning(
"PIL Failed to resize file to below {0}B", max_filesize
)
return path_out
else:
return path_out
except OSError:
log.error(
"PIL cannot create thumbnail for '{0}'",
displayable_path(path_in),
)
return path_in
def get_size(self, path_in):
from PIL import Image
try:
im = Image.open(syspath(path_in))
return im.size
except OSError as exc:
log.error(
"PIL could not read file {}: {}", displayable_path(path_in), exc
)
return None
def deinterlace(self, path_in, path_out=None):
if not path_out:
path_out = get_temp_filename(__name__, "deinterlace_PIL_", path_in)
from PIL import Image
try:
im = Image.open(syspath(path_in))
im.save(os.fsdecode(path_out), progressive=False)
return path_out
except OSError:
# FIXME: Should probably issue a warning?
return path_in
def get_format(self, filepath):
from PIL import Image, UnidentifiedImageError
try:
with Image.open(syspath(filepath)) as im:
return im.format
except (
ValueError,
TypeError,
UnidentifiedImageError,
FileNotFoundError,
):
log.exception("failed to detect image format for {}", filepath)
return None
def convert_format(self, source, target, deinterlaced):
from PIL import Image, UnidentifiedImageError
try:
with Image.open(syspath(source)) as im:
im.save(os.fsdecode(target), progressive=not deinterlaced)
return target
except (
ValueError,
TypeError,
UnidentifiedImageError,
FileNotFoundError,
OSError,
):
log.exception("failed to convert image {} -> {}", source, target)
return source
@property
def can_compare(self):
return False
def compare(self, im1, im2, compare_threshold):
# It is an error to call this when ArtResizer.can_compare is not True.
raise NotImplementedError()
@property
def can_write_metadata(self):
return True
def write_metadata(self, file, metadata):
from PIL import Image, PngImagePlugin
# FIXME: Detect and handle other file types (currently, the only user
# is the thumbnails plugin, which generates PNG images).
im = Image.open(syspath(file))
meta = PngImagePlugin.PngInfo()
for k, v in metadata.items():
meta.add_text(k, v, 0)
im.save(os.fsdecode(file), "PNG", pnginfo=meta)
class Shareable(type):
@@ -319,28 +524,36 @@ class Shareable(type):
return cls._instance
BACKEND_CLASSES = [
IMBackend,
PILBackend,
]
class ArtResizer(metaclass=Shareable):
"""A singleton class that performs image resizes.
"""
"""A singleton class that performs image resizes."""
def __init__(self):
"""Create a resizer object with an inferred method.
"""
self.method = self._check_method()
log.debug("artresizer: method is {0}", self.method)
self.can_compare = self._can_compare()
"""Create a resizer object with an inferred method."""
# Check if a local backend is available, and store an instance of the
# backend class. Otherwise, fallback to the web proxy.
for backend_cls in BACKEND_CLASSES:
try:
self.local_method = backend_cls()
log.debug(f"artresizer: method is {self.local_method.NAME}")
break
except LocalBackendNotAvailableError:
continue
else:
log.debug("artresizer: method is WEBPROXY")
self.local_method = None
# Use ImageMagick's magick binary when it's available. If it's
# not, fall back to the older, separate convert and identify
# commands.
if self.method[0] == IMAGEMAGICK:
self.im_legacy = self.method[2]
if self.im_legacy:
self.im_convert_cmd = ['convert']
self.im_identify_cmd = ['identify']
else:
self.im_convert_cmd = ['magick']
self.im_identify_cmd = ['magick', 'identify']
@property
def method(self):
if self.local:
return self.local_method.NAME
else:
return "WEBPROXY"
def resize(
self, maxwidth, path_in, path_out=None, quality=0, max_filesize=0
@@ -351,17 +564,26 @@ class ArtResizer(metaclass=Shareable):
For WEBPROXY, returns `path_in` unmodified.
"""
if self.local:
func = BACKEND_FUNCS[self.method[0]]
return func(maxwidth, path_in, path_out,
quality=quality, max_filesize=max_filesize)
return self.local_method.resize(
maxwidth,
path_in,
path_out,
quality=quality,
max_filesize=max_filesize,
)
else:
# Handled by `proxy_url` already.
return path_in
def deinterlace(self, path_in, path_out=None):
"""Deinterlace an image.
Only available locally.
"""
if self.local:
func = DEINTERLACE_FUNCS[self.method[0]]
return func(path_in, path_out)
return self.local_method.deinterlace(path_in, path_out)
else:
# FIXME: Should probably issue a warning?
return path_in
def proxy_url(self, maxwidth, url, quality=0):
@@ -370,6 +592,7 @@ class ArtResizer(metaclass=Shareable):
Otherwise, the URL is returned unmodified.
"""
if self.local:
# Going to be handled by `resize()`.
return url
else:
return resize_url(url, maxwidth, quality)
@@ -379,7 +602,7 @@ class ArtResizer(metaclass=Shareable):
"""A boolean indicating whether the resizing method is performed
locally (i.e., PIL or ImageMagick).
"""
return self.method[0] in BACKEND_FUNCS
return self.local_method is not None
def get_size(self, path_in):
"""Return the size of an image file as an int couple (width, height)
@@ -388,8 +611,10 @@ class ArtResizer(metaclass=Shareable):
Only available locally.
"""
if self.local:
func = BACKEND_GET_SIZE[self.method[0]]
return func(path_in)
return self.local_method.get_size(path_in)
else:
# FIXME: Should probably issue a warning?
return path_in
def get_format(self, path_in):
"""Returns the format of the image as a string.
@@ -397,8 +622,10 @@ class ArtResizer(metaclass=Shareable):
Only available locally.
"""
if self.local:
func = BACKEND_GET_FORMAT[self.method[0]]
return func(path_in)
return self.local_method.get_format(path_in)
else:
# FIXME: Should probably issue a warning?
return None
def reformat(self, path_in, new_format, deinterlaced=True):
"""Converts image to desired format, updating its extension, but
@@ -407,86 +634,66 @@ class ArtResizer(metaclass=Shareable):
Only available locally.
"""
if not self.local:
# FIXME: Should probably issue a warning?
return path_in
new_format = new_format.lower()
# A nonexhaustive map of image "types" to extensions overrides
new_format = {
'jpeg': 'jpg',
"jpeg": "jpg",
}.get(new_format, new_format)
fname, ext = os.path.splitext(path_in)
path_new = fname + b'.' + new_format.encode('utf8')
func = BACKEND_CONVERT_IMAGE_FORMAT[self.method[0]]
path_new = fname + b"." + new_format.encode("utf8")
# allows the exception to propagate, while still making sure a changed
# file path was removed
result_path = path_in
try:
result_path = func(path_in, path_new, deinterlaced)
result_path = self.local_method.convert_format(
path_in, path_new, deinterlaced
)
finally:
if result_path != path_in:
os.unlink(path_in)
return result_path
def _can_compare(self):
@property
def can_compare(self):
"""A boolean indicating whether image comparison is available"""
return self.method[0] == IMAGEMAGICK and self.method[1] > (6, 8, 7)
@staticmethod
def _check_method():
"""Return a tuple indicating an available method and its version.
The result has at least two elements:
- The method, eitehr WEBPROXY, PIL, or IMAGEMAGICK.
- The version.
If the method is IMAGEMAGICK, there is also a third element: a
bool flag indicating whether to use the `magick` binary or
legacy single-purpose executables (`convert`, `identify`, etc.)
"""
version = get_im_version()
if version:
version, legacy = version
return IMAGEMAGICK, version, legacy
version = get_pil_version()
if version:
return PIL, version
return WEBPROXY, (0)
def get_im_version():
"""Get the ImageMagick version and legacy flag as a pair. Or return
None if ImageMagick is not available.
"""
for cmd_name, legacy in ((['magick'], False), (['convert'], True)):
cmd = cmd_name + ['--version']
try:
out = util.command_output(cmd).stdout
except (subprocess.CalledProcessError, OSError) as exc:
log.debug('ImageMagick version check failed: {}', exc)
if self.local:
return self.local_method.can_compare
else:
if b'imagemagick' in out.lower():
pattern = br".+ (\d+)\.(\d+)\.(\d+).*"
match = re.search(pattern, out)
if match:
version = (int(match.group(1)),
int(match.group(2)),
int(match.group(3)))
return version, legacy
return False
return None
def compare(self, im1, im2, compare_threshold):
"""Return a boolean indicating whether two images are similar.
Only available locally.
"""
if self.local:
return self.local_method.compare(im1, im2, compare_threshold)
else:
# FIXME: Should probably issue a warning?
return None
def get_pil_version():
"""Get the PIL/Pillow version, or None if it is unavailable.
"""
try:
__import__('PIL', fromlist=['Image'])
return (0,)
except ImportError:
return None
@property
def can_write_metadata(self):
"""A boolean indicating whether writing image metadata is supported."""
if self.local:
return self.local_method.can_write_metadata
else:
return False
def write_metadata(self, file, metadata):
"""Write key-value metadata to the image file.
Only available locally. Currently, expects the image to be a PNG file.
"""
if self.local:
self.local_method.write_metadata(file, metadata)
else:
# FIXME: Should probably issue a warning?
pass
lib/beets/util/bluelet.py
+60 -41
View File
@@ -6,23 +6,24 @@ asyncore.
Bluelet: easy concurrency without all the messy parallelism.
"""
import socket
import select
import sys
import types
import errno
import traceback
import time
import collections
import errno
import select
import socket
import sys
import time
import traceback
import types
# Basic events used for thread scheduling.
class Event:
"""Just a base class identifying Bluelet events. An event is an
object yielded from a Bluelet thread coroutine to suspend operation
and communicate with the scheduler.
"""
pass
@@ -31,6 +32,7 @@ class WaitableEvent(Event):
waited for using a select() call. That is, it's an event with an
associated file descriptor.
"""
def waitables(self):
"""Return "waitable" objects to pass to select(). Should return
three iterables for input readiness, output readiness, and
@@ -48,18 +50,21 @@ class WaitableEvent(Event):
class ValueEvent(Event):
"""An event that does nothing but return a fixed value."""
def __init__(self, value):
self.value = value
class ExceptionEvent(Event):
"""Raise an exception at the yield point. Used internally."""
def __init__(self, exc_info):
self.exc_info = exc_info
class SpawnEvent(Event):
"""Add a new coroutine thread to the scheduler."""
def __init__(self, coro):
self.spawned = coro
@@ -68,12 +73,14 @@ class JoinEvent(Event):
"""Suspend the thread until the specified child thread has
completed.
"""
def __init__(self, child):
self.child = child
class KillEvent(Event):
"""Unschedule a child thread."""
def __init__(self, child):
self.child = child
@@ -83,6 +90,7 @@ class DelegationEvent(Event):
once the child thread finished, return control to the parent
thread.
"""
def __init__(self, coro):
self.spawned = coro
@@ -91,13 +99,14 @@ class ReturnEvent(Event):
"""Return a value the current thread's delegator at the point of
delegation. Ends the current (delegate) thread.
"""
def __init__(self, value):
self.value = value
class SleepEvent(WaitableEvent):
"""Suspend the thread for a given duration.
"""
"""Suspend the thread for a given duration."""
def __init__(self, duration):
self.wakeup_time = time.time() + duration
@@ -107,6 +116,7 @@ class SleepEvent(WaitableEvent):
class ReadEvent(WaitableEvent):
"""Reads from a file-like object."""
def __init__(self, fd, bufsize):
self.fd = fd
self.bufsize = bufsize
@@ -120,6 +130,7 @@ class ReadEvent(WaitableEvent):
class WriteEvent(WaitableEvent):
"""Writes to a file-like object."""
def __init__(self, fd, data):
self.fd = fd
self.data = data
@@ -133,6 +144,7 @@ class WriteEvent(WaitableEvent):
# Core logic for executing and scheduling threads.
def _event_select(events):
"""Perform a select() over all the Events provided, returning the
ones ready to be fired. Only WaitableEvents (including SleepEvents)
@@ -154,11 +166,11 @@ def _event_select(events):
wlist += w
xlist += x
for waitable in r:
waitable_to_event[('r', waitable)] = event
waitable_to_event[("r", waitable)] = event
for waitable in w:
waitable_to_event[('w', waitable)] = event
waitable_to_event[("w", waitable)] = event
for waitable in x:
waitable_to_event[('x', waitable)] = event
waitable_to_event[("x", waitable)] = event
# If we have a any sleeping threads, determine how long to sleep.
if earliest_wakeup:
@@ -177,11 +189,11 @@ def _event_select(events):
# Gather ready events corresponding to the ready waitables.
ready_events = set()
for ready in rready:
ready_events.add(waitable_to_event[('r', ready)])
ready_events.add(waitable_to_event[("r", ready)])
for ready in wready:
ready_events.add(waitable_to_event[('w', ready)])
ready_events.add(waitable_to_event[("w", ready)])
for ready in xready:
ready_events.add(waitable_to_event[('x', ready)])
ready_events.add(waitable_to_event[("x", ready)])
# Gather any finished sleeps.
for event in events:
@@ -207,6 +219,7 @@ class Delegated(Event):
"""Placeholder indicating that a thread has delegated execution to a
different thread.
"""
def __init__(self, child):
self.child = child
@@ -277,8 +290,7 @@ def run(root_coro):
threads[coro] = next_event
def kill_thread(coro):
"""Unschedule this thread and its (recursive) delegates.
"""
"""Unschedule this thread and its (recursive) delegates."""
# Collect all coroutines in the delegation stack.
coros = [coro]
while isinstance(threads[coro], Delegated):
@@ -338,12 +350,16 @@ def run(root_coro):
try:
value = event.fire()
except OSError as exc:
if isinstance(exc.args, tuple) and \
exc.args[0] == errno.EPIPE:
if (
isinstance(exc.args, tuple)
and exc.args[0] == errno.EPIPE
):
# Broken pipe. Remote host disconnected.
pass
elif isinstance(exc.args, tuple) and \
exc.args[0] == errno.ECONNRESET:
elif (
isinstance(exc.args, tuple)
and exc.args[0] == errno.ECONNRESET
):
# Connection was reset by peer.
pass
else:
@@ -382,16 +398,16 @@ def run(root_coro):
# Sockets and their associated events.
class SocketClosedError(Exception):
pass
class Listener:
"""A socket wrapper object for listening sockets.
"""
"""A socket wrapper object for listening sockets."""
def __init__(self, host, port):
"""Create a listening socket on the given hostname and port.
"""
"""Create a listening socket on the given hostname and port."""
self._closed = False
self.host = host
self.port = port
@@ -410,19 +426,18 @@ class Listener:
return AcceptEvent(self)
def close(self):
"""Immediately close the listening socket. (Not an event.)
"""
"""Immediately close the listening socket. (Not an event.)"""
self._closed = True
self.sock.close()
class Connection:
"""A socket wrapper object for connected sockets.
"""
"""A socket wrapper object for connected sockets."""
def __init__(self, sock, addr):
self.sock = sock
self.addr = addr
self._buf = b''
self._buf = b""
self._closed = False
def close(self):
@@ -473,7 +488,7 @@ class Connection:
self._buf += data
else:
line = self._buf
self._buf = b''
self._buf = b""
yield ReturnEvent(line)
break
@@ -482,6 +497,7 @@ class AcceptEvent(WaitableEvent):
"""An event for Listener objects (listening sockets) that suspends
execution until the socket gets a connection.
"""
def __init__(self, listener):
self.listener = listener
@@ -497,6 +513,7 @@ class ReceiveEvent(WaitableEvent):
"""An event for Connection objects (connected sockets) for
asynchronously reading data.
"""
def __init__(self, conn, bufsize):
self.conn = conn
self.bufsize = bufsize
@@ -512,6 +529,7 @@ class SendEvent(WaitableEvent):
"""An event for Connection objects (connected sockets) for
asynchronously writing data.
"""
def __init__(self, conn, data, sendall=False):
self.conn = conn
self.data = data
@@ -530,9 +548,9 @@ class SendEvent(WaitableEvent):
# Public interface for threads; each returns an event object that
# can immediately be "yield"ed.
def null():
"""Event: yield to the scheduler without doing anything special.
"""
"""Event: yield to the scheduler without doing anything special."""
return ValueEvent(None)
@@ -541,7 +559,7 @@ def spawn(coro):
and child coroutines run concurrently.
"""
if not isinstance(coro, types.GeneratorType):
raise ValueError('%s is not a coroutine' % coro)
raise ValueError("%s is not a coroutine" % coro)
return SpawnEvent(coro)
@@ -551,7 +569,7 @@ def call(coro):
returns a value using end(), then this event returns that value.
"""
if not isinstance(coro, types.GeneratorType):
raise ValueError('%s is not a coroutine' % coro)
raise ValueError("%s is not a coroutine" % coro)
return DelegationEvent(coro)
@@ -573,7 +591,8 @@ def read(fd, bufsize=None):
if not data:
break
buf.append(data)
yield ReturnEvent(''.join(buf))
yield ReturnEvent("".join(buf))
return DelegationEvent(reader())
else:
@@ -595,8 +614,7 @@ def connect(host, port):
def sleep(duration):
"""Event: suspend the thread for ``duration`` seconds.
"""
"""Event: suspend the thread for ``duration`` seconds."""
return SleepEvent(duration)
@@ -608,19 +626,20 @@ def join(coro):
def kill(coro):
"""Halt the execution of a different `spawn`ed thread.
"""
"""Halt the execution of a different `spawn`ed thread."""
return KillEvent(coro)
# Convenience function for running socket servers.
def server(host, port, func):
"""A coroutine that runs a network server. Host and port specify the
listening address. func should be a coroutine that takes a single
parameter, a Connection object. The coroutine is invoked for every
incoming connection on the listening socket.
"""
def handler(conn):
try:
yield func(conn)
lib/beets/util/enumeration.py
+1
View File
@@ -20,6 +20,7 @@ class OrderedEnum(Enum):
"""
An Enum subclass that allows comparison of members.
"""
def __ge__(self, other):
if self.__class__ is other.__class__:
return self.value >= other.value
lib/beets/util/functemplate.py
+105 -116
View File
@@ -27,22 +27,21 @@ engine like Jinja2 or Mustache.
"""
import re
import ast
import dis
import types
import sys
import functools
import re
import types
SYMBOL_DELIM = '$'
FUNC_DELIM = '%'
GROUP_OPEN = '{'
GROUP_CLOSE = '}'
ARG_SEP = ','
ESCAPE_CHAR = '$'
SYMBOL_DELIM = "$"
FUNC_DELIM = "%"
GROUP_OPEN = "{"
GROUP_CLOSE = "}"
ARG_SEP = ","
ESCAPE_CHAR = "$"
VARIABLE_PREFIX = '__var_'
FUNCTION_PREFIX = '__func_'
VARIABLE_PREFIX = "__var_"
FUNCTION_PREFIX = "__func_"
class Environment:
@@ -57,10 +56,6 @@ class Environment:
# Code generation helpers.
def ex_lvalue(name):
"""A variable load expression."""
return ast.Name(name, ast.Store())
def ex_rvalue(name):
"""A variable store expression."""
@@ -74,15 +69,6 @@ def ex_literal(val):
return ast.Constant(val)
def ex_varassign(name, expr):
"""Assign an expression into a single variable. The expression may
either be an `ast.expr` object or a value to be used as a literal.
"""
if not isinstance(expr, ast.expr):
expr = ex_literal(expr)
return ast.Assign([ex_lvalue(name)], expr)
def ex_call(func, args):
"""A function-call expression with only positional parameters. The
function may be an expression or the name of a function. Each
@@ -99,19 +85,18 @@ def ex_call(func, args):
return ast.Call(func, args, [])
def compile_func(arg_names, statements, name='_the_func', debug=False):
def compile_func(arg_names, statements, name="_the_func", debug=False):
"""Compile a list of statements as the body of a function and return
the resulting Python function. If `debug`, then print out the
bytecode of the compiled function.
"""
args_fields = {
'args': [ast.arg(arg=n, annotation=None) for n in arg_names],
'kwonlyargs': [],
'kw_defaults': [],
'defaults': [ex_literal(None) for _ in arg_names],
"args": [ast.arg(arg=n, annotation=None) for n in arg_names],
"kwonlyargs": [],
"kw_defaults": [],
"defaults": [ex_literal(None) for _ in arg_names],
}
if 'posonlyargs' in ast.arguments._fields: # Added in Python 3.8.
args_fields['posonlyargs'] = []
args_fields["posonlyargs"] = []
args = ast.arguments(**args_fields)
func_def = ast.FunctionDef(
@@ -123,14 +108,11 @@ def compile_func(arg_names, statements, name='_the_func', debug=False):
# The ast.Module signature changed in 3.8 to accept a list of types to
# ignore.
if sys.version_info >= (3, 8):
mod = ast.Module([func_def], [])
else:
mod = ast.Module([func_def])
mod = ast.Module([func_def], [])
ast.fix_missing_locations(mod)
prog = compile(mod, '<generated>', 'exec')
prog = compile(mod, "<generated>", "exec")
# Debug: show bytecode.
if debug:
@@ -146,6 +128,7 @@ def compile_func(arg_names, statements, name='_the_func', debug=False):
# AST nodes for the template language.
class Symbol:
"""A variable-substitution symbol in a template."""
@@ -154,7 +137,7 @@ class Symbol:
self.original = original
def __repr__(self):
return 'Symbol(%s)' % repr(self.ident)
return "Symbol(%s)" % repr(self.ident)
def evaluate(self, env):
"""Evaluate the symbol in the environment, returning a Unicode
@@ -183,8 +166,9 @@ class Call:
self.original = original
def __repr__(self):
return 'Call({}, {}, {})'.format(repr(self.ident), repr(self.args),
repr(self.original))
return "Call({}, {}, {})".format(
repr(self.ident), repr(self.args), repr(self.original)
)
def evaluate(self, env):
"""Evaluate the function call in the environment, returning a
@@ -197,7 +181,7 @@ class Call:
except Exception as exc:
# Function raised exception! Maybe inlining the name of
# the exception will help debug.
return '<%s>' % str(exc)
return "<%s>" % str(exc)
return str(out)
else:
return self.original
@@ -215,21 +199,22 @@ class Call:
# Create a subexpression that joins the result components of
# the arguments.
arg_exprs.append(ex_call(
ast.Attribute(ex_literal(''), 'join', ast.Load()),
[ex_call(
'map',
arg_exprs.append(
ex_call(
ast.Attribute(ex_literal(""), "join", ast.Load()),
[
ex_rvalue(str.__name__),
ast.List(subexprs, ast.Load()),
]
)],
))
ex_call(
"map",
[
ex_rvalue(str.__name__),
ast.List(subexprs, ast.Load()),
],
)
],
)
)
subexpr_call = ex_call(
FUNCTION_PREFIX + self.ident,
arg_exprs
)
subexpr_call = ex_call(FUNCTION_PREFIX + self.ident, arg_exprs)
return [subexpr_call], varnames, funcnames
@@ -242,7 +227,7 @@ class Expression:
self.parts = parts
def __repr__(self):
return 'Expression(%s)' % (repr(self.parts))
return "Expression(%s)" % (repr(self.parts))
def evaluate(self, env):
"""Evaluate the entire expression in the environment, returning
@@ -254,7 +239,7 @@ class Expression:
out.append(part)
else:
out.append(part.evaluate(env))
return ''.join(map(str, out))
return "".join(map(str, out))
def translate(self):
"""Compile the expression to a list of Python AST expressions, a
@@ -276,6 +261,7 @@ class Expression:
# Parser.
class ParseError(Exception):
pass
@@ -295,7 +281,7 @@ class Parser:
"""
def __init__(self, string, in_argument=False):
""" Create a new parser.
"""Create a new parser.
:param in_arguments: boolean that indicates the parser is to be
used for parsing function arguments, ie. considering commas
(`ARG_SEP`) a special character
@@ -306,10 +292,16 @@ class Parser:
self.parts = []
# Common parsing resources.
special_chars = (SYMBOL_DELIM, FUNC_DELIM, GROUP_OPEN, GROUP_CLOSE,
ESCAPE_CHAR)
special_char_re = re.compile(r'[%s]|\Z' %
''.join(re.escape(c) for c in special_chars))
special_chars = (
SYMBOL_DELIM,
FUNC_DELIM,
GROUP_OPEN,
GROUP_CLOSE,
ESCAPE_CHAR,
)
special_char_re = re.compile(
r"[%s]|\Z" % "".join(re.escape(c) for c in special_chars)
)
escapable_chars = (SYMBOL_DELIM, FUNC_DELIM, GROUP_CLOSE, ARG_SEP)
terminator_chars = (GROUP_CLOSE,)
@@ -326,9 +318,10 @@ class Parser:
if self.in_argument:
extra_special_chars = (ARG_SEP,)
special_char_re = re.compile(
r'[%s]|\Z' % ''.join(
re.escape(c) for c in
self.special_chars + extra_special_chars
r"[%s]|\Z"
% "".join(
re.escape(c)
for c in self.special_chars + extra_special_chars
)
)
@@ -341,10 +334,10 @@ class Parser:
# A non-special character. Skip to the next special
# character, treating the interstice as literal text.
next_pos = (
special_char_re.search(
self.string[self.pos:]).start() + self.pos
special_char_re.search(self.string[self.pos :]).start()
+ self.pos
)
text_parts.append(self.string[self.pos:next_pos])
text_parts.append(self.string[self.pos : next_pos])
self.pos = next_pos
continue
@@ -358,8 +351,9 @@ class Parser:
break
next_char = self.string[self.pos + 1]
if char == ESCAPE_CHAR and next_char in (self.escapable_chars +
extra_special_chars):
if char == ESCAPE_CHAR and next_char in (
self.escapable_chars + extra_special_chars
):
# An escaped special character ($$, $}, etc.). Note that
# ${ is not an escape sequence: this is ambiguous with
# the start of a symbol and it's not necessary (just
@@ -370,7 +364,7 @@ class Parser:
# Shift all characters collected so far into a single string.
if text_parts:
self.parts.append(''.join(text_parts))
self.parts.append("".join(text_parts))
text_parts = []
if char == SYMBOL_DELIM:
@@ -392,7 +386,7 @@ class Parser:
# If any parsed characters remain, shift them into a string.
if text_parts:
self.parts.append(''.join(text_parts))
self.parts.append("".join(text_parts))
def parse_symbol(self):
"""Parse a variable reference (like ``$foo`` or ``${foo}``)
@@ -419,21 +413,23 @@ class Parser:
closer = self.string.find(GROUP_CLOSE, self.pos)
if closer == -1 or closer == self.pos:
# No closing brace found or identifier is empty.
self.parts.append(self.string[start_pos:self.pos])
self.parts.append(self.string[start_pos : self.pos])
else:
# Closer found.
ident = self.string[self.pos:closer]
ident = self.string[self.pos : closer]
self.pos = closer + 1
self.parts.append(Symbol(ident,
self.string[start_pos:self.pos]))
self.parts.append(
Symbol(ident, self.string[start_pos : self.pos])
)
else:
# A bare-word symbol.
ident = self._parse_ident()
if ident:
# Found a real symbol.
self.parts.append(Symbol(ident,
self.string[start_pos:self.pos]))
self.parts.append(
Symbol(ident, self.string[start_pos : self.pos])
)
else:
# A standalone $.
self.parts.append(SYMBOL_DELIM)
@@ -457,25 +453,24 @@ class Parser:
if self.pos >= len(self.string):
# Identifier terminates string.
self.parts.append(self.string[start_pos:self.pos])
self.parts.append(self.string[start_pos : self.pos])
return
if self.string[self.pos] != GROUP_OPEN:
# Argument list not opened.
self.parts.append(self.string[start_pos:self.pos])
self.parts.append(self.string[start_pos : self.pos])
return
# Skip past opening brace and try to parse an argument list.
self.pos += 1
args = self.parse_argument_list()
if self.pos >= len(self.string) or \
self.string[self.pos] != GROUP_CLOSE:
if self.pos >= len(self.string) or self.string[self.pos] != GROUP_CLOSE:
# Arguments unclosed.
self.parts.append(self.string[start_pos:self.pos])
self.parts.append(self.string[start_pos : self.pos])
return
self.pos += 1 # Move past closing brace.
self.parts.append(Call(ident, args, self.string[start_pos:self.pos]))
self.parts.append(Call(ident, args, self.string[start_pos : self.pos]))
def parse_argument_list(self):
"""Parse a list of arguments starting at ``pos``, returning a
@@ -487,15 +482,17 @@ class Parser:
expressions = []
while self.pos < len(self.string):
subparser = Parser(self.string[self.pos:], in_argument=True)
subparser = Parser(self.string[self.pos :], in_argument=True)
subparser.parse_expression()
# Extract and advance past the parsed expression.
expressions.append(Expression(subparser.parts))
self.pos += subparser.pos
if self.pos >= len(self.string) or \
self.string[self.pos] == GROUP_CLOSE:
if (
self.pos >= len(self.string)
or self.string[self.pos] == GROUP_CLOSE
):
# Argument list terminated by EOF or closing brace.
break
@@ -510,8 +507,8 @@ class Parser:
"""Parse an identifier and return it (possibly an empty string).
Updates ``pos``.
"""
remainder = self.string[self.pos:]
ident = re.match(r'\w*', remainder).group(0)
remainder = self.string[self.pos :]
ident = re.match(r"\w*", remainder).group(0)
self.pos += len(ident)
return ident
@@ -524,32 +521,20 @@ def _parse(template):
parser.parse_expression()
parts = parser.parts
remainder = parser.string[parser.pos:]
remainder = parser.string[parser.pos :]
if remainder:
parts.append(remainder)
return Expression(parts)
def cached(func):
"""Like the `functools.lru_cache` decorator, but works (as a no-op)
on Python < 3.2.
"""
if hasattr(functools, 'lru_cache'):
return functools.lru_cache(maxsize=128)(func)
else:
# Do nothing when lru_cache is not available.
return func
@cached
@functools.lru_cache(maxsize=128)
def template(fmt):
return Template(fmt)
# External interface.
class Template:
"""A string template, including text, Symbols, and Calls.
"""
"""A string template, including text, Symbols, and Calls."""
def __init__(self, template):
self.expr = _parse(template)
@@ -568,8 +553,7 @@ class Template:
return self.expr.evaluate(Environment(values, functions))
def substitute(self, values={}, functions={}):
"""Evaluate the template given the values and functions.
"""
"""Evaluate the template given the values and functions."""
try:
res = self.compiled(values, functions)
except Exception: # Handle any exceptions thrown by compiled version.
@@ -599,24 +583,29 @@ class Template:
for funcname in funcnames:
args[FUNCTION_PREFIX + funcname] = functions[funcname]
parts = func(**args)
return ''.join(parts)
return "".join(parts)
return wrapper_func
# Performance tests.
if __name__ == '__main__':
if __name__ == "__main__":
import timeit
_tmpl = Template('foo $bar %baz{foozle $bar barzle} $bar')
_vars = {'bar': 'qux'}
_funcs = {'baz': str.upper}
interp_time = timeit.timeit('_tmpl.interpret(_vars, _funcs)',
'from __main__ import _tmpl, _vars, _funcs',
number=10000)
_tmpl = Template("foo $bar %baz{foozle $bar barzle} $bar")
_vars = {"bar": "qux"}
_funcs = {"baz": str.upper}
interp_time = timeit.timeit(
"_tmpl.interpret(_vars, _funcs)",
"from __main__ import _tmpl, _vars, _funcs",
number=10000,
)
print(interp_time)
comp_time = timeit.timeit('_tmpl.substitute(_vars, _funcs)',
'from __main__ import _tmpl, _vars, _funcs',
number=10000)
comp_time = timeit.timeit(
"_tmpl.substitute(_vars, _funcs)",
"from __main__ import _tmpl, _vars, _funcs",
number=10000,
)
print(comp_time)
print('Speedup:', interp_time / comp_time)
print("Speedup:", interp_time / comp_time)
lib/beets/util/hidden.py
+30 -51
View File
@@ -1,5 +1,6 @@
# This file is part of beets.
# Copyright 2016, Adrian Sampson.
# Copyright 2024, Arav K.
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
@@ -14,71 +15,49 @@
"""Simple library to work out if a file is hidden on different platforms."""
import ctypes
import os
import stat
import ctypes
import sys
import beets.util
from pathlib import Path
from typing import Union
def _is_hidden_osx(path):
"""Return whether or not a file is hidden on OS X.
This uses os.lstat to work out if a file has the "hidden" flag.
def is_hidden(path: Union[bytes, Path]) -> bool:
"""
file_stat = os.lstat(beets.util.syspath(path))
if hasattr(file_stat, 'st_flags') and hasattr(stat, 'UF_HIDDEN'):
return bool(file_stat.st_flags & stat.UF_HIDDEN)
else:
return False
def _is_hidden_win(path):
"""Return whether or not a file is hidden on Windows.
This uses GetFileAttributes to work out if a file has the "hidden" flag
(FILE_ATTRIBUTE_HIDDEN).
Determine whether the given path is treated as a 'hidden file' by the OS.
"""
# FILE_ATTRIBUTE_HIDDEN = 2 (0x2) from GetFileAttributes documentation.
hidden_mask = 2
# Retrieve the attributes for the file.
attrs = ctypes.windll.kernel32.GetFileAttributesW(beets.util.syspath(path))
if isinstance(path, bytes):
path = Path(os.fsdecode(path))
# Ensure we have valid attribues and compare them against the mask.
return attrs >= 0 and attrs & hidden_mask
# TODO: Avoid doing a platform check on every invocation of the function.
# TODO: Stop supporting 'bytes' inputs once 'pathlib' is fully integrated.
if sys.platform == "win32":
# On Windows, we check for an FS-provided attribute.
def _is_hidden_dot(path):
"""Return whether or not a file starts with a dot.
# FILE_ATTRIBUTE_HIDDEN = 2 (0x2) from GetFileAttributes documentation.
hidden_mask = 2
Files starting with a dot are seen as "hidden" files on Unix-based OSes.
"""
return os.path.basename(path).startswith(b'.')
# Retrieve the attributes for the file.
attrs = ctypes.windll.kernel32.GetFileAttributesW(str(path))
# Ensure the attribute mask is valid.
if attrs < 0:
return False
def is_hidden(path):
"""Return whether or not a file is hidden. `path` should be a
bytestring filename.
# Check for the hidden attribute.
return attrs & hidden_mask
This method works differently depending on the platform it is called on.
# On OS X, we check for an FS-provided attribute.
if sys.platform == "darwin":
if hasattr(os.stat_result, "st_flags") and hasattr(stat, "UF_HIDDEN"):
if path.lstat().st_flags & stat.UF_HIDDEN:
return True
On OS X, it uses both the result of `is_hidden_osx` and `is_hidden_dot` to
work out if a file is hidden.
# On all non-Windows platforms, we check for a '.'-prefixed file name.
if path.name.startswith("."):
return True
On Windows, it uses the result of `is_hidden_win` to work out if a file is
hidden.
On any other operating systems (i.e. Linux), it uses `is_hidden_dot` to
work out if a file is hidden.
"""
# Run platform specific functions depending on the platform
if sys.platform == 'darwin':
return _is_hidden_osx(path) or _is_hidden_dot(path)
elif sys.platform == 'win32':
return _is_hidden_win(path)
else:
return _is_hidden_dot(path)
__all__ = ['is_hidden']
return False
lib/beets/util/id_extractors.py
+65
View File
@@ -0,0 +1,65 @@
# This file is part of beets.
# Copyright 2016, Adrian Sampson.
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
"""Helpers around the extraction of album/track ID's from metadata sources."""
import re
# Spotify IDs consist of 22 alphanumeric characters
# (zero-left-padded base62 representation of randomly generated UUID4)
spotify_id_regex = {
"pattern": r"(^|open\.spotify\.com/{}/)([0-9A-Za-z]{{22}})",
"match_group": 2,
}
deezer_id_regex = {
"pattern": r"(^|deezer\.com/)([a-z]*/)?({}/)?(\d+)",
"match_group": 4,
}
beatport_id_regex = {
"pattern": r"(^|beatport\.com/release/.+/)(\d+)$",
"match_group": 2,
}
# A note on Bandcamp: There is no such thing as a Bandcamp album or artist ID,
# the URL can be used as the identifier. The Bandcamp metadata source plugin
# works that way - https://github.com/snejus/beetcamp. Bandcamp album
# URLs usually look like: https://nameofartist.bandcamp.com/album/nameofalbum
def extract_discogs_id_regex(album_id):
"""Returns the Discogs_id or None."""
# Discogs-IDs are simple integers. In order to avoid confusion with
# other metadata plugins, we only look for very specific formats of the
# input string:
# - plain integer, optionally wrapped in brackets and prefixed by an
# 'r', as this is how discogs displays the release ID on its webpage.
# - legacy url format: discogs.com/<name of release>/release/<id>
# - legacy url short format: discogs.com/release/<id>
# - current url format: discogs.com/release/<id>-<name of release>
# See #291, #4080 and #4085 for the discussions leading up to these
# patterns.
# Regex has been tested here https://regex101.com/r/TOu7kw/1
for pattern in [
r"^\[?r?(?P<id>\d+)\]?$",
r"discogs\.com/release/(?P<id>\d+)-?",
r"discogs\.com/[^/]+/release/(?P<id>\d+)",
]:
match = re.search(pattern, album_id)
if match:
return int(match.group("id"))
return None
lib/beets/util/m3u.py
+97
View File
@@ -0,0 +1,97 @@
# This file is part of beets.
# Copyright 2022, J0J0 Todos.
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
"""Provides utilities to read, write and manipulate m3u playlist files."""
import traceback
from beets.util import FilesystemError, mkdirall, normpath, syspath
class EmptyPlaylistError(Exception):
"""Raised when a playlist file without media files is saved or loaded."""
pass
class M3UFile:
"""Reads and writes m3u or m3u8 playlist files."""
def __init__(self, path):
"""``path`` is the absolute path to the playlist file.
The playlist file type, m3u or m3u8 is determined by 1) the ending
being m3u8 and 2) the file paths contained in the list being utf-8
encoded. Since the list is passed from the outside, this is currently
out of control of this class.
"""
self.path = path
self.extm3u = False
self.media_list = []
def load(self):
"""Reads the m3u file from disk and sets the object's attributes."""
pl_normpath = normpath(self.path)
try:
with open(syspath(pl_normpath), "rb") as pl_file:
raw_contents = pl_file.readlines()
except OSError as exc:
raise FilesystemError(
exc, "read", (pl_normpath,), traceback.format_exc()
)
self.extm3u = True if raw_contents[0].rstrip() == b"#EXTM3U" else False
for line in raw_contents[1:]:
if line.startswith(b"#"):
# Support for specific EXTM3U comments could be added here.
continue
self.media_list.append(normpath(line.rstrip()))
if not self.media_list:
raise EmptyPlaylistError
def set_contents(self, media_list, extm3u=True):
"""Sets self.media_list to a list of media file paths.
Also sets additional flags, changing the final m3u-file's format.
``media_list`` is a list of paths to media files that should be added
to the playlist (relative or absolute paths, that's the responsibility
of the caller). By default the ``extm3u`` flag is set, to ensure a
save-operation writes an m3u-extended playlist (comment "#EXTM3U" at
the top of the file).
"""
self.media_list = media_list
self.extm3u = extm3u
def write(self):
"""Writes the m3u file to disk.
Handles the creation of potential parent directories.
"""
header = [b"#EXTM3U"] if self.extm3u else []
if not self.media_list:
raise EmptyPlaylistError
contents = header + self.media_list
pl_normpath = normpath(self.path)
mkdirall(pl_normpath)
try:
with open(syspath(pl_normpath), "wb") as pl_file:
for line in contents:
pl_file.write(line + b"\n")
pl_file.write(b"\n") # Final linefeed to prevent noeol file.
except OSError as exc:
raise FilesystemError(
exc, "create", (pl_normpath,), traceback.format_exc()
)
lib/beets/util/pipeline.py
+29 -28
View File
@@ -33,11 +33,11 @@ in place of any single coroutine.
import queue
from threading import Thread, Lock
import sys
from threading import Lock, Thread
BUBBLE = '__PIPELINE_BUBBLE__'
POISON = '__PIPELINE_POISON__'
BUBBLE = "__PIPELINE_BUBBLE__"
POISON = "__PIPELINE_POISON__"
DEFAULT_QUEUE_SIZE = 16
@@ -48,6 +48,7 @@ def _invalidate_queue(q, val=None, sync=True):
which defaults to None. `sync` controls whether a lock is
required (because it's not reentrant!).
"""
def _qsize(len=len):
return 1
@@ -75,8 +76,8 @@ def _invalidate_queue(q, val=None, sync=True):
q._qsize = _qsize
q._put = _put
q._get = _get
q.not_empty.notifyAll()
q.not_full.notifyAll()
q.not_empty.notify_all()
q.not_full.notify_all()
finally:
if sync:
@@ -168,6 +169,7 @@ def stage(func):
while True:
task = yield task
task = func(*(args + (task,)))
return coro
@@ -191,6 +193,7 @@ def mutator_stage(func):
while True:
task = yield task
func(*(args + (task,)))
return coro
@@ -218,20 +221,18 @@ class PipelineThread(Thread):
self.exc_info = None
def abort(self):
"""Shut down the thread at the next chance possible.
"""
"""Shut down the thread at the next chance possible."""
with self.abort_lock:
self.abort_flag = True
# Ensure that we are not blocking on a queue read or write.
if hasattr(self, 'in_queue'):
if hasattr(self, "in_queue"):
_invalidate_queue(self.in_queue, POISON)
if hasattr(self, 'out_queue'):
if hasattr(self, "out_queue"):
_invalidate_queue(self.out_queue, POISON)
def abort_all(self, exc_info):
"""Abort all other threads in the system for an exception.
"""
"""Abort all other threads in the system for an exception."""
self.exc_info = exc_info
for thread in self.all_threads:
thread.abort()
@@ -373,7 +374,7 @@ class Pipeline:
be at least two stages.
"""
if len(stages) < 2:
raise ValueError('pipeline must have at least two stages')
raise ValueError("pipeline must have at least two stages")
self.stages = []
for stage in stages:
if isinstance(stage, (list, tuple)):
@@ -405,15 +406,15 @@ class Pipeline:
# Middle stages.
for i in range(1, queue_count):
for coro in self.stages[i]:
threads.append(MiddlePipelineThread(
coro, queues[i - 1], queues[i], threads
))
threads.append(
MiddlePipelineThread(
coro, queues[i - 1], queues[i], threads
)
)
# Last stage.
for coro in self.stages[-1]:
threads.append(
LastPipelineThread(coro, queues[-1], threads)
)
threads.append(LastPipelineThread(coro, queues[-1], threads))
# Start threads.
for thread in threads:
@@ -472,21 +473,21 @@ class Pipeline:
# Smoke test.
if __name__ == '__main__':
if __name__ == "__main__":
import time
# Test a normally-terminating pipeline both in sequence and
# in parallel.
def produce():
for i in range(5):
print('generating %i' % i)
print("generating %i" % i)
time.sleep(1)
yield i
def work():
num = yield
while True:
print('processing %i' % num)
print("processing %i" % num)
time.sleep(2)
num = yield num * 2
@@ -494,7 +495,7 @@ if __name__ == '__main__':
while True:
num = yield
time.sleep(1)
print('received %i' % num)
print("received %i" % num)
ts_start = time.time()
Pipeline([produce(), work(), consume()]).run_sequential()
@@ -503,22 +504,22 @@ if __name__ == '__main__':
ts_par = time.time()
Pipeline([produce(), (work(), work()), consume()]).run_parallel()
ts_end = time.time()
print('Sequential time:', ts_seq - ts_start)
print('Parallel time:', ts_par - ts_seq)
print('Multiply-parallel time:', ts_end - ts_par)
print("Sequential time:", ts_seq - ts_start)
print("Parallel time:", ts_par - ts_seq)
print("Multiply-parallel time:", ts_end - ts_par)
print()
# Test a pipeline that raises an exception.
def exc_produce():
for i in range(10):
print('generating %i' % i)
print("generating %i" % i)
time.sleep(1)
yield i
def exc_work():
num = yield
while True:
print('processing %i' % num)
print("processing %i" % num)
time.sleep(3)
if num == 3:
raise Exception()
@@ -527,6 +528,6 @@ if __name__ == '__main__':
def exc_consume():
while True:
num = yield
print('received %i' % num)
print("received %i" % num)
Pipeline([exc_produce(), exc_work(), exc_consume()]).run_parallel(1)
lib/beets/vfs.py
+2 -1
View File
@@ -17,9 +17,10 @@ libraries.
"""
from collections import namedtuple
from beets import util
Node = namedtuple('Node', ['files', 'dirs'])
Node = namedtuple("Node", ["files", "dirs"])
def _insert(node, path, itemid):