gronod/headphones
1
0
Fork 0
mirror of https://github.com/rembo10/headphones.git synced 2026-07-19 23:44:01 +01:00
Code Issues Packages Projects Releases Wiki Activity

update Beets

Browse Source
This commit is contained in:
AdeHub
2024-08-24 16:44:41 +12:00
parent a63098a919
commit 046d4d82b4
116 changed files with 17353 additions and 9964 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/beets/util/__init__.py
+365 -351
View File
File diff suppressed because it is too large Load Diff
lib/beets/util/artresizer.py
+522 -315
View File
@@ -16,23 +16,20 @@
public resizing proxy if neither is available.
"""
import subprocess
import os
import os.path
import platform
import re
from tempfile import NamedTemporaryFile
import subprocess
from itertools import chain
from urllib.parse import urlencode
from beets import logging
from beets import util
# Resizing methods
PIL = 1
IMAGEMAGICK = 2
WEBPROXY = 3
from beets import logging, util
from beets.util import displayable_path, get_temp_filename, syspath
PROXY_URL = 'https://images.weserv.nl/'
PROXY_URL = "https://images.weserv.nl/"
log = logging.getLogger('beets')
log = logging.getLogger("beets")
def resize_url(url, maxwidth, quality=0):
@@ -40,265 +37,473 @@ def resize_url(url, maxwidth, quality=0):
maxwidth (preserving aspect ratio).
"""
params = {
'url': url.replace('http://', ''),
'w': maxwidth,
"url": url.replace("http://", ""),
"w": maxwidth,
}
if quality > 0:
params['q'] = quality
params["q"] = quality
return '{}?{}'.format(PROXY_URL, urlencode(params))
return "{}?{}".format(PROXY_URL, urlencode(params))
def temp_file_for(path):
"""Return an unused filename with the same extension as the
specified path.
"""
ext = os.path.splitext(path)[1]
with NamedTemporaryFile(suffix=util.py3_path(ext), delete=False) as f:
return util.bytestring_path(f.name)
class LocalBackendNotAvailableError(Exception):
pass
def pil_resize(maxwidth, path_in, path_out=None, quality=0, max_filesize=0):
"""Resize using Python Imaging Library (PIL). Return the output path
of resized image.
"""
path_out = path_out or temp_file_for(path_in)
from PIL import Image
_NOT_AVAILABLE = object()
log.debug('artresizer: PIL resizing {0} to {1}',
util.displayable_path(path_in), util.displayable_path(path_out))
try:
im = Image.open(util.syspath(path_in))
size = maxwidth, maxwidth
im.thumbnail(size, Image.ANTIALIAS)
class LocalBackend:
@classmethod
def available(cls):
try:
cls.version()
return True
except LocalBackendNotAvailableError:
return False
if quality == 0:
# Use PIL's default quality.
quality = -1
# progressive=False only affects JPEGs and is the default,
# but we include it here for explicitness.
im.save(util.py3_path(path_out), quality=quality, progressive=False)
class IMBackend(LocalBackend):
NAME = "ImageMagick"
if max_filesize > 0:
# If maximum filesize is set, we attempt to lower the quality of
# jpeg conversion by a proportional amount, up to 3 attempts
# First, set the maximum quality to either provided, or 95
if quality > 0:
lower_qual = quality
else:
lower_qual = 95
for i in range(5):
# 5 attempts is an abitrary choice
filesize = os.stat(util.syspath(path_out)).st_size
log.debug("PIL Pass {0} : Output size: {1}B", i, filesize)
if filesize <= max_filesize:
return path_out
# The relationship between filesize & quality will be
# image dependent.
lower_qual -= 10
# Restrict quality dropping below 10
if lower_qual < 10:
lower_qual = 10
# Use optimize flag to improve filesize decrease
im.save(util.py3_path(path_out), quality=lower_qual,
optimize=True, progressive=False)
log.warning("PIL Failed to resize file to below {0}B",
max_filesize)
return path_out
# These fields are used as a cache for `version()`. `_legacy` indicates
# whether the modern `magick` binary is available or whether to fall back
# to the old-style `convert`, `identify`, etc. commands.
_version = None
_legacy = None
@classmethod
def version(cls):
"""Obtain and cache ImageMagick version.
Raises `LocalBackendNotAvailableError` if not available.
"""
if cls._version is None:
for cmd_name, legacy in (("magick", False), ("convert", True)):
try:
out = util.command_output([cmd_name, "--version"]).stdout
except (subprocess.CalledProcessError, OSError) as exc:
log.debug("ImageMagick version check failed: {}", exc)
cls._version = _NOT_AVAILABLE
else:
if b"imagemagick" in out.lower():
pattern = rb".+ (\d+)\.(\d+)\.(\d+).*"
match = re.search(pattern, out)
if match:
cls._version = (
int(match.group(1)),
int(match.group(2)),
int(match.group(3)),
)
cls._legacy = legacy
if cls._version is _NOT_AVAILABLE:
raise LocalBackendNotAvailableError()
else:
return path_out
except OSError:
log.error("PIL cannot create thumbnail for '{0}'",
util.displayable_path(path_in))
return path_in
return cls._version
def __init__(self):
"""Initialize a wrapper around ImageMagick for local image operations.
def im_resize(maxwidth, path_in, path_out=None, quality=0, max_filesize=0):
"""Resize using ImageMagick.
Stores the ImageMagick version and legacy flag. If ImageMagick is not
available, raise an Exception.
"""
self.version()
Use the ``magick`` program or ``convert`` on older versions. Return
the output path of resized image.
"""
path_out = path_out or temp_file_for(path_in)
log.debug('artresizer: ImageMagick resizing {0} to {1}',
util.displayable_path(path_in), util.displayable_path(path_out))
# Use ImageMagick's magick binary when it's available.
# If it's not, fall back to the older, separate convert
# and identify commands.
if self._legacy:
self.convert_cmd = ["convert"]
self.identify_cmd = ["identify"]
self.compare_cmd = ["compare"]
else:
self.convert_cmd = ["magick"]
self.identify_cmd = ["magick", "identify"]
self.compare_cmd = ["magick", "compare"]
# "-resize WIDTHx>" shrinks images with the width larger
# than the given width while maintaining the aspect ratio
# with regards to the height.
# ImageMagick already seems to default to no interlace, but we include it
# here for the sake of explicitness.
cmd = ArtResizer.shared.im_convert_cmd + [
util.syspath(path_in, prefix=False),
'-resize', f'{maxwidth}x>',
'-interlace', 'none',
]
def resize(
self, maxwidth, path_in, path_out=None, quality=0, max_filesize=0
):
"""Resize using ImageMagick.
if quality > 0:
cmd += ['-quality', f'{quality}']
Use the ``magick`` program or ``convert`` on older versions. Return
the output path of resized image.
"""
if not path_out:
path_out = get_temp_filename(__name__, "resize_IM_", path_in)
# "-define jpeg:extent=SIZEb" sets the target filesize for imagemagick to
# SIZE in bytes.
if max_filesize > 0:
cmd += ['-define', f'jpeg:extent={max_filesize}b']
cmd.append(util.syspath(path_out, prefix=False))
try:
util.command_output(cmd)
except subprocess.CalledProcessError:
log.warning('artresizer: IM convert failed for {0}',
util.displayable_path(path_in))
return path_in
return path_out
BACKEND_FUNCS = {
PIL: pil_resize,
IMAGEMAGICK: im_resize,
}
def pil_getsize(path_in):
from PIL import Image
try:
im = Image.open(util.syspath(path_in))
return im.size
except OSError as exc:
log.error("PIL could not read file {}: {}",
util.displayable_path(path_in), exc)
def im_getsize(path_in):
cmd = ArtResizer.shared.im_identify_cmd + \
['-format', '%w %h', util.syspath(path_in, prefix=False)]
try:
out = util.command_output(cmd).stdout
except subprocess.CalledProcessError as exc:
log.warning('ImageMagick size query failed')
log.debug(
'`convert` exited with (status {}) when '
'getting size with command {}:\n{}',
exc.returncode, cmd, exc.output.strip()
"artresizer: ImageMagick resizing {0} to {1}",
displayable_path(path_in),
displayable_path(path_out),
)
return
try:
return tuple(map(int, out.split(b' ')))
except IndexError:
log.warning('Could not understand IM output: {0!r}', out)
# "-resize WIDTHx>" shrinks images with the width larger
# than the given width while maintaining the aspect ratio
# with regards to the height.
# ImageMagick already seems to default to no interlace, but we include
# it here for the sake of explicitness.
cmd = self.convert_cmd + [
syspath(path_in, prefix=False),
"-resize",
f"{maxwidth}x>",
"-interlace",
"none",
]
BACKEND_GET_SIZE = {
PIL: pil_getsize,
IMAGEMAGICK: im_getsize,
}
if quality > 0:
cmd += ["-quality", f"{quality}"]
# "-define jpeg:extent=SIZEb" sets the target filesize for imagemagick
# to SIZE in bytes.
if max_filesize > 0:
cmd += ["-define", f"jpeg:extent={max_filesize}b"]
def pil_deinterlace(path_in, path_out=None):
path_out = path_out or temp_file_for(path_in)
from PIL import Image
cmd.append(syspath(path_out, prefix=False))
try:
util.command_output(cmd)
except subprocess.CalledProcessError:
log.warning(
"artresizer: IM convert failed for {0}",
displayable_path(path_in),
)
return path_in
try:
im = Image.open(util.syspath(path_in))
im.save(util.py3_path(path_out), progressive=False)
return path_out
except IOError:
return path_in
def get_size(self, path_in):
cmd = self.identify_cmd + [
"-format",
"%w %h",
syspath(path_in, prefix=False),
]
def im_deinterlace(path_in, path_out=None):
path_out = path_out or temp_file_for(path_in)
try:
out = util.command_output(cmd).stdout
except subprocess.CalledProcessError as exc:
log.warning("ImageMagick size query failed")
log.debug(
"`convert` exited with (status {}) when "
"getting size with command {}:\n{}",
exc.returncode,
cmd,
exc.output.strip(),
)
return None
try:
return tuple(map(int, out.split(b" ")))
except IndexError:
log.warning("Could not understand IM output: {0!r}", out)
return None
cmd = ArtResizer.shared.im_convert_cmd + [
util.syspath(path_in, prefix=False),
'-interlace', 'none',
util.syspath(path_out, prefix=False),
]
def deinterlace(self, path_in, path_out=None):
if not path_out:
path_out = get_temp_filename(__name__, "deinterlace_IM_", path_in)
try:
util.command_output(cmd)
return path_out
except subprocess.CalledProcessError:
return path_in
cmd = self.convert_cmd + [
syspath(path_in, prefix=False),
"-interlace",
"none",
syspath(path_out, prefix=False),
]
try:
util.command_output(cmd)
return path_out
except subprocess.CalledProcessError:
# FIXME: Should probably issue a warning?
return path_in
DEINTERLACE_FUNCS = {
PIL: pil_deinterlace,
IMAGEMAGICK: im_deinterlace,
}
def get_format(self, filepath):
cmd = self.identify_cmd + ["-format", "%[magick]", syspath(filepath)]
try:
return util.command_output(cmd).stdout
except subprocess.CalledProcessError:
# FIXME: Should probably issue a warning?
return None
def im_get_format(filepath):
cmd = ArtResizer.shared.im_identify_cmd + [
'-format', '%[magick]',
util.syspath(filepath)
]
def convert_format(self, source, target, deinterlaced):
cmd = self.convert_cmd + [
syspath(source),
*(["-interlace", "none"] if deinterlaced else []),
syspath(target),
]
try:
return util.command_output(cmd).stdout
except subprocess.CalledProcessError:
return None
def pil_get_format(filepath):
from PIL import Image, UnidentifiedImageError
try:
with Image.open(util.syspath(filepath)) as im:
return im.format
except (ValueError, TypeError, UnidentifiedImageError, FileNotFoundError):
log.exception("failed to detect image format for {}", filepath)
return None
BACKEND_GET_FORMAT = {
PIL: pil_get_format,
IMAGEMAGICK: im_get_format,
}
def im_convert_format(source, target, deinterlaced):
cmd = ArtResizer.shared.im_convert_cmd + [
util.syspath(source),
*(["-interlace", "none"] if deinterlaced else []),
util.syspath(target),
]
try:
subprocess.check_call(
cmd,
stderr=subprocess.DEVNULL,
stdout=subprocess.DEVNULL
)
return target
except subprocess.CalledProcessError:
return source
def pil_convert_format(source, target, deinterlaced):
from PIL import Image, UnidentifiedImageError
try:
with Image.open(util.syspath(source)) as im:
im.save(util.py3_path(target), progressive=not deinterlaced)
try:
subprocess.check_call(
cmd, stderr=subprocess.DEVNULL, stdout=subprocess.DEVNULL
)
return target
except (ValueError, TypeError, UnidentifiedImageError, FileNotFoundError,
OSError):
log.exception("failed to convert image {} -> {}", source, target)
return source
except subprocess.CalledProcessError:
# FIXME: Should probably issue a warning?
return source
@property
def can_compare(self):
return self.version() > (6, 8, 7)
def compare(self, im1, im2, compare_threshold):
is_windows = platform.system() == "Windows"
# Converting images to grayscale tends to minimize the weight
# of colors in the diff score. So we first convert both images
# to grayscale and then pipe them into the `compare` command.
# On Windows, ImageMagick doesn't support the magic \\?\ prefix
# on paths, so we pass `prefix=False` to `syspath`.
convert_cmd = self.convert_cmd + [
syspath(im2, prefix=False),
syspath(im1, prefix=False),
"-colorspace",
"gray",
"MIFF:-",
]
compare_cmd = self.compare_cmd + [
"-define",
"phash:colorspaces=sRGB,HCLp",
"-metric",
"PHASH",
"-",
"null:",
]
log.debug(
"comparing images with pipeline {} | {}", convert_cmd, compare_cmd
)
convert_proc = subprocess.Popen(
convert_cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
close_fds=not is_windows,
)
compare_proc = subprocess.Popen(
compare_cmd,
stdin=convert_proc.stdout,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
close_fds=not is_windows,
)
# Check the convert output. We're not interested in the
# standard output; that gets piped to the next stage.
convert_proc.stdout.close()
convert_stderr = convert_proc.stderr.read()
convert_proc.stderr.close()
convert_proc.wait()
if convert_proc.returncode:
log.debug(
"ImageMagick convert failed with status {}: {!r}",
convert_proc.returncode,
convert_stderr,
)
return None
# Check the compare output.
stdout, stderr = compare_proc.communicate()
if compare_proc.returncode:
if compare_proc.returncode != 1:
log.debug(
"ImageMagick compare failed: {0}, {1}",
displayable_path(im2),
displayable_path(im1),
)
return None
out_str = stderr
else:
out_str = stdout
try:
phash_diff = float(out_str)
except ValueError:
log.debug("IM output is not a number: {0!r}", out_str)
return None
log.debug("ImageMagick compare score: {0}", phash_diff)
return phash_diff <= compare_threshold
@property
def can_write_metadata(self):
return True
def write_metadata(self, file, metadata):
assignments = list(
chain.from_iterable(("-set", k, v) for k, v in metadata.items())
)
command = self.convert_cmd + [file, *assignments, file]
util.command_output(command)
BACKEND_CONVERT_IMAGE_FORMAT = {
PIL: pil_convert_format,
IMAGEMAGICK: im_convert_format,
}
class PILBackend(LocalBackend):
NAME = "PIL"
@classmethod
def version(cls):
try:
__import__("PIL", fromlist=["Image"])
except ImportError:
raise LocalBackendNotAvailableError()
def __init__(self):
"""Initialize a wrapper around PIL for local image operations.
If PIL is not available, raise an Exception.
"""
self.version()
def resize(
self, maxwidth, path_in, path_out=None, quality=0, max_filesize=0
):
"""Resize using Python Imaging Library (PIL). Return the output path
of resized image.
"""
if not path_out:
path_out = get_temp_filename(__name__, "resize_PIL_", path_in)
from PIL import Image
log.debug(
"artresizer: PIL resizing {0} to {1}",
displayable_path(path_in),
displayable_path(path_out),
)
try:
im = Image.open(syspath(path_in))
size = maxwidth, maxwidth
im.thumbnail(size, Image.Resampling.LANCZOS)
if quality == 0:
# Use PIL's default quality.
quality = -1
# progressive=False only affects JPEGs and is the default,
# but we include it here for explicitness.
im.save(os.fsdecode(path_out), quality=quality, progressive=False)
if max_filesize > 0:
# If maximum filesize is set, we attempt to lower the quality
# of jpeg conversion by a proportional amount, up to 3 attempts
# First, set the maximum quality to either provided, or 95
if quality > 0:
lower_qual = quality
else:
lower_qual = 95
for i in range(5):
# 5 attempts is an arbitrary choice
filesize = os.stat(syspath(path_out)).st_size
log.debug("PIL Pass {0} : Output size: {1}B", i, filesize)
if filesize <= max_filesize:
return path_out
# The relationship between filesize & quality will be
# image dependent.
lower_qual -= 10
# Restrict quality dropping below 10
if lower_qual < 10:
lower_qual = 10
# Use optimize flag to improve filesize decrease
im.save(
os.fsdecode(path_out),
quality=lower_qual,
optimize=True,
progressive=False,
)
log.warning(
"PIL Failed to resize file to below {0}B", max_filesize
)
return path_out
else:
return path_out
except OSError:
log.error(
"PIL cannot create thumbnail for '{0}'",
displayable_path(path_in),
)
return path_in
def get_size(self, path_in):
from PIL import Image
try:
im = Image.open(syspath(path_in))
return im.size
except OSError as exc:
log.error(
"PIL could not read file {}: {}", displayable_path(path_in), exc
)
return None
def deinterlace(self, path_in, path_out=None):
if not path_out:
path_out = get_temp_filename(__name__, "deinterlace_PIL_", path_in)
from PIL import Image
try:
im = Image.open(syspath(path_in))
im.save(os.fsdecode(path_out), progressive=False)
return path_out
except OSError:
# FIXME: Should probably issue a warning?
return path_in
def get_format(self, filepath):
from PIL import Image, UnidentifiedImageError
try:
with Image.open(syspath(filepath)) as im:
return im.format
except (
ValueError,
TypeError,
UnidentifiedImageError,
FileNotFoundError,
):
log.exception("failed to detect image format for {}", filepath)
return None
def convert_format(self, source, target, deinterlaced):
from PIL import Image, UnidentifiedImageError
try:
with Image.open(syspath(source)) as im:
im.save(os.fsdecode(target), progressive=not deinterlaced)
return target
except (
ValueError,
TypeError,
UnidentifiedImageError,
FileNotFoundError,
OSError,
):
log.exception("failed to convert image {} -> {}", source, target)
return source
@property
def can_compare(self):
return False
def compare(self, im1, im2, compare_threshold):
# It is an error to call this when ArtResizer.can_compare is not True.
raise NotImplementedError()
@property
def can_write_metadata(self):
return True
def write_metadata(self, file, metadata):
from PIL import Image, PngImagePlugin
# FIXME: Detect and handle other file types (currently, the only user
# is the thumbnails plugin, which generates PNG images).
im = Image.open(syspath(file))
meta = PngImagePlugin.PngInfo()
for k, v in metadata.items():
meta.add_text(k, v, 0)
im.save(os.fsdecode(file), "PNG", pnginfo=meta)
class Shareable(type):
@@ -319,28 +524,36 @@ class Shareable(type):
return cls._instance
BACKEND_CLASSES = [
IMBackend,
PILBackend,
]
class ArtResizer(metaclass=Shareable):
"""A singleton class that performs image resizes.
"""
"""A singleton class that performs image resizes."""
def __init__(self):
"""Create a resizer object with an inferred method.
"""
self.method = self._check_method()
log.debug("artresizer: method is {0}", self.method)
self.can_compare = self._can_compare()
"""Create a resizer object with an inferred method."""
# Check if a local backend is available, and store an instance of the
# backend class. Otherwise, fallback to the web proxy.
for backend_cls in BACKEND_CLASSES:
try:
self.local_method = backend_cls()
log.debug(f"artresizer: method is {self.local_method.NAME}")
break
except LocalBackendNotAvailableError:
continue
else:
log.debug("artresizer: method is WEBPROXY")
self.local_method = None
# Use ImageMagick's magick binary when it's available. If it's
# not, fall back to the older, separate convert and identify
# commands.
if self.method[0] == IMAGEMAGICK:
self.im_legacy = self.method[2]
if self.im_legacy:
self.im_convert_cmd = ['convert']
self.im_identify_cmd = ['identify']
else:
self.im_convert_cmd = ['magick']
self.im_identify_cmd = ['magick', 'identify']
@property
def method(self):
if self.local:
return self.local_method.NAME
else:
return "WEBPROXY"
def resize(
self, maxwidth, path_in, path_out=None, quality=0, max_filesize=0
@@ -351,17 +564,26 @@ class ArtResizer(metaclass=Shareable):
For WEBPROXY, returns `path_in` unmodified.
"""
if self.local:
func = BACKEND_FUNCS[self.method[0]]
return func(maxwidth, path_in, path_out,
quality=quality, max_filesize=max_filesize)
return self.local_method.resize(
maxwidth,
path_in,
path_out,
quality=quality,
max_filesize=max_filesize,
)
else:
# Handled by `proxy_url` already.
return path_in
def deinterlace(self, path_in, path_out=None):
"""Deinterlace an image.
Only available locally.
"""
if self.local:
func = DEINTERLACE_FUNCS[self.method[0]]
return func(path_in, path_out)
return self.local_method.deinterlace(path_in, path_out)
else:
# FIXME: Should probably issue a warning?
return path_in
def proxy_url(self, maxwidth, url, quality=0):
@@ -370,6 +592,7 @@ class ArtResizer(metaclass=Shareable):
Otherwise, the URL is returned unmodified.
"""
if self.local:
# Going to be handled by `resize()`.
return url
else:
return resize_url(url, maxwidth, quality)
@@ -379,7 +602,7 @@ class ArtResizer(metaclass=Shareable):
"""A boolean indicating whether the resizing method is performed
locally (i.e., PIL or ImageMagick).
"""
return self.method[0] in BACKEND_FUNCS
return self.local_method is not None
def get_size(self, path_in):
"""Return the size of an image file as an int couple (width, height)
@@ -388,8 +611,10 @@ class ArtResizer(metaclass=Shareable):
Only available locally.
"""
if self.local:
func = BACKEND_GET_SIZE[self.method[0]]
return func(path_in)
return self.local_method.get_size(path_in)
else:
# FIXME: Should probably issue a warning?
return path_in
def get_format(self, path_in):
"""Returns the format of the image as a string.
@@ -397,8 +622,10 @@ class ArtResizer(metaclass=Shareable):
Only available locally.
"""
if self.local:
func = BACKEND_GET_FORMAT[self.method[0]]
return func(path_in)
return self.local_method.get_format(path_in)
else:
# FIXME: Should probably issue a warning?
return None
def reformat(self, path_in, new_format, deinterlaced=True):
"""Converts image to desired format, updating its extension, but
@@ -407,86 +634,66 @@ class ArtResizer(metaclass=Shareable):
Only available locally.
"""
if not self.local:
# FIXME: Should probably issue a warning?
return path_in
new_format = new_format.lower()
# A nonexhaustive map of image "types" to extensions overrides
new_format = {
'jpeg': 'jpg',
"jpeg": "jpg",
}.get(new_format, new_format)
fname, ext = os.path.splitext(path_in)
path_new = fname + b'.' + new_format.encode('utf8')
func = BACKEND_CONVERT_IMAGE_FORMAT[self.method[0]]
path_new = fname + b"." + new_format.encode("utf8")
# allows the exception to propagate, while still making sure a changed
# file path was removed
result_path = path_in
try:
result_path = func(path_in, path_new, deinterlaced)
result_path = self.local_method.convert_format(
path_in, path_new, deinterlaced
)
finally:
if result_path != path_in:
os.unlink(path_in)
return result_path
def _can_compare(self):
@property
def can_compare(self):
"""A boolean indicating whether image comparison is available"""
return self.method[0] == IMAGEMAGICK and self.method[1] > (6, 8, 7)
@staticmethod
def _check_method():
"""Return a tuple indicating an available method and its version.
The result has at least two elements:
- The method, eitehr WEBPROXY, PIL, or IMAGEMAGICK.
- The version.
If the method is IMAGEMAGICK, there is also a third element: a
bool flag indicating whether to use the `magick` binary or
legacy single-purpose executables (`convert`, `identify`, etc.)
"""
version = get_im_version()
if version:
version, legacy = version
return IMAGEMAGICK, version, legacy
version = get_pil_version()
if version:
return PIL, version
return WEBPROXY, (0)
def get_im_version():
"""Get the ImageMagick version and legacy flag as a pair. Or return
None if ImageMagick is not available.
"""
for cmd_name, legacy in ((['magick'], False), (['convert'], True)):
cmd = cmd_name + ['--version']
try:
out = util.command_output(cmd).stdout
except (subprocess.CalledProcessError, OSError) as exc:
log.debug('ImageMagick version check failed: {}', exc)
if self.local:
return self.local_method.can_compare
else:
if b'imagemagick' in out.lower():
pattern = br".+ (\d+)\.(\d+)\.(\d+).*"
match = re.search(pattern, out)
if match:
version = (int(match.group(1)),
int(match.group(2)),
int(match.group(3)))
return version, legacy
return False
return None
def compare(self, im1, im2, compare_threshold):
"""Return a boolean indicating whether two images are similar.
Only available locally.
"""
if self.local:
return self.local_method.compare(im1, im2, compare_threshold)
else:
# FIXME: Should probably issue a warning?
return None
def get_pil_version():
"""Get the PIL/Pillow version, or None if it is unavailable.
"""
try:
__import__('PIL', fromlist=['Image'])
return (0,)
except ImportError:
return None
@property
def can_write_metadata(self):
"""A boolean indicating whether writing image metadata is supported."""
if self.local:
return self.local_method.can_write_metadata
else:
return False
def write_metadata(self, file, metadata):
"""Write key-value metadata to the image file.
Only available locally. Currently, expects the image to be a PNG file.
"""
if self.local:
self.local_method.write_metadata(file, metadata)
else:
# FIXME: Should probably issue a warning?
pass
lib/beets/util/bluelet.py
+60 -41
View File
@@ -6,23 +6,24 @@ asyncore.
Bluelet: easy concurrency without all the messy parallelism.
"""
import socket
import select
import sys
import types
import errno
import traceback
import time
import collections
import errno
import select
import socket
import sys
import time
import traceback
import types
# Basic events used for thread scheduling.
class Event:
"""Just a base class identifying Bluelet events. An event is an
object yielded from a Bluelet thread coroutine to suspend operation
and communicate with the scheduler.
"""
pass
@@ -31,6 +32,7 @@ class WaitableEvent(Event):
waited for using a select() call. That is, it's an event with an
associated file descriptor.
"""
def waitables(self):
"""Return "waitable" objects to pass to select(). Should return
three iterables for input readiness, output readiness, and
@@ -48,18 +50,21 @@ class WaitableEvent(Event):
class ValueEvent(Event):
"""An event that does nothing but return a fixed value."""
def __init__(self, value):
self.value = value
class ExceptionEvent(Event):
"""Raise an exception at the yield point. Used internally."""
def __init__(self, exc_info):
self.exc_info = exc_info
class SpawnEvent(Event):
"""Add a new coroutine thread to the scheduler."""
def __init__(self, coro):
self.spawned = coro
@@ -68,12 +73,14 @@ class JoinEvent(Event):
"""Suspend the thread until the specified child thread has
completed.
"""
def __init__(self, child):
self.child = child
class KillEvent(Event):
"""Unschedule a child thread."""
def __init__(self, child):
self.child = child
@@ -83,6 +90,7 @@ class DelegationEvent(Event):
once the child thread finished, return control to the parent
thread.
"""
def __init__(self, coro):
self.spawned = coro
@@ -91,13 +99,14 @@ class ReturnEvent(Event):
"""Return a value the current thread's delegator at the point of
delegation. Ends the current (delegate) thread.
"""
def __init__(self, value):
self.value = value
class SleepEvent(WaitableEvent):
"""Suspend the thread for a given duration.
"""
"""Suspend the thread for a given duration."""
def __init__(self, duration):
self.wakeup_time = time.time() + duration
@@ -107,6 +116,7 @@ class SleepEvent(WaitableEvent):
class ReadEvent(WaitableEvent):
"""Reads from a file-like object."""
def __init__(self, fd, bufsize):
self.fd = fd
self.bufsize = bufsize
@@ -120,6 +130,7 @@ class ReadEvent(WaitableEvent):
class WriteEvent(WaitableEvent):
"""Writes to a file-like object."""
def __init__(self, fd, data):
self.fd = fd
self.data = data
@@ -133,6 +144,7 @@ class WriteEvent(WaitableEvent):
# Core logic for executing and scheduling threads.
def _event_select(events):
"""Perform a select() over all the Events provided, returning the
ones ready to be fired. Only WaitableEvents (including SleepEvents)
@@ -154,11 +166,11 @@ def _event_select(events):
wlist += w
xlist += x
for waitable in r:
waitable_to_event[('r', waitable)] = event
waitable_to_event[("r", waitable)] = event
for waitable in w:
waitable_to_event[('w', waitable)] = event
waitable_to_event[("w", waitable)] = event
for waitable in x:
waitable_to_event[('x', waitable)] = event
waitable_to_event[("x", waitable)] = event
# If we have a any sleeping threads, determine how long to sleep.
if earliest_wakeup:
@@ -177,11 +189,11 @@ def _event_select(events):
# Gather ready events corresponding to the ready waitables.
ready_events = set()
for ready in rready:
ready_events.add(waitable_to_event[('r', ready)])
ready_events.add(waitable_to_event[("r", ready)])
for ready in wready:
ready_events.add(waitable_to_event[('w', ready)])
ready_events.add(waitable_to_event[("w", ready)])
for ready in xready:
ready_events.add(waitable_to_event[('x', ready)])
ready_events.add(waitable_to_event[("x", ready)])
# Gather any finished sleeps.
for event in events:
@@ -207,6 +219,7 @@ class Delegated(Event):
"""Placeholder indicating that a thread has delegated execution to a
different thread.
"""
def __init__(self, child):
self.child = child
@@ -277,8 +290,7 @@ def run(root_coro):
threads[coro] = next_event
def kill_thread(coro):
"""Unschedule this thread and its (recursive) delegates.
"""
"""Unschedule this thread and its (recursive) delegates."""
# Collect all coroutines in the delegation stack.
coros = [coro]
while isinstance(threads[coro], Delegated):
@@ -338,12 +350,16 @@ def run(root_coro):
try:
value = event.fire()
except OSError as exc:
if isinstance(exc.args, tuple) and \
exc.args[0] == errno.EPIPE:
if (
isinstance(exc.args, tuple)
and exc.args[0] == errno.EPIPE
):
# Broken pipe. Remote host disconnected.
pass
elif isinstance(exc.args, tuple) and \
exc.args[0] == errno.ECONNRESET:
elif (
isinstance(exc.args, tuple)
and exc.args[0] == errno.ECONNRESET
):
# Connection was reset by peer.
pass
else:
@@ -382,16 +398,16 @@ def run(root_coro):
# Sockets and their associated events.
class SocketClosedError(Exception):
pass
class Listener:
"""A socket wrapper object for listening sockets.
"""
"""A socket wrapper object for listening sockets."""
def __init__(self, host, port):
"""Create a listening socket on the given hostname and port.
"""
"""Create a listening socket on the given hostname and port."""
self._closed = False
self.host = host
self.port = port
@@ -410,19 +426,18 @@ class Listener:
return AcceptEvent(self)
def close(self):
"""Immediately close the listening socket. (Not an event.)
"""
"""Immediately close the listening socket. (Not an event.)"""
self._closed = True
self.sock.close()
class Connection:
"""A socket wrapper object for connected sockets.
"""
"""A socket wrapper object for connected sockets."""
def __init__(self, sock, addr):
self.sock = sock
self.addr = addr
self._buf = b''
self._buf = b""
self._closed = False
def close(self):
@@ -473,7 +488,7 @@ class Connection:
self._buf += data
else:
line = self._buf
self._buf = b''
self._buf = b""
yield ReturnEvent(line)
break
@@ -482,6 +497,7 @@ class AcceptEvent(WaitableEvent):
"""An event for Listener objects (listening sockets) that suspends
execution until the socket gets a connection.
"""
def __init__(self, listener):
self.listener = listener
@@ -497,6 +513,7 @@ class ReceiveEvent(WaitableEvent):
"""An event for Connection objects (connected sockets) for
asynchronously reading data.
"""
def __init__(self, conn, bufsize):
self.conn = conn
self.bufsize = bufsize
@@ -512,6 +529,7 @@ class SendEvent(WaitableEvent):
"""An event for Connection objects (connected sockets) for
asynchronously writing data.
"""
def __init__(self, conn, data, sendall=False):
self.conn = conn
self.data = data
@@ -530,9 +548,9 @@ class SendEvent(WaitableEvent):
# Public interface for threads; each returns an event object that
# can immediately be "yield"ed.
def null():
"""Event: yield to the scheduler without doing anything special.
"""
"""Event: yield to the scheduler without doing anything special."""
return ValueEvent(None)
@@ -541,7 +559,7 @@ def spawn(coro):
and child coroutines run concurrently.
"""
if not isinstance(coro, types.GeneratorType):
raise ValueError('%s is not a coroutine' % coro)
raise ValueError("%s is not a coroutine" % coro)
return SpawnEvent(coro)
@@ -551,7 +569,7 @@ def call(coro):
returns a value using end(), then this event returns that value.
"""
if not isinstance(coro, types.GeneratorType):
raise ValueError('%s is not a coroutine' % coro)
raise ValueError("%s is not a coroutine" % coro)
return DelegationEvent(coro)
@@ -573,7 +591,8 @@ def read(fd, bufsize=None):
if not data:
break
buf.append(data)
yield ReturnEvent(''.join(buf))
yield ReturnEvent("".join(buf))
return DelegationEvent(reader())
else:
@@ -595,8 +614,7 @@ def connect(host, port):
def sleep(duration):
"""Event: suspend the thread for ``duration`` seconds.
"""
"""Event: suspend the thread for ``duration`` seconds."""
return SleepEvent(duration)
@@ -608,19 +626,20 @@ def join(coro):
def kill(coro):
"""Halt the execution of a different `spawn`ed thread.
"""
"""Halt the execution of a different `spawn`ed thread."""
return KillEvent(coro)
# Convenience function for running socket servers.
def server(host, port, func):
"""A coroutine that runs a network server. Host and port specify the
listening address. func should be a coroutine that takes a single
parameter, a Connection object. The coroutine is invoked for every
incoming connection on the listening socket.
"""
def handler(conn):
try:
yield func(conn)
lib/beets/util/confit.py
-28
View File
@@ -1,28 +0,0 @@
# This file is part of beets.
# Copyright 2016-2019, Adrian Sampson.
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
import confuse
import warnings
warnings.warn("beets.util.confit is deprecated; use confuse instead")
# Import everything from the confuse module into this module.
for key, value in confuse.__dict__.items():
if key not in ['__name__']:
globals()[key] = value
# Cleanup namespace.
del key, value, warnings, confuse
lib/beets/util/enumeration.py
+1
View File
@@ -20,6 +20,7 @@ class OrderedEnum(Enum):
"""
An Enum subclass that allows comparison of members.
"""
def __ge__(self, other):
if self.__class__ is other.__class__:
return self.value >= other.value
lib/beets/util/functemplate.py
+105 -116
View File
@@ -27,22 +27,21 @@ engine like Jinja2 or Mustache.
"""
import re
import ast
import dis
import types
import sys
import functools
import re
import types
SYMBOL_DELIM = '$'
FUNC_DELIM = '%'
GROUP_OPEN = '{'
GROUP_CLOSE = '}'
ARG_SEP = ','
ESCAPE_CHAR = '$'
SYMBOL_DELIM = "$"
FUNC_DELIM = "%"
GROUP_OPEN = "{"
GROUP_CLOSE = "}"
ARG_SEP = ","
ESCAPE_CHAR = "$"
VARIABLE_PREFIX = '__var_'
FUNCTION_PREFIX = '__func_'
VARIABLE_PREFIX = "__var_"
FUNCTION_PREFIX = "__func_"
class Environment:
@@ -57,10 +56,6 @@ class Environment:
# Code generation helpers.
def ex_lvalue(name):
"""A variable load expression."""
return ast.Name(name, ast.Store())
def ex_rvalue(name):
"""A variable store expression."""
@@ -74,15 +69,6 @@ def ex_literal(val):
return ast.Constant(val)
def ex_varassign(name, expr):
"""Assign an expression into a single variable. The expression may
either be an `ast.expr` object or a value to be used as a literal.
"""
if not isinstance(expr, ast.expr):
expr = ex_literal(expr)
return ast.Assign([ex_lvalue(name)], expr)
def ex_call(func, args):
"""A function-call expression with only positional parameters. The
function may be an expression or the name of a function. Each
@@ -99,19 +85,18 @@ def ex_call(func, args):
return ast.Call(func, args, [])
def compile_func(arg_names, statements, name='_the_func', debug=False):
def compile_func(arg_names, statements, name="_the_func", debug=False):
"""Compile a list of statements as the body of a function and return
the resulting Python function. If `debug`, then print out the
bytecode of the compiled function.
"""
args_fields = {
'args': [ast.arg(arg=n, annotation=None) for n in arg_names],
'kwonlyargs': [],
'kw_defaults': [],
'defaults': [ex_literal(None) for _ in arg_names],
"args": [ast.arg(arg=n, annotation=None) for n in arg_names],
"kwonlyargs": [],
"kw_defaults": [],
"defaults": [ex_literal(None) for _ in arg_names],
}
if 'posonlyargs' in ast.arguments._fields: # Added in Python 3.8.
args_fields['posonlyargs'] = []
args_fields["posonlyargs"] = []
args = ast.arguments(**args_fields)
func_def = ast.FunctionDef(
@@ -123,14 +108,11 @@ def compile_func(arg_names, statements, name='_the_func', debug=False):
# The ast.Module signature changed in 3.8 to accept a list of types to
# ignore.
if sys.version_info >= (3, 8):
mod = ast.Module([func_def], [])
else:
mod = ast.Module([func_def])
mod = ast.Module([func_def], [])
ast.fix_missing_locations(mod)
prog = compile(mod, '<generated>', 'exec')
prog = compile(mod, "<generated>", "exec")
# Debug: show bytecode.
if debug:
@@ -146,6 +128,7 @@ def compile_func(arg_names, statements, name='_the_func', debug=False):
# AST nodes for the template language.
class Symbol:
"""A variable-substitution symbol in a template."""
@@ -154,7 +137,7 @@ class Symbol:
self.original = original
def __repr__(self):
return 'Symbol(%s)' % repr(self.ident)
return "Symbol(%s)" % repr(self.ident)
def evaluate(self, env):
"""Evaluate the symbol in the environment, returning a Unicode
@@ -183,8 +166,9 @@ class Call:
self.original = original
def __repr__(self):
return 'Call({}, {}, {})'.format(repr(self.ident), repr(self.args),
repr(self.original))
return "Call({}, {}, {})".format(
repr(self.ident), repr(self.args), repr(self.original)
)
def evaluate(self, env):
"""Evaluate the function call in the environment, returning a
@@ -197,7 +181,7 @@ class Call:
except Exception as exc:
# Function raised exception! Maybe inlining the name of
# the exception will help debug.
return '<%s>' % str(exc)
return "<%s>" % str(exc)
return str(out)
else:
return self.original
@@ -215,21 +199,22 @@ class Call:
# Create a subexpression that joins the result components of
# the arguments.
arg_exprs.append(ex_call(
ast.Attribute(ex_literal(''), 'join', ast.Load()),
[ex_call(
'map',
arg_exprs.append(
ex_call(
ast.Attribute(ex_literal(""), "join", ast.Load()),
[
ex_rvalue(str.__name__),
ast.List(subexprs, ast.Load()),
]
)],
))
ex_call(
"map",
[
ex_rvalue(str.__name__),
ast.List(subexprs, ast.Load()),
],
)
],
)
)
subexpr_call = ex_call(
FUNCTION_PREFIX + self.ident,
arg_exprs
)
subexpr_call = ex_call(FUNCTION_PREFIX + self.ident, arg_exprs)
return [subexpr_call], varnames, funcnames
@@ -242,7 +227,7 @@ class Expression:
self.parts = parts
def __repr__(self):
return 'Expression(%s)' % (repr(self.parts))
return "Expression(%s)" % (repr(self.parts))
def evaluate(self, env):
"""Evaluate the entire expression in the environment, returning
@@ -254,7 +239,7 @@ class Expression:
out.append(part)
else:
out.append(part.evaluate(env))
return ''.join(map(str, out))
return "".join(map(str, out))
def translate(self):
"""Compile the expression to a list of Python AST expressions, a
@@ -276,6 +261,7 @@ class Expression:
# Parser.
class ParseError(Exception):
pass
@@ -295,7 +281,7 @@ class Parser:
"""
def __init__(self, string, in_argument=False):
""" Create a new parser.
"""Create a new parser.
:param in_arguments: boolean that indicates the parser is to be
used for parsing function arguments, ie. considering commas
(`ARG_SEP`) a special character
@@ -306,10 +292,16 @@ class Parser:
self.parts = []
# Common parsing resources.
special_chars = (SYMBOL_DELIM, FUNC_DELIM, GROUP_OPEN, GROUP_CLOSE,
ESCAPE_CHAR)
special_char_re = re.compile(r'[%s]|\Z' %
''.join(re.escape(c) for c in special_chars))
special_chars = (
SYMBOL_DELIM,
FUNC_DELIM,
GROUP_OPEN,
GROUP_CLOSE,
ESCAPE_CHAR,
)
special_char_re = re.compile(
r"[%s]|\Z" % "".join(re.escape(c) for c in special_chars)
)
escapable_chars = (SYMBOL_DELIM, FUNC_DELIM, GROUP_CLOSE, ARG_SEP)
terminator_chars = (GROUP_CLOSE,)
@@ -326,9 +318,10 @@ class Parser:
if self.in_argument:
extra_special_chars = (ARG_SEP,)
special_char_re = re.compile(
r'[%s]|\Z' % ''.join(
re.escape(c) for c in
self.special_chars + extra_special_chars
r"[%s]|\Z"
% "".join(
re.escape(c)
for c in self.special_chars + extra_special_chars
)
)
@@ -341,10 +334,10 @@ class Parser:
# A non-special character. Skip to the next special
# character, treating the interstice as literal text.
next_pos = (
special_char_re.search(
self.string[self.pos:]).start() + self.pos
special_char_re.search(self.string[self.pos :]).start()
+ self.pos
)
text_parts.append(self.string[self.pos:next_pos])
text_parts.append(self.string[self.pos : next_pos])
self.pos = next_pos
continue
@@ -358,8 +351,9 @@ class Parser:
break
next_char = self.string[self.pos + 1]
if char == ESCAPE_CHAR and next_char in (self.escapable_chars +
extra_special_chars):
if char == ESCAPE_CHAR and next_char in (
self.escapable_chars + extra_special_chars
):
# An escaped special character ($$, $}, etc.). Note that
# ${ is not an escape sequence: this is ambiguous with
# the start of a symbol and it's not necessary (just
@@ -370,7 +364,7 @@ class Parser:
# Shift all characters collected so far into a single string.
if text_parts:
self.parts.append(''.join(text_parts))
self.parts.append("".join(text_parts))
text_parts = []
if char == SYMBOL_DELIM:
@@ -392,7 +386,7 @@ class Parser:
# If any parsed characters remain, shift them into a string.
if text_parts:
self.parts.append(''.join(text_parts))
self.parts.append("".join(text_parts))
def parse_symbol(self):
"""Parse a variable reference (like ``$foo`` or ``${foo}``)
@@ -419,21 +413,23 @@ class Parser:
closer = self.string.find(GROUP_CLOSE, self.pos)
if closer == -1 or closer == self.pos:
# No closing brace found or identifier is empty.
self.parts.append(self.string[start_pos:self.pos])
self.parts.append(self.string[start_pos : self.pos])
else:
# Closer found.
ident = self.string[self.pos:closer]
ident = self.string[self.pos : closer]
self.pos = closer + 1
self.parts.append(Symbol(ident,
self.string[start_pos:self.pos]))
self.parts.append(
Symbol(ident, self.string[start_pos : self.pos])
)
else:
# A bare-word symbol.
ident = self._parse_ident()
if ident:
# Found a real symbol.
self.parts.append(Symbol(ident,
self.string[start_pos:self.pos]))
self.parts.append(
Symbol(ident, self.string[start_pos : self.pos])
)
else:
# A standalone $.
self.parts.append(SYMBOL_DELIM)
@@ -457,25 +453,24 @@ class Parser:
if self.pos >= len(self.string):
# Identifier terminates string.
self.parts.append(self.string[start_pos:self.pos])
self.parts.append(self.string[start_pos : self.pos])
return
if self.string[self.pos] != GROUP_OPEN:
# Argument list not opened.
self.parts.append(self.string[start_pos:self.pos])
self.parts.append(self.string[start_pos : self.pos])
return
# Skip past opening brace and try to parse an argument list.
self.pos += 1
args = self.parse_argument_list()
if self.pos >= len(self.string) or \
self.string[self.pos] != GROUP_CLOSE:
if self.pos >= len(self.string) or self.string[self.pos] != GROUP_CLOSE:
# Arguments unclosed.
self.parts.append(self.string[start_pos:self.pos])
self.parts.append(self.string[start_pos : self.pos])
return
self.pos += 1 # Move past closing brace.
self.parts.append(Call(ident, args, self.string[start_pos:self.pos]))
self.parts.append(Call(ident, args, self.string[start_pos : self.pos]))
def parse_argument_list(self):
"""Parse a list of arguments starting at ``pos``, returning a
@@ -487,15 +482,17 @@ class Parser:
expressions = []
while self.pos < len(self.string):
subparser = Parser(self.string[self.pos:], in_argument=True)
subparser = Parser(self.string[self.pos :], in_argument=True)
subparser.parse_expression()
# Extract and advance past the parsed expression.
expressions.append(Expression(subparser.parts))
self.pos += subparser.pos
if self.pos >= len(self.string) or \
self.string[self.pos] == GROUP_CLOSE:
if (
self.pos >= len(self.string)
or self.string[self.pos] == GROUP_CLOSE
):
# Argument list terminated by EOF or closing brace.
break
@@ -510,8 +507,8 @@ class Parser:
"""Parse an identifier and return it (possibly an empty string).
Updates ``pos``.
"""
remainder = self.string[self.pos:]
ident = re.match(r'\w*', remainder).group(0)
remainder = self.string[self.pos :]
ident = re.match(r"\w*", remainder).group(0)
self.pos += len(ident)
return ident
@@ -524,32 +521,20 @@ def _parse(template):
parser.parse_expression()
parts = parser.parts
remainder = parser.string[parser.pos:]
remainder = parser.string[parser.pos :]
if remainder:
parts.append(remainder)
return Expression(parts)
def cached(func):
"""Like the `functools.lru_cache` decorator, but works (as a no-op)
on Python < 3.2.
"""
if hasattr(functools, 'lru_cache'):
return functools.lru_cache(maxsize=128)(func)
else:
# Do nothing when lru_cache is not available.
return func
@cached
@functools.lru_cache(maxsize=128)
def template(fmt):
return Template(fmt)
# External interface.
class Template:
"""A string template, including text, Symbols, and Calls.
"""
"""A string template, including text, Symbols, and Calls."""
def __init__(self, template):
self.expr = _parse(template)
@@ -568,8 +553,7 @@ class Template:
return self.expr.evaluate(Environment(values, functions))
def substitute(self, values={}, functions={}):
"""Evaluate the template given the values and functions.
"""
"""Evaluate the template given the values and functions."""
try:
res = self.compiled(values, functions)
except Exception: # Handle any exceptions thrown by compiled version.
@@ -599,24 +583,29 @@ class Template:
for funcname in funcnames:
args[FUNCTION_PREFIX + funcname] = functions[funcname]
parts = func(**args)
return ''.join(parts)
return "".join(parts)
return wrapper_func
# Performance tests.
if __name__ == '__main__':
if __name__ == "__main__":
import timeit
_tmpl = Template('foo $bar %baz{foozle $bar barzle} $bar')
_vars = {'bar': 'qux'}
_funcs = {'baz': str.upper}
interp_time = timeit.timeit('_tmpl.interpret(_vars, _funcs)',
'from __main__ import _tmpl, _vars, _funcs',
number=10000)
_tmpl = Template("foo $bar %baz{foozle $bar barzle} $bar")
_vars = {"bar": "qux"}
_funcs = {"baz": str.upper}
interp_time = timeit.timeit(
"_tmpl.interpret(_vars, _funcs)",
"from __main__ import _tmpl, _vars, _funcs",
number=10000,
)
print(interp_time)
comp_time = timeit.timeit('_tmpl.substitute(_vars, _funcs)',
'from __main__ import _tmpl, _vars, _funcs',
number=10000)
comp_time = timeit.timeit(
"_tmpl.substitute(_vars, _funcs)",
"from __main__ import _tmpl, _vars, _funcs",
number=10000,
)
print(comp_time)
print('Speedup:', interp_time / comp_time)
print("Speedup:", interp_time / comp_time)
lib/beets/util/hidden.py
+30 -51
View File
@@ -1,5 +1,6 @@
# This file is part of beets.
# Copyright 2016, Adrian Sampson.
# Copyright 2024, Arav K.
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
@@ -14,71 +15,49 @@
"""Simple library to work out if a file is hidden on different platforms."""
import ctypes
import os
import stat
import ctypes
import sys
import beets.util
from pathlib import Path
from typing import Union
def _is_hidden_osx(path):
"""Return whether or not a file is hidden on OS X.
This uses os.lstat to work out if a file has the "hidden" flag.
def is_hidden(path: Union[bytes, Path]) -> bool:
"""
file_stat = os.lstat(beets.util.syspath(path))
if hasattr(file_stat, 'st_flags') and hasattr(stat, 'UF_HIDDEN'):
return bool(file_stat.st_flags & stat.UF_HIDDEN)
else:
return False
def _is_hidden_win(path):
"""Return whether or not a file is hidden on Windows.
This uses GetFileAttributes to work out if a file has the "hidden" flag
(FILE_ATTRIBUTE_HIDDEN).
Determine whether the given path is treated as a 'hidden file' by the OS.
"""
# FILE_ATTRIBUTE_HIDDEN = 2 (0x2) from GetFileAttributes documentation.
hidden_mask = 2
# Retrieve the attributes for the file.
attrs = ctypes.windll.kernel32.GetFileAttributesW(beets.util.syspath(path))
if isinstance(path, bytes):
path = Path(os.fsdecode(path))
# Ensure we have valid attribues and compare them against the mask.
return attrs >= 0 and attrs & hidden_mask
# TODO: Avoid doing a platform check on every invocation of the function.
# TODO: Stop supporting 'bytes' inputs once 'pathlib' is fully integrated.
if sys.platform == "win32":
# On Windows, we check for an FS-provided attribute.
def _is_hidden_dot(path):
"""Return whether or not a file starts with a dot.
# FILE_ATTRIBUTE_HIDDEN = 2 (0x2) from GetFileAttributes documentation.
hidden_mask = 2
Files starting with a dot are seen as "hidden" files on Unix-based OSes.
"""
return os.path.basename(path).startswith(b'.')
# Retrieve the attributes for the file.
attrs = ctypes.windll.kernel32.GetFileAttributesW(str(path))
# Ensure the attribute mask is valid.
if attrs < 0:
return False
def is_hidden(path):
"""Return whether or not a file is hidden. `path` should be a
bytestring filename.
# Check for the hidden attribute.
return attrs & hidden_mask
This method works differently depending on the platform it is called on.
# On OS X, we check for an FS-provided attribute.
if sys.platform == "darwin":
if hasattr(os.stat_result, "st_flags") and hasattr(stat, "UF_HIDDEN"):
if path.lstat().st_flags & stat.UF_HIDDEN:
return True
On OS X, it uses both the result of `is_hidden_osx` and `is_hidden_dot` to
work out if a file is hidden.
# On all non-Windows platforms, we check for a '.'-prefixed file name.
if path.name.startswith("."):
return True
On Windows, it uses the result of `is_hidden_win` to work out if a file is
hidden.
On any other operating systems (i.e. Linux), it uses `is_hidden_dot` to
work out if a file is hidden.
"""
# Run platform specific functions depending on the platform
if sys.platform == 'darwin':
return _is_hidden_osx(path) or _is_hidden_dot(path)
elif sys.platform == 'win32':
return _is_hidden_win(path)
else:
return _is_hidden_dot(path)
__all__ = ['is_hidden']
return False
lib/beets/util/id_extractors.py
+65
View File
@@ -0,0 +1,65 @@
# This file is part of beets.
# Copyright 2016, Adrian Sampson.
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
"""Helpers around the extraction of album/track ID's from metadata sources."""
import re
# Spotify IDs consist of 22 alphanumeric characters
# (zero-left-padded base62 representation of randomly generated UUID4)
spotify_id_regex = {
"pattern": r"(^|open\.spotify\.com/{}/)([0-9A-Za-z]{{22}})",
"match_group": 2,
}
deezer_id_regex = {
"pattern": r"(^|deezer\.com/)([a-z]*/)?({}/)?(\d+)",
"match_group": 4,
}
beatport_id_regex = {
"pattern": r"(^|beatport\.com/release/.+/)(\d+)$",
"match_group": 2,
}
# A note on Bandcamp: There is no such thing as a Bandcamp album or artist ID,
# the URL can be used as the identifier. The Bandcamp metadata source plugin
# works that way - https://github.com/snejus/beetcamp. Bandcamp album
# URLs usually look like: https://nameofartist.bandcamp.com/album/nameofalbum
def extract_discogs_id_regex(album_id):
"""Returns the Discogs_id or None."""
# Discogs-IDs are simple integers. In order to avoid confusion with
# other metadata plugins, we only look for very specific formats of the
# input string:
# - plain integer, optionally wrapped in brackets and prefixed by an
# 'r', as this is how discogs displays the release ID on its webpage.
# - legacy url format: discogs.com/<name of release>/release/<id>
# - legacy url short format: discogs.com/release/<id>
# - current url format: discogs.com/release/<id>-<name of release>
# See #291, #4080 and #4085 for the discussions leading up to these
# patterns.
# Regex has been tested here https://regex101.com/r/TOu7kw/1
for pattern in [
r"^\[?r?(?P<id>\d+)\]?$",
r"discogs\.com/release/(?P<id>\d+)-?",
r"discogs\.com/[^/]+/release/(?P<id>\d+)",
]:
match = re.search(pattern, album_id)
if match:
return int(match.group("id"))
return None
lib/beets/util/m3u.py
+97
View File
@@ -0,0 +1,97 @@
# This file is part of beets.
# Copyright 2022, J0J0 Todos.
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
"""Provides utilities to read, write and manipulate m3u playlist files."""
import traceback
from beets.util import FilesystemError, mkdirall, normpath, syspath
class EmptyPlaylistError(Exception):
"""Raised when a playlist file without media files is saved or loaded."""
pass
class M3UFile:
"""Reads and writes m3u or m3u8 playlist files."""
def __init__(self, path):
"""``path`` is the absolute path to the playlist file.
The playlist file type, m3u or m3u8 is determined by 1) the ending
being m3u8 and 2) the file paths contained in the list being utf-8
encoded. Since the list is passed from the outside, this is currently
out of control of this class.
"""
self.path = path
self.extm3u = False
self.media_list = []
def load(self):
"""Reads the m3u file from disk and sets the object's attributes."""
pl_normpath = normpath(self.path)
try:
with open(syspath(pl_normpath), "rb") as pl_file:
raw_contents = pl_file.readlines()
except OSError as exc:
raise FilesystemError(
exc, "read", (pl_normpath,), traceback.format_exc()
)
self.extm3u = True if raw_contents[0].rstrip() == b"#EXTM3U" else False
for line in raw_contents[1:]:
if line.startswith(b"#"):
# Support for specific EXTM3U comments could be added here.
continue
self.media_list.append(normpath(line.rstrip()))
if not self.media_list:
raise EmptyPlaylistError
def set_contents(self, media_list, extm3u=True):
"""Sets self.media_list to a list of media file paths.
Also sets additional flags, changing the final m3u-file's format.
``media_list`` is a list of paths to media files that should be added
to the playlist (relative or absolute paths, that's the responsibility
of the caller). By default the ``extm3u`` flag is set, to ensure a
save-operation writes an m3u-extended playlist (comment "#EXTM3U" at
the top of the file).
"""
self.media_list = media_list
self.extm3u = extm3u
def write(self):
"""Writes the m3u file to disk.
Handles the creation of potential parent directories.
"""
header = [b"#EXTM3U"] if self.extm3u else []
if not self.media_list:
raise EmptyPlaylistError
contents = header + self.media_list
pl_normpath = normpath(self.path)
mkdirall(pl_normpath)
try:
with open(syspath(pl_normpath), "wb") as pl_file:
for line in contents:
pl_file.write(line + b"\n")
pl_file.write(b"\n") # Final linefeed to prevent noeol file.
except OSError as exc:
raise FilesystemError(
exc, "create", (pl_normpath,), traceback.format_exc()
)
lib/beets/util/pipeline.py
+29 -28
View File
@@ -33,11 +33,11 @@ in place of any single coroutine.
import queue
from threading import Thread, Lock
import sys
from threading import Lock, Thread
BUBBLE = '__PIPELINE_BUBBLE__'
POISON = '__PIPELINE_POISON__'
BUBBLE = "__PIPELINE_BUBBLE__"
POISON = "__PIPELINE_POISON__"
DEFAULT_QUEUE_SIZE = 16
@@ -48,6 +48,7 @@ def _invalidate_queue(q, val=None, sync=True):
which defaults to None. `sync` controls whether a lock is
required (because it's not reentrant!).
"""
def _qsize(len=len):
return 1
@@ -75,8 +76,8 @@ def _invalidate_queue(q, val=None, sync=True):
q._qsize = _qsize
q._put = _put
q._get = _get
q.not_empty.notifyAll()
q.not_full.notifyAll()
q.not_empty.notify_all()
q.not_full.notify_all()
finally:
if sync:
@@ -168,6 +169,7 @@ def stage(func):
while True:
task = yield task
task = func(*(args + (task,)))
return coro
@@ -191,6 +193,7 @@ def mutator_stage(func):
while True:
task = yield task
func(*(args + (task,)))
return coro
@@ -218,20 +221,18 @@ class PipelineThread(Thread):
self.exc_info = None
def abort(self):
"""Shut down the thread at the next chance possible.
"""
"""Shut down the thread at the next chance possible."""
with self.abort_lock:
self.abort_flag = True
# Ensure that we are not blocking on a queue read or write.
if hasattr(self, 'in_queue'):
if hasattr(self, "in_queue"):
_invalidate_queue(self.in_queue, POISON)
if hasattr(self, 'out_queue'):
if hasattr(self, "out_queue"):
_invalidate_queue(self.out_queue, POISON)
def abort_all(self, exc_info):
"""Abort all other threads in the system for an exception.
"""
"""Abort all other threads in the system for an exception."""
self.exc_info = exc_info
for thread in self.all_threads:
thread.abort()
@@ -373,7 +374,7 @@ class Pipeline:
be at least two stages.
"""
if len(stages) < 2:
raise ValueError('pipeline must have at least two stages')
raise ValueError("pipeline must have at least two stages")
self.stages = []
for stage in stages:
if isinstance(stage, (list, tuple)):
@@ -405,15 +406,15 @@ class Pipeline:
# Middle stages.
for i in range(1, queue_count):
for coro in self.stages[i]:
threads.append(MiddlePipelineThread(
coro, queues[i - 1], queues[i], threads
))
threads.append(
MiddlePipelineThread(
coro, queues[i - 1], queues[i], threads
)
)
# Last stage.
for coro in self.stages[-1]:
threads.append(
LastPipelineThread(coro, queues[-1], threads)
)
threads.append(LastPipelineThread(coro, queues[-1], threads))
# Start threads.
for thread in threads:
@@ -472,21 +473,21 @@ class Pipeline:
# Smoke test.
if __name__ == '__main__':
if __name__ == "__main__":
import time
# Test a normally-terminating pipeline both in sequence and
# in parallel.
def produce():
for i in range(5):
print('generating %i' % i)
print("generating %i" % i)
time.sleep(1)
yield i
def work():
num = yield
while True:
print('processing %i' % num)
print("processing %i" % num)
time.sleep(2)
num = yield num * 2
@@ -494,7 +495,7 @@ if __name__ == '__main__':
while True:
num = yield
time.sleep(1)
print('received %i' % num)
print("received %i" % num)
ts_start = time.time()
Pipeline([produce(), work(), consume()]).run_sequential()
@@ -503,22 +504,22 @@ if __name__ == '__main__':
ts_par = time.time()
Pipeline([produce(), (work(), work()), consume()]).run_parallel()
ts_end = time.time()
print('Sequential time:', ts_seq - ts_start)
print('Parallel time:', ts_par - ts_seq)
print('Multiply-parallel time:', ts_end - ts_par)
print("Sequential time:", ts_seq - ts_start)
print("Parallel time:", ts_par - ts_seq)
print("Multiply-parallel time:", ts_end - ts_par)
print()
# Test a pipeline that raises an exception.
def exc_produce():
for i in range(10):
print('generating %i' % i)
print("generating %i" % i)
time.sleep(1)
yield i
def exc_work():
num = yield
while True:
print('processing %i' % num)
print("processing %i" % num)
time.sleep(3)
if num == 3:
raise Exception()
@@ -527,6 +528,6 @@ if __name__ == '__main__':
def exc_consume():
while True:
num = yield
print('received %i' % num)
print("received %i" % num)
Pipeline([exc_produce(), exc_work(), exc_consume()]).run_parallel(1)