gronod/headphones
1
0
Fork 0
mirror of https://github.com/rembo10/headphones.git synced 2026-07-22 08:53:59 +01:00
Code Issues Packages Projects Releases Wiki Activity

mutagen, musicbrainzngs, beets lib updates

Browse Source
This commit is contained in:
Ade
2017-03-25 11:47:04 +13:00
parent f3020207c0
commit 4095bf213e
110 changed files with 17022 additions and 6397 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/beets/util/__init__.py
Regular → Executable
+415 -108
View File
@@ -1,5 +1,6 @@
# -*- coding: utf-8 -*-
# This file is part of beets.
# Copyright 2013, Adrian Sampson.
# Copyright 2016, Adrian Sampson.
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
@@ -13,21 +14,28 @@
# included in all copies or substantial portions of the Software.
"""Miscellaneous utility functions."""
from __future__ import division
from __future__ import division, absolute_import, print_function
import os
import sys
import errno
import locale
import re
import shutil
import fnmatch
from collections import defaultdict
from collections import Counter
import traceback
import subprocess
import platform
import shlex
from beets.util import hidden
import six
from unidecode import unidecode
MAX_FILENAME_LENGTH = 200
WINDOWS_MAGIC_PREFIX = u'\\\\?\\'
SNI_SUPPORTED = sys.version_info >= (2, 7, 9)
class HumanReadableException(Exception):
@@ -54,22 +62,22 @@ class HumanReadableException(Exception):
def _gerund(self):
"""Generate a (likely) gerund form of the English verb.
"""
if ' ' in self.verb:
if u' ' in self.verb:
return self.verb
gerund = self.verb[:-1] if self.verb.endswith('e') else self.verb
gerund += 'ing'
gerund = self.verb[:-1] if self.verb.endswith(u'e') else self.verb
gerund += u'ing'
return gerund
def _reasonstr(self):
"""Get the reason as a string."""
if isinstance(self.reason, unicode):
if isinstance(self.reason, six.text_type):
return self.reason
elif isinstance(self.reason, basestring): # Byte string.
return self.reason.decode('utf8', 'ignore')
elif isinstance(self.reason, bytes):
return self.reason.decode('utf-8', 'ignore')
elif hasattr(self.reason, 'strerror'): # i.e., EnvironmentError
return self.reason.strerror
else:
return u'"{0}"'.format(unicode(self.reason))
return u'"{0}"'.format(six.text_type(self.reason))
def get_message(self):
"""Create the human-readable description of the error, sans
@@ -83,7 +91,7 @@ class HumanReadableException(Exception):
"""
if self.tb:
logger.debug(self.tb)
logger.error(u'{0}: {1}'.format(self.error_kind, self.args[0]))
logger.error(u'{0}: {1}', self.error_kind, self.args[0])
class FilesystemError(HumanReadableException):
@@ -149,21 +157,22 @@ def ancestry(path):
return out
def sorted_walk(path, ignore=(), logger=None):
def sorted_walk(path, ignore=(), ignore_hidden=False, logger=None):
"""Like `os.walk`, but yields things in case-insensitive sorted,
breadth-first order. Directory and file names matching any glob
pattern in `ignore` are skipped. If `logger` is provided, then
warning messages are logged there when a directory cannot be listed.
"""
# Make sure the path isn't a Unicode string.
# Make sure the pathes aren't Unicode strings.
path = bytestring_path(path)
ignore = [bytestring_path(i) for i in ignore]
# Get all the directories and files at this level.
try:
contents = os.listdir(syspath(path))
except OSError as exc:
if logger:
logger.warn(u'could not list directory {0}: {1}'.format(
logger.warning(u'could not list directory {0}: {1}'.format(
displayable_path(path), exc.strerror
))
return
@@ -183,10 +192,11 @@ def sorted_walk(path, ignore=(), logger=None):
# Add to output as either a file or a directory.
cur = os.path.join(path, base)
if os.path.isdir(syspath(cur)):
dirs.append(base)
else:
files.append(base)
if (ignore_hidden and not hidden.is_hidden(cur)) or not ignore_hidden:
if os.path.isdir(syspath(cur)):
dirs.append(base)
else:
files.append(base)
# Sort lists (case-insensitive) and yield the current level.
dirs.sort(key=bytes.lower)
@@ -197,7 +207,7 @@ def sorted_walk(path, ignore=(), logger=None):
for base in dirs:
cur = os.path.join(path, base)
# yield from sorted_walk(...)
for res in sorted_walk(cur, ignore, logger):
for res in sorted_walk(cur, ignore, ignore_hidden, logger):
yield res
@@ -260,7 +270,9 @@ def prune_dirs(path, root=None, clutter=('.DS_Store', 'Thumbs.db')):
if not os.path.exists(directory):
# Directory gone already.
continue
if fnmatch_all(os.listdir(directory), clutter):
clutter = [bytestring_path(c) for c in clutter]
match_paths = [bytestring_path(d) for d in os.listdir(directory)]
if fnmatch_all(match_paths, clutter):
# Directory contains only clutter (or nothing).
try:
shutil.rmtree(directory)
@@ -294,6 +306,18 @@ def components(path):
return comps
def arg_encoding():
"""Get the encoding for command-line arguments (and other OS
locale-sensitive strings).
"""
try:
return locale.getdefaultlocale()[1] or 'utf-8'
except ValueError:
# Invalid locale environment variable setting. To avoid
# failing entirely for no good reason, assume UTF-8.
return 'utf-8'
def _fsencoding():
"""Get the system's filesystem encoding. On Windows, this is always
UTF-8 (not MBCS).
@@ -305,16 +329,16 @@ def _fsencoding():
# for Windows paths, so the encoding is actually immaterial so
# we can avoid dealing with this nastiness. We arbitrarily
# choose UTF-8.
encoding = 'utf8'
encoding = 'utf-8'
return encoding
def bytestring_path(path):
"""Given a path, which is either a str or a unicode, returns a str
"""Given a path, which is either a bytes or a unicode, returns a str
path (ensuring that we never deal with Unicode pathnames).
"""
# Pass through bytestrings.
if isinstance(path, str):
if isinstance(path, bytes):
return path
# On Windows, remove the magic prefix added by `syspath`. This makes
@@ -323,11 +347,14 @@ def bytestring_path(path):
if os.path.__name__ == 'ntpath' and path.startswith(WINDOWS_MAGIC_PREFIX):
path = path[len(WINDOWS_MAGIC_PREFIX):]
# Try to encode with default encodings, but fall back to UTF8.
# Try to encode with default encodings, but fall back to utf-8.
try:
return path.encode(_fsencoding())
except (UnicodeError, LookupError):
return path.encode('utf8')
return path.encode('utf-8')
PATH_SEP = bytestring_path(os.sep)
def displayable_path(path, separator=u'; '):
@@ -337,16 +364,16 @@ def displayable_path(path, separator=u'; '):
"""
if isinstance(path, (list, tuple)):
return separator.join(displayable_path(p) for p in path)
elif isinstance(path, unicode):
elif isinstance(path, six.text_type):
return path
elif not isinstance(path, str):
elif not isinstance(path, bytes):
# A non-string object: just get its unicode representation.
return unicode(path)
return six.text_type(path)
try:
return path.decode(_fsencoding(), 'ignore')
except (UnicodeError, LookupError):
return path.decode('utf8', 'ignore')
return path.decode('utf-8', 'ignore')
def syspath(path, prefix=True):
@@ -360,12 +387,12 @@ def syspath(path, prefix=True):
if os.path.__name__ != 'ntpath':
return path
if not isinstance(path, unicode):
if not isinstance(path, six.text_type):
# Beets currently represents Windows paths internally with UTF-8
# arbitrarily. But earlier versions used MBCS because it is
# reported as the FS encoding by Windows. Try both.
try:
path = path.decode('utf8')
path = path.decode('utf-8')
except UnicodeError:
# The encoding should always be MBCS, Windows' broken
# Unicode representation.
@@ -412,7 +439,7 @@ def copy(path, dest, replace=False):
path = syspath(path)
dest = syspath(dest)
if not replace and os.path.exists(dest):
raise FilesystemError('file exists', 'copy', (path, dest))
raise FilesystemError(u'file exists', 'copy', (path, dest))
try:
shutil.copyfile(path, dest)
except (OSError, IOError) as exc:
@@ -433,8 +460,7 @@ def move(path, dest, replace=False):
path = syspath(path)
dest = syspath(dest)
if os.path.exists(dest) and not replace:
raise FilesystemError('file exists', 'rename', (path, dest),
traceback.format_exc())
raise FilesystemError(u'file exists', 'rename', (path, dest))
# First, try renaming the file.
try:
@@ -452,23 +478,52 @@ def move(path, dest, replace=False):
def link(path, dest, replace=False):
"""Create a symbolic link from path to `dest`. Raises an OSError if
`dest` already exists, unless `replace` is True. Does nothing if
`path` == `dest`."""
if (samefile(path, dest)):
`path` == `dest`.
"""
if samefile(path, dest):
return
path = syspath(path)
dest = syspath(dest)
if os.path.exists(dest) and not replace:
raise FilesystemError('file exists', 'rename', (path, dest),
traceback.format_exc())
if os.path.exists(syspath(dest)) and not replace:
raise FilesystemError(u'file exists', 'rename', (path, dest))
try:
os.symlink(path, dest)
except OSError:
raise FilesystemError('Operating system does not support symbolic '
'links.', 'link', (path, dest),
os.symlink(syspath(path), syspath(dest))
except NotImplementedError:
# raised on python >= 3.2 and Windows versions before Vista
raise FilesystemError(u'OS does not support symbolic links.'
'link', (path, dest), traceback.format_exc())
except OSError as exc:
# TODO: Windows version checks can be removed for python 3
if hasattr('sys', 'getwindowsversion'):
if sys.getwindowsversion()[0] < 6: # is before Vista
exc = u'OS does not support symbolic links.'
raise FilesystemError(exc, 'link', (path, dest),
traceback.format_exc())
def hardlink(path, dest, replace=False):
"""Create a hard link from path to `dest`. Raises an OSError if
`dest` already exists, unless `replace` is True. Does nothing if
`path` == `dest`.
"""
if samefile(path, dest):
return
if os.path.exists(syspath(dest)) and not replace:
raise FilesystemError(u'file exists', 'rename', (path, dest))
try:
os.link(syspath(path), syspath(dest))
except NotImplementedError:
raise FilesystemError(u'OS does not support hard links.'
'link', (path, dest), traceback.format_exc())
except OSError as exc:
if exc.errno == errno.EXDEV:
raise FilesystemError(u'Cannot hard link across devices.'
'link', (path, dest), traceback.format_exc())
else:
raise FilesystemError(exc, 'link', (path, dest),
traceback.format_exc())
def unique_path(path):
"""Returns a version of ``path`` that does not exist on the
filesystem. Specifically, if ``path` itself already exists, then
@@ -478,7 +533,7 @@ def unique_path(path):
return path
base, ext = os.path.splitext(path)
match = re.search(r'\.(\d)+$', base)
match = re.search(br'\.(\d)+$', base)
if match:
num = int(match.group(1))
base = base[:match.start()]
@@ -486,7 +541,8 @@ def unique_path(path):
num = 0
while True:
num += 1
new_path = '%s.%i%s' % (base, num, ext)
suffix = u'.{}'.format(num).encode() + ext
new_path = base + suffix
if not os.path.exists(new_path):
return new_path
@@ -495,12 +551,12 @@ def unique_path(path):
# shares, which are sufficiently common as to cause frequent problems.
# http://msdn.microsoft.com/en-us/library/windows/desktop/aa365247.aspx
CHAR_REPLACE = [
(re.compile(ur'[\\/]'), u'_'), # / and \ -- forbidden everywhere.
(re.compile(ur'^\.'), u'_'), # Leading dot (hidden files on Unix).
(re.compile(ur'[\x00-\x1f]'), u''), # Control characters.
(re.compile(ur'[<>:"\?\*\|]'), u'_'), # Windows "reserved characters".
(re.compile(ur'\.$'), u'_'), # Trailing dots.
(re.compile(ur'\s+$'), u''), # Trailing whitespace.
(re.compile(r'[\\/]'), u'_'), # / and \ -- forbidden everywhere.
(re.compile(r'^\.'), u'_'), # Leading dot (hidden files on Unix).
(re.compile(r'[\x00-\x1f]'), u''), # Control characters.
(re.compile(r'[<>:"\?\*\|]'), u'_'), # Windows "reserved characters".
(re.compile(r'\.$'), u'_'), # Trailing dots.
(re.compile(r'\s+$'), u''), # Trailing whitespace.
]
@@ -542,73 +598,142 @@ def truncate_path(path, length=MAX_FILENAME_LENGTH):
return os.path.join(*out)
def _legalize_stage(path, replacements, length, extension, fragment):
"""Perform a single round of path legalization steps
(sanitation/replacement, encoding from Unicode to bytes,
extension-appending, and truncation). Return the path (Unicode if
`fragment` is set, `bytes` otherwise) and whether truncation was
required.
"""
# Perform an initial sanitization including user replacements.
path = sanitize_path(path, replacements)
# Encode for the filesystem.
if not fragment:
path = bytestring_path(path)
# Preserve extension.
path += extension.lower()
# Truncate too-long components.
pre_truncate_path = path
path = truncate_path(path, length)
return path, path != pre_truncate_path
def legalize_path(path, replacements, length, extension, fragment):
"""Given a path-like Unicode string, produce a legal path. Return
the path and a flag indicating whether some replacements had to be
ignored (see below).
The legalization process (see `_legalize_stage`) consists of
applying the sanitation rules in `replacements`, encoding the string
to bytes (unless `fragment` is set), truncating components to
`length`, appending the `extension`.
This function performs up to three calls to `_legalize_stage` in
case truncation conflicts with replacements (as can happen when
truncation creates whitespace at the end of the string, for
example). The limited number of iterations iterations avoids the
possibility of an infinite loop of sanitation and truncation
operations, which could be caused by replacement rules that make the
string longer. The flag returned from this function indicates that
the path has to be truncated twice (indicating that replacements
made the string longer again after it was truncated); the
application should probably log some sort of warning.
"""
if fragment:
# Outputting Unicode.
extension = extension.decode('utf-8', 'ignore')
first_stage_path, _ = _legalize_stage(
path, replacements, length, extension, fragment
)
# Convert back to Unicode with extension removed.
first_stage_path, _ = os.path.splitext(displayable_path(first_stage_path))
# Re-sanitize following truncation (including user replacements).
second_stage_path, retruncated = _legalize_stage(
first_stage_path, replacements, length, extension, fragment
)
# If the path was once again truncated, discard user replacements
# and run through one last legalization stage.
if retruncated:
second_stage_path, _ = _legalize_stage(
first_stage_path, None, length, extension, fragment
)
return second_stage_path, retruncated
def py3_path(path):
"""Convert a bytestring path to Unicode on Python 3 only. On Python
2, return the bytestring path unchanged.
This helps deal with APIs on Python 3 that *only* accept Unicode
(i.e., `str` objects). I philosophically disagree with this
decision, because paths are sadly bytes on Unix, but that's the way
it is. So this function helps us "smuggle" the true bytes data
through APIs that took Python 3's Unicode mandate too seriously.
"""
if isinstance(path, six.text_type):
return path
assert isinstance(path, bytes)
if six.PY2:
return path
return os.fsdecode(path)
def str2bool(value):
"""Returns a boolean reflecting a human-entered string."""
if value.lower() in ('yes', '1', 'true', 't', 'y'):
return True
else:
return False
return value.lower() in (u'yes', u'1', u'true', u't', u'y')
def as_string(value):
"""Convert a value to a Unicode object for matching with a query.
None becomes the empty string. Bytestrings are silently decoded.
"""
if six.PY2:
buffer_types = buffer, memoryview # noqa: F821
else:
buffer_types = memoryview
if value is None:
return u''
elif isinstance(value, buffer):
return str(value).decode('utf8', 'ignore')
elif isinstance(value, str):
return value.decode('utf8', 'ignore')
elif isinstance(value, buffer_types):
return bytes(value).decode('utf-8', 'ignore')
elif isinstance(value, bytes):
return value.decode('utf-8', 'ignore')
else:
return unicode(value)
return six.text_type(value)
def levenshtein(s1, s2):
"""A nice DP edit distance implementation from Wikibooks:
http://en.wikibooks.org/wiki/Algorithm_implementation/Strings/
Levenshtein_distance#Python
def text_string(value, encoding='utf-8'):
"""Convert a string, which can either be bytes or unicode, to
unicode.
Text (unicode) is left untouched; bytes are decoded. This is useful
to convert from a "native string" (bytes on Python 2, str on Python
3) to a consistently unicode value.
"""
if len(s1) < len(s2):
return levenshtein(s2, s1)
if not s1:
return len(s2)
previous_row = xrange(len(s2) + 1)
for i, c1 in enumerate(s1):
current_row = [i + 1]
for j, c2 in enumerate(s2):
insertions = previous_row[j + 1] + 1
deletions = current_row[j] + 1
substitutions = previous_row[j] + (c1 != c2)
current_row.append(min(insertions, deletions, substitutions))
previous_row = current_row
return previous_row[-1]
if isinstance(value, bytes):
return value.decode(encoding)
return value
def plurality(objs):
"""Given a sequence of comparable objects, returns the object that
is most common in the set and the frequency of that object. The
"""Given a sequence of hashble objects, returns the object that
is most common in the set and the its number of appearance. The
sequence must contain at least one object.
"""
# Calculate frequencies.
freqs = defaultdict(int)
for obj in objs:
freqs[obj] += 1
if not freqs:
raise ValueError('sequence must be non-empty')
# Find object with maximum frequency.
max_freq = 0
res = None
for obj, freq in freqs.items():
if freq > max_freq:
max_freq = freq
res = obj
return res, max_freq
c = Counter(objs)
if not c:
raise ValueError(u'sequence must be non-empty')
return c.most_common(1)[0]
def cpu_count():
@@ -624,8 +749,8 @@ def cpu_count():
num = 0
elif sys.platform == 'darwin':
try:
num = int(command_output(['sysctl', '-n', 'hw.ncpu']))
except ValueError:
num = int(command_output(['/usr/sbin/sysctl', '-n', 'hw.ncpu']))
except (ValueError, OSError, subprocess.CalledProcessError):
num = 0
else:
try:
@@ -638,21 +763,40 @@ def cpu_count():
return 1
def convert_command_args(args):
"""Convert command arguments to bytestrings on Python 2 and
surrogate-escaped strings on Python 3."""
assert isinstance(args, list)
def convert(arg):
if six.PY2:
if isinstance(arg, six.text_type):
arg = arg.encode(arg_encoding())
else:
if isinstance(arg, bytes):
arg = arg.decode(arg_encoding(), 'surrogateescape')
return arg
return [convert(a) for a in args]
def command_output(cmd, shell=False):
"""Runs the command and returns its output after it has exited.
``cmd`` is a list of arguments starting with the command names. If
``shell`` is true, ``cmd`` is assumed to be a string and passed to a
``cmd`` is a list of arguments starting with the command names. The
arguments are bytes on Unix and strings on Windows.
If ``shell`` is true, ``cmd`` is assumed to be a string and passed to a
shell to execute.
If the process exits with a non-zero return code
``subprocess.CalledProcessError`` is raised. May also raise
``OSError``.
This replaces `subprocess.check_output`, which isn't available in
Python 2.6 and which can have problems if lots of output is sent to
stderr.
This replaces `subprocess.check_output` which can have problems if lots of
output is sent to stderr.
"""
cmd = convert_command_args(cmd)
proc = subprocess.Popen(
cmd,
stdout=subprocess.PIPE,
@@ -665,6 +809,7 @@ def command_output(cmd, shell=False):
raise subprocess.CalledProcessError(
returncode=proc.returncode,
cmd=' '.join(cmd),
output=stdout + stderr,
)
return stdout
@@ -684,3 +829,165 @@ def max_filename_length(path, limit=MAX_FILENAME_LENGTH):
return min(res[9], limit)
else:
return limit
def open_anything():
"""Return the system command that dispatches execution to the correct
program.
"""
sys_name = platform.system()
if sys_name == 'Darwin':
base_cmd = 'open'
elif sys_name == 'Windows':
base_cmd = 'start'
else: # Assume Unix
base_cmd = 'xdg-open'
return base_cmd
def editor_command():
"""Get a command for opening a text file.
Use the `EDITOR` environment variable by default. If it is not
present, fall back to `open_anything()`, the platform-specific tool
for opening files in general.
"""
editor = os.environ.get('EDITOR')
if editor:
return editor
return open_anything()
def shlex_split(s):
"""Split a Unicode or bytes string according to shell lexing rules.
Raise `ValueError` if the string is not a well-formed shell string.
This is a workaround for a bug in some versions of Python.
"""
if not six.PY2 or isinstance(s, bytes): # Shlex works fine.
return shlex.split(s)
elif isinstance(s, six.text_type):
# Work around a Python bug.
# http://bugs.python.org/issue6988
bs = s.encode('utf-8')
return [c.decode('utf-8') for c in shlex.split(bs)]
else:
raise TypeError(u'shlex_split called with non-string')
def interactive_open(targets, command):
"""Open the files in `targets` by `exec`ing a new `command`, given
as a Unicode string. (The new program takes over, and Python
execution ends: this does not fork a subprocess.)
Can raise `OSError`.
"""
assert command
# Split the command string into its arguments.
try:
args = shlex_split(command)
except ValueError: # Malformed shell tokens.
args = [command]
args.insert(0, args[0]) # for argv[0]
args += targets
return os.execlp(*args)
def _windows_long_path_name(short_path):
"""Use Windows' `GetLongPathNameW` via ctypes to get the canonical,
long path given a short filename.
"""
if not isinstance(short_path, six.text_type):
short_path = short_path.decode(_fsencoding())
import ctypes
buf = ctypes.create_unicode_buffer(260)
get_long_path_name_w = ctypes.windll.kernel32.GetLongPathNameW
return_value = get_long_path_name_w(short_path, buf, 260)
if return_value == 0 or return_value > 260:
# An error occurred
return short_path
else:
long_path = buf.value
# GetLongPathNameW does not change the case of the drive
# letter.
if len(long_path) > 1 and long_path[1] == ':':
long_path = long_path[0].upper() + long_path[1:]
return long_path
def case_sensitive(path):
"""Check whether the filesystem at the given path is case sensitive.
To work best, the path should point to a file or a directory. If the path
does not exist, assume a case sensitive file system on every platform
except Windows.
"""
# A fallback in case the path does not exist.
if not os.path.exists(syspath(path)):
# By default, the case sensitivity depends on the platform.
return platform.system() != 'Windows'
# If an upper-case version of the path exists but a lower-case
# version does not, then the filesystem must be case-sensitive.
# (Otherwise, we have more work to do.)
if not (os.path.exists(syspath(path.lower())) and
os.path.exists(syspath(path.upper()))):
return True
# Both versions of the path exist on the file system. Check whether
# they refer to different files by their inodes. Alas,
# `os.path.samefile` is only available on Unix systems on Python 2.
if platform.system() != 'Windows':
return not os.path.samefile(syspath(path.lower()),
syspath(path.upper()))
# On Windows, we check whether the canonical, long filenames for the
# files are the same.
lower = _windows_long_path_name(path.lower())
upper = _windows_long_path_name(path.upper())
return lower != upper
def raw_seconds_short(string):
"""Formats a human-readable M:SS string as a float (number of seconds).
Raises ValueError if the conversion cannot take place due to `string` not
being in the right format.
"""
match = re.match(r'^(\d+):([0-5]\d)$', string)
if not match:
raise ValueError(u'String not in M:SS format')
minutes, seconds = map(int, match.groups())
return float(minutes * 60 + seconds)
def asciify_path(path, sep_replace):
"""Decodes all unicode characters in a path into ASCII equivalents.
Substitutions are provided by the unidecode module. Path separators in the
input are preserved.
Keyword arguments:
path -- The path to be asciified.
sep_replace -- the string to be used to replace extraneous path separators.
"""
# if this platform has an os.altsep, change it to os.sep.
if os.altsep:
path = path.replace(os.altsep, os.sep)
path_components = path.split(os.sep)
for index, item in enumerate(path_components):
path_components[index] = unidecode(item).replace(os.sep, sep_replace)
if os.altsep:
path_components[index] = unidecode(item).replace(
os.altsep,
sep_replace
)
return os.sep.join(path_components)
lib/beets/util/artresizer.py
Regular → Executable
+120 -69
View File
@@ -1,5 +1,6 @@
# -*- coding: utf-8 -*-
# This file is part of beets.
# Copyright 2014, Fabrice Laporte
# Copyright 2016, Fabrice Laporte
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
@@ -15,20 +16,26 @@
"""Abstraction layer to resize images using PIL, ImageMagick, or a
public resizing proxy if neither is available.
"""
import urllib
from __future__ import division, absolute_import, print_function
import subprocess
import os
import re
from tempfile import NamedTemporaryFile
import logging
from six.moves.urllib.parse import urlencode
from beets import logging
from beets import util
import six
# Resizing methods
PIL = 1
IMAGEMAGICK = 2
WEBPROXY = 3
PROXY_URL = 'http://images.weserv.nl/'
if util.SNI_SUPPORTED:
PROXY_URL = 'https://images.weserv.nl/'
else:
PROXY_URL = 'http://images.weserv.nl/'
log = logging.getLogger('beets')
@@ -37,9 +44,9 @@ def resize_url(url, maxwidth):
"""Return a proxied image URL that resizes the original image to
maxwidth (preserving aspect ratio).
"""
return '{0}?{1}'.format(PROXY_URL, urllib.urlencode({
return '{0}?{1}'.format(PROXY_URL, urlencode({
'url': url.replace('http://', ''),
'w': str(maxwidth),
'w': maxwidth,
}))
@@ -48,8 +55,8 @@ def temp_file_for(path):
specified path.
"""
ext = os.path.splitext(path)[1]
with NamedTemporaryFile(suffix=ext, delete=False) as f:
return f.name
with NamedTemporaryFile(suffix=util.py3_path(ext), delete=False) as f:
return util.bytestring_path(f.name)
def pil_resize(maxwidth, path_in, path_out=None):
@@ -58,9 +65,8 @@ def pil_resize(maxwidth, path_in, path_out=None):
"""
path_out = path_out or temp_file_for(path_in)
from PIL import Image
log.debug(u'artresizer: PIL resizing {0} to {1}'.format(
util.displayable_path(path_in), util.displayable_path(path_out)
))
log.debug(u'artresizer: PIL resizing {0} to {1}',
util.displayable_path(path_in), util.displayable_path(path_out))
try:
im = Image.open(util.syspath(path_in))
@@ -69,9 +75,8 @@ def pil_resize(maxwidth, path_in, path_out=None):
im.save(path_out)
return path_out
except IOError:
log.error(u"PIL cannot create thumbnail for '{0}'".format(
util.displayable_path(path_in)
))
log.error(u"PIL cannot create thumbnail for '{0}'",
util.displayable_path(path_in))
return path_in
@@ -80,9 +85,8 @@ def im_resize(maxwidth, path_in, path_out=None):
Return the output path of resized image.
"""
path_out = path_out or temp_file_for(path_in)
log.debug(u'artresizer: ImageMagick resizing {0} to {1}'.format(
util.displayable_path(path_in), util.displayable_path(path_out)
))
log.debug(u'artresizer: ImageMagick resizing {0} to {1}',
util.displayable_path(path_in), util.displayable_path(path_out))
# "-resize widthxheight>" shrinks images with dimension(s) larger
# than the corresponding width and/or height dimension(s). The >
@@ -90,13 +94,13 @@ def im_resize(maxwidth, path_in, path_out=None):
# compatibility.
try:
util.command_output([
'convert', util.syspath(path_in),
'-resize', '{0}x^>'.format(maxwidth), path_out
'convert', util.syspath(path_in, prefix=False),
'-resize', '{0}x^>'.format(maxwidth),
util.syspath(path_out, prefix=False),
])
except subprocess.CalledProcessError:
log.warn(u'artresizer: IM convert failed for {0}'.format(
util.displayable_path(path_in)
))
log.warning(u'artresizer: IM convert failed for {0}',
util.displayable_path(path_in))
return path_in
return path_out
@@ -107,34 +111,67 @@ BACKEND_FUNCS = {
}
def pil_getsize(path_in):
from PIL import Image
try:
im = Image.open(util.syspath(path_in))
return im.size
except IOError as exc:
log.error(u"PIL could not read file {}: {}",
util.displayable_path(path_in), exc)
def im_getsize(path_in):
cmd = ['identify', '-format', '%w %h',
util.syspath(path_in, prefix=False)]
try:
out = util.command_output(cmd)
except subprocess.CalledProcessError as exc:
log.warning(u'ImageMagick size query failed')
log.debug(
u'`convert` exited with (status {}) when '
u'getting size with command {}:\n{}',
exc.returncode, cmd, exc.output.strip()
)
return
try:
return tuple(map(int, out.split(b' ')))
except IndexError:
log.warning(u'Could not understand IM output: {0!r}', out)
BACKEND_GET_SIZE = {
PIL: pil_getsize,
IMAGEMAGICK: im_getsize,
}
class Shareable(type):
"""A pseudo-singleton metaclass that allows both shared and
non-shared instances. The ``MyClass.shared`` property holds a
lazily-created shared instance of ``MyClass`` while calling
``MyClass()`` to construct a new object works as usual.
"""
def __init__(cls, name, bases, dict):
super(Shareable, cls).__init__(name, bases, dict)
cls._instance = None
def __init__(self, name, bases, dict):
super(Shareable, self).__init__(name, bases, dict)
self._instance = None
@property
def shared(cls):
if cls._instance is None:
cls._instance = cls()
return cls._instance
def shared(self):
if self._instance is None:
self._instance = self()
return self._instance
class ArtResizer(object):
class ArtResizer(six.with_metaclass(Shareable, object)):
"""A singleton class that performs image resizes.
"""
__metaclass__ = Shareable
def __init__(self, method=None):
"""Create a resizer object for the given method or, if none is
specified, with an inferred method.
def __init__(self):
"""Create a resizer object with an inferred method.
"""
self.method = self._check_method(method)
log.debug(u"artresizer: method is {0}".format(self.method))
self.method = self._check_method()
log.debug(u"artresizer: method is {0}", self.method)
self.can_compare = self._can_compare()
def resize(self, maxwidth, path_in, path_out=None):
@@ -165,47 +202,61 @@ class ArtResizer(object):
"""
return self.method[0] in BACKEND_FUNCS
def get_size(self, path_in):
"""Return the size of an image file as an int couple (width, height)
in pixels.
Only available locally
"""
if self.local:
func = BACKEND_GET_SIZE[self.method[0]]
return func(path_in)
def _can_compare(self):
"""A boolean indicating whether image comparison is available"""
return self.method[0] == IMAGEMAGICK and self.method[1] > (6, 8, 7)
@staticmethod
def _check_method(method=None):
"""A tuple indicating whether current method is available and its
version. If no method is given, it returns a supported one.
"""
# Guess available method
if not method:
for m in [IMAGEMAGICK, PIL]:
_, version = ArtResizer._check_method(m)
if version:
return (m, version)
return (WEBPROXY, (0))
def _check_method():
"""Return a tuple indicating an available method and its version."""
version = get_im_version()
if version:
return IMAGEMAGICK, version
if method == IMAGEMAGICK:
version = get_pil_version()
if version:
return PIL, version
# Try invoking ImageMagick's "convert".
try:
out = util.command_output(['identify', '--version'])
return WEBPROXY, (0)
if 'imagemagick' in out.lower():
pattern = r".+ (\d+)\.(\d+)\.(\d+).*"
match = re.search(pattern, out)
if match:
return (IMAGEMAGICK,
(int(match.group(1)),
int(match.group(2)),
int(match.group(3))))
return (IMAGEMAGICK, (0))
except (subprocess.CalledProcessError, OSError):
return (IMAGEMAGICK, None)
def get_im_version():
"""Return Image Magick version or None if it is unavailable
Try invoking ImageMagick's "convert".
"""
try:
out = util.command_output(['convert', '--version'])
if method == PIL:
# Try importing PIL.
try:
__import__('PIL', fromlist=['Image'])
return (PIL, (0))
except ImportError:
return (PIL, None)
if b'imagemagick' in out.lower():
pattern = br".+ (\d+)\.(\d+)\.(\d+).*"
match = re.search(pattern, out)
if match:
return (int(match.group(1)),
int(match.group(2)),
int(match.group(3)))
return (0,)
except (subprocess.CalledProcessError, OSError) as exc:
log.debug(u'ImageMagick check `convert --version` failed: {}', exc)
return None
def get_pil_version():
"""Return Image Magick version or None if it is unavailable
Try importing PIL."""
try:
__import__('PIL', fromlist=[str('Image')])
return (0,)
except ImportError:
return None
lib/beets/util/bluelet.py
Regular → Executable
+8 -17
View File
@@ -1,3 +1,5 @@
# -*- coding: utf-8 -*-
"""Extremely simple pure-Python implementation of coroutine-style
asynchronous socket I/O. Inspired by, but inferior to, Eventlet.
Bluelet can also be thought of as a less-terrible replacement for
@@ -5,6 +7,9 @@ asyncore.
Bluelet: easy concurrency without all the messy parallelism.
"""
from __future__ import division, absolute_import, print_function
import six
import socket
import select
import sys
@@ -15,20 +20,6 @@ import time
import collections
# A little bit of "six" (Python 2/3 compatibility): cope with PEP 3109 syntax
# changes.
PY3 = sys.version_info[0] == 3
if PY3:
def _reraise(typ, exc, tb):
raise exc.with_traceback(tb)
else:
exec("""
def _reraise(typ, exc, tb):
raise typ, exc, tb
""")
# Basic events used for thread scheduling.
class Event(object):
@@ -210,7 +201,7 @@ class ThreadException(Exception):
self.exc_info = exc_info
def reraise(self):
_reraise(self.exc_info[0], self.exc_info[1], self.exc_info[2])
six.reraise(self.exc_info[0], self.exc_info[1], self.exc_info[2])
SUSPENDED = Event() # Special sentinel placeholder for suspended threads.
@@ -550,7 +541,7 @@ def spawn(coro):
and child coroutines run concurrently.
"""
if not isinstance(coro, types.GeneratorType):
raise ValueError('%s is not a coroutine' % str(coro))
raise ValueError(u'%s is not a coroutine' % coro)
return SpawnEvent(coro)
@@ -560,7 +551,7 @@ def call(coro):
returns a value using end(), then this event returns that value.
"""
if not isinstance(coro, types.GeneratorType):
raise ValueError('%s is not a coroutine' % str(coro))
raise ValueError(u'%s is not a coroutine' % coro)
return DelegationEvent(coro)
lib/beets/util/confit.py
Regular → Executable
+268 -90
View File
@@ -1,5 +1,6 @@
# This file is part of Confit.
# Copyright 2014, Adrian Sampson.
# -*- coding: utf-8 -*-
# This file is part of Confuse.
# Copyright 2016, Adrian Sampson.
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
@@ -14,19 +15,16 @@
"""Worry-free YAML configuration files.
"""
from __future__ import unicode_literals
from __future__ import division, absolute_import, print_function
import platform
import os
import pkgutil
import sys
import yaml
import types
import collections
import re
try:
from collections import OrderedDict
except ImportError:
from ordereddict import OrderedDict
from collections import OrderedDict
UNIX_DIR_VAR = 'XDG_CONFIG_HOME'
UNIX_DIR_FALLBACK = '~/.config'
@@ -40,14 +38,15 @@ ROOT_NAME = 'root'
YAML_TAB_PROBLEM = "found character '\\t' that cannot start any token"
REDACTED_TOMBSTONE = 'REDACTED'
# Utilities.
PY3 = sys.version_info[0] == 3
STRING = str if PY3 else unicode
BASESTRING = str if PY3 else basestring
NUMERIC_TYPES = (int, float) if PY3 else (int, float, long)
TYPE_TYPES = (type,) if PY3 else (type, types.ClassType)
STRING = str if PY3 else unicode # noqa: F821
BASESTRING = str if PY3 else basestring # noqa: F821
NUMERIC_TYPES = (int, float) if PY3 else (int, float, long) # noqa: F821
def iter_first(sequence):
@@ -56,10 +55,7 @@ def iter_first(sequence):
"""
it = iter(sequence)
try:
if PY3:
return next(it)
else:
return it.next()
return next(it)
except StopIteration:
raise ValueError()
@@ -96,17 +92,17 @@ class ConfigReadError(ConfigError):
self.filename = filename
self.reason = reason
message = 'file {0} could not be read'.format(filename)
message = u'file {0} could not be read'.format(filename)
if isinstance(reason, yaml.scanner.ScannerError) and \
reason.problem == YAML_TAB_PROBLEM:
# Special-case error message for tab indentation in YAML markup.
message += ': found tab character at line {0}, column {1}'.format(
message += u': found tab character at line {0}, column {1}'.format(
reason.problem_mark.line + 1,
reason.problem_mark.column + 1,
)
elif reason:
# Generic error message uses exception's message.
message += ': {0}'.format(reason)
message += u': {0}'.format(reason)
super(ConfigReadError, self).__init__(message)
@@ -120,19 +116,19 @@ class ConfigSource(dict):
def __init__(self, value, filename=None, default=False):
super(ConfigSource, self).__init__(value)
if filename is not None and not isinstance(filename, BASESTRING):
raise TypeError('filename must be a string or None')
raise TypeError(u'filename must be a string or None')
self.filename = filename
self.default = default
def __repr__(self):
return 'ConfigSource({0}, {1}, {2})'.format(
super(ConfigSource, self).__repr__(),
repr(self.filename),
repr(self.default)
return 'ConfigSource({0!r}, {1!r}, {2!r})'.format(
super(ConfigSource, self),
self.filename,
self.default,
)
@classmethod
def of(self, value):
def of(cls, value):
"""Given either a dictionary or a `ConfigSource` object, return
a `ConfigSource` object. This lets a function accept either type
of object as an argument.
@@ -142,7 +138,7 @@ class ConfigSource(dict):
elif isinstance(value, dict):
return ConfigSource(value)
else:
raise TypeError('source value must be a dict')
raise TypeError(u'source value must be a dict')
class ConfigView(object):
@@ -177,7 +173,7 @@ class ConfigView(object):
try:
return iter_first(pairs)
except ValueError:
raise NotFoundError("{0} not found".format(self.name))
raise NotFoundError(u"{0} not found".format(self.name))
def exists(self):
"""Determine whether the view has a setting in any source.
@@ -208,7 +204,31 @@ class ConfigView(object):
raise NotImplementedError
def __repr__(self):
return '<ConfigView: %s>' % self.name
return '<{}: {}>'.format(self.__class__.__name__, self.name)
def __iter__(self):
"""Iterate over the keys of a dictionary view or the *subviews*
of a list view.
"""
# Try getting the keys, if this is a dictionary view.
try:
keys = self.keys()
for key in keys:
yield key
except ConfigTypeError:
# Otherwise, try iterating over a list.
collection = self.get()
if not isinstance(collection, (list, tuple)):
raise ConfigTypeError(
u'{0} must be a dictionary or a list, not {1}'.format(
self.name, type(collection).__name__
)
)
# Yield all the indices in the list.
for index in range(len(collection)):
yield self[index]
def __getitem__(self, key):
"""Get a subview of this view."""
@@ -225,10 +245,15 @@ class ConfigView(object):
def set_args(self, namespace):
"""Overlay parsed command-line arguments, generated by a library
like argparse or optparse, onto this view's value.
like argparse or optparse, onto this view's value. ``namespace``
can be a ``dict`` or namespace object.
"""
args = {}
for key, value in namespace.__dict__.items():
if isinstance(namespace, dict):
items = namespace.items()
else:
items = namespace.__dict__.items()
for key, value in items:
if value is not None: # Avoid unset options.
args[key] = value
self.set(args)
@@ -239,14 +264,17 @@ class ConfigView(object):
# just say ``bool(view)`` or use ``view`` in a conditional.
def __str__(self):
"""Gets the value for this view as a byte string."""
return str(self.get())
"""Get the value for this view as a bytestring.
"""
if PY3:
return self.__unicode__()
else:
return bytes(self.get())
def __unicode__(self):
"""Gets the value for this view as a unicode string. (Python 2
only.)
"""Get the value for this view as a Unicode string.
"""
return unicode(self.get())
return STRING(self.get())
def __nonzero__(self):
"""Gets the value for this view as a boolean. (Python 2 only.)
@@ -276,7 +304,7 @@ class ConfigView(object):
cur_keys = dic.keys()
except AttributeError:
raise ConfigTypeError(
'{0} must be a dict, not {1}'.format(
u'{0} must be a dict, not {1}'.format(
self.name, type(dic).__name__
)
)
@@ -317,7 +345,7 @@ class ConfigView(object):
it = iter(collection)
except TypeError:
raise ConfigTypeError(
'{0} must be an iterable, not {1}'.format(
u'{0} must be an iterable, not {1}'.format(
self.name, type(collection).__name__
)
)
@@ -326,17 +354,23 @@ class ConfigView(object):
# Validation and conversion.
def flatten(self):
def flatten(self, redact=False):
"""Create a hierarchy of OrderedDicts containing the data from
this view, recursively reifying all views to get their
represented values.
If `redact` is set, then sensitive values are replaced with
the string "REDACTED".
"""
od = OrderedDict()
for key, view in self.items():
try:
od[key] = view.flatten()
except ConfigTypeError:
od[key] = view.get()
if redact and view.redact:
od[key] = REDACTED_TOMBSTONE
else:
try:
od[key] = view.flatten(redact=redact)
except ConfigTypeError:
od[key] = view.get()
return od
def get(self, template=None):
@@ -354,19 +388,60 @@ class ConfigView(object):
"""
return as_template(template).value(self, template)
# Old validation methods (deprecated).
# Shortcuts for common templates.
def as_filename(self):
"""Get the value as a path. Equivalent to `get(Filename())`.
"""
return self.get(Filename())
def as_choice(self, choices):
"""Get the value from a list of choices. Equivalent to
`get(Choice(choices))`.
"""
return self.get(Choice(choices))
def as_number(self):
"""Get the value as any number type: int or float. Equivalent to
`get(Number())`.
"""
return self.get(Number())
def as_str_seq(self):
return self.get(StrSeq())
def as_str_seq(self, split=True):
"""Get the value as a sequence of strings. Equivalent to
`get(StrSeq())`.
"""
return self.get(StrSeq(split=split))
def as_str(self):
"""Get the value as a (Unicode) string. Equivalent to
`get(unicode)` on Python 2 and `get(str)` on Python 3.
"""
return self.get(String())
# Redaction.
@property
def redact(self):
"""Whether the view contains sensitive information and should be
redacted from output.
"""
return () in self.get_redactions()
@redact.setter
def redact(self, flag):
self.set_redaction((), flag)
def set_redaction(self, path, flag):
"""Add or remove a redaction for a key path, which should be an
iterable of keys.
"""
raise NotImplementedError()
def get_redactions(self):
"""Get the set of currently-redacted sub-key-paths at this view.
"""
raise NotImplementedError()
class RootView(ConfigView):
@@ -380,6 +455,7 @@ class RootView(ConfigView):
"""
self.sources = list(sources)
self.name = ROOT_NAME
self.redactions = set()
def add(self, obj):
self.sources.append(ConfigSource.of(obj))
@@ -391,12 +467,24 @@ class RootView(ConfigView):
return ((dict(s), s) for s in self.sources)
def clear(self):
"""Remove all sources from this configuration."""
"""Remove all sources (and redactions) from this
configuration.
"""
del self.sources[:]
self.redactions.clear()
def root(self):
return self
def set_redaction(self, path, flag):
if flag:
self.redactions.add(path)
elif path in self.redactions:
self.redactions.remove(path)
def get_redactions(self):
return self.redactions
class Subview(ConfigView):
"""A subview accessed via a subscript of a parent view."""
@@ -414,11 +502,13 @@ class Subview(ConfigView):
if not isinstance(self.key, int):
self.name += '.'
if isinstance(self.key, int):
self.name += '#{0}'.format(self.key)
elif isinstance(self.key, BASESTRING):
self.name += '{0}'.format(self.key)
self.name += u'#{0}'.format(self.key)
elif isinstance(self.key, bytes):
self.name += self.key.decode('utf-8')
elif isinstance(self.key, STRING):
self.name += self.key
else:
self.name += '{0}'.format(repr(self.key))
self.name += repr(self.key)
def resolve(self):
for collection, source in self.parent.resolve():
@@ -433,7 +523,7 @@ class Subview(ConfigView):
except TypeError:
# Not subscriptable.
raise ConfigTypeError(
"{0} must be a collection, not {1}".format(
u"{0} must be a collection, not {1}".format(
self.parent.name, type(collection).__name__
)
)
@@ -448,6 +538,13 @@ class Subview(ConfigView):
def root(self):
return self.parent.root()
def set_redaction(self, path, flag):
self.parent.set_redaction((self.key,) + path, flag)
def get_redactions(self):
return (kp[1:] for kp in self.parent.get_redactions()
if kp and kp[0] == self.key)
# Config file paths, including platform-specific paths and in-package
# defaults.
@@ -536,7 +633,7 @@ class Loader(yaml.SafeLoader):
else:
raise yaml.constructor.ConstructorError(
None, None,
'expected a mapping node, but found %s' % node.id,
u'expected a mapping node, but found %s' % node.id,
node.start_mark
)
@@ -547,7 +644,7 @@ class Loader(yaml.SafeLoader):
hash(key)
except TypeError as exc:
raise yaml.constructor.ConstructorError(
'while constructing a mapping',
u'while constructing a mapping',
node.start_mark, 'found unacceptable key (%s)' % exc,
key_node.start_mark
)
@@ -595,11 +692,11 @@ class Dumper(yaml.SafeDumper):
for item_key, item_value in mapping:
node_key = self.represent_data(item_key)
node_value = self.represent_data(item_value)
if not (isinstance(node_key, yaml.ScalarNode)
and not node_key.style):
if not (isinstance(node_key, yaml.ScalarNode) and
not node_key.style):
best_style = False
if not (isinstance(node_value, yaml.ScalarNode)
and not node_value.style):
if not (isinstance(node_value, yaml.ScalarNode) and
not node_value.style):
best_style = False
value.append((node_key, node_value))
if flow_style is None:
@@ -625,9 +722,9 @@ class Dumper(yaml.SafeDumper):
"""Represent bool as 'yes' or 'no' instead of 'true' or 'false'.
"""
if data:
value = 'yes'
value = u'yes'
else:
value = 'no'
value = u'no'
return self.represent_scalar('tag:yaml.org,2002:bool', value)
def represent_none(self, data):
@@ -752,7 +849,7 @@ class Configuration(RootView):
appdir = os.environ[self._env_var]
appdir = os.path.abspath(os.path.expanduser(appdir))
if os.path.isfile(appdir):
raise ConfigError('{0} must be a directory'.format(
raise ConfigError(u'{0} must be a directory'.format(
self._env_var
))
@@ -776,7 +873,7 @@ class Configuration(RootView):
filename = os.path.abspath(filename)
self.set(ConfigSource(load_yaml(filename), filename))
def dump(self, full=True):
def dump(self, full=True, redact=False):
"""Dump the Configuration object to a YAML file.
The order of the keys is determined from the default
@@ -788,13 +885,17 @@ class Configuration(RootView):
:type filename: unicode
:param full: Dump settings that don't differ from the defaults
as well
:param redact: Remove sensitive information (views with the `redact`
flag set) from the output
"""
if full:
out_dict = self.flatten()
out_dict = self.flatten(redact=redact)
else:
# Exclude defaults when flattening.
sources = [s for s in self.sources if not s.default]
out_dict = RootView(sources).flatten()
temp_root = RootView(sources)
temp_root.redactions = self.redactions
out_dict = temp_root.flatten(redact=redact)
yaml_out = yaml.dump(out_dict, Dumper=Dumper,
default_flow_style=None, indent=4,
@@ -806,7 +907,7 @@ class Configuration(RootView):
if source.default:
default_source = source
break
if default_source:
if default_source and default_source.filename:
with open(default_source.filename, 'r') as fp:
default_data = fp.read()
yaml_out = restore_yaml_comments(yaml_out, default_data)
@@ -853,7 +954,7 @@ class LazyConfig(Configuration):
def clear(self):
"""Remove all sources from this configuration."""
del self.sources[:]
super(LazyConfig, self).clear()
self._lazy_suffix = []
self._lazy_prefix = []
@@ -870,7 +971,7 @@ should be raised when the value is missing.
class Template(object):
"""A value template for configuration fields.
The template works like a type and instructs Confit about how to
The template works like a type and instructs Confuse about how to
interpret a deserialized YAML value. This includes type conversions,
providing a default value, and validating for errors. For example, a
filepath type might expand tildes and check that the file exists.
@@ -901,7 +1002,7 @@ class Template(object):
return self.convert(value, view)
elif self.default is REQUIRED:
# Missing required value. This is an error.
raise NotFoundError("{0} not found".format(view.name))
raise NotFoundError(u"{0} not found".format(view.name))
else:
# Missing value, but not required.
return self.default
@@ -926,7 +1027,7 @@ class Template(object):
"""
exc_class = ConfigTypeError if type_error else ConfigValueError
raise exc_class(
'{0}: {1}'.format(view.name, message)
u'{0}: {1}'.format(view.name, message)
)
def __repr__(self):
@@ -947,7 +1048,7 @@ class Integer(Template):
elif isinstance(value, float):
return int(value)
else:
self.fail('must be a number', view, True)
self.fail(u'must be a number', view, True)
class Number(Template):
@@ -960,7 +1061,7 @@ class Number(Template):
return value
else:
self.fail(
'must be numeric, not {0}'.format(type(value).__name__),
u'must be numeric, not {0}'.format(type(value).__name__),
view,
True
)
@@ -1005,18 +1106,29 @@ class String(Template):
if pattern:
self.regex = re.compile(pattern)
def __repr__(self):
args = []
if self.default is not REQUIRED:
args.append(repr(self.default))
if self.pattern is not None:
args.append('pattern=' + repr(self.pattern))
return 'String({0})'.format(', '.join(args))
def convert(self, value, view):
"""Check that the value is a string and matches the pattern.
"""
if isinstance(value, BASESTRING):
if self.pattern and not self.regex.match(value):
self.fail(
"must match the pattern {0}".format(self.pattern),
u"must match the pattern {0}".format(self.pattern),
view
)
return value
else:
self.fail('must be a string', view, True)
self.fail(u'must be a string', view, True)
class Choice(Template):
@@ -1037,7 +1149,7 @@ class Choice(Template):
"""
if value not in self.choices:
self.fail(
'must be one of {0}, not {1}'.format(
u'must be one of {0}, not {1}'.format(
repr(list(self.choices)), repr(value)
),
view
@@ -1052,6 +1164,67 @@ class Choice(Template):
return 'Choice({0!r})'.format(self.choices)
class OneOf(Template):
"""A template that permits values complying to one of the given templates.
"""
def __init__(self, allowed, default=REQUIRED):
super(OneOf, self).__init__(default)
self.allowed = list(allowed)
def __repr__(self):
args = []
if self.allowed is not None:
args.append('allowed=' + repr(self.allowed))
if self.default is not REQUIRED:
args.append(repr(self.default))
return 'OneOf({0})'.format(', '.join(args))
def value(self, view, template):
self.template = template
return super(OneOf, self).value(view, template)
def convert(self, value, view):
"""Ensure that the value follows at least one template.
"""
is_mapping = isinstance(self.template, MappingTemplate)
for candidate in self.allowed:
try:
if is_mapping:
if isinstance(candidate, Filename) and \
candidate.relative_to:
next_template = candidate.template_with_relatives(
view,
self.template
)
next_template.subtemplates[view.key] = as_template(
candidate
)
else:
next_template = MappingTemplate({view.key: candidate})
return view.parent.get(next_template)[view.key]
else:
return view.get(candidate)
except ConfigTemplateError:
raise
except ConfigError:
pass
except ValueError as exc:
raise ConfigTemplateError(exc)
self.fail(
u'must be one of {0}, not {1}'.format(
repr(self.allowed), repr(value)
),
view
)
class StrSeq(Template):
"""A template for values that are lists of strings.
@@ -1070,7 +1243,7 @@ class StrSeq(Template):
def convert(self, value, view):
if isinstance(value, bytes):
value = value.decode('utf8', 'ignore')
value = value.decode('utf-8', 'ignore')
if isinstance(value, STRING):
if self.split:
@@ -1081,17 +1254,17 @@ class StrSeq(Template):
try:
value = list(value)
except TypeError:
self.fail('must be a whitespace-separated string or a list',
self.fail(u'must be a whitespace-separated string or a list',
view, True)
def convert(x):
if isinstance(x, unicode):
if isinstance(x, STRING):
return x
elif isinstance(x, BASESTRING):
return x.decode('utf8', 'ignore')
elif isinstance(x, bytes):
return x.decode('utf-8', 'ignore')
else:
self.fail('must be a list of strings', view, True)
return map(convert, value)
self.fail(u'must be a list of strings', view, True)
return list(map(convert, value))
class Filename(Template):
@@ -1107,7 +1280,7 @@ class Filename(Template):
"""
def __init__(self, default=REQUIRED, cwd=None, relative_to=None,
in_app_dir=False):
""" `relative_to` is the name of a sibling value that is
"""`relative_to` is the name of a sibling value that is
being validated at the same time.
`in_app_dir` indicates whether the path should be resolved
@@ -1140,19 +1313,19 @@ class Filename(Template):
if not isinstance(template, (collections.Mapping, MappingTemplate)):
# disallow config.get(Filename(relative_to='foo'))
raise ConfigTemplateError(
'relative_to may only be used when getting multiple values.'
u'relative_to may only be used when getting multiple values.'
)
elif self.relative_to == view.key:
raise ConfigTemplateError(
'{0} is relative to itself'.format(view.name)
u'{0} is relative to itself'.format(view.name)
)
elif self.relative_to not in view.parent.keys():
# self.relative_to is not in the config
self.fail(
(
'needs sibling value "{0}" to expand relative path'
u'needs sibling value "{0}" to expand relative path'
).format(self.relative_to),
view
)
@@ -1174,12 +1347,12 @@ class Filename(Template):
if next_relative in template.subtemplates:
# we encountered this config key previously
raise ConfigTemplateError((
'{0} and {1} are recursively relative'
u'{0} and {1} are recursively relative'
).format(view.name, self.relative_to))
else:
raise ConfigTemplateError((
'missing template for {0}, needed to expand {1}\'s' +
'relative path'
u'missing template for {0}, needed to expand {1}\'s' +
u'relative path'
).format(self.relative_to, view.name))
next_template.subtemplates[next_relative] = rel_to_template
@@ -1191,7 +1364,7 @@ class Filename(Template):
path, source = view.first()
if not isinstance(path, BASESTRING):
self.fail(
'must be a filename, not {0}'.format(type(path).__name__),
u'must be a filename, not {0}'.format(type(path).__name__),
view,
True
)
@@ -1229,7 +1402,7 @@ class TypeTemplate(Template):
def convert(self, value, view):
if not isinstance(value, self.typ):
self.fail(
'must be a {0}, not {1}'.format(
u'must be a {0}, not {1}'.format(
self.typ.__name__,
type(value).__name__,
),
@@ -1267,6 +1440,11 @@ def as_template(value):
return String()
elif isinstance(value, BASESTRING):
return String(value)
elif isinstance(value, set):
# convert to list to avoid hash related problems
return Choice(list(value))
elif isinstance(value, list):
return OneOf(value)
elif value is float:
return Number()
elif value is None:
@@ -1278,4 +1456,4 @@ def as_template(value):
elif isinstance(value, type):
return TypeTemplate(value)
else:
raise ValueError('cannot convert to template: {0!r}'.format(value))
raise ValueError(u'cannot convert to template: {0!r}'.format(value))
lib/beets/util/enumeration.py
Regular → Executable
+4 -1
View File
@@ -1,5 +1,6 @@
# -*- coding: utf-8 -*-
# This file is part of beets.
# Copyright 2013, Adrian Sampson.
# Copyright 2016, Adrian Sampson.
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
@@ -12,6 +13,8 @@
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
from __future__ import division, absolute_import, print_function
from enum import Enum
lib/beets/util/functemplate.py
Regular → Executable
+94 -45
View File
@@ -1,5 +1,6 @@
# -*- coding: utf-8 -*-
# This file is part of beets.
# Copyright 2013, Adrian Sampson.
# Copyright 2016, Adrian Sampson.
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
@@ -25,12 +26,15 @@ library: unknown symbols are left intact.
This is sort of like a tiny, horrible degeneration of a real templating
engine like Jinja2 or Mustache.
"""
from __future__ import print_function
from __future__ import division, absolute_import, print_function
import re
import ast
import dis
import types
import sys
import six
SYMBOL_DELIM = u'$'
FUNC_DELIM = u'%'
@@ -70,13 +74,13 @@ def ex_literal(val):
"""
if val is None:
return ast.Name('None', ast.Load())
elif isinstance(val, (int, float, long)):
elif isinstance(val, six.integer_types):
return ast.Num(val)
elif isinstance(val, bool):
return ast.Name(str(val), ast.Load())
elif isinstance(val, basestring):
return ast.Name(bytes(val), ast.Load())
elif isinstance(val, six.string_types):
return ast.Str(val)
raise TypeError('no literal for {0}'.format(type(val)))
raise TypeError(u'no literal for {0}'.format(type(val)))
def ex_varassign(name, expr):
@@ -93,7 +97,7 @@ def ex_call(func, args):
function may be an expression or the name of a function. Each
argument may be an expression or a value to be used as a literal.
"""
if isinstance(func, basestring):
if isinstance(func, six.string_types):
func = ex_rvalue(func)
args = list(args)
@@ -101,7 +105,10 @@ def ex_call(func, args):
if not isinstance(args[i], ast.expr):
args[i] = ex_literal(args[i])
return ast.Call(func, args, [], None, None)
if sys.version_info[:2] < (3, 5):
return ast.Call(func, args, [], None, None)
else:
return ast.Call(func, args, [])
def compile_func(arg_names, statements, name='_the_func', debug=False):
@@ -109,16 +116,31 @@ def compile_func(arg_names, statements, name='_the_func', debug=False):
the resulting Python function. If `debug`, then print out the
bytecode of the compiled function.
"""
func_def = ast.FunctionDef(
name,
ast.arguments(
[ast.Name(n, ast.Param()) for n in arg_names],
None, None,
[ex_literal(None) for _ in arg_names],
),
statements,
[],
)
if six.PY2:
func_def = ast.FunctionDef(
name=name.encode('utf-8'),
args=ast.arguments(
args=[ast.Name(n, ast.Param()) for n in arg_names],
vararg=None,
kwarg=None,
defaults=[ex_literal(None) for _ in arg_names],
),
body=statements,
decorator_list=[],
)
else:
func_def = ast.FunctionDef(
name=name,
args=ast.arguments(
args=[ast.arg(arg=n, annotation=None) for n in arg_names],
kwonlyargs=[],
kw_defaults=[],
defaults=[ex_literal(None) for _ in arg_names],
),
body=statements,
decorator_list=[],
)
mod = ast.Module([func_def])
ast.fix_missing_locations(mod)
@@ -132,7 +154,7 @@ def compile_func(arg_names, statements, name='_the_func', debug=False):
dis.dis(const)
the_locals = {}
exec prog in {}, the_locals
exec(prog, {}, the_locals)
return the_locals[name]
@@ -160,8 +182,12 @@ class Symbol(object):
def translate(self):
"""Compile the variable lookup."""
expr = ex_rvalue(VARIABLE_PREFIX + self.ident.encode('utf8'))
return [expr], set([self.ident.encode('utf8')]), set()
if six.PY2:
ident = self.ident.encode('utf-8')
else:
ident = self.ident
expr = ex_rvalue(VARIABLE_PREFIX + ident)
return [expr], set([ident]), set()
class Call(object):
@@ -186,15 +212,19 @@ class Call(object):
except Exception as exc:
# Function raised exception! Maybe inlining the name of
# the exception will help debug.
return u'<%s>' % unicode(exc)
return unicode(out)
return u'<%s>' % six.text_type(exc)
return six.text_type(out)
else:
return self.original
def translate(self):
"""Compile the function call."""
varnames = set()
funcnames = set([self.ident.encode('utf8')])
if six.PY2:
ident = self.ident.encode('utf-8')
else:
ident = self.ident
funcnames = set([ident])
arg_exprs = []
for arg in self.args:
@@ -209,14 +239,14 @@ class Call(object):
[ex_call(
'map',
[
ex_rvalue('unicode'),
ex_rvalue(six.text_type.__name__),
ast.List(subexprs, ast.Load()),
]
)],
))
subexpr_call = ex_call(
FUNCTION_PREFIX + self.ident.encode('utf8'),
FUNCTION_PREFIX + ident,
arg_exprs
)
return [subexpr_call], varnames, funcnames
@@ -238,11 +268,11 @@ class Expression(object):
"""
out = []
for part in self.parts:
if isinstance(part, basestring):
if isinstance(part, six.string_types):
out.append(part)
else:
out.append(part.evaluate(env))
return u''.join(map(unicode, out))
return u''.join(map(six.text_type, out))
def translate(self):
"""Compile the expression to a list of Python AST expressions, a
@@ -252,7 +282,7 @@ class Expression(object):
varnames = set()
funcnames = set()
for part in self.parts:
if isinstance(part, basestring):
if isinstance(part, six.string_types):
expressions.append(ex_literal(part))
else:
e, v, f = part.translate()
@@ -281,16 +311,24 @@ class Parser(object):
replaced with a real, accepted parsing technique (PEG, parser
generator, etc.).
"""
def __init__(self, string):
def __init__(self, string, in_argument=False):
""" Create a new parser.
:param in_arguments: boolean that indicates the parser is to be
used for parsing function arguments, ie. considering commas
(`ARG_SEP`) a special character
"""
self.string = string
self.in_argument = in_argument
self.pos = 0
self.parts = []
# Common parsing resources.
special_chars = (SYMBOL_DELIM, FUNC_DELIM, GROUP_OPEN, GROUP_CLOSE,
ARG_SEP, ESCAPE_CHAR)
special_char_re = re.compile(ur'[%s]|$' %
ESCAPE_CHAR)
special_char_re = re.compile(r'[%s]|$' %
u''.join(re.escape(c) for c in special_chars))
escapable_chars = (SYMBOL_DELIM, FUNC_DELIM, GROUP_CLOSE, ARG_SEP)
terminator_chars = (GROUP_CLOSE,)
def parse_expression(self):
"""Parse a template expression starting at ``pos``. Resulting
@@ -298,17 +336,27 @@ class Parser(object):
the ``parts`` field, a list. The ``pos`` field is updated to be
the next character after the expression.
"""
# Append comma (ARG_SEP) to the list of special characters only when
# parsing function arguments.
extra_special_chars = ()
special_char_re = self.special_char_re
if self.in_argument:
extra_special_chars = (ARG_SEP,)
special_char_re = re.compile(
r'[%s]|$' % u''.join(re.escape(c) for c in
self.special_chars + extra_special_chars))
text_parts = []
while self.pos < len(self.string):
char = self.string[self.pos]
if char not in self.special_chars:
if char not in self.special_chars + extra_special_chars:
# A non-special character. Skip to the next special
# character, treating the interstice as literal text.
next_pos = (
self.special_char_re.search(self.string[self.pos:]).start()
+ self.pos
special_char_re.search(
self.string[self.pos:]).start() + self.pos
)
text_parts.append(self.string[self.pos:next_pos])
self.pos = next_pos
@@ -318,14 +366,14 @@ class Parser(object):
# The last character can never begin a structure, so we
# just interpret it as a literal character (unless it
# terminates the expression, as with , and }).
if char not in (GROUP_CLOSE, ARG_SEP):
if char not in self.terminator_chars + extra_special_chars:
text_parts.append(char)
self.pos += 1
break
next_char = self.string[self.pos + 1]
if char == ESCAPE_CHAR and next_char in \
(SYMBOL_DELIM, FUNC_DELIM, GROUP_CLOSE, ARG_SEP):
if char == ESCAPE_CHAR and next_char in (self.escapable_chars +
extra_special_chars):
# An escaped special character ($$, $}, etc.). Note that
# ${ is not an escape sequence: this is ambiguous with
# the start of a symbol and it's not necessary (just
@@ -345,7 +393,7 @@ class Parser(object):
elif char == FUNC_DELIM:
# Parse a function call.
self.parse_call()
elif char in (GROUP_CLOSE, ARG_SEP):
elif char in self.terminator_chars + extra_special_chars:
# Template terminated.
break
elif char == GROUP_OPEN:
@@ -453,7 +501,7 @@ class Parser(object):
expressions = []
while self.pos < len(self.string):
subparser = Parser(self.string[self.pos:])
subparser = Parser(self.string[self.pos:], in_argument=True)
subparser.parse_expression()
# Extract and advance past the parsed expression.
@@ -477,7 +525,7 @@ class Parser(object):
Updates ``pos``.
"""
remainder = self.string[self.pos:]
ident = re.match(ur'\w*', remainder).group(0)
ident = re.match(r'\w*', remainder).group(0)
self.pos += len(ident)
return ident
@@ -524,6 +572,7 @@ class Template(object):
res = self.compiled(values, functions)
except: # Handle any exceptions thrown by compiled version.
res = self.interpret(values, functions)
return res
def translate(self):
@@ -532,9 +581,9 @@ class Template(object):
argnames = []
for varname in varnames:
argnames.append(VARIABLE_PREFIX.encode('utf8') + varname)
argnames.append(VARIABLE_PREFIX + varname)
for funcname in funcnames:
argnames.append(FUNCTION_PREFIX.encode('utf8') + funcname)
argnames.append(FUNCTION_PREFIX + funcname)
func = compile_func(
argnames,
@@ -559,7 +608,7 @@ if __name__ == '__main__':
import timeit
_tmpl = Template(u'foo $bar %baz{foozle $bar barzle} $bar')
_vars = {'bar': 'qux'}
_funcs = {'baz': unicode.upper}
_funcs = {'baz': six.text_type.upper}
interp_time = timeit.timeit('_tmpl.interpret(_vars, _funcs)',
'from __main__ import _tmpl, _vars, _funcs',
number=10000)
@@ -568,4 +617,4 @@ if __name__ == '__main__':
'from __main__ import _tmpl, _vars, _funcs',
number=10000)
print(comp_time)
print('Speedup:', interp_time / comp_time)
print(u'Speedup:', interp_time / comp_time)
lib/beets/util/hidden.py
Executable
+86
View File
@@ -0,0 +1,86 @@
# -*- coding: utf-8 -*-
# This file is part of beets.
# Copyright 2016, Adrian Sampson.
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
"""Simple library to work out if a file is hidden on different platforms."""
from __future__ import division, absolute_import, print_function
import os
import stat
import ctypes
import sys
import beets.util
def _is_hidden_osx(path):
"""Return whether or not a file is hidden on OS X.
This uses os.lstat to work out if a file has the "hidden" flag.
"""
file_stat = os.lstat(beets.util.syspath(path))
if hasattr(file_stat, 'st_flags') and hasattr(stat, 'UF_HIDDEN'):
return bool(file_stat.st_flags & stat.UF_HIDDEN)
else:
return False
def _is_hidden_win(path):
"""Return whether or not a file is hidden on Windows.
This uses GetFileAttributes to work out if a file has the "hidden" flag
(FILE_ATTRIBUTE_HIDDEN).
"""
# FILE_ATTRIBUTE_HIDDEN = 2 (0x2) from GetFileAttributes documentation.
hidden_mask = 2
# Retrieve the attributes for the file.
attrs = ctypes.windll.kernel32.GetFileAttributesW(beets.util.syspath(path))
# Ensure we have valid attribues and compare them against the mask.
return attrs >= 0 and attrs & hidden_mask
def _is_hidden_dot(path):
"""Return whether or not a file starts with a dot.
Files starting with a dot are seen as "hidden" files on Unix-based OSes.
"""
return os.path.basename(path).startswith(b'.')
def is_hidden(path):
"""Return whether or not a file is hidden. `path` should be a
bytestring filename.
This method works differently depending on the platform it is called on.
On OS X, it uses both the result of `is_hidden_osx` and `is_hidden_dot` to
work out if a file is hidden.
On Windows, it uses the result of `is_hidden_win` to work out if a file is
hidden.
On any other operating systems (i.e. Linux), it uses `is_hidden_dot` to
work out if a file is hidden.
"""
# Run platform specific functions depending on the platform
if sys.platform == 'darwin':
return _is_hidden_osx(path) or _is_hidden_dot(path)
elif sys.platform == 'win32':
return _is_hidden_win(path)
else:
return _is_hidden_dot(path)
__all__ = ['is_hidden']
lib/beets/util/pipeline.py
Regular → Executable
+35 -22
View File
@@ -1,5 +1,6 @@
# -*- coding: utf-8 -*-
# This file is part of beets.
# Copyright 2013, Adrian Sampson.
# Copyright 2016, Adrian Sampson.
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
@@ -30,11 +31,13 @@ up a bottleneck stage by dividing its work among multiple threads.
To do so, pass an iterable of coroutines to the Pipeline constructor
in place of any single coroutine.
"""
from __future__ import print_function
import Queue
from __future__ import division, absolute_import, print_function
from six.moves import queue
from threading import Thread, Lock
import sys
import six
BUBBLE = '__PIPELINE_BUBBLE__'
POISON = '__PIPELINE_POISON__'
@@ -61,7 +64,17 @@ def _invalidate_queue(q, val=None, sync=True):
q.mutex.acquire()
try:
q.maxsize = 0
# Originally, we set `maxsize` to 0 here, which is supposed to mean
# an unlimited queue size. However, there is a race condition since
# Python 3.2 when this attribute is changed while another thread is
# waiting in put()/get() due to a full/empty queue.
# Setting it to 2 is still hacky because Python does not give any
# guarantee what happens if Queue methods/attributes are overwritten
# when it is already in use. However, because of our dummy _put()
# and _get() methods, it provides a workaround to let the queue appear
# to be never empty or full.
# See issue https://github.com/beetbox/beets/issues/2078
q.maxsize = 2
q._qsize = _qsize
q._put = _put
q._get = _get
@@ -73,13 +86,13 @@ def _invalidate_queue(q, val=None, sync=True):
q.mutex.release()
class CountedQueue(Queue.Queue):
class CountedQueue(queue.Queue):
"""A queue that keeps track of the number of threads that are
still feeding into it. The queue is poisoned when all threads are
finished with the queue.
"""
def __init__(self, maxsize=0):
Queue.Queue.__init__(self, maxsize)
queue.Queue.__init__(self, maxsize)
self.nthreads = 0
self.poisoned = False
@@ -246,7 +259,7 @@ class FirstPipelineThread(PipelineThread):
# Get the value from the generator.
try:
msg = self.coro.next()
msg = next(self.coro)
except StopIteration:
break
@@ -279,7 +292,7 @@ class MiddlePipelineThread(PipelineThread):
def run(self):
try:
# Prime the coroutine.
self.coro.next()
next(self.coro)
while True:
with self.abort_lock:
@@ -324,7 +337,7 @@ class LastPipelineThread(PipelineThread):
def run(self):
# Prime the coroutine.
self.coro.next()
next(self.coro)
try:
while True:
@@ -359,7 +372,7 @@ class Pipeline(object):
be at least two stages.
"""
if len(stages) < 2:
raise ValueError('pipeline must have at least two stages')
raise ValueError(u'pipeline must have at least two stages')
self.stages = []
for stage in stages:
if isinstance(stage, (list, tuple)):
@@ -409,7 +422,7 @@ class Pipeline(object):
try:
# Using a timeout allows us to receive KeyboardInterrupt
# exceptions during the join().
while threads[-1].isAlive():
while threads[-1].is_alive():
threads[-1].join(1)
except:
@@ -429,7 +442,7 @@ class Pipeline(object):
exc_info = thread.exc_info
if exc_info:
# Make the exception appear as it was raised originally.
raise exc_info[0], exc_info[1], exc_info[2]
six.reraise(exc_info[0], exc_info[1], exc_info[2])
def pull(self):
"""Yield elements from the end of the pipeline. Runs the stages
@@ -442,7 +455,7 @@ class Pipeline(object):
# "Prime" the coroutines.
for coro in coros[1:]:
coro.next()
next(coro)
# Begin the pipeline.
for out in coros[0]:
@@ -464,14 +477,14 @@ if __name__ == '__main__':
# in parallel.
def produce():
for i in range(5):
print('generating %i' % i)
print(u'generating %i' % i)
time.sleep(1)
yield i
def work():
num = yield
while True:
print('processing %i' % num)
print(u'processing %i' % num)
time.sleep(2)
num = yield num * 2
@@ -479,7 +492,7 @@ if __name__ == '__main__':
while True:
num = yield
time.sleep(1)
print('received %i' % num)
print(u'received %i' % num)
ts_start = time.time()
Pipeline([produce(), work(), consume()]).run_sequential()
@@ -488,22 +501,22 @@ if __name__ == '__main__':
ts_par = time.time()
Pipeline([produce(), (work(), work()), consume()]).run_parallel()
ts_end = time.time()
print('Sequential time:', ts_seq - ts_start)
print('Parallel time:', ts_par - ts_seq)
print('Multiply-parallel time:', ts_end - ts_par)
print(u'Sequential time:', ts_seq - ts_start)
print(u'Parallel time:', ts_par - ts_seq)
print(u'Multiply-parallel time:', ts_end - ts_par)
print()
# Test a pipeline that raises an exception.
def exc_produce():
for i in range(10):
print('generating %i' % i)
print(u'generating %i' % i)
time.sleep(1)
yield i
def exc_work():
num = yield
while True:
print('processing %i' % num)
print(u'processing %i' % num)
time.sleep(3)
if num == 3:
raise Exception()
@@ -512,6 +525,6 @@ if __name__ == '__main__':
def exc_consume():
while True:
num = yield
print('received %i' % num)
print(u'received %i' % num)
Pipeline([exc_produce(), exc_work(), exc_consume()]).run_parallel(1)