gronod/headphones
1
0
Fork 0
mirror of https://github.com/rembo10/headphones.git synced 2026-07-21 00:14:02 +01:00
Code Issues Packages Projects Releases Wiki Activity

Upgraded beets to 1.3.10, including patches

Browse Source
This commit is contained in:
Bas Stottelaar
2015-01-27 22:26:35 +01:00
parent ea842a95ca
commit cf6a6a876f
31 changed files with 4792 additions and 3002 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/beets/util/__init__.py
+81 -13
View File
@@ -23,10 +23,13 @@ import fnmatch
from collections import defaultdict
import traceback
import subprocess
import platform
MAX_FILENAME_LENGTH = 200
WINDOWS_MAGIC_PREFIX = u'\\\\?\\'
class HumanReadableException(Exception):
"""An Exception that can include a human-readable error message to
be logged without a traceback. Can preserve a traceback for
@@ -82,6 +85,7 @@ class HumanReadableException(Exception):
logger.debug(self.tb)
logger.error(u'{0}: {1}'.format(self.error_kind, self.args[0]))
class FilesystemError(HumanReadableException):
"""An error that occurred while performing a filesystem manipulation
via a function in this module. The `paths` field is a sequence of
@@ -111,6 +115,7 @@ class FilesystemError(HumanReadableException):
return u'{0} {1}'.format(self._reasonstr(), clause)
def normpath(path):
"""Provide the canonical form of the path suitable for storing in
the database.
@@ -119,6 +124,7 @@ def normpath(path):
path = os.path.normpath(os.path.abspath(os.path.expanduser(path)))
return bytestring_path(path)
def ancestry(path):
"""Return a list consisting of path's parent directory, its
grandparent, and so on. For instance:
@@ -137,10 +143,12 @@ def ancestry(path):
break
last_path = path
if path: # don't yield ''
if path:
# don't yield ''
out.insert(0, path)
return out
def sorted_walk(path, ignore=(), logger=None):
"""Like `os.walk`, but yields things in case-insensitive sorted,
breadth-first order. Directory and file names matching any glob
@@ -192,6 +200,7 @@ def sorted_walk(path, ignore=(), logger=None):
for res in sorted_walk(cur, ignore, logger):
yield res
def mkdirall(path):
"""Make all the enclosing directories of path (like mkdir -p on the
parent).
@@ -204,6 +213,7 @@ def mkdirall(path):
raise FilesystemError(exc, 'create', (ancestor,),
traceback.format_exc())
def fnmatch_all(names, patterns):
"""Determine whether all strings in `names` match at least one of
the `patterns`, which should be shell glob expressions.
@@ -218,6 +228,7 @@ def fnmatch_all(names, patterns):
return False
return True
def prune_dirs(path, root=None, clutter=('.DS_Store', 'Thumbs.db')):
"""If path is an empty directory, then remove it. Recursively remove
path's ancestry up to root (which is never removed) where there are
@@ -236,7 +247,7 @@ def prune_dirs(path, root=None, clutter=('.DS_Store', 'Thumbs.db')):
ancestors = []
elif root in ancestors:
# Only remove directories below the root.
ancestors = ancestors[ancestors.index(root)+1:]
ancestors = ancestors[ancestors.index(root) + 1:]
else:
# Remove nothing.
return
@@ -258,6 +269,7 @@ def prune_dirs(path, root=None, clutter=('.DS_Store', 'Thumbs.db')):
else:
break
def components(path):
"""Return a list of the path components in path. For instance:
@@ -281,6 +293,7 @@ def components(path):
return comps
def _fsencoding():
"""Get the system's filesystem encoding. On Windows, this is always
UTF-8 (not MBCS).
@@ -295,6 +308,7 @@ def _fsencoding():
encoding = 'utf8'
return encoding
def bytestring_path(path):
"""Given a path, which is either a str or a unicode, returns a str
path (ensuring that we never deal with Unicode pathnames).
@@ -315,6 +329,7 @@ def bytestring_path(path):
except (UnicodeError, LookupError):
return path.encode('utf8')
def displayable_path(path, separator=u'; '):
"""Attempts to decode a bytestring path to a unicode object for the
purpose of displaying it to the user. If the `path` argument is a
@@ -333,6 +348,7 @@ def displayable_path(path, separator=u'; '):
except (UnicodeError, LookupError):
return path.decode('utf8', 'ignore')
def syspath(path, prefix=True):
"""Convert a path for use by the operating system. In particular,
paths on Windows must receive a magic prefix and must be converted
@@ -356,16 +372,22 @@ def syspath(path, prefix=True):
encoding = sys.getfilesystemencoding() or sys.getdefaultencoding()
path = path.decode(encoding, 'replace')
# Add the magic prefix if it isn't already there
# Add the magic prefix if it isn't already there.
# http://msdn.microsoft.com/en-us/library/windows/desktop/aa365247.aspx
if prefix and not path.startswith(WINDOWS_MAGIC_PREFIX):
if path.startswith(u'\\\\'):
# UNC path. Final path should look like \\?\UNC\...
path = u'UNC' + path[1:]
path = WINDOWS_MAGIC_PREFIX + path
return path
def samefile(p1, p2):
"""Safer equality for paths."""
return shutil._samefile(syspath(p1), syspath(p2))
def remove(path, soft=True):
"""Remove the file. If `soft`, then no error will be raised if the
file does not exist.
@@ -378,6 +400,7 @@ def remove(path, soft=True):
except (OSError, IOError) as exc:
raise FilesystemError(exc, 'delete', (path,), traceback.format_exc())
def copy(path, dest, replace=False):
"""Copy a plain file. Permissions are not copied. If `dest` already
exists, raises a FilesystemError unless `replace` is True. Has no
@@ -396,6 +419,7 @@ def copy(path, dest, replace=False):
raise FilesystemError(exc, 'copy', (path, dest),
traceback.format_exc())
def move(path, dest, replace=False):
"""Rename a file. `dest` may not be a directory. If `dest` already
exists, raises an OSError unless `replace` is True. Has no effect if
@@ -424,6 +448,27 @@ def move(path, dest, replace=False):
raise FilesystemError(exc, 'move', (path, dest),
traceback.format_exc())
def link(path, dest, replace=False):
"""Create a symbolic link from path to `dest`. Raises an OSError if
`dest` already exists, unless `replace` is True. Does nothing if
`path` == `dest`."""
if (samefile(path, dest)):
return
path = syspath(path)
dest = syspath(dest)
if os.path.exists(dest) and not replace:
raise FilesystemError('file exists', 'rename', (path, dest),
traceback.format_exc())
try:
os.symlink(path, dest)
except OSError:
raise FilesystemError('Operating system does not support symbolic '
'links.', 'link', (path, dest),
traceback.format_exc())
def unique_path(path):
"""Returns a version of ``path`` that does not exist on the
filesystem. Specifically, if ``path` itself already exists, then
@@ -457,6 +502,8 @@ CHAR_REPLACE = [
(re.compile(ur'\.$'), u'_'), # Trailing dots.
(re.compile(ur'\s+$'), u''), # Trailing whitespace.
]
def sanitize_path(path, replacements=None):
"""Takes a path (as a Unicode string) and makes sure that it is
legal. Returns a new path. Only works with fragments; won't work
@@ -477,6 +524,7 @@ def sanitize_path(path, replacements=None):
comps[i] = comp
return os.path.join(*comps)
def truncate_path(path, length=MAX_FILENAME_LENGTH):
"""Given a bytestring path or a Unicode path fragment, truncate the
components to a legal length. In the last component, the extension
@@ -493,6 +541,7 @@ def truncate_path(path, length=MAX_FILENAME_LENGTH):
return os.path.join(*out)
def str2bool(value):
"""Returns a boolean reflecting a human-entered string."""
if value.lower() in ('yes', '1', 'true', 't', 'y'):
@@ -500,6 +549,7 @@ def str2bool(value):
else:
return False
def as_string(value):
"""Convert a value to a Unicode object for matching with a query.
None becomes the empty string. Bytestrings are silently decoded.
@@ -513,6 +563,7 @@ def as_string(value):
else:
return unicode(value)
def levenshtein(s1, s2):
"""A nice DP edit distance implementation from Wikibooks:
http://en.wikibooks.org/wiki/Algorithm_implementation/Strings/
@@ -535,6 +586,7 @@ def levenshtein(s1, s2):
return previous_row[-1]
def plurality(objs):
"""Given a sequence of comparable objects, returns the object that
is most common in the set and the frequency of that object. The
@@ -558,6 +610,7 @@ def plurality(objs):
return res, max_freq
def cpu_count():
"""Return the number of hardware thread contexts (cores or SMT
threads) in the system.
@@ -571,7 +624,7 @@ def cpu_count():
num = 0
elif sys.platform == 'darwin':
try:
num = int(os.popen('sysctl -n hw.ncpu').read())
num = int(command_output(['sysctl', '-n', 'hw.ncpu']))
except ValueError:
num = 0
else:
@@ -584,23 +637,38 @@ def cpu_count():
else:
return 1
def command_output(cmd):
"""Wraps the `subprocess` module to invoke a command (given as a
list of arguments starting with the command name) and collect
stdout. The stderr stream is ignored. May raise
`subprocess.CalledProcessError` or an `OSError`.
def command_output(cmd, shell=False):
"""Runs the command and returns its output after it has exited.
``cmd`` is a list of arguments starting with the command names. If
``shell`` is true, ``cmd`` is assumed to be a string and passed to a
shell to execute.
If the process exits with a non-zero return code
``subprocess.CalledProcessError`` is raised. May also raise
``OSError``.
This replaces `subprocess.check_output`, which isn't available in
Python 2.6 and which can have problems if lots of output is sent to
stderr.
"""
with open(os.devnull, 'w') as devnull:
proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=devnull)
stdout, _ = proc.communicate()
proc = subprocess.Popen(
cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
close_fds=platform.system() != 'Windows',
shell=shell
)
stdout, stderr = proc.communicate()
if proc.returncode:
raise subprocess.CalledProcessError(proc.returncode, cmd)
raise subprocess.CalledProcessError(
returncode=proc.returncode,
cmd=' '.join(cmd),
)
return stdout
def max_filename_length(path, limit=MAX_FILENAME_LENGTH):
"""Attempt to determine the maximum filename length for the
filesystem containing `path`. If the value is greater than `limit`,
lib/beets/util/artresizer.py
+49 -26
View File
@@ -1,5 +1,5 @@
# This file is part of beets.
# Copyright 2013, Fabrice Laporte
# Copyright 2014, Fabrice Laporte
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
@@ -18,6 +18,7 @@ public resizing proxy if neither is available.
import urllib
import subprocess
import os
import re
from tempfile import NamedTemporaryFile
import logging
from beets import util
@@ -37,7 +38,7 @@ def resize_url(url, maxwidth):
maxwidth (preserving aspect ratio).
"""
return '{0}?{1}'.format(PROXY_URL, urllib.urlencode({
'url': url.replace('http://',''),
'url': url.replace('http://', ''),
'w': str(maxwidth),
}))
@@ -76,7 +77,7 @@ def pil_resize(maxwidth, path_in, path_out=None):
def im_resize(maxwidth, path_in, path_out=None):
"""Resize using ImageMagick's ``convert`` tool.
tool. Return the output path of resized image.
Return the output path of resized image.
"""
path_out = path_out or temp_file_for(path_in)
log.debug(u'artresizer: ImageMagick resizing {0} to {1}'.format(
@@ -132,8 +133,9 @@ class ArtResizer(object):
"""Create a resizer object for the given method or, if none is
specified, with an inferred method.
"""
self.method = method or self._guess_method()
self.method = self._check_method(method)
log.debug(u"artresizer: method is {0}".format(self.method))
self.can_compare = self._can_compare()
def resize(self, maxwidth, path_in, path_out=None):
"""Manipulate an image file according to the method, returning a
@@ -141,7 +143,7 @@ class ArtResizer(object):
temporary file. For WEBPROXY, returns `path_in` unmodified.
"""
if self.local:
func = BACKEND_FUNCS[self.method]
func = BACKEND_FUNCS[self.method[0]]
return func(maxwidth, path_in, path_out)
else:
return path_in
@@ -159,30 +161,51 @@ class ArtResizer(object):
@property
def local(self):
"""A boolean indicating whether the resizing method is performed
locally (i.e., PIL or IMAGEMAGICK).
locally (i.e., PIL or ImageMagick).
"""
return self.method in BACKEND_FUNCS
return self.method[0] in BACKEND_FUNCS
def _can_compare(self):
"""A boolean indicating whether image comparison is available"""
return self.method[0] == IMAGEMAGICK and self.method[1] > (6, 8, 7)
@staticmethod
def _guess_method():
"""Determine which resizing method to use. Returns PIL,
IMAGEMAGICK, or WEBPROXY depending on available dependencies.
def _check_method(method=None):
"""A tuple indicating whether current method is available and its
version. If no method is given, it returns a supported one.
"""
# Try importing PIL.
try:
__import__('PIL', fromlist=['Image'])
return PIL
except ImportError:
pass
# Guess available method
if not method:
for m in [IMAGEMAGICK, PIL]:
_, version = ArtResizer._check_method(m)
if version:
return (m, version)
return (WEBPROXY, (0))
# Try invoking ImageMagick's "convert".
try:
out = util.command_output(['convert', '--version'])
if 'imagemagick' in out.lower():
# system32/convert.exe may be interfering
return IMAGEMAGICK
except (subprocess.CalledProcessError, OSError):
pass
if method == IMAGEMAGICK:
# Fall back to Web proxy method.
return WEBPROXY
# Try invoking ImageMagick's "convert".
try:
out = util.command_output(['identify', '--version'])
if 'imagemagick' in out.lower():
pattern = r".+ (\d+)\.(\d+)\.(\d+).*"
match = re.search(pattern, out)
if match:
return (IMAGEMAGICK,
(int(match.group(1)),
int(match.group(2)),
int(match.group(3))))
return (IMAGEMAGICK, (0))
except (subprocess.CalledProcessError, OSError):
return (IMAGEMAGICK, None)
if method == PIL:
# Try importing PIL.
try:
__import__('PIL', fromlist=['Image'])
return (PIL, (0))
except ImportError:
return (PIL, None)
lib/beets/util/bluelet.py
+31 -1
View File
@@ -38,6 +38,7 @@ class Event(object):
"""
pass
class WaitableEvent(Event):
"""A waitable event is one encapsulating an action that can be
waited for using a select() call. That is, it's an event with an
@@ -57,21 +58,25 @@ class WaitableEvent(Event):
"""
pass
class ValueEvent(Event):
"""An event that does nothing but return a fixed value."""
def __init__(self, value):
self.value = value
class ExceptionEvent(Event):
"""Raise an exception at the yield point. Used internally."""
def __init__(self, exc_info):
self.exc_info = exc_info
class SpawnEvent(Event):
"""Add a new coroutine thread to the scheduler."""
def __init__(self, coro):
self.spawned = coro
class JoinEvent(Event):
"""Suspend the thread until the specified child thread has
completed.
@@ -79,11 +84,13 @@ class JoinEvent(Event):
def __init__(self, child):
self.child = child
class KillEvent(Event):
"""Unschedule a child thread."""
def __init__(self, child):
self.child = child
class DelegationEvent(Event):
"""Suspend execution of the current thread, start a new thread and,
once the child thread finished, return control to the parent
@@ -92,6 +99,7 @@ class DelegationEvent(Event):
def __init__(self, coro):
self.spawned = coro
class ReturnEvent(Event):
"""Return a value the current thread's delegator at the point of
delegation. Ends the current (delegate) thread.
@@ -99,6 +107,7 @@ class ReturnEvent(Event):
def __init__(self, value):
self.value = value
class SleepEvent(WaitableEvent):
"""Suspend the thread for a given duration.
"""
@@ -108,6 +117,7 @@ class SleepEvent(WaitableEvent):
def time_left(self):
return max(self.wakeup_time - time.time(), 0.0)
class ReadEvent(WaitableEvent):
"""Reads from a file-like object."""
def __init__(self, fd, bufsize):
@@ -120,6 +130,7 @@ class ReadEvent(WaitableEvent):
def fire(self):
return self.fd.read(self.bufsize)
class WriteEvent(WaitableEvent):
"""Writes to a file-like object."""
def __init__(self, fd, data):
@@ -192,15 +203,19 @@ def _event_select(events):
return ready_events
class ThreadException(Exception):
def __init__(self, coro, exc_info):
self.coro = coro
self.exc_info = exc_info
def reraise(self):
_reraise(self.exc_info[0], self.exc_info[1], self.exc_info[2])
SUSPENDED = Event() # Special sentinel placeholder for suspended threads.
class Delegated(Event):
"""Placeholder indicating that a thread has delegated execution to a
different thread.
@@ -208,6 +223,7 @@ class Delegated(Event):
def __init__(self, child):
self.child = child
def run(root_coro):
"""Schedules a coroutine, running it to completion. This
encapsulates the Bluelet scheduler, which the root coroutine can
@@ -329,7 +345,7 @@ def run(root_coro):
break
# Wait and fire.
event2coro = dict((v,k) for k,v in threads.items())
event2coro = dict((v, k) for k, v in threads.items())
for event in _event_select(threads.values()):
# Run the IO operation, but catch socket errors.
try:
@@ -378,6 +394,7 @@ def run(root_coro):
class SocketClosedError(Exception):
pass
class Listener(object):
"""A socket wrapper object for listening sockets.
"""
@@ -407,6 +424,7 @@ class Listener(object):
self._closed = True
self.sock.close()
class Connection(object):
"""A socket wrapper object for connected sockets.
"""
@@ -468,6 +486,7 @@ class Connection(object):
yield ReturnEvent(line)
break
class AcceptEvent(WaitableEvent):
"""An event for Listener objects (listening sockets) that suspends
execution until the socket gets a connection.
@@ -482,6 +501,7 @@ class AcceptEvent(WaitableEvent):
sock, addr = self.listener.sock.accept()
return Connection(sock, addr)
class ReceiveEvent(WaitableEvent):
"""An event for Connection objects (connected sockets) for
asynchronously reading data.
@@ -496,6 +516,7 @@ class ReceiveEvent(WaitableEvent):
def fire(self):
return self.conn.sock.recv(self.bufsize)
class SendEvent(WaitableEvent):
"""An event for Connection objects (connected sockets) for
asynchronously writing data.
@@ -523,6 +544,7 @@ def null():
"""
return ValueEvent(None)
def spawn(coro):
"""Event: add another coroutine to the scheduler. Both the parent
and child coroutines run concurrently.
@@ -531,6 +553,7 @@ def spawn(coro):
raise ValueError('%s is not a coroutine' % str(coro))
return SpawnEvent(coro)
def call(coro):
"""Event: delegate to another coroutine. The current coroutine
is resumed once the sub-coroutine finishes. If the sub-coroutine
@@ -540,12 +563,14 @@ def call(coro):
raise ValueError('%s is not a coroutine' % str(coro))
return DelegationEvent(coro)
def end(value=None):
"""Event: ends the coroutine and returns a value to its
delegator.
"""
return ReturnEvent(value)
def read(fd, bufsize=None):
"""Event: read from a file descriptor asynchronously."""
if bufsize is None:
@@ -563,10 +588,12 @@ def read(fd, bufsize=None):
else:
return ReadEvent(fd, bufsize)
def write(fd, data):
"""Event: write to a file descriptor asynchronously."""
return WriteEvent(fd, data)
def connect(host, port):
"""Event: connect to a network address and return a Connection
object for communicating on the socket.
@@ -575,17 +602,20 @@ def connect(host, port):
sock = socket.create_connection(addr)
return ValueEvent(Connection(sock, addr))
def sleep(duration):
"""Event: suspend the thread for ``duration`` seconds.
"""
return SleepEvent(duration)
def join(coro):
"""Suspend the thread until another, previously `spawn`ed thread
completes.
"""
return JoinEvent(coro)
def kill(coro):
"""Halt the execution of a different `spawn`ed thread.
"""
lib/beets/util/confit.py
+476 -95
View File
@@ -21,6 +21,8 @@ import pkgutil
import sys
import yaml
import types
import collections
import re
try:
from collections import OrderedDict
except ImportError:
@@ -47,6 +49,7 @@ BASESTRING = str if PY3 else basestring
NUMERIC_TYPES = (int, float) if PY3 else (int, float, long)
TYPE_TYPES = (type,) if PY3 else (type, types.ClassType)
def iter_first(sequence):
"""Get the first element from an iterable or raise a ValueError if
the iterator generates no values.
@@ -67,16 +70,25 @@ class ConfigError(Exception):
"""Base class for exceptions raised when querying a configuration.
"""
class NotFoundError(ConfigError):
"""A requested value could not be found in the configuration trees.
"""
class ConfigTypeError(ConfigError, TypeError):
class ConfigValueError(ConfigError):
"""The value in the configuration is illegal."""
class ConfigTypeError(ConfigValueError):
"""The value in the configuration did not match the expected type.
"""
class ConfigValueError(ConfigError, ValueError):
"""The value in the configuration is illegal."""
class ConfigTemplateError(ConfigError):
"""Base class for exceptions raised because of an invalid template.
"""
class ConfigReadError(ConfigError):
"""A configuration file could not be read."""
@@ -132,6 +144,7 @@ class ConfigSource(dict):
else:
raise TypeError('source value must be a dict')
class ConfigView(object):
"""A configuration "view" is a query into a program's configuration
data. A view represents a hypothetical location in the configuration
@@ -207,6 +220,9 @@ class ConfigView(object):
"""
self.set({key: value})
def __contains__(self, key):
return self[key].exists()
def set_args(self, namespace):
"""Overlay parsed command-line arguments, generated by a library
like argparse or optparse, onto this view's value.
@@ -310,98 +326,6 @@ class ConfigView(object):
# Validation and conversion.
def get(self, typ=None):
"""Returns the canonical value for the view, checked against the
passed-in type. If the value is not an instance of the given
type, a ConfigTypeError is raised. May also raise a
NotFoundError.
"""
value, _ = self.first()
if typ is not None:
if not isinstance(typ, TYPE_TYPES):
raise TypeError('argument to get() must be a type')
if not isinstance(value, typ):
raise ConfigTypeError(
"{0} must be of type {1}, not {2}".format(
self.name, typ.__name__, type(value).__name__
)
)
return value
def as_filename(self):
"""Get a string as a normalized as an absolute, tilde-free path.
Relative paths are relative to the configuration directory (see
the `config_dir` method) if they come from a file. Otherwise,
they are relative to the current working directory. This helps
attain the expected behavior when using command-line options.
"""
path, source = self.first()
if not isinstance(path, BASESTRING):
raise ConfigTypeError('{0} must be a filename, not {1}'.format(
self.name, type(path).__name__
))
path = os.path.expanduser(STRING(path))
if not os.path.isabs(path) and source.filename:
# From defaults: relative to the app's directory.
path = os.path.join(self.root().config_dir(), path)
return os.path.abspath(path)
def as_choice(self, choices):
"""Ensure that the value is among a collection of choices and
return it. If `choices` is a dictionary, then return the
corresponding value rather than the value itself (the key).
"""
value = self.get()
if value not in choices:
raise ConfigValueError(
'{0} must be one of {1}, not {2}'.format(
self.name, repr(list(choices)), repr(value)
)
)
if isinstance(choices, dict):
return choices[value]
else:
return value
def as_number(self):
"""Ensure that a value is of numeric type."""
value = self.get()
if isinstance(value, NUMERIC_TYPES):
return value
raise ConfigTypeError(
'{0} must be numeric, not {1}'.format(
self.name, type(value).__name__
)
)
def as_str_seq(self):
"""Get the value as a list of strings. The underlying configured
value can be a sequence or a single string. In the latter case,
the string is treated as a white-space separated list of words.
"""
value = self.get()
if isinstance(value, bytes):
value = value.decode('utf8', 'ignore')
if isinstance(value, STRING):
return value.split()
else:
try:
return list(value)
except TypeError:
raise ConfigTypeError(
'{0} must be a whitespace-separated string or '
'a list'.format(self.name)
)
def flatten(self):
"""Create a hierarchy of OrderedDicts containing the data from
this view, recursively reifying all views to get their
@@ -415,6 +339,35 @@ class ConfigView(object):
od[key] = view.get()
return od
def get(self, template=None):
"""Retrieve the value for this view according to the template.
The `template` against which the values are checked can be
anything convertible to a `Template` using `as_template`. This
means you can pass in a default integer or string value, for
example, or a type to just check that something matches the type
you expect.
May raise a `ConfigValueError` (or its subclass,
`ConfigTypeError`) or a `NotFoundError` when the configuration
doesn't satisfy the template.
"""
return as_template(template).value(self, template)
# Old validation methods (deprecated).
def as_filename(self):
return self.get(Filename())
def as_choice(self, choices):
return self.get(Choice(choices))
def as_number(self):
return self.get(Number())
def as_str_seq(self):
return self.get(StrSeq())
class RootView(ConfigView):
"""The base of a view hierarchy. This view keeps track of the
@@ -518,6 +471,7 @@ def _package_path(name):
return os.path.dirname(os.path.abspath(filepath))
def config_dirs():
"""Return a platform-specific list of candidates for user
configuration directories on the system.
@@ -606,10 +560,12 @@ class Loader(yaml.SafeLoader):
plain = super(Loader, self).check_plain()
return plain or self.peek() == '%'
Loader.add_constructor('tag:yaml.org,2002:str', Loader._construct_unicode)
Loader.add_constructor('tag:yaml.org,2002:map', Loader.construct_yaml_map)
Loader.add_constructor('tag:yaml.org,2002:omap', Loader.construct_yaml_map)
def load_yaml(filename):
"""Read a YAML document from a file. If the file cannot be read or
parsed, a ConfigReadError is raised.
@@ -679,11 +635,13 @@ class Dumper(yaml.SafeDumper):
"""
return self.represent_scalar('tag:yaml.org,2002:null', '')
Dumper.add_representer(OrderedDict, Dumper.represent_dict)
Dumper.add_representer(bool, Dumper.represent_bool)
Dumper.add_representer(type(None), Dumper.represent_none)
Dumper.add_representer(list, Dumper.represent_list)
def restore_yaml_comments(data, default_data):
"""Scan default_data for comments (we include empty lines in our
definition of comments) and place them before the same keys in data.
@@ -898,3 +856,426 @@ class LazyConfig(Configuration):
del self.sources[:]
self._lazy_suffix = []
self._lazy_prefix = []
# "Validated" configuration views: experimental!
REQUIRED = object()
"""A sentinel indicating that there is no default value and an exception
should be raised when the value is missing.
"""
class Template(object):
"""A value template for configuration fields.
The template works like a type and instructs Confit about how to
interpret a deserialized YAML value. This includes type conversions,
providing a default value, and validating for errors. For example, a
filepath type might expand tildes and check that the file exists.
"""
def __init__(self, default=REQUIRED):
"""Create a template with a given default value.
If `default` is the sentinel `REQUIRED` (as it is by default),
then an error will be raised when a value is missing. Otherwise,
missing values will instead return `default`.
"""
self.default = default
def __call__(self, view):
"""Invoking a template on a view gets the view's value according
to the template.
"""
return self.value(view, self)
def value(self, view, template=None):
"""Get the value for a `ConfigView`.
May raise a `NotFoundError` if the value is missing (and the
template requires it) or a `ConfigValueError` for invalid values.
"""
if view.exists():
value, _ = view.first()
return self.convert(value, view)
elif self.default is REQUIRED:
# Missing required value. This is an error.
raise NotFoundError("{0} not found".format(view.name))
else:
# Missing value, but not required.
return self.default
def convert(self, value, view):
"""Convert the YAML-deserialized value to a value of the desired
type.
Subclasses should override this to provide useful conversions.
May raise a `ConfigValueError` when the configuration is wrong.
"""
# Default implementation does no conversion.
return value
def fail(self, message, view, type_error=False):
"""Raise an exception indicating that a value cannot be
accepted.
`type_error` indicates whether the error is due to a type
mismatch rather than a malformed value. In this case, a more
specific exception is raised.
"""
exc_class = ConfigTypeError if type_error else ConfigValueError
raise exc_class(
'{0}: {1}'.format(view.name, message)
)
def __repr__(self):
return '{0}({1})'.format(
type(self).__name__,
'' if self.default is REQUIRED else repr(self.default),
)
class Integer(Template):
"""An integer configuration value template.
"""
def convert(self, value, view):
"""Check that the value is an integer. Floats are rounded.
"""
if isinstance(value, int):
return value
elif isinstance(value, float):
return int(value)
else:
self.fail('must be a number', view, True)
class Number(Template):
"""A numeric type: either an integer or a floating-point number.
"""
def convert(self, value, view):
"""Check that the value is an int or a float.
"""
if isinstance(value, NUMERIC_TYPES):
return value
else:
self.fail(
'must be numeric, not {0}'.format(type(value).__name__),
view,
True
)
class MappingTemplate(Template):
"""A template that uses a dictionary to specify other types for the
values for a set of keys and produce a validated `AttrDict`.
"""
def __init__(self, mapping):
"""Create a template according to a dict (mapping). The
mapping's values should themselves either be Types or
convertible to Types.
"""
subtemplates = {}
for key, typ in mapping.items():
subtemplates[key] = as_template(typ)
self.subtemplates = subtemplates
def value(self, view, template=None):
"""Get a dict with the same keys as the template and values
validated according to the value types.
"""
out = AttrDict()
for key, typ in self.subtemplates.items():
out[key] = typ.value(view[key], self)
return out
def __repr__(self):
return 'MappingTemplate({0})'.format(repr(self.subtemplates))
class String(Template):
"""A string configuration value template.
"""
def __init__(self, default=REQUIRED, pattern=None):
"""Create a template with the added optional `pattern` argument,
a regular expression string that the value should match.
"""
super(String, self).__init__(default)
self.pattern = pattern
if pattern:
self.regex = re.compile(pattern)
def convert(self, value, view):
"""Check that the value is a string and matches the pattern.
"""
if isinstance(value, BASESTRING):
if self.pattern and not self.regex.match(value):
self.fail(
"must match the pattern {0}".format(self.pattern),
view
)
return value
else:
self.fail('must be a string', view, True)
class Choice(Template):
"""A template that permits values from a sequence of choices.
"""
def __init__(self, choices):
"""Create a template that validates any of the values from the
iterable `choices`.
If `choices` is a map, then the corresponding value is emitted.
Otherwise, the value itself is emitted.
"""
self.choices = choices
def convert(self, value, view):
"""Ensure that the value is among the choices (and remap if the
choices are a mapping).
"""
if value not in self.choices:
self.fail(
'must be one of {0}, not {1}'.format(
repr(list(self.choices)), repr(value)
),
view
)
if isinstance(self.choices, collections.Mapping):
return self.choices[value]
else:
return value
def __repr__(self):
return 'Choice({0!r})'.format(self.choices)
class StrSeq(Template):
"""A template for values that are lists of strings.
Validates both actual YAML string lists and single strings. Strings
can optionally be split on whitespace.
"""
def __init__(self, split=True):
"""Create a new template.
`split` indicates whether, when the underlying value is a single
string, it should be split on whitespace. Otherwise, the
resulting value is a list containing a single string.
"""
super(StrSeq, self).__init__()
self.split = split
def convert(self, value, view):
if isinstance(value, bytes):
value = value.decode('utf8', 'ignore')
if isinstance(value, STRING):
if self.split:
return value.split()
else:
return [value]
try:
value = list(value)
except TypeError:
self.fail('must be a whitespace-separated string or a list',
view, True)
def convert(x):
if isinstance(x, unicode):
return x
elif isinstance(x, BASESTRING):
return x.decode('utf8', 'ignore')
else:
self.fail('must be a list of strings', view, True)
return map(convert, value)
class Filename(Template):
"""A template that validates strings as filenames.
Filenames are returned as absolute, tilde-free paths.
Relative paths are relative to the template's `cwd` argument
when it is specified, then the configuration directory (see
the `config_dir` method) if they come from a file. Otherwise,
they are relative to the current working directory. This helps
attain the expected behavior when using command-line options.
"""
def __init__(self, default=REQUIRED, cwd=None, relative_to=None,
in_app_dir=False):
""" `relative_to` is the name of a sibling value that is
being validated at the same time.
`in_app_dir` indicates whether the path should be resolved
inside the application's config directory (even when the setting
does not come from a file).
"""
super(Filename, self).__init__(default)
self.cwd = cwd
self.relative_to = relative_to
self.in_app_dir = in_app_dir
def __repr__(self):
args = []
if self.default is not REQUIRED:
args.append(repr(self.default))
if self.cwd is not None:
args.append('cwd=' + repr(self.cwd))
if self.relative_to is not None:
args.append('relative_to=' + repr(self.relative_to))
if self.in_app_dir:
args.append('in_app_dir=True')
return 'Filename({0})'.format(', '.join(args))
def resolve_relative_to(self, view, template):
if not isinstance(template, (collections.Mapping, MappingTemplate)):
# disallow config.get(Filename(relative_to='foo'))
raise ConfigTemplateError(
'relative_to may only be used when getting multiple values.'
)
elif self.relative_to == view.key:
raise ConfigTemplateError(
'{0} is relative to itself'.format(view.name)
)
elif self.relative_to not in view.parent.keys():
# self.relative_to is not in the config
self.fail(
(
'needs sibling value "{0}" to expand relative path'
).format(self.relative_to),
view
)
old_template = {}
old_template.update(template.subtemplates)
# save time by skipping MappingTemplate's init loop
next_template = MappingTemplate({})
next_relative = self.relative_to
# gather all the needed templates and nothing else
while next_relative is not None:
try:
# pop to avoid infinite loop because of recursive
# relative paths
rel_to_template = old_template.pop(next_relative)
except KeyError:
if next_relative in template.subtemplates:
# we encountered this config key previously
raise ConfigTemplateError((
'{0} and {1} are recursively relative'
).format(view.name, self.relative_to))
else:
raise ConfigTemplateError((
'missing template for {0}, needed to expand {1}\'s' +
'relative path'
).format(self.relative_to, view.name))
next_template.subtemplates[next_relative] = rel_to_template
next_relative = rel_to_template.relative_to
return view.parent.get(next_template)[self.relative_to]
def value(self, view, template=None):
path, source = view.first()
if not isinstance(path, BASESTRING):
self.fail(
'must be a filename, not {0}'.format(type(path).__name__),
view,
True
)
path = os.path.expanduser(STRING(path))
if not os.path.isabs(path):
if self.cwd is not None:
# relative to the template's argument
path = os.path.join(self.cwd, path)
elif self.relative_to is not None:
path = os.path.join(
self.resolve_relative_to(view, template),
path,
)
elif source.filename or self.in_app_dir:
# From defaults: relative to the app's directory.
path = os.path.join(view.root().config_dir(), path)
return os.path.abspath(path)
class TypeTemplate(Template):
"""A simple template that checks that a value is an instance of a
desired Python type.
"""
def __init__(self, typ, default=REQUIRED):
"""Create a template that checks that the value is an instance
of `typ`.
"""
super(TypeTemplate, self).__init__(default)
self.typ = typ
def convert(self, value, view):
if not isinstance(value, self.typ):
self.fail(
'must be a {0}, not {1}'.format(
self.typ.__name__,
type(value).__name__,
),
view,
True
)
return value
class AttrDict(dict):
"""A `dict` subclass that can be accessed via attributes (dot
notation) for convenience.
"""
def __getattr__(self, key):
if key in self:
return self[key]
else:
raise AttributeError(key)
def as_template(value):
"""Convert a simple "shorthand" Python value to a `Template`.
"""
if isinstance(value, Template):
# If it's already a Template, pass it through.
return value
elif isinstance(value, collections.Mapping):
# Dictionaries work as templates.
return MappingTemplate(value)
elif value is int:
return Integer()
elif isinstance(value, int):
return Integer(value)
elif isinstance(value, type) and issubclass(value, BASESTRING):
return String()
elif isinstance(value, BASESTRING):
return String(value)
elif value is float:
return Number()
elif value is None:
return Template()
elif value is dict:
return TypeTemplate(collections.Mapping)
elif value is list:
return TypeTemplate(collections.Sequence)
elif isinstance(value, type):
return TypeTemplate(value)
else:
raise ValueError('cannot convert to template: {0!r}'.format(value))
lib/beets/util/enumeration.py
+19 -157
View File
@@ -12,167 +12,29 @@
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
"""A metaclass for enumerated types that really are types.
from enum import Enum
You can create enumerations with `enum(values, [name])` and they work
how you would expect them to.
>>> from enumeration import enum
>>> Direction = enum('north east south west', name='Direction')
>>> Direction.west
Direction.west
>>> Direction.west == Direction.west
True
>>> Direction.west == Direction.east
False
>>> isinstance(Direction.west, Direction)
True
>>> Direction[3]
Direction.west
>>> Direction['west']
Direction.west
>>> Direction.west.name
'west'
>>> Direction.north < Direction.west
True
Enumerations are classes; their instances represent the possible values
of the enumeration. Because Python classes must have names, you may
provide a `name` parameter to `enum`; if you don't, a meaningless one
will be chosen for you.
"""
import random
class Enumeration(type):
"""A metaclass whose classes are enumerations.
The `values` attribute of the class is used to populate the
enumeration. Values may either be a list of enumerated names or a
string containing a space-separated list of names. When the class
is created, it is instantiated for each name value in `values`.
Each such instance is the name of the enumerated item as the sole
argument.
The `Enumerated` class is a good choice for a superclass.
class OrderedEnum(Enum):
"""
def __init__(cls, name, bases, dic):
super(Enumeration, cls).__init__(name, bases, dic)
if 'values' not in dic:
# Do nothing if no values are provided (i.e., with
# Enumerated itself).
return
# May be called with a single string, in which case we split on
# whitespace for convenience.
values = dic['values']
if isinstance(values, basestring):
values = values.split()
# Create the Enumerated instances for each value. We have to use
# super's __setattr__ here because we disallow setattr below.
super(Enumeration, cls).__setattr__('_items_dict', {})
super(Enumeration, cls).__setattr__('_items_list', [])
for value in values:
item = cls(value, len(cls._items_list))
cls._items_dict[value] = item
cls._items_list.append(item)
def __getattr__(cls, key):
try:
return cls._items_dict[key]
except KeyError:
raise AttributeError("enumeration '" + cls.__name__ +
"' has no item '" + key + "'")
def __setattr__(cls, key, val):
raise TypeError("enumerations do not support attribute assignment")
def __getitem__(cls, key):
if isinstance(key, int):
return cls._items_list[key]
else:
return getattr(cls, key)
def __len__(cls):
return len(cls._items_list)
def __iter__(cls):
return iter(cls._items_list)
def __nonzero__(cls):
# Ensures that __len__ doesn't get called before __init__ by
# pydoc.
return True
class Enumerated(object):
"""An item in an enumeration.
Contains instance methods inherited by enumerated objects. The
metaclass is preset to `Enumeration` for your convenience.
Instance attributes:
name -- The name of the item.
index -- The index of the item in its enumeration.
>>> from enumeration import Enumerated
>>> class Garment(Enumerated):
... values = 'hat glove belt poncho lederhosen suspenders'
... def wear(self):
... print('now wearing a ' + self.name)
...
>>> Garment.poncho.wear()
now wearing a poncho
An Enum subclass that allows comparison of members.
"""
def __ge__(self, other):
if self.__class__ is other.__class__:
return self.value >= other.value
return NotImplemented
__metaclass__ = Enumeration
def __gt__(self, other):
if self.__class__ is other.__class__:
return self.value > other.value
return NotImplemented
def __init__(self, name, index):
self.name = name
self.index = index
def __le__(self, other):
if self.__class__ is other.__class__:
return self.value <= other.value
return NotImplemented
def __str__(self):
return type(self).__name__ + '.' + self.name
def __repr__(self):
return str(self)
def __cmp__(self, other):
if type(self) is type(other):
# Note that we're assuming that the items are direct
# instances of the same Enumeration (i.e., no fancy
# subclassing), which is probably okay.
return cmp(self.index, other.index)
else:
return NotImplemented
def enum(*values, **kwargs):
"""Shorthand for creating a new Enumeration class.
Call with enumeration values as a list, a space-delimited string, or
just an argument list. To give the class a name, pass it as the
`name` keyword argument. Otherwise, a name will be chosen for you.
The following are all equivalent:
enum('pinkie ring middle index thumb')
enum('pinkie', 'ring', 'middle', 'index', 'thumb')
enum(['pinkie', 'ring', 'middle', 'index', 'thumb'])
"""
if ('name' not in kwargs) or kwargs['name'] is None:
# Create a probably-unique name. It doesn't really have to be
# unique, but getting distinct names each time helps with
# identification in debugging.
name = 'Enumeration' + hex(random.randint(0,0xfffffff))[2:].upper()
else:
name = kwargs['name']
if len(values) == 1:
# If there's only one value, we have a couple of alternate calling
# styles.
if isinstance(values[0], basestring) or hasattr(values[0], '__iter__'):
values = values[0]
return type(name, (Enumerated,), {'values': values})
def __lt__(self, other):
if self.__class__ is other.__class__:
return self.value < other.value
return NotImplemented
lib/beets/util/functemplate.py
+14 -4
View File
@@ -42,6 +42,7 @@ ESCAPE_CHAR = u'$'
VARIABLE_PREFIX = '__var_'
FUNCTION_PREFIX = '__func_'
class Environment(object):
"""Contains the values and functions to be substituted into a
template.
@@ -57,10 +58,12 @@ def ex_lvalue(name):
"""A variable load expression."""
return ast.Name(name, ast.Store())
def ex_rvalue(name):
"""A variable store expression."""
return ast.Name(name, ast.Load())
def ex_literal(val):
"""An int, float, long, bool, string, or None literal with the given
value.
@@ -75,6 +78,7 @@ def ex_literal(val):
return ast.Str(val)
raise TypeError('no literal for {0}'.format(type(val)))
def ex_varassign(name, expr):
"""Assign an expression into a single variable. The expression may
either be an `ast.expr` object or a value to be used as a literal.
@@ -83,6 +87,7 @@ def ex_varassign(name, expr):
expr = ex_literal(expr)
return ast.Assign([ex_lvalue(name)], expr)
def ex_call(func, args):
"""A function-call expression with only positional parameters. The
function may be an expression or the name of a function. Each
@@ -98,6 +103,7 @@ def ex_call(func, args):
return ast.Call(func, args, [], None, None)
def compile_func(arg_names, statements, name='_the_func', debug=False):
"""Compile a list of statements as the body of a function and return
the resulting Python function. If `debug`, then print out the
@@ -157,6 +163,7 @@ class Symbol(object):
expr = ex_rvalue(VARIABLE_PREFIX + self.ident.encode('utf8'))
return [expr], set([self.ident.encode('utf8')]), set()
class Call(object):
"""A function call in a template."""
def __init__(self, ident, args, original):
@@ -214,6 +221,7 @@ class Call(object):
)
return [subexpr_call], varnames, funcnames
class Expression(object):
"""Top-level template construct: contains a list of text blobs,
Symbols, and Calls.
@@ -259,6 +267,7 @@ class Expression(object):
class ParseError(Exception):
pass
class Parser(object):
"""Parses a template expression string. Instantiate the class with
the template source and call ``parse_expression``. The ``pos`` field
@@ -316,13 +325,13 @@ class Parser(object):
next_char = self.string[self.pos + 1]
if char == ESCAPE_CHAR and next_char in \
(SYMBOL_DELIM, FUNC_DELIM, GROUP_CLOSE, ARG_SEP):
(SYMBOL_DELIM, FUNC_DELIM, GROUP_CLOSE, ARG_SEP):
# An escaped special character ($$, $}, etc.). Note that
# ${ is not an escape sequence: this is ambiguous with
# the start of a symbol and it's not necessary (just
# using { suffices in all cases).
text_parts.append(next_char)
self.pos += 2 # Skip the next character.
self.pos += 2 # Skip the next character.
continue
# Shift all characters collected so far into a single string.
@@ -372,7 +381,7 @@ class Parser(object):
if next_char == GROUP_OPEN:
# A symbol like ${this}.
self.pos += 1 # Skip opening.
self.pos += 1 # Skip opening.
closer = self.string.find(GROUP_CLOSE, self.pos)
if closer == -1 or closer == self.pos:
# No closing brace found or identifier is empty.
@@ -431,7 +440,7 @@ class Parser(object):
self.parts.append(self.string[start_pos:self.pos])
return
self.pos += 1 # Move past closing brace.
self.pos += 1 # Move past closing brace.
self.parts.append(Call(ident, args, self.string[start_pos:self.pos]))
def parse_argument_list(self):
@@ -472,6 +481,7 @@ class Parser(object):
self.pos += len(ident)
return ident
def _parse(template):
"""Parse a top-level template string Expression. Any extraneous text
is considered literal text.
lib/beets/util/pipeline.py
+70 -7
View File
@@ -35,13 +35,13 @@ from __future__ import print_function
import Queue
from threading import Thread, Lock
import sys
import types
BUBBLE = '__PIPELINE_BUBBLE__'
POISON = '__PIPELINE_POISON__'
DEFAULT_QUEUE_SIZE = 16
def _invalidate_queue(q, val=None, sync=True):
"""Breaks a Queue such that it never blocks, always has size 1,
and has no maximum size. get()ing from the queue returns `val`,
@@ -50,8 +50,10 @@ def _invalidate_queue(q, val=None, sync=True):
"""
def _qsize(len=len):
return 1
def _put(item):
pass
def _get():
return val
@@ -70,6 +72,7 @@ def _invalidate_queue(q, val=None, sync=True):
if sync:
q.mutex.release()
class CountedQueue(Queue.Queue):
"""A queue that keeps track of the number of threads that are
still feeding into it. The queue is poisoned when all threads are
@@ -104,6 +107,7 @@ class CountedQueue(Queue.Queue):
# Replacement _get invalidates when no items remain.
_old_get = self._get
def _get():
out = _old_get()
if not self.queue:
@@ -117,18 +121,67 @@ class CountedQueue(Queue.Queue):
# No items. Invalidate immediately.
_invalidate_queue(self, POISON, False)
class MultiMessage(object):
"""A message yielded by a pipeline stage encapsulating multiple
values to be sent to the next stage.
"""
def __init__(self, messages):
self.messages = messages
def multiple(messages):
"""Yield multiple([message, ..]) from a pipeline stage to send
multiple values to the next pipeline stage.
"""
return MultiMessage(messages)
def stage(func):
"""Decorate a function to become a simple stage.
>>> @stage
... def add(n, i):
... return i + n
>>> pipe = Pipeline([
... iter([1, 2, 3]),
... add(2),
... ])
>>> list(pipe.pull())
[3, 4, 5]
"""
def coro(*args):
task = None
while True:
task = yield task
task = func(*(args + (task,)))
return coro
def mutator_stage(func):
"""Decorate a function that manipulates items in a coroutine to
become a simple stage.
>>> @mutator_stage
... def setkey(key, item):
... item[key] = True
>>> pipe = Pipeline([
... iter([{'x': False}, {'a': False}]),
... setkey('x'),
... ])
>>> list(pipe.pull())
[{'x': True}, {'a': False, 'x': True}]
"""
def coro(*args):
task = None
while True:
task = yield task
func(*(args + (task,)))
return coro
def _allmsgs(obj):
"""Returns a list of all the messages encapsulated in obj. If obj
is a MultiMessage, returns its enclosed messages. If obj is BUBBLE,
@@ -141,6 +194,7 @@ def _allmsgs(obj):
else:
return [obj]
class PipelineThread(Thread):
"""Abstract base class for pipeline-stage threads."""
def __init__(self, all_threads):
@@ -169,6 +223,7 @@ class PipelineThread(Thread):
for thread in self.all_threads:
thread.abort()
class FirstPipelineThread(PipelineThread):
"""The thread running the first stage in a parallel pipeline setup.
The coroutine should just be a generator.
@@ -209,6 +264,7 @@ class FirstPipelineThread(PipelineThread):
# Generator finished; shut down the pipeline.
self.out_queue.release()
class MiddlePipelineThread(PipelineThread):
"""A thread running any stage in the pipeline except the first or
last.
@@ -256,6 +312,7 @@ class MiddlePipelineThread(PipelineThread):
# Pipeline is shutting down normally.
self.out_queue.release()
class LastPipelineThread(PipelineThread):
"""A thread running the last stage in a pipeline. The coroutine
should yield nothing.
@@ -291,6 +348,7 @@ class LastPipelineThread(PipelineThread):
self.abort_all(sys.exc_info())
return
class Pipeline(object):
"""Represents a staged pattern of work. Each stage in the pipeline
is a coroutine that receives messages from the previous stage and
@@ -322,7 +380,8 @@ class Pipeline(object):
messages between the stages are stored in queues of the given
size.
"""
queues = [CountedQueue(queue_size) for i in range(len(self.stages)-1)]
queue_count = len(self.stages) - 1
queues = [CountedQueue(queue_size) for i in range(queue_count)]
threads = []
# Set up first stage.
@@ -330,10 +389,10 @@ class Pipeline(object):
threads.append(FirstPipelineThread(coro, queues[0], threads))
# Middle stages.
for i in range(1, len(self.stages)-1):
for i in range(1, queue_count):
for coro in self.stages[i]:
threads.append(MiddlePipelineThread(
coro, queues[i-1], queues[i], threads
coro, queues[i - 1], queues[i], threads
))
# Last stage.
@@ -408,17 +467,20 @@ if __name__ == '__main__':
print('generating %i' % i)
time.sleep(1)
yield i
def work():
num = yield
while True:
print('processing %i' % num)
time.sleep(2)
num = yield num*2
num = yield num * 2
def consume():
while True:
num = yield
time.sleep(1)
print('received %i' % num)
ts_start = time.time()
Pipeline([produce(), work(), consume()]).run_sequential()
ts_seq = time.time()
@@ -437,6 +499,7 @@ if __name__ == '__main__':
print('generating %i' % i)
time.sleep(1)
yield i
def exc_work():
num = yield
while True:
@@ -445,10 +508,10 @@ if __name__ == '__main__':
if num == 3:
raise Exception()
num = yield num * 2
def exc_consume():
while True:
num = yield
#if num == 4:
# raise Exception()
print('received %i' % num)
Pipeline([exc_produce(), exc_work(), exc_consume()]).run_parallel(1)