Initial python3 changes

Mostly just updating libraries, removing string encoding/decoding,
fixing some edge cases. No new functionality was added in this
commit.

lib/beets/util/__init__.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # This file is part of beets.
 # Copyright 2016, Adrian Sampson.
 #
@@ -15,27 +14,28 @@
 
 """Miscellaneous utility functions."""
 
-from __future__ import division, absolute_import, print_function
 import os
 import sys
 import errno
 import locale
 import re
+import tempfile
 import shutil
 import fnmatch
-from collections import Counter
+import functools
+from collections import Counter, namedtuple
+from multiprocessing.pool import ThreadPool
 import traceback
 import subprocess
 import platform
 import shlex
 from beets.util import hidden
-import six
 from unidecode import unidecode
+from enum import Enum
 
 
 MAX_FILENAME_LENGTH = 200
-WINDOWS_MAGIC_PREFIX = u'\\\\?\\'
-SNI_SUPPORTED = sys.version_info >= (2, 7, 9)
+WINDOWS_MAGIC_PREFIX = '\\\\?\\'
 
 
 class HumanReadableException(Exception):
@@ -57,27 +57,27 @@ class HumanReadableException(Exception):
         self.reason = reason
         self.verb = verb
         self.tb = tb
-        super(HumanReadableException, self).__init__(self.get_message())
+        super().__init__(self.get_message())
 
     def _gerund(self):
         """Generate a (likely) gerund form of the English verb.
         """
-        if u' ' in self.verb:
+        if ' ' in self.verb:
             return self.verb
-        gerund = self.verb[:-1] if self.verb.endswith(u'e') else self.verb
-        gerund += u'ing'
+        gerund = self.verb[:-1] if self.verb.endswith('e') else self.verb
+        gerund += 'ing'
         return gerund
 
     def _reasonstr(self):
         """Get the reason as a string."""
-        if isinstance(self.reason, six.text_type):
+        if isinstance(self.reason, str):
             return self.reason
         elif isinstance(self.reason, bytes):
             return self.reason.decode('utf-8', 'ignore')
         elif hasattr(self.reason, 'strerror'):  # i.e., EnvironmentError
             return self.reason.strerror
         else:
-            return u'"{0}"'.format(six.text_type(self.reason))
+            return '"{}"'.format(str(self.reason))
 
     def get_message(self):
         """Create the human-readable description of the error, sans
@@ -91,7 +91,7 @@ class HumanReadableException(Exception):
         """
         if self.tb:
             logger.debug(self.tb)
-        logger.error(u'{0}: {1}', self.error_kind, self.args[0])
+        logger.error('{0}: {1}', self.error_kind, self.args[0])
 
 
 class FilesystemError(HumanReadableException):
@@ -99,29 +99,41 @@ class FilesystemError(HumanReadableException):
     via a function in this module. The `paths` field is a sequence of
     pathnames involved in the operation.
     """
+
     def __init__(self, reason, verb, paths, tb=None):
         self.paths = paths
-        super(FilesystemError, self).__init__(reason, verb, tb)
+        super().__init__(reason, verb, tb)
 
     def get_message(self):
         # Use a nicer English phrasing for some specific verbs.
         if self.verb in ('move', 'copy', 'rename'):
-            clause = u'while {0} {1} to {2}'.format(
+            clause = 'while {} {} to {}'.format(
                 self._gerund(),
                 displayable_path(self.paths[0]),
                 displayable_path(self.paths[1])
             )
         elif self.verb in ('delete', 'write', 'create', 'read'):
-            clause = u'while {0} {1}'.format(
+            clause = 'while {} {}'.format(
                 self._gerund(),
                 displayable_path(self.paths[0])
             )
         else:
-            clause = u'during {0} of paths {1}'.format(
-                self.verb, u', '.join(displayable_path(p) for p in self.paths)
+            clause = 'during {} of paths {}'.format(
+                self.verb, ', '.join(displayable_path(p) for p in self.paths)
             )
 
-        return u'{0} {1}'.format(self._reasonstr(), clause)
+        return f'{self._reasonstr()} {clause}'
+
+
+class MoveOperation(Enum):
+    """The file operations that e.g. various move functions can carry out.
+    """
+    MOVE = 0
+    COPY = 1
+    LINK = 2
+    HARDLINK = 3
+    REFLINK = 4
+    REFLINK_AUTO = 5
 
 
 def normpath(path):
@@ -172,7 +184,7 @@ def sorted_walk(path, ignore=(), ignore_hidden=False, logger=None):
         contents = os.listdir(syspath(path))
     except OSError as exc:
         if logger:
-            logger.warning(u'could not list directory {0}: {1}'.format(
+            logger.warning('could not list directory {}: {}'.format(
                 displayable_path(path), exc.strerror
             ))
         return
@@ -185,6 +197,10 @@ def sorted_walk(path, ignore=(), ignore_hidden=False, logger=None):
         skip = False
         for pat in ignore:
             if fnmatch.fnmatch(base, pat):
+                if logger:
+                    logger.debug('ignoring {} due to ignore rule {}'.format(
+                        base, pat
+                    ))
                 skip = True
                 break
         if skip:
@@ -207,8 +223,14 @@ def sorted_walk(path, ignore=(), ignore_hidden=False, logger=None):
     for base in dirs:
         cur = os.path.join(path, base)
         # yield from sorted_walk(...)
-        for res in sorted_walk(cur, ignore, ignore_hidden, logger):
-            yield res
+        yield from sorted_walk(cur, ignore, ignore_hidden, logger)
+
+
+def path_as_posix(path):
+    """Return the string representation of the path with forward (/)
+    slashes.
+    """
+    return path.replace(b'\\', b'/')
 
 
 def mkdirall(path):
@@ -219,7 +241,7 @@ def mkdirall(path):
         if not os.path.isdir(syspath(ancestor)):
             try:
                 os.mkdir(syspath(ancestor))
-            except (OSError, IOError) as exc:
+            except OSError as exc:
                 raise FilesystemError(exc, 'create', (ancestor,),
                                       traceback.format_exc())
 
@@ -272,13 +294,13 @@ def prune_dirs(path, root=None, clutter=('.DS_Store', 'Thumbs.db')):
             continue
         clutter = [bytestring_path(c) for c in clutter]
         match_paths = [bytestring_path(d) for d in os.listdir(directory)]
-        if fnmatch_all(match_paths, clutter):
-            # Directory contains only clutter (or nothing).
-            try:
+        try:
+            if fnmatch_all(match_paths, clutter):
+                # Directory contains only clutter (or nothing).
                 shutil.rmtree(directory)
-            except OSError:
+            else:
                 break
-        else:
+        except OSError:
             break
 
 
@@ -357,18 +379,18 @@ def bytestring_path(path):
 PATH_SEP = bytestring_path(os.sep)
 
 
-def displayable_path(path, separator=u'; '):
+def displayable_path(path, separator='; '):
     """Attempts to decode a bytestring path to a unicode object for the
     purpose of displaying it to the user. If the `path` argument is a
     list or a tuple, the elements are joined with `separator`.
     """
     if isinstance(path, (list, tuple)):
         return separator.join(displayable_path(p) for p in path)
-    elif isinstance(path, six.text_type):
+    elif isinstance(path, str):
         return path
     elif not isinstance(path, bytes):
         # A non-string object: just get its unicode representation.
-        return six.text_type(path)
+        return str(path)
 
     try:
         return path.decode(_fsencoding(), 'ignore')
@@ -387,7 +409,7 @@ def syspath(path, prefix=True):
     if os.path.__name__ != 'ntpath':
         return path
 
-    if not isinstance(path, six.text_type):
+    if not isinstance(path, str):
         # Beets currently represents Windows paths internally with UTF-8
         # arbitrarily. But earlier versions used MBCS because it is
         # reported as the FS encoding by Windows. Try both.
@@ -400,11 +422,11 @@ def syspath(path, prefix=True):
             path = path.decode(encoding, 'replace')
 
     # Add the magic prefix if it isn't already there.
-    # http://msdn.microsoft.com/en-us/library/windows/desktop/aa365247.aspx
+    # https://msdn.microsoft.com/en-us/library/windows/desktop/aa365247.aspx
     if prefix and not path.startswith(WINDOWS_MAGIC_PREFIX):
-        if path.startswith(u'\\\\'):
+        if path.startswith('\\\\'):
             # UNC path. Final path should look like \\?\UNC\...
-            path = u'UNC' + path[1:]
+            path = 'UNC' + path[1:]
         path = WINDOWS_MAGIC_PREFIX + path
 
     return path
@@ -412,6 +434,8 @@ def syspath(path, prefix=True):
 
 def samefile(p1, p2):
     """Safer equality for paths."""
+    if p1 == p2:
+        return True
     return shutil._samefile(syspath(p1), syspath(p2))
 
 
@@ -424,7 +448,7 @@ def remove(path, soft=True):
         return
     try:
         os.remove(path)
-    except (OSError, IOError) as exc:
+    except OSError as exc:
         raise FilesystemError(exc, 'delete', (path,), traceback.format_exc())
 
 
@@ -439,10 +463,10 @@ def copy(path, dest, replace=False):
     path = syspath(path)
     dest = syspath(dest)
     if not replace and os.path.exists(dest):
-        raise FilesystemError(u'file exists', 'copy', (path, dest))
+        raise FilesystemError('file exists', 'copy', (path, dest))
     try:
         shutil.copyfile(path, dest)
-    except (OSError, IOError) as exc:
+    except OSError as exc:
         raise FilesystemError(exc, 'copy', (path, dest),
                               traceback.format_exc())
 
@@ -455,24 +479,37 @@ def move(path, dest, replace=False):
     instead, in which case metadata will *not* be preserved. Paths are
     translated to system paths.
     """
+    if os.path.isdir(path):
+        raise FilesystemError(u'source is directory', 'move', (path, dest))
+    if os.path.isdir(dest):
+        raise FilesystemError(u'destination is directory', 'move',
+                              (path, dest))
     if samefile(path, dest):
         return
     path = syspath(path)
     dest = syspath(dest)
     if os.path.exists(dest) and not replace:
-        raise FilesystemError(u'file exists', 'rename', (path, dest))
+        raise FilesystemError('file exists', 'rename', (path, dest))
 
     # First, try renaming the file.
     try:
-        os.rename(path, dest)
+        os.replace(path, dest)
     except OSError:
-        # Otherwise, copy and delete the original.
+        tmp = tempfile.mktemp(suffix='.beets',
+                              prefix=py3_path(b'.' + os.path.basename(dest)),
+                              dir=py3_path(os.path.dirname(dest)))
+        tmp = syspath(tmp)
         try:
-            shutil.copyfile(path, dest)
+            shutil.copyfile(path, tmp)
+            os.replace(tmp, dest)
+            tmp = None
             os.remove(path)
-        except (OSError, IOError) as exc:
+        except OSError as exc:
             raise FilesystemError(exc, 'move', (path, dest),
                                   traceback.format_exc())
+        finally:
+            if tmp is not None:
+                os.remove(tmp)
 
 
 def link(path, dest, replace=False):
@@ -484,18 +521,18 @@ def link(path, dest, replace=False):
         return
 
     if os.path.exists(syspath(dest)) and not replace:
-        raise FilesystemError(u'file exists', 'rename', (path, dest))
+        raise FilesystemError('file exists', 'rename', (path, dest))
     try:
         os.symlink(syspath(path), syspath(dest))
     except NotImplementedError:
         # raised on python >= 3.2 and Windows versions before Vista
-        raise FilesystemError(u'OS does not support symbolic links.'
+        raise FilesystemError('OS does not support symbolic links.'
                               'link', (path, dest), traceback.format_exc())
     except OSError as exc:
         # TODO: Windows version checks can be removed for python 3
         if hasattr('sys', 'getwindowsversion'):
             if sys.getwindowsversion()[0] < 6:  # is before Vista
-                exc = u'OS does not support symbolic links.'
+                exc = 'OS does not support symbolic links.'
         raise FilesystemError(exc, 'link', (path, dest),
                               traceback.format_exc())
 
@@ -509,21 +546,50 @@ def hardlink(path, dest, replace=False):
         return
 
     if os.path.exists(syspath(dest)) and not replace:
-        raise FilesystemError(u'file exists', 'rename', (path, dest))
+        raise FilesystemError('file exists', 'rename', (path, dest))
     try:
         os.link(syspath(path), syspath(dest))
     except NotImplementedError:
-        raise FilesystemError(u'OS does not support hard links.'
+        raise FilesystemError('OS does not support hard links.'
                               'link', (path, dest), traceback.format_exc())
     except OSError as exc:
         if exc.errno == errno.EXDEV:
-            raise FilesystemError(u'Cannot hard link across devices.'
+            raise FilesystemError('Cannot hard link across devices.'
                                   'link', (path, dest), traceback.format_exc())
         else:
             raise FilesystemError(exc, 'link', (path, dest),
                                   traceback.format_exc())
 
 
+def reflink(path, dest, replace=False, fallback=False):
+    """Create a reflink from `dest` to `path`.
+
+    Raise an `OSError` if `dest` already exists, unless `replace` is
+    True. If `path` == `dest`, then do nothing.
+
+    If reflinking fails and `fallback` is enabled, try copying the file
+    instead. Otherwise, raise an error without trying a plain copy.
+
+    May raise an `ImportError` if the `reflink` module is not available.
+    """
+    import reflink as pyreflink
+
+    if samefile(path, dest):
+        return
+
+    if os.path.exists(syspath(dest)) and not replace:
+        raise FilesystemError('file exists', 'rename', (path, dest))
+
+    try:
+        pyreflink.reflink(path, dest)
+    except (NotImplementedError, pyreflink.ReflinkImpossibleError):
+        if fallback:
+            copy(path, dest, replace)
+        else:
+            raise FilesystemError('OS/filesystem does not support reflinks.',
+                                  'link', (path, dest), traceback.format_exc())
+
+
 def unique_path(path):
     """Returns a version of ``path`` that does not exist on the
     filesystem. Specifically, if ``path` itself already exists, then
@@ -541,22 +607,23 @@ def unique_path(path):
         num = 0
     while True:
         num += 1
-        suffix = u'.{}'.format(num).encode() + ext
+        suffix = f'.{num}'.encode() + ext
         new_path = base + suffix
         if not os.path.exists(new_path):
             return new_path
 
+
 # Note: The Windows "reserved characters" are, of course, allowed on
 # Unix. They are forbidden here because they cause problems on Samba
 # shares, which are sufficiently common as to cause frequent problems.
-# http://msdn.microsoft.com/en-us/library/windows/desktop/aa365247.aspx
+# https://msdn.microsoft.com/en-us/library/windows/desktop/aa365247.aspx
 CHAR_REPLACE = [
-    (re.compile(r'[\\/]'), u'_'),  # / and \ -- forbidden everywhere.
-    (re.compile(r'^\.'), u'_'),  # Leading dot (hidden files on Unix).
-    (re.compile(r'[\x00-\x1f]'), u''),  # Control characters.
-    (re.compile(r'[<>:"\?\*\|]'), u'_'),  # Windows "reserved characters".
-    (re.compile(r'\.$'), u'_'),  # Trailing dots.
-    (re.compile(r'\s+$'), u''),  # Trailing whitespace.
+    (re.compile(r'[\\/]'), '_'),  # / and \ -- forbidden everywhere.
+    (re.compile(r'^\.'), '_'),  # Leading dot (hidden files on Unix).
+    (re.compile(r'[\x00-\x1f]'), ''),  # Control characters.
+    (re.compile(r'[<>:"\?\*\|]'), '_'),  # Windows "reserved characters".
+    (re.compile(r'\.$'), '_'),  # Trailing dots.
+    (re.compile(r'\s+$'), ''),  # Trailing whitespace.
 ]
 
 
@@ -680,36 +747,29 @@ def py3_path(path):
     it is. So this function helps us "smuggle" the true bytes data
     through APIs that took Python 3's Unicode mandate too seriously.
     """
-    if isinstance(path, six.text_type):
+    if isinstance(path, str):
         return path
     assert isinstance(path, bytes)
-    if six.PY2:
-        return path
     return os.fsdecode(path)
 
 
 def str2bool(value):
     """Returns a boolean reflecting a human-entered string."""
-    return value.lower() in (u'yes', u'1', u'true', u't', u'y')
+    return value.lower() in ('yes', '1', 'true', 't', 'y')
 
 
 def as_string(value):
     """Convert a value to a Unicode object for matching with a query.
     None becomes the empty string. Bytestrings are silently decoded.
     """
-    if six.PY2:
-        buffer_types = buffer, memoryview  # noqa: F821
-    else:
-        buffer_types = memoryview
-
     if value is None:
-        return u''
-    elif isinstance(value, buffer_types):
+        return ''
+    elif isinstance(value, memoryview):
         return bytes(value).decode('utf-8', 'ignore')
     elif isinstance(value, bytes):
         return value.decode('utf-8', 'ignore')
     else:
-        return six.text_type(value)
+        return str(value)
 
 
 def text_string(value, encoding='utf-8'):
@@ -732,7 +792,7 @@ def plurality(objs):
     """
     c = Counter(objs)
     if not c:
-        raise ValueError(u'sequence must be non-empty')
+        raise ValueError('sequence must be non-empty')
     return c.most_common(1)[0]
 
 
@@ -749,7 +809,11 @@ def cpu_count():
             num = 0
     elif sys.platform == 'darwin':
         try:
-            num = int(command_output(['/usr/sbin/sysctl', '-n', 'hw.ncpu']))
+            num = int(command_output([
+                '/usr/sbin/sysctl',
+                '-n',
+                'hw.ncpu',
+            ]).stdout)
         except (ValueError, OSError, subprocess.CalledProcessError):
             num = 0
     else:
@@ -769,20 +833,23 @@ def convert_command_args(args):
     assert isinstance(args, list)
 
     def convert(arg):
-        if six.PY2:
-            if isinstance(arg, six.text_type):
-                arg = arg.encode(arg_encoding())
-        else:
-            if isinstance(arg, bytes):
-                arg = arg.decode(arg_encoding(), 'surrogateescape')
+        if isinstance(arg, bytes):
+            arg = arg.decode(arg_encoding(), 'surrogateescape')
         return arg
 
     return [convert(a) for a in args]
 
 
+# stdout and stderr as bytes
+CommandOutput = namedtuple("CommandOutput", ("stdout", "stderr"))
+
+
 def command_output(cmd, shell=False):
     """Runs the command and returns its output after it has exited.
 
+    Returns a CommandOutput. The attributes ``stdout`` and ``stderr`` contain
+    byte strings of the respective output streams.
+
     ``cmd`` is a list of arguments starting with the command names. The
     arguments are bytes on Unix and strings on Windows.
     If ``shell`` is true, ``cmd`` is assumed to be a string and passed to a
@@ -797,10 +864,16 @@ def command_output(cmd, shell=False):
     """
     cmd = convert_command_args(cmd)
 
+    try:  # python >= 3.3
+        devnull = subprocess.DEVNULL
+    except AttributeError:
+        devnull = open(os.devnull, 'r+b')
+
     proc = subprocess.Popen(
         cmd,
         stdout=subprocess.PIPE,
         stderr=subprocess.PIPE,
+        stdin=devnull,
         close_fds=platform.system() != 'Windows',
         shell=shell
     )
@@ -811,7 +884,7 @@ def command_output(cmd, shell=False):
             cmd=' '.join(cmd),
             output=stdout + stderr,
         )
-    return stdout
+    return CommandOutput(stdout, stderr)
 
 
 def max_filename_length(path, limit=MAX_FILENAME_LENGTH):
@@ -858,25 +931,6 @@ def editor_command():
     return open_anything()
 
 
-def shlex_split(s):
-    """Split a Unicode or bytes string according to shell lexing rules.
-
-    Raise `ValueError` if the string is not a well-formed shell string.
-    This is a workaround for a bug in some versions of Python.
-    """
-    if not six.PY2 or isinstance(s, bytes):  # Shlex works fine.
-        return shlex.split(s)
-
-    elif isinstance(s, six.text_type):
-        # Work around a Python bug.
-        # http://bugs.python.org/issue6988
-        bs = s.encode('utf-8')
-        return [c.decode('utf-8') for c in shlex.split(bs)]
-
-    else:
-        raise TypeError(u'shlex_split called with non-string')
-
-
 def interactive_open(targets, command):
     """Open the files in `targets` by `exec`ing a new `command`, given
     as a Unicode string. (The new program takes over, and Python
@@ -888,7 +942,7 @@ def interactive_open(targets, command):
 
     # Split the command string into its arguments.
     try:
-        args = shlex_split(command)
+        args = shlex.split(command)
     except ValueError:  # Malformed shell tokens.
         args = [command]
 
@@ -903,7 +957,7 @@ def _windows_long_path_name(short_path):
     """Use Windows' `GetLongPathNameW` via ctypes to get the canonical,
     long path given a short filename.
     """
-    if not isinstance(short_path, six.text_type):
+    if not isinstance(short_path, str):
         short_path = short_path.decode(_fsencoding())
 
     import ctypes
@@ -964,7 +1018,7 @@ def raw_seconds_short(string):
     """
     match = re.match(r'^(\d+):([0-5]\d)$', string)
     if not match:
-        raise ValueError(u'String not in M:SS format')
+        raise ValueError('String not in M:SS format')
     minutes, seconds = map(int, match.groups())
     return float(minutes * 60 + seconds)
 
@@ -991,3 +1045,59 @@ def asciify_path(path, sep_replace):
                 sep_replace
             )
     return os.sep.join(path_components)
+
+
+def par_map(transform, items):
+    """Apply the function `transform` to all the elements in the
+    iterable `items`, like `map(transform, items)` but with no return
+    value. The map *might* happen in parallel: it's parallel on Python 3
+    and sequential on Python 2.
+
+    The parallelism uses threads (not processes), so this is only useful
+    for IO-bound `transform`s.
+    """
+    pool = ThreadPool()
+    pool.map(transform, items)
+    pool.close()
+    pool.join()
+
+
+def lazy_property(func):
+    """A decorator that creates a lazily evaluated property. On first access,
+    the property is assigned the return value of `func`. This first value is
+    stored, so that future accesses do not have to evaluate `func` again.
+
+    This behaviour is useful when `func` is expensive to evaluate, and it is
+    not certain that the result will be needed.
+    """
+    field_name = '_' + func.__name__
+
+    @property
+    @functools.wraps(func)
+    def wrapper(self):
+        if hasattr(self, field_name):
+            return getattr(self, field_name)
+
+        value = func(self)
+        setattr(self, field_name, value)
+        return value
+
+    return wrapper
+
+
+def decode_commandline_path(path):
+    """Prepare a path for substitution into commandline template.
+
+    On Python 3, we need to construct the subprocess commands to invoke as a
+    Unicode string. On Unix, this is a little unfortunate---the OS is
+    expecting bytes---so we use surrogate escaping and decode with the
+    argument encoding, which is the same encoding that will then be
+    *reversed* to recover the same bytes before invoking the OS. On
+    Windows, we want to preserve the Unicode filename "as is."
+    """
+    # On Python 3, the template is a Unicode string, which only supports
+    # substitution of Unicode variables.
+    if platform.system() == 'Windows':
+        return path.decode(_fsencoding())
+    else:
+        return path.decode(arg_encoding(), 'surrogateescape')

lib/beets/util/artresizer.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # This file is part of beets.
 # Copyright 2016, Fabrice Laporte
 #
@@ -16,38 +15,39 @@
 """Abstraction layer to resize images using PIL, ImageMagick, or a
 public resizing proxy if neither is available.
 """
-from __future__ import division, absolute_import, print_function
 
 import subprocess
 import os
+import os.path
 import re
 from tempfile import NamedTemporaryFile
-from six.moves.urllib.parse import urlencode
+from urllib.parse import urlencode
 from beets import logging
 from beets import util
-import six
 
 # Resizing methods
 PIL = 1
 IMAGEMAGICK = 2
 WEBPROXY = 3
 
-if util.SNI_SUPPORTED:
-    PROXY_URL = 'https://images.weserv.nl/'
-else:
-    PROXY_URL = 'http://images.weserv.nl/'
+PROXY_URL = 'https://images.weserv.nl/'
 
 log = logging.getLogger('beets')
 
 
-def resize_url(url, maxwidth):
+def resize_url(url, maxwidth, quality=0):
     """Return a proxied image URL that resizes the original image to
     maxwidth (preserving aspect ratio).
     """
-    return '{0}?{1}'.format(PROXY_URL, urlencode({
+    params = {
         'url': url.replace('http://', ''),
         'w': maxwidth,
-    }))
+    }
+
+    if quality > 0:
+        params['q'] = quality
+
+    return '{}?{}'.format(PROXY_URL, urlencode(params))
 
 
 def temp_file_for(path):
@@ -59,49 +59,102 @@ def temp_file_for(path):
         return util.bytestring_path(f.name)
 
 
-def pil_resize(maxwidth, path_in, path_out=None):
+def pil_resize(maxwidth, path_in, path_out=None, quality=0, max_filesize=0):
     """Resize using Python Imaging Library (PIL).  Return the output path
     of resized image.
     """
     path_out = path_out or temp_file_for(path_in)
     from PIL import Image
-    log.debug(u'artresizer: PIL resizing {0} to {1}',
+
+    log.debug('artresizer: PIL resizing {0} to {1}',
               util.displayable_path(path_in), util.displayable_path(path_out))
 
     try:
         im = Image.open(util.syspath(path_in))
         size = maxwidth, maxwidth
         im.thumbnail(size, Image.ANTIALIAS)
-        im.save(path_out)
-        return path_out
-    except IOError:
-        log.error(u"PIL cannot create thumbnail for '{0}'",
+
+        if quality == 0:
+            # Use PIL's default quality.
+            quality = -1
+
+        # progressive=False only affects JPEGs and is the default,
+        # but we include it here for explicitness.
+        im.save(util.py3_path(path_out), quality=quality, progressive=False)
+
+        if max_filesize > 0:
+            # If maximum filesize is set, we attempt to lower the quality of
+            # jpeg conversion by a proportional amount, up to 3 attempts
+            # First, set the maximum quality to either provided, or 95
+            if quality > 0:
+                lower_qual = quality
+            else:
+                lower_qual = 95
+            for i in range(5):
+                # 5 attempts is an abitrary choice
+                filesize = os.stat(util.syspath(path_out)).st_size
+                log.debug("PIL Pass {0} : Output size: {1}B", i, filesize)
+                if filesize <= max_filesize:
+                    return path_out
+                # The relationship between filesize & quality will be
+                # image dependent.
+                lower_qual -= 10
+                # Restrict quality dropping below 10
+                if lower_qual < 10:
+                    lower_qual = 10
+                # Use optimize flag to improve filesize decrease
+                im.save(util.py3_path(path_out), quality=lower_qual,
+                        optimize=True, progressive=False)
+            log.warning("PIL Failed to resize file to below {0}B",
+                        max_filesize)
+            return path_out
+
+        else:
+            return path_out
+    except OSError:
+        log.error("PIL cannot create thumbnail for '{0}'",
                   util.displayable_path(path_in))
         return path_in
 
 
-def im_resize(maxwidth, path_in, path_out=None):
-    """Resize using ImageMagick's ``convert`` tool.
-    Return the output path of resized image.
+def im_resize(maxwidth, path_in, path_out=None, quality=0, max_filesize=0):
+    """Resize using ImageMagick.
+
+    Use the ``magick`` program or ``convert`` on older versions. Return
+    the output path of resized image.
     """
     path_out = path_out or temp_file_for(path_in)
-    log.debug(u'artresizer: ImageMagick resizing {0} to {1}',
+    log.debug('artresizer: ImageMagick resizing {0} to {1}',
               util.displayable_path(path_in), util.displayable_path(path_out))
 
-    # "-resize widthxheight>" shrinks images with dimension(s) larger
-    # than the corresponding width and/or height dimension(s). The >
-    # "only shrink" flag is prefixed by ^ escape char for Windows
-    # compatibility.
+    # "-resize WIDTHx>" shrinks images with the width larger
+    # than the given width while maintaining the aspect ratio
+    # with regards to the height.
+    # ImageMagick already seems to default to no interlace, but we include it
+    # here for the sake of explicitness.
+    cmd = ArtResizer.shared.im_convert_cmd + [
+        util.syspath(path_in, prefix=False),
+        '-resize', f'{maxwidth}x>',
+        '-interlace', 'none',
+    ]
+
+    if quality > 0:
+        cmd += ['-quality', f'{quality}']
+
+    # "-define jpeg:extent=SIZEb" sets the target filesize for imagemagick to
+    # SIZE in bytes.
+    if max_filesize > 0:
+        cmd += ['-define', f'jpeg:extent={max_filesize}b']
+
+    cmd.append(util.syspath(path_out, prefix=False))
+
     try:
-        util.command_output([
-            'convert', util.syspath(path_in, prefix=False),
-            '-resize', '{0}x^>'.format(maxwidth),
-            util.syspath(path_out, prefix=False),
-        ])
+        util.command_output(cmd)
     except subprocess.CalledProcessError:
-        log.warning(u'artresizer: IM convert failed for {0}',
+        log.warning('artresizer: IM convert failed for {0}',
                     util.displayable_path(path_in))
         return path_in
+
     return path_out
 
 
@@ -113,31 +166,33 @@ BACKEND_FUNCS = {
 
 def pil_getsize(path_in):
     from PIL import Image
+
     try:
         im = Image.open(util.syspath(path_in))
         return im.size
-    except IOError as exc:
-        log.error(u"PIL could not read file {}: {}",
+    except OSError as exc:
+        log.error("PIL could not read file {}: {}",
                   util.displayable_path(path_in), exc)
 
 
 def im_getsize(path_in):
-    cmd = ['identify', '-format', '%w %h',
-           util.syspath(path_in, prefix=False)]
+    cmd = ArtResizer.shared.im_identify_cmd + \
+        ['-format', '%w %h', util.syspath(path_in, prefix=False)]
+
     try:
-        out = util.command_output(cmd)
+        out = util.command_output(cmd).stdout
     except subprocess.CalledProcessError as exc:
-        log.warning(u'ImageMagick size query failed')
+        log.warning('ImageMagick size query failed')
         log.debug(
-            u'`convert` exited with (status {}) when '
-            u'getting size with command {}:\n{}',
+            '`convert` exited with (status {}) when '
+            'getting size with command {}:\n{}',
             exc.returncode, cmd, exc.output.strip()
         )
         return
     try:
         return tuple(map(int, out.split(b' ')))
     except IndexError:
-        log.warning(u'Could not understand IM output: {0!r}', out)
+        log.warning('Could not understand IM output: {0!r}', out)
 
 
 BACKEND_GET_SIZE = {
@@ -146,24 +201,125 @@ BACKEND_GET_SIZE = {
 }
 
 
+def pil_deinterlace(path_in, path_out=None):
+    path_out = path_out or temp_file_for(path_in)
+    from PIL import Image
+
+    try:
+        im = Image.open(util.syspath(path_in))
+        im.save(util.py3_path(path_out), progressive=False)
+        return path_out
+    except IOError:
+        return path_in
+
+
+def im_deinterlace(path_in, path_out=None):
+    path_out = path_out or temp_file_for(path_in)
+
+    cmd = ArtResizer.shared.im_convert_cmd + [
+        util.syspath(path_in, prefix=False),
+        '-interlace', 'none',
+        util.syspath(path_out, prefix=False),
+    ]
+
+    try:
+        util.command_output(cmd)
+        return path_out
+    except subprocess.CalledProcessError:
+        return path_in
+
+
+DEINTERLACE_FUNCS = {
+    PIL: pil_deinterlace,
+    IMAGEMAGICK: im_deinterlace,
+}
+
+
+def im_get_format(filepath):
+    cmd = ArtResizer.shared.im_identify_cmd + [
+        '-format', '%[magick]',
+        util.syspath(filepath)
+    ]
+
+    try:
+        return util.command_output(cmd).stdout
+    except subprocess.CalledProcessError:
+        return None
+
+
+def pil_get_format(filepath):
+    from PIL import Image, UnidentifiedImageError
+
+    try:
+        with Image.open(util.syspath(filepath)) as im:
+            return im.format
+    except (ValueError, TypeError, UnidentifiedImageError, FileNotFoundError):
+        log.exception("failed to detect image format for {}", filepath)
+        return None
+
+
+BACKEND_GET_FORMAT = {
+    PIL: pil_get_format,
+    IMAGEMAGICK: im_get_format,
+}
+
+
+def im_convert_format(source, target, deinterlaced):
+    cmd = ArtResizer.shared.im_convert_cmd + [
+        util.syspath(source),
+        *(["-interlace", "none"] if deinterlaced else []),
+        util.syspath(target),
+    ]
+
+    try:
+        subprocess.check_call(
+            cmd,
+            stderr=subprocess.DEVNULL,
+            stdout=subprocess.DEVNULL
+        )
+        return target
+    except subprocess.CalledProcessError:
+        return source
+
+
+def pil_convert_format(source, target, deinterlaced):
+    from PIL import Image, UnidentifiedImageError
+
+    try:
+        with Image.open(util.syspath(source)) as im:
+            im.save(util.py3_path(target), progressive=not deinterlaced)
+            return target
+    except (ValueError, TypeError, UnidentifiedImageError, FileNotFoundError,
+            OSError):
+        log.exception("failed to convert image {} -> {}", source, target)
+        return source
+
+
+BACKEND_CONVERT_IMAGE_FORMAT = {
+    PIL: pil_convert_format,
+    IMAGEMAGICK: im_convert_format,
+}
+
+
 class Shareable(type):
     """A pseudo-singleton metaclass that allows both shared and
     non-shared instances. The ``MyClass.shared`` property holds a
     lazily-created shared instance of ``MyClass`` while calling
     ``MyClass()`` to construct a new object works as usual.
     """
-    def __init__(self, name, bases, dict):
-        super(Shareable, self).__init__(name, bases, dict)
-        self._instance = None
+
+    def __init__(cls, name, bases, dict):
+        super().__init__(name, bases, dict)
+        cls._instance = None
 
     @property
-    def shared(self):
-        if self._instance is None:
-            self._instance = self()
-        return self._instance
+    def shared(cls):
+        if cls._instance is None:
+            cls._instance = cls()
+        return cls._instance
 
 
-class ArtResizer(six.with_metaclass(Shareable, object)):
+class ArtResizer(metaclass=Shareable):
     """A singleton class that performs image resizes.
     """
 
@@ -171,21 +327,44 @@ class ArtResizer(six.with_metaclass(Shareable, object)):
         """Create a resizer object with an inferred method.
         """
         self.method = self._check_method()
-        log.debug(u"artresizer: method is {0}", self.method)
+        log.debug("artresizer: method is {0}", self.method)
         self.can_compare = self._can_compare()
 
-    def resize(self, maxwidth, path_in, path_out=None):
+        # Use ImageMagick's magick binary when it's available. If it's
+        # not, fall back to the older, separate convert and identify
+        # commands.
+        if self.method[0] == IMAGEMAGICK:
+            self.im_legacy = self.method[2]
+            if self.im_legacy:
+                self.im_convert_cmd = ['convert']
+                self.im_identify_cmd = ['identify']
+            else:
+                self.im_convert_cmd = ['magick']
+                self.im_identify_cmd = ['magick', 'identify']
+
+    def resize(
+        self, maxwidth, path_in, path_out=None, quality=0, max_filesize=0
+    ):
         """Manipulate an image file according to the method, returning a
         new path. For PIL or IMAGEMAGIC methods, resizes the image to a
-        temporary file. For WEBPROXY, returns `path_in` unmodified.
+        temporary file and encodes with the specified quality level.
+        For WEBPROXY, returns `path_in` unmodified.
         """
         if self.local:
             func = BACKEND_FUNCS[self.method[0]]
-            return func(maxwidth, path_in, path_out)
+            return func(maxwidth, path_in, path_out,
+                        quality=quality, max_filesize=max_filesize)
         else:
             return path_in
 
-    def proxy_url(self, maxwidth, url):
+    def deinterlace(self, path_in, path_out=None):
+        if self.local:
+            func = DEINTERLACE_FUNCS[self.method[0]]
+            return func(path_in, path_out)
+        else:
+            return path_in
+
+    def proxy_url(self, maxwidth, url, quality=0):
         """Modifies an image URL according the method, returning a new
         URL. For WEBPROXY, a URL on the proxy server is returned.
         Otherwise, the URL is returned unmodified.
@@ -193,7 +372,7 @@ class ArtResizer(six.with_metaclass(Shareable, object)):
         if self.local:
             return url
         else:
-            return resize_url(url, maxwidth)
+            return resize_url(url, maxwidth, quality)
 
     @property
     def local(self):
@@ -206,12 +385,50 @@ class ArtResizer(six.with_metaclass(Shareable, object)):
         """Return the size of an image file as an int couple (width, height)
         in pixels.
 
-        Only available locally
+        Only available locally.
         """
         if self.local:
             func = BACKEND_GET_SIZE[self.method[0]]
             return func(path_in)
 
+    def get_format(self, path_in):
+        """Returns the format of the image as a string.
+
+        Only available locally.
+        """
+        if self.local:
+            func = BACKEND_GET_FORMAT[self.method[0]]
+            return func(path_in)
+
+    def reformat(self, path_in, new_format, deinterlaced=True):
+        """Converts image to desired format, updating its extension, but
+        keeping the same filename.
+
+        Only available locally.
+        """
+        if not self.local:
+            return path_in
+
+        new_format = new_format.lower()
+        # A nonexhaustive map of image "types" to extensions overrides
+        new_format = {
+            'jpeg': 'jpg',
+        }.get(new_format, new_format)
+
+        fname, ext = os.path.splitext(path_in)
+        path_new = fname + b'.' + new_format.encode('utf8')
+        func = BACKEND_CONVERT_IMAGE_FORMAT[self.method[0]]
+
+        # allows the exception to propagate, while still making sure a changed
+        # file path was removed
+        result_path = path_in
+        try:
+            result_path = func(path_in, path_new, deinterlaced)
+        finally:
+            if result_path != path_in:
+                os.unlink(path_in)
+        return result_path
+
     def _can_compare(self):
         """A boolean indicating whether image comparison is available"""
 
@@ -219,10 +436,20 @@ class ArtResizer(six.with_metaclass(Shareable, object)):
 
     @staticmethod
     def _check_method():
-        """Return a tuple indicating an available method and its version."""
+        """Return a tuple indicating an available method and its version.
+
+        The result has at least two elements:
+        - The method, eitehr WEBPROXY, PIL, or IMAGEMAGICK.
+        - The version.
+
+        If the method is IMAGEMAGICK, there is also a third element: a
+        bool flag indicating whether to use the `magick` binary or
+        legacy single-purpose executables (`convert`, `identify`, etc.)
+        """
         version = get_im_version()
         if version:
-            return IMAGEMAGICK, version
+            version, legacy = version
+            return IMAGEMAGICK, version, legacy
 
         version = get_pil_version()
         if version:
@@ -232,31 +459,34 @@ class ArtResizer(six.with_metaclass(Shareable, object)):
 
 
 def get_im_version():
-    """Return Image Magick version or None if it is unavailable
-    Try invoking ImageMagick's "convert".
+    """Get the ImageMagick version and legacy flag as a pair. Or return
+    None if ImageMagick is not available.
     """
-    try:
-        out = util.command_output(['convert', '--version'])
+    for cmd_name, legacy in ((['magick'], False), (['convert'], True)):
+        cmd = cmd_name + ['--version']
 
-        if b'imagemagick' in out.lower():
-            pattern = br".+ (\d+)\.(\d+)\.(\d+).*"
-            match = re.search(pattern, out)
-            if match:
-                return (int(match.group(1)),
-                        int(match.group(2)),
-                        int(match.group(3)))
-            return (0,)
+        try:
+            out = util.command_output(cmd).stdout
+        except (subprocess.CalledProcessError, OSError) as exc:
+            log.debug('ImageMagick version check failed: {}', exc)
+        else:
+            if b'imagemagick' in out.lower():
+                pattern = br".+ (\d+)\.(\d+)\.(\d+).*"
+                match = re.search(pattern, out)
+                if match:
+                    version = (int(match.group(1)),
+                               int(match.group(2)),
+                               int(match.group(3)))
+                    return version, legacy
 
-    except (subprocess.CalledProcessError, OSError) as exc:
-        log.debug(u'ImageMagick check `convert --version` failed: {}', exc)
-        return None
+    return None
 
 
 def get_pil_version():
-    """Return Image Magick version or None if it is unavailable
-    Try importing PIL."""
+    """Get the PIL/Pillow version, or None if it is unavailable.
+    """
     try:
-        __import__('PIL', fromlist=[str('Image')])
+        __import__('PIL', fromlist=['Image'])
         return (0,)
     except ImportError:
         return None

lib/beets/util/bluelet.py

@@ -1,5 +1,3 @@
-# -*- coding: utf-8 -*-
-
 """Extremely simple pure-Python implementation of coroutine-style
 asynchronous socket I/O. Inspired by, but inferior to, Eventlet.
 Bluelet can also be thought of as a less-terrible replacement for
@@ -7,9 +5,7 @@ asyncore.
 
 Bluelet: easy concurrency without all the messy parallelism.
 """
-from __future__ import division, absolute_import, print_function
 
-import six
 import socket
 import select
 import sys
@@ -22,7 +18,7 @@ import collections
 
 # Basic events used for thread scheduling.
 
-class Event(object):
+class Event:
     """Just a base class identifying Bluelet events. An event is an
     object yielded from a Bluelet thread coroutine to suspend operation
     and communicate with the scheduler.
@@ -201,7 +197,7 @@ class ThreadException(Exception):
         self.exc_info = exc_info
 
     def reraise(self):
-        six.reraise(self.exc_info[0], self.exc_info[1], self.exc_info[2])
+        raise self.exc_info[1].with_traceback(self.exc_info[2])
 
 
 SUSPENDED = Event()  # Special sentinel placeholder for suspended threads.
@@ -269,7 +265,7 @@ def run(root_coro):
         except StopIteration:
             # Thread is done.
             complete_thread(coro, None)
-        except:
+        except BaseException:
             # Thread raised some other exception.
             del threads[coro]
             raise ThreadException(coro, sys.exc_info())
@@ -336,16 +332,20 @@ def run(root_coro):
                     break
 
             # Wait and fire.
-            event2coro = dict((v, k) for k, v in threads.items())
+            event2coro = {v: k for k, v in threads.items()}
             for event in _event_select(threads.values()):
                 # Run the IO operation, but catch socket errors.
                 try:
                     value = event.fire()
-                except socket.error as exc:
+                except OSError as exc:
                     if isinstance(exc.args, tuple) and \
                             exc.args[0] == errno.EPIPE:
                         # Broken pipe. Remote host disconnected.
                         pass
+                    elif isinstance(exc.args, tuple) and \
+                            exc.args[0] == errno.ECONNRESET:
+                        # Connection was reset by peer.
+                        pass
                     else:
                         traceback.print_exc()
                     # Abort the coroutine.
@@ -366,7 +366,7 @@ def run(root_coro):
                 exit_te = te
                 break
 
-        except:
+        except BaseException:
             # For instance, KeyboardInterrupt during select(). Raise
             # into root thread and terminate others.
             threads = {root_coro: ExceptionEvent(sys.exc_info())}
@@ -386,7 +386,7 @@ class SocketClosedError(Exception):
     pass
 
 
-class Listener(object):
+class Listener:
     """A socket wrapper object for listening sockets.
     """
     def __init__(self, host, port):
@@ -416,7 +416,7 @@ class Listener(object):
         self.sock.close()
 
 
-class Connection(object):
+class Connection:
     """A socket wrapper object for connected sockets.
     """
     def __init__(self, sock, addr):
@@ -541,7 +541,7 @@ def spawn(coro):
     and child coroutines run concurrently.
     """
     if not isinstance(coro, types.GeneratorType):
-        raise ValueError(u'%s is not a coroutine' % coro)
+        raise ValueError('%s is not a coroutine' % coro)
     return SpawnEvent(coro)
 
 
@@ -551,7 +551,7 @@ def call(coro):
     returns a value using end(), then this event returns that value.
     """
     if not isinstance(coro, types.GeneratorType):
-        raise ValueError(u'%s is not a coroutine' % coro)
+        raise ValueError('%s is not a coroutine' % coro)
     return DelegationEvent(coro)
 
 

lib/beets/util/enumeration.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # This file is part of beets.
 # Copyright 2016, Adrian Sampson.
 #
@@ -13,7 +12,6 @@
 # The above copyright notice and this permission notice shall be
 # included in all copies or substantial portions of the Software.
 
-from __future__ import division, absolute_import, print_function
 
 from enum import Enum
 

lib/beets/util/functemplate.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # This file is part of beets.
 # Copyright 2016, Adrian Sampson.
 #
@@ -27,30 +26,30 @@ This is sort of like a tiny, horrible degeneration of a real templating
 engine like Jinja2 or Mustache.
 """
 
-from __future__ import division, absolute_import, print_function
 
 import re
 import ast
 import dis
 import types
 import sys
-import six
+import functools
 
-SYMBOL_DELIM = u'$'
-FUNC_DELIM = u'%'
-GROUP_OPEN = u'{'
-GROUP_CLOSE = u'}'
-ARG_SEP = u','
-ESCAPE_CHAR = u'$'
+SYMBOL_DELIM = '$'
+FUNC_DELIM = '%'
+GROUP_OPEN = '{'
+GROUP_CLOSE = '}'
+ARG_SEP = ','
+ESCAPE_CHAR = '$'
 
 VARIABLE_PREFIX = '__var_'
 FUNCTION_PREFIX = '__func_'
 
 
-class Environment(object):
+class Environment:
     """Contains the values and functions to be substituted into a
     template.
     """
+
     def __init__(self, values, functions):
         self.values = values
         self.functions = functions
@@ -72,15 +71,7 @@ def ex_literal(val):
     """An int, float, long, bool, string, or None literal with the given
     value.
     """
-    if val is None:
-        return ast.Name('None', ast.Load())
-    elif isinstance(val, six.integer_types):
-        return ast.Num(val)
-    elif isinstance(val, bool):
-        return ast.Name(bytes(val), ast.Load())
-    elif isinstance(val, six.string_types):
-        return ast.Str(val)
-    raise TypeError(u'no literal for {0}'.format(type(val)))
+    return ast.Constant(val)
 
 
 def ex_varassign(name, expr):
@@ -97,7 +88,7 @@ def ex_call(func, args):
     function may be an expression or the name of a function. Each
     argument may be an expression or a value to be used as a literal.
     """
-    if isinstance(func, six.string_types):
+    if isinstance(func, str):
         func = ex_rvalue(func)
 
     args = list(args)
@@ -105,10 +96,7 @@ def ex_call(func, args):
         if not isinstance(args[i], ast.expr):
             args[i] = ex_literal(args[i])
 
-    if sys.version_info[:2] < (3, 5):
-        return ast.Call(func, args, [], None, None)
-    else:
-        return ast.Call(func, args, [])
+    return ast.Call(func, args, [])
 
 
 def compile_func(arg_names, statements, name='_the_func', debug=False):
@@ -116,32 +104,30 @@ def compile_func(arg_names, statements, name='_the_func', debug=False):
     the resulting Python function. If `debug`, then print out the
     bytecode of the compiled function.
     """
-    if six.PY2:
-        func_def = ast.FunctionDef(
-            name=name.encode('utf-8'),
-            args=ast.arguments(
-                args=[ast.Name(n, ast.Param()) for n in arg_names],
-                vararg=None,
-                kwarg=None,
-                defaults=[ex_literal(None) for _ in arg_names],
-            ),
-            body=statements,
-            decorator_list=[],
-        )
-    else:
-        func_def = ast.FunctionDef(
-            name=name,
-            args=ast.arguments(
-                args=[ast.arg(arg=n, annotation=None) for n in arg_names],
-                kwonlyargs=[],
-                kw_defaults=[],
-                defaults=[ex_literal(None) for _ in arg_names],
-            ),
-            body=statements,
-            decorator_list=[],
-        )
+    args_fields = {
+        'args': [ast.arg(arg=n, annotation=None) for n in arg_names],
+        'kwonlyargs': [],
+        'kw_defaults': [],
+        'defaults': [ex_literal(None) for _ in arg_names],
+    }
+    if 'posonlyargs' in ast.arguments._fields:  # Added in Python 3.8.
+        args_fields['posonlyargs'] = []
+    args = ast.arguments(**args_fields)
+
+    func_def = ast.FunctionDef(
+        name=name,
+        args=args,
+        body=statements,
+        decorator_list=[],
+    )
+
+    # The ast.Module signature changed in 3.8 to accept a list of types to
+    # ignore.
+    if sys.version_info >= (3, 8):
+        mod = ast.Module([func_def], [])
+    else:
+        mod = ast.Module([func_def])
 
-    mod = ast.Module([func_def])
     ast.fix_missing_locations(mod)
 
     prog = compile(mod, '<generated>', 'exec')
@@ -160,14 +146,15 @@ def compile_func(arg_names, statements, name='_the_func', debug=False):
 
 # AST nodes for the template language.
 
-class Symbol(object):
+class Symbol:
     """A variable-substitution symbol in a template."""
+
     def __init__(self, ident, original):
         self.ident = ident
         self.original = original
 
     def __repr__(self):
-        return u'Symbol(%s)' % repr(self.ident)
+        return 'Symbol(%s)' % repr(self.ident)
 
     def evaluate(self, env):
         """Evaluate the symbol in the environment, returning a Unicode
@@ -182,24 +169,22 @@ class Symbol(object):
 
     def translate(self):
         """Compile the variable lookup."""
-        if six.PY2:
-            ident = self.ident.encode('utf-8')
-        else:
-            ident = self.ident
+        ident = self.ident
         expr = ex_rvalue(VARIABLE_PREFIX + ident)
-        return [expr], set([ident]), set()
+        return [expr], {ident}, set()
 
 
-class Call(object):
+class Call:
     """A function call in a template."""
+
     def __init__(self, ident, args, original):
         self.ident = ident
         self.args = args
         self.original = original
 
     def __repr__(self):
-        return u'Call(%s, %s, %s)' % (repr(self.ident), repr(self.args),
-                                      repr(self.original))
+        return 'Call({}, {}, {})'.format(repr(self.ident), repr(self.args),
+                                         repr(self.original))
 
     def evaluate(self, env):
         """Evaluate the function call in the environment, returning a
@@ -212,19 +197,15 @@ class Call(object):
             except Exception as exc:
                 # Function raised exception! Maybe inlining the name of
                 # the exception will help debug.
-                return u'<%s>' % six.text_type(exc)
-            return six.text_type(out)
+                return '<%s>' % str(exc)
+            return str(out)
         else:
             return self.original
 
     def translate(self):
         """Compile the function call."""
         varnames = set()
-        if six.PY2:
-            ident = self.ident.encode('utf-8')
-        else:
-            ident = self.ident
-        funcnames = set([ident])
+        funcnames = {self.ident}
 
         arg_exprs = []
         for arg in self.args:
@@ -235,32 +216,33 @@ class Call(object):
             # Create a subexpression that joins the result components of
             # the arguments.
             arg_exprs.append(ex_call(
-                ast.Attribute(ex_literal(u''), 'join', ast.Load()),
+                ast.Attribute(ex_literal(''), 'join', ast.Load()),
                 [ex_call(
                     'map',
                     [
-                        ex_rvalue(six.text_type.__name__),
+                        ex_rvalue(str.__name__),
                         ast.List(subexprs, ast.Load()),
                     ]
                 )],
             ))
 
         subexpr_call = ex_call(
-            FUNCTION_PREFIX + ident,
+            FUNCTION_PREFIX + self.ident,
             arg_exprs
         )
         return [subexpr_call], varnames, funcnames
 
 
-class Expression(object):
+class Expression:
     """Top-level template construct: contains a list of text blobs,
     Symbols, and Calls.
     """
+
     def __init__(self, parts):
         self.parts = parts
 
     def __repr__(self):
-        return u'Expression(%s)' % (repr(self.parts))
+        return 'Expression(%s)' % (repr(self.parts))
 
     def evaluate(self, env):
         """Evaluate the entire expression in the environment, returning
@@ -268,11 +250,11 @@ class Expression(object):
         """
         out = []
         for part in self.parts:
-            if isinstance(part, six.string_types):
+            if isinstance(part, str):
                 out.append(part)
             else:
                 out.append(part.evaluate(env))
-        return u''.join(map(six.text_type, out))
+        return ''.join(map(str, out))
 
     def translate(self):
         """Compile the expression to a list of Python AST expressions, a
@@ -282,7 +264,7 @@ class Expression(object):
         varnames = set()
         funcnames = set()
         for part in self.parts:
-            if isinstance(part, six.string_types):
+            if isinstance(part, str):
                 expressions.append(ex_literal(part))
             else:
                 e, v, f = part.translate()
@@ -298,7 +280,7 @@ class ParseError(Exception):
     pass
 
 
-class Parser(object):
+class Parser:
     """Parses a template expression string. Instantiate the class with
     the template source and call ``parse_expression``. The ``pos`` field
     will indicate the character after the expression finished and
@@ -311,6 +293,7 @@ class Parser(object):
     replaced with a real, accepted parsing technique (PEG, parser
     generator, etc.).
     """
+
     def __init__(self, string, in_argument=False):
         """ Create a new parser.
         :param in_arguments: boolean that indicates the parser is to be
@@ -325,8 +308,8 @@ class Parser(object):
     # Common parsing resources.
     special_chars = (SYMBOL_DELIM, FUNC_DELIM, GROUP_OPEN, GROUP_CLOSE,
                      ESCAPE_CHAR)
-    special_char_re = re.compile(r'[%s]|$' %
-                                 u''.join(re.escape(c) for c in special_chars))
+    special_char_re = re.compile(r'[%s]|\Z' %
+                                 ''.join(re.escape(c) for c in special_chars))
     escapable_chars = (SYMBOL_DELIM, FUNC_DELIM, GROUP_CLOSE, ARG_SEP)
     terminator_chars = (GROUP_CLOSE,)
 
@@ -343,8 +326,11 @@ class Parser(object):
         if self.in_argument:
             extra_special_chars = (ARG_SEP,)
             special_char_re = re.compile(
-                r'[%s]|$' % u''.join(re.escape(c) for c in
-                                     self.special_chars + extra_special_chars))
+                r'[%s]|\Z' % ''.join(
+                    re.escape(c) for c in
+                    self.special_chars + extra_special_chars
+                )
+            )
 
         text_parts = []
 
@@ -384,7 +370,7 @@ class Parser(object):
 
             # Shift all characters collected so far into a single string.
             if text_parts:
-                self.parts.append(u''.join(text_parts))
+                self.parts.append(''.join(text_parts))
                 text_parts = []
 
             if char == SYMBOL_DELIM:
@@ -406,7 +392,7 @@ class Parser(object):
 
         # If any parsed characters remain, shift them into a string.
         if text_parts:
-            self.parts.append(u''.join(text_parts))
+            self.parts.append(''.join(text_parts))
 
     def parse_symbol(self):
         """Parse a variable reference (like ``$foo`` or ``${foo}``)
@@ -544,11 +530,27 @@ def _parse(template):
     return Expression(parts)
 
 
-# External interface.
+def cached(func):
+    """Like the `functools.lru_cache` decorator, but works (as a no-op)
+    on Python < 3.2.
+    """
+    if hasattr(functools, 'lru_cache'):
+        return functools.lru_cache(maxsize=128)(func)
+    else:
+        # Do nothing when lru_cache is not available.
+        return func
 
-class Template(object):
+
+@cached
+def template(fmt):
+    return Template(fmt)
+
+
+# External interface.
+class Template:
     """A string template, including text, Symbols, and Calls.
     """
+
     def __init__(self, template):
         self.expr = _parse(template)
         self.original = template
@@ -570,7 +572,7 @@ class Template(object):
         """
         try:
             res = self.compiled(values, functions)
-        except:  # Handle any exceptions thrown by compiled version.
+        except Exception:  # Handle any exceptions thrown by compiled version.
             res = self.interpret(values, functions)
 
         return res
@@ -597,7 +599,7 @@ class Template(object):
             for funcname in funcnames:
                 args[FUNCTION_PREFIX + funcname] = functions[funcname]
             parts = func(**args)
-            return u''.join(parts)
+            return ''.join(parts)
 
         return wrapper_func
 
@@ -606,9 +608,9 @@ class Template(object):
 
 if __name__ == '__main__':
     import timeit
-    _tmpl = Template(u'foo $bar %baz{foozle $bar barzle} $bar')
+    _tmpl = Template('foo $bar %baz{foozle $bar barzle} $bar')
     _vars = {'bar': 'qux'}
-    _funcs = {'baz': six.text_type.upper}
+    _funcs = {'baz': str.upper}
     interp_time = timeit.timeit('_tmpl.interpret(_vars, _funcs)',
                                 'from __main__ import _tmpl, _vars, _funcs',
                                 number=10000)
@@ -617,4 +619,4 @@ if __name__ == '__main__':
                               'from __main__ import _tmpl, _vars, _funcs',
                               number=10000)
     print(comp_time)
-    print(u'Speedup:', interp_time / comp_time)
+    print('Speedup:', interp_time / comp_time)

lib/beets/util/hidden.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # This file is part of beets.
 # Copyright 2016, Adrian Sampson.
 #
@@ -14,7 +13,6 @@
 # included in all copies or substantial portions of the Software.
 
 """Simple library to work out if a file is hidden on different platforms."""
-from __future__ import division, absolute_import, print_function
 
 import os
 import stat

lib/beets/util/pipeline.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # This file is part of beets.
 # Copyright 2016, Adrian Sampson.
 #
@@ -32,12 +31,10 @@ To do so, pass an iterable of coroutines to the Pipeline constructor
 in place of any single coroutine.
 """
 
-from __future__ import division, absolute_import, print_function
 
-from six.moves import queue
+import queue
 from threading import Thread, Lock
 import sys
-import six
 
 BUBBLE = '__PIPELINE_BUBBLE__'
 POISON = '__PIPELINE_POISON__'
@@ -91,6 +88,7 @@ class CountedQueue(queue.Queue):
     still feeding into it. The queue is poisoned when all threads are
     finished with the queue.
     """
+
     def __init__(self, maxsize=0):
         queue.Queue.__init__(self, maxsize)
         self.nthreads = 0
@@ -135,10 +133,11 @@ class CountedQueue(queue.Queue):
                     _invalidate_queue(self, POISON, False)
 
 
-class MultiMessage(object):
+class MultiMessage:
     """A message yielded by a pipeline stage encapsulating multiple
     values to be sent to the next stage.
     """
+
     def __init__(self, messages):
         self.messages = messages
 
@@ -210,8 +209,9 @@ def _allmsgs(obj):
 
 class PipelineThread(Thread):
     """Abstract base class for pipeline-stage threads."""
+
     def __init__(self, all_threads):
-        super(PipelineThread, self).__init__()
+        super().__init__()
         self.abort_lock = Lock()
         self.abort_flag = False
         self.all_threads = all_threads
@@ -241,15 +241,13 @@ class FirstPipelineThread(PipelineThread):
     """The thread running the first stage in a parallel pipeline setup.
     The coroutine should just be a generator.
     """
+
     def __init__(self, coro, out_queue, all_threads):
-        super(FirstPipelineThread, self).__init__(all_threads)
+        super().__init__(all_threads)
         self.coro = coro
         self.out_queue = out_queue
         self.out_queue.acquire()
 
-        self.abort_lock = Lock()
-        self.abort_flag = False
-
     def run(self):
         try:
             while True:
@@ -270,7 +268,7 @@ class FirstPipelineThread(PipelineThread):
                             return
                     self.out_queue.put(msg)
 
-        except:
+        except BaseException:
             self.abort_all(sys.exc_info())
             return
 
@@ -282,8 +280,9 @@ class MiddlePipelineThread(PipelineThread):
     """A thread running any stage in the pipeline except the first or
     last.
     """
+
     def __init__(self, coro, in_queue, out_queue, all_threads):
-        super(MiddlePipelineThread, self).__init__(all_threads)
+        super().__init__(all_threads)
         self.coro = coro
         self.in_queue = in_queue
         self.out_queue = out_queue
@@ -318,7 +317,7 @@ class MiddlePipelineThread(PipelineThread):
                             return
                     self.out_queue.put(msg)
 
-        except:
+        except BaseException:
             self.abort_all(sys.exc_info())
             return
 
@@ -330,8 +329,9 @@ class LastPipelineThread(PipelineThread):
     """A thread running the last stage in a pipeline. The coroutine
     should yield nothing.
     """
+
     def __init__(self, coro, in_queue, all_threads):
-        super(LastPipelineThread, self).__init__(all_threads)
+        super().__init__(all_threads)
         self.coro = coro
         self.in_queue = in_queue
 
@@ -357,22 +357,23 @@ class LastPipelineThread(PipelineThread):
                 # Send to consumer.
                 self.coro.send(msg)
 
-        except:
+        except BaseException:
             self.abort_all(sys.exc_info())
             return
 
 
-class Pipeline(object):
+class Pipeline:
     """Represents a staged pattern of work. Each stage in the pipeline
     is a coroutine that receives messages from the previous stage and
     yields messages to be sent to the next stage.
     """
+
     def __init__(self, stages):
         """Makes a new pipeline from a list of coroutines. There must
         be at least two stages.
         """
         if len(stages) < 2:
-            raise ValueError(u'pipeline must have at least two stages')
+            raise ValueError('pipeline must have at least two stages')
         self.stages = []
         for stage in stages:
             if isinstance(stage, (list, tuple)):
@@ -425,7 +426,7 @@ class Pipeline(object):
             while threads[-1].is_alive():
                 threads[-1].join(1)
 
-        except:
+        except BaseException:
             # Stop all the threads immediately.
             for thread in threads:
                 thread.abort()
@@ -442,7 +443,7 @@ class Pipeline(object):
             exc_info = thread.exc_info
             if exc_info:
                 # Make the exception appear as it was raised originally.
-                six.reraise(exc_info[0], exc_info[1], exc_info[2])
+                raise exc_info[1].with_traceback(exc_info[2])
 
     def pull(self):
         """Yield elements from the end of the pipeline. Runs the stages
@@ -469,6 +470,7 @@ class Pipeline(object):
             for msg in msgs:
                 yield msg
 
+
 # Smoke test.
 if __name__ == '__main__':
     import time
@@ -477,14 +479,14 @@ if __name__ == '__main__':
     # in parallel.
     def produce():
         for i in range(5):
-            print(u'generating %i' % i)
+            print('generating %i' % i)
             time.sleep(1)
             yield i
 
     def work():
         num = yield
         while True:
-            print(u'processing %i' % num)
+            print('processing %i' % num)
             time.sleep(2)
             num = yield num * 2
 
@@ -492,7 +494,7 @@ if __name__ == '__main__':
         while True:
             num = yield
             time.sleep(1)
-            print(u'received %i' % num)
+            print('received %i' % num)
 
     ts_start = time.time()
     Pipeline([produce(), work(), consume()]).run_sequential()
@@ -501,22 +503,22 @@ if __name__ == '__main__':
     ts_par = time.time()
     Pipeline([produce(), (work(), work()), consume()]).run_parallel()
     ts_end = time.time()
-    print(u'Sequential time:', ts_seq - ts_start)
-    print(u'Parallel time:', ts_par - ts_seq)
-    print(u'Multiply-parallel time:', ts_end - ts_par)
+    print('Sequential time:', ts_seq - ts_start)
+    print('Parallel time:', ts_par - ts_seq)
+    print('Multiply-parallel time:', ts_end - ts_par)
     print()
 
     # Test a pipeline that raises an exception.
     def exc_produce():
         for i in range(10):
-            print(u'generating %i' % i)
+            print('generating %i' % i)
             time.sleep(1)
             yield i
 
     def exc_work():
         num = yield
         while True:
-            print(u'processing %i' % num)
+            print('processing %i' % num)
             time.sleep(3)
             if num == 3:
                 raise Exception()
@@ -525,6 +527,6 @@ if __name__ == '__main__':
     def exc_consume():
         while True:
             num = yield
-            print(u'received %i' % num)
+            print('received %i' % num)
 
     Pipeline([exc_produce(), exc_work(), exc_consume()]).run_parallel(1)