Open blackhole nzb file as binary

headphones/config.py

@@ -157,7 +157,7 @@ _CONFIG_DEFINITIONS = {
     'LASTFM_USERNAME': (str, 'General', ''),
     'LAUNCH_BROWSER': (int, 'General', 1),
     'LIBRARYSCAN': (int, 'General', 1),
-    'LIBRARYSCAN_INTERVAL': (int, 'General', 300),
+    'LIBRARYSCAN_INTERVAL': (int, 'General', 24),
     'LMS_ENABLED': (int, 'LMS', 0),
     'LMS_HOST': (str, 'LMS', ''),
     'LOG_DIR': (path, 'General', ''),
@@ -327,7 +327,7 @@ class Config(object):
     def __init__(self, config_file):
         """ Initialize the config with values from a file """
         self._config_file = config_file
-        self._config = ConfigParser()
+        self._config = ConfigParser(interpolation=None)
         self._config.read(self._config_file)
         for key in list(_CONFIG_DEFINITIONS.keys()):
             self.check_setting(key)
@@ -376,7 +376,7 @@ class Config(object):
 
     def write(self):
         """ Make a copy of the stored config and write it to the configured file """
-        new_config = ConfigParser()
+        new_config = ConfigParser(interpolation=None)
 
         # first copy over everything from the old config, even if it is not
         # correctly defined to keep from losing data

headphones/db.py

@@ -117,7 +117,7 @@ class DBConnection:
                     break
 
             except sqlite3.OperationalError as e:
-                if "unable to open database file" in e.message or "database is locked" in e.message:
+                if "unable to open database file" in str(e) or "database is locked" in str(e):
                     dberror = e
                     if args is None:
                         logger.debug('Database error: %s. Query: %s', e, query)

headphones/helpers.py

@@ -14,25 +14,25 @@
 #  You should have received a copy of the GNU General Public License
 #  along with Headphones.  If not, see <http://www.gnu.org/licenses/>.
 
-from operator import itemgetter
-import unicodedata
-import datetime
+import os
+import re
 import shutil
-import time
 import sys
 import tempfile
-import glob
+import time
+import unicodedata
+from contextlib import contextmanager
+from datetime import datetime, date
+from fnmatch import fnmatch
+from functools import cmp_to_key
+from glob import glob
+from operator import itemgetter
 
 from beets import logging as beetslogging
-import six
-from contextlib import contextmanager
-
-import fnmatch
-import functools
-import re
-import os
 from mediafile import MediaFile, FileTypeError, UnreadableFileError
+from six import text_type
 from unidecode import unidecode
+
 import headphones
 
 
@@ -74,7 +74,7 @@ def multikeysort(items, columns):
         else:
             return 0
 
-    return sorted(items, key=functools.cmp_to_key(comparer))
+    return sorted(items, key=cmp_to_key(comparer))
 
 
 def checked(variable):
@@ -156,21 +156,19 @@ def convert_seconds(s):
 
 
 def today():
-    today = datetime.date.today()
-    yyyymmdd = datetime.date.isoformat(today)
-    return yyyymmdd
+    return date.isoformat(date.today())
 
 
 def now():
-    now = datetime.datetime.now()
+    now = datetime.now()
     return now.strftime("%Y-%m-%d %H:%M:%S")
 
 
-def is_valid_date(date):
-    if not date:
+def is_valid_date(d):
+    if not d:
         return False
     else:
-        return bool(re.match(r'\d{4}-\d{2}-\d{2}', date))
+        return bool(re.match(r'\d{4}-\d{2}-\d{2}', d))
 
 
 def age(d):
@@ -508,7 +506,7 @@ def path_match_patterns(path, patterns):
     """
 
     for pattern in patterns:
-        if fnmatch.fnmatch(path, pattern):
+        if fnmatch(path, pattern):
             return True
 
     # No match
@@ -714,7 +712,7 @@ def preserve_torrent_directory(albumpath, forced=False, single=False):
             workdir = os.path.join(tempdir, prefix)
             workdir = re.sub(r'\[', '[[]', workdir)
             workdir = re.sub(r'(?<!\[)\]', '[]]', workdir)
-            if len(glob.glob(workdir + '*/')) >= 3:
+            if len(glob(workdir + '*/')) >= 3:
                 logger.error(
                     "Looks like a temp directory has previously been created "
                     "for this albumpath, not continuing "
@@ -1033,7 +1031,7 @@ class BeetsLogCapture(beetslogging.Handler):
         self.messages = []
 
     def emit(self, record):
-        self.messages.append(six.text_type(record.msg))
+        self.messages.append(text_type(record.msg))
 
 
 @contextmanager
@@ -1045,3 +1043,10 @@ def capture_beets_log(logger='beets'):
         yield capture.messages
     finally:
         log.removeHandler(capture)
+
+def have_pct_have_total(db_artist):
+    have_tracks = db_artist['HaveTracks'] or 0
+    total_tracks = db_artist['TotalTracks'] or 0 
+    have_pct = have_tracks / total_tracks if total_tracks else 0
+    return (have_pct, total_tracks)
+

headphones/importer.py

@@ -102,12 +102,13 @@ def artistlist_to_mbids(artistlist, forced=False):
             myDB.action('DELETE from newartists WHERE ArtistName=?', [artist])
 
     # Update the similar artist tag cloud:
-    logger.info('Updating artist information from Last.fm')
+    # TODO: Fix last.fm api
+    # logger.info('Updating artist information from Last.fm')
 
-    try:
-        lastfm.getSimilar()
-    except Exception as e:
-        logger.warn('Failed to update artist information from Last.fm: %s' % e)
+    # try:
+    #     lastfm.getSimilar()
+    # except Exception as e:
+    #     logger.warn('Failed to update artist information from Last.fm: %s' % e)
 
 
 def addArtistIDListToDB(artistidlist):
@@ -274,9 +275,9 @@ def addArtisttoDB(artistid, extrasonly=False, forcefull=False, type="artist"):
                     if len(check_release_date) == 10:
                         release_date = check_release_date
                     elif len(check_release_date) == 7:
-                        release_date = check_release_date + "-31"
+                        release_date = check_release_date + "-27"
                     elif len(check_release_date) == 4:
-                        release_date = check_release_date + "-12-31"
+                        release_date = check_release_date + "-12-27"
                     else:
                         release_date = today
                     if helpers.age(release_date) < ignore_age:

headphones/lastfm.py

@@ -63,12 +63,12 @@ def request_lastfm(method, **kwargs):
 
 def getSimilar():
     myDB = db.DBConnection()
-    results = myDB.select("SELECT ArtistID from artists ORDER BY HaveTracks DESC")
+    results = myDB.select("SELECT ArtistID from artists ORDER BY HaveTracks DESC LIMIT 10")
 
     logger.info("Fetching similar artists from Last.FM for tag cloud")
     artistlist = []
 
-    for result in results[:12]:
+    for result in results:
         data = request_lastfm("artist.getsimilar", mbid=result["ArtistId"])
 
         if data and "similarartists" in data:

headphones/librarysync.py

@@ -442,8 +442,9 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None,
     # if not append:
     #     update_album_status()
 
-    if not append and not artistScan:
-        lastfm.getSimilar()
+    # TODO: Fix last.fm api calls
+    #if not append and not artistScan:
+        #lastfm.getSimilar()
 
     if ArtistName:
         logger.info('Scanning complete for artist: %s', ArtistName)

headphones/mb.py

@@ -14,20 +14,14 @@
 #  along with Headphones.  If not, see <http://www.gnu.org/licenses/>.
 
 
-from headphones import logger, db, helpers
+from collections import OrderedDict
+
+import musicbrainzngs
 
 import headphones
-import musicbrainzngs
 import headphones.lock
+from headphones import logger, db, helpers
 
-try:
-    # pylint:disable=E0611
-    # ignore this error because we are catching the ImportError
-    from collections import OrderedDict
-    # pylint:enable=E0611
-except ImportError:
-    # Python 2.6.x fallback, from libs
-    from ordereddict import OrderedDict
 
 mb_lock = headphones.lock.TimedLock(0)
 
@@ -97,7 +91,7 @@ def findArtist(name, limit=1):
         try:
             artistResults = musicbrainzngs.search_artists(limit=limit, **criteria)['artist-list']
         except ValueError as e:
-            if "at least one query term is required" in e.message:
+            if "at least one query term is required" in str(e):
                 logger.error(
                     "Tried to search without a term, or an empty one. Provided artist (probably emtpy): %s",
                     name)

headphones/searcher.py

@@ -809,7 +809,7 @@ def send_to_downloader(data, bestqual, album):
             try:
                 prev = os.umask(headphones.UMASK)
 
-                with open(download_path, 'w') as fp:
+                with open(download_path, 'wb') as fp:
                     fp.write(data)
 
                 os.umask(prev)

headphones/webserve.py

@@ -15,34 +15,44 @@
 
 # NZBGet support added by CurlyMo <curlymoo1@gmail.com> as a part of XBian - XBMC on the Raspberry Pi
 
-from operator import itemgetter
-import threading
-import secrets
-import random
-import urllib.request, urllib.parse, urllib.error
 import json
-import time
-import sys
-from html import escape as html_escape
-import urllib.request, urllib.error, urllib.parse
-
 import os
+import random
 import re
-from headphones import logger, searcher, db, importer, mb, lastfm, librarysync, helpers, notifiers, crier
-from headphones.helpers import checked, radio, today, clean_name
-from mako.lookup import TemplateLookup
-from mako import exceptions
-import headphones
-import cherrypy
+import secrets
+import sys
+import threading
+import time
+from collections import OrderedDict
+from html import escape as html_escape
+from operator import itemgetter
+from urllib import parse
 
-try:
-    # pylint:disable=E0611
-    # ignore this error because we are catching the ImportError
-    from collections import OrderedDict
-    # pylint:enable=E0611
-except ImportError:
-    # Python 2.6.x fallback, from libs
-    from ordereddict import OrderedDict
+import cherrypy
+from mako import exceptions
+from mako.lookup import TemplateLookup
+
+import headphones
+from headphones import (
+    crier,
+    db,
+    importer,
+    lastfm,
+    librarysync,
+    logger,
+    mb,
+    notifiers,
+    searcher,
+)
+from headphones.helpers import (
+    checked,
+    clean_name,
+    have_pct_have_total,
+    pattern_substitute,
+    radio,
+    replace_illegal_chars,
+    today,
+)
 
 
 def serve_template(templatename, **kwargs):
@@ -326,9 +336,9 @@ class WebInterface(object):
                   '$first': firstchar.lower(),
                   }
 
-        folder = helpers.pattern_substitute(folder_format.strip(), values, normalize=True)
+        folder = pattern_substitute(folder_format.strip(), values, normalize=True)
 
-        folder = helpers.replace_illegal_chars(folder, type="folder")
+        folder = replace_illegal_chars(folder, type="folder")
         folder = folder.replace('./', '_/').replace('/.', '/_')
 
         if folder.endswith('.'):
@@ -461,7 +471,7 @@ class WebInterface(object):
     def download_specific_release(self, AlbumID, title, size, url, provider, kind, **kwargs):
         # Handle situations where the torrent url contains arguments that are parsed
         if kwargs:
-            url = urllib.parse.quote(url, safe=":?/=&") + '&' + urllib.parse.urlencode(kwargs)
+            url = parse.quote(url, safe=":?/=&") + '&' + parse.urlencode(kwargs)
         try:
             result = [(title, int(size), url, provider, kind)]
         except ValueError:
@@ -586,7 +596,7 @@ class WebInterface(object):
         for albums in have_albums:
             # Have to skip over manually matched tracks
             if albums['ArtistName'] and albums['AlbumTitle'] and albums['TrackTitle']:
-                original_clean = helpers.clean_name(
+                original_clean = clean_name(
                     albums['ArtistName'] + " " + albums['AlbumTitle'] + " " + albums['TrackTitle'])
                 # else:
                 #     original_clean = None
@@ -633,8 +643,8 @@ class WebInterface(object):
                 (artist, album))
 
         elif action == "matchArtist":
-            existing_artist_clean = helpers.clean_name(existing_artist).lower()
-            new_artist_clean = helpers.clean_name(new_artist).lower()
+            existing_artist_clean = clean_name(existing_artist).lower()
+            new_artist_clean = clean_name(new_artist).lower()
             if new_artist_clean != existing_artist_clean:
                 have_tracks = myDB.action(
                     'SELECT Matched, CleanName, Location, BitRate, Format FROM have WHERE ArtistName=?',
@@ -678,10 +688,10 @@ class WebInterface(object):
                     "Artist %s already named appropriately; nothing to modify" % existing_artist)
 
         elif action == "matchAlbum":
-            existing_artist_clean = helpers.clean_name(existing_artist).lower()
-            new_artist_clean = helpers.clean_name(new_artist).lower()
-            existing_album_clean = helpers.clean_name(existing_album).lower()
-            new_album_clean = helpers.clean_name(new_album).lower()
+            existing_artist_clean = clean_name(existing_artist).lower()
+            new_artist_clean = clean_name(new_artist).lower()
+            existing_album_clean = clean_name(existing_album).lower()
+            new_album_clean = clean_name(new_album).lower()
             existing_clean_string = existing_artist_clean + " " + existing_album_clean
             new_clean_string = new_artist_clean + " " + new_album_clean
             if existing_clean_string != new_clean_string:
@@ -737,7 +747,7 @@ class WebInterface(object):
             'SELECT ArtistName, AlbumTitle, TrackTitle, CleanName, Matched from have')
         for albums in manualalbums:
             if albums['ArtistName'] and albums['AlbumTitle'] and albums['TrackTitle']:
-                original_clean = helpers.clean_name(
+                original_clean = clean_name(
                     albums['ArtistName'] + " " + albums['AlbumTitle'] + " " + albums['TrackTitle'])
                 if albums['Matched'] == "Ignored" or albums['Matched'] == "Manual" or albums[
                         'CleanName'] != original_clean:
@@ -778,7 +788,7 @@ class WebInterface(object):
                 [artist])
             update_count = 0
             for tracks in update_clean:
-                original_clean = helpers.clean_name(
+                original_clean = clean_name(
                     tracks['ArtistName'] + " " + tracks['AlbumTitle'] + " " + tracks[
                         'TrackTitle']).lower()
                 album = tracks['AlbumTitle']
@@ -810,7 +820,7 @@ class WebInterface(object):
                 (artist, album))
             update_count = 0
             for tracks in update_clean:
-                original_clean = helpers.clean_name(
+                original_clean = clean_name(
                     tracks['ArtistName'] + " " + tracks['AlbumTitle'] + " " + tracks[
                         'TrackTitle']).lower()
                 track_title = tracks['TrackTitle']
@@ -1018,9 +1028,7 @@ class WebInterface(object):
             totalcount = myDB.select('SELECT COUNT(*) from artists')[0][0]
 
         if sortbyhavepercent:
-            filtered.sort(key=lambda x: (
-                float(x['HaveTracks']) / x['TotalTracks'] if x['TotalTracks'] > 0 else 0.0,
-                x['HaveTracks'] if x['HaveTracks'] else 0.0), reverse=sSortDir_0 == "asc")
+            filtered.sort(key=have_pct_have_total, reverse=sSortDir_0 == "asc")
 
         # can't figure out how to change the datatables default sorting order when its using an ajax datasource so ill
         # just reverse it here and the first click on the "Latest Album" header will sort by descending release date

lib/importlib_resources/__init__.py

@@ -0,0 +1,36 @@
+"""Read resources contained within a package."""
+
+from ._common import (
+    as_file,
+    files,
+    Package,
+)
+
+from ._legacy import (
+    contents,
+    open_binary,
+    read_binary,
+    open_text,
+    read_text,
+    is_resource,
+    path,
+    Resource,
+)
+
+from importlib_resources.abc import ResourceReader
+
+
+__all__ = [
+    'Package',
+    'Resource',
+    'ResourceReader',
+    'as_file',
+    'contents',
+    'files',
+    'is_resource',
+    'open_binary',
+    'open_text',
+    'path',
+    'read_binary',
+    'read_text',
+]

lib/importlib_resources/_adapters.py

@@ -0,0 +1,170 @@
+from contextlib import suppress
+from io import TextIOWrapper
+
+from . import abc
+
+
+class SpecLoaderAdapter:
+    """
+    Adapt a package spec to adapt the underlying loader.
+    """
+
+    def __init__(self, spec, adapter=lambda spec: spec.loader):
+        self.spec = spec
+        self.loader = adapter(spec)
+
+    def __getattr__(self, name):
+        return getattr(self.spec, name)
+
+
+class TraversableResourcesLoader:
+    """
+    Adapt a loader to provide TraversableResources.
+    """
+
+    def __init__(self, spec):
+        self.spec = spec
+
+    def get_resource_reader(self, name):
+        return CompatibilityFiles(self.spec)._native()
+
+
+def _io_wrapper(file, mode='r', *args, **kwargs):
+    if mode == 'r':
+        return TextIOWrapper(file, *args, **kwargs)
+    elif mode == 'rb':
+        return file
+    raise ValueError(
+        "Invalid mode value '{}', only 'r' and 'rb' are supported".format(mode)
+    )
+
+
+class CompatibilityFiles:
+    """
+    Adapter for an existing or non-existent resource reader
+    to provide a compatibility .files().
+    """
+
+    class SpecPath(abc.Traversable):
+        """
+        Path tied to a module spec.
+        Can be read and exposes the resource reader children.
+        """
+
+        def __init__(self, spec, reader):
+            self._spec = spec
+            self._reader = reader
+
+        def iterdir(self):
+            if not self._reader:
+                return iter(())
+            return iter(
+                CompatibilityFiles.ChildPath(self._reader, path)
+                for path in self._reader.contents()
+            )
+
+        def is_file(self):
+            return False
+
+        is_dir = is_file
+
+        def joinpath(self, other):
+            if not self._reader:
+                return CompatibilityFiles.OrphanPath(other)
+            return CompatibilityFiles.ChildPath(self._reader, other)
+
+        @property
+        def name(self):
+            return self._spec.name
+
+        def open(self, mode='r', *args, **kwargs):
+            return _io_wrapper(self._reader.open_resource(None), mode, *args, **kwargs)
+
+    class ChildPath(abc.Traversable):
+        """
+        Path tied to a resource reader child.
+        Can be read but doesn't expose any meaningful children.
+        """
+
+        def __init__(self, reader, name):
+            self._reader = reader
+            self._name = name
+
+        def iterdir(self):
+            return iter(())
+
+        def is_file(self):
+            return self._reader.is_resource(self.name)
+
+        def is_dir(self):
+            return not self.is_file()
+
+        def joinpath(self, other):
+            return CompatibilityFiles.OrphanPath(self.name, other)
+
+        @property
+        def name(self):
+            return self._name
+
+        def open(self, mode='r', *args, **kwargs):
+            return _io_wrapper(
+                self._reader.open_resource(self.name), mode, *args, **kwargs
+            )
+
+    class OrphanPath(abc.Traversable):
+        """
+        Orphan path, not tied to a module spec or resource reader.
+        Can't be read and doesn't expose any meaningful children.
+        """
+
+        def __init__(self, *path_parts):
+            if len(path_parts) < 1:
+                raise ValueError('Need at least one path part to construct a path')
+            self._path = path_parts
+
+        def iterdir(self):
+            return iter(())
+
+        def is_file(self):
+            return False
+
+        is_dir = is_file
+
+        def joinpath(self, other):
+            return CompatibilityFiles.OrphanPath(*self._path, other)
+
+        @property
+        def name(self):
+            return self._path[-1]
+
+        def open(self, mode='r', *args, **kwargs):
+            raise FileNotFoundError("Can't open orphan path")
+
+    def __init__(self, spec):
+        self.spec = spec
+
+    @property
+    def _reader(self):
+        with suppress(AttributeError):
+            return self.spec.loader.get_resource_reader(self.spec.name)
+
+    def _native(self):
+        """
+        Return the native reader if it supports files().
+        """
+        reader = self._reader
+        return reader if hasattr(reader, 'files') else self
+
+    def __getattr__(self, attr):
+        return getattr(self._reader, attr)
+
+    def files(self):
+        return CompatibilityFiles.SpecPath(self.spec, self._reader)
+
+
+def wrap_spec(package):
+    """
+    Construct a package spec with traversable compatibility
+    on the spec/loader/reader.
+    """
+    return SpecLoaderAdapter(package.__spec__, TraversableResourcesLoader)

lib/importlib_resources/_common.py

@@ -0,0 +1,104 @@
+import os
+import pathlib
+import tempfile
+import functools
+import contextlib
+import types
+import importlib
+
+from typing import Union, Optional
+from .abc import ResourceReader, Traversable
+
+from ._compat import wrap_spec
+
+Package = Union[types.ModuleType, str]
+
+
+def files(package):
+    # type: (Package) -> Traversable
+    """
+    Get a Traversable resource from a package
+    """
+    return from_package(get_package(package))
+
+
+def get_resource_reader(package):
+    # type: (types.ModuleType) -> Optional[ResourceReader]
+    """
+    Return the package's loader if it's a ResourceReader.
+    """
+    # We can't use
+    # a issubclass() check here because apparently abc.'s __subclasscheck__()
+    # hook wants to create a weak reference to the object, but
+    # zipimport.zipimporter does not support weak references, resulting in a
+    # TypeError.  That seems terrible.
+    spec = package.__spec__
+    reader = getattr(spec.loader, 'get_resource_reader', None)  # type: ignore
+    if reader is None:
+        return None
+    return reader(spec.name)  # type: ignore
+
+
+def resolve(cand):
+    # type: (Package) -> types.ModuleType
+    return cand if isinstance(cand, types.ModuleType) else importlib.import_module(cand)
+
+
+def get_package(package):
+    # type: (Package) -> types.ModuleType
+    """Take a package name or module object and return the module.
+
+    Raise an exception if the resolved module is not a package.
+    """
+    resolved = resolve(package)
+    if wrap_spec(resolved).submodule_search_locations is None:
+        raise TypeError(f'{package!r} is not a package')
+    return resolved
+
+
+def from_package(package):
+    """
+    Return a Traversable object for the given package.
+
+    """
+    spec = wrap_spec(package)
+    reader = spec.loader.get_resource_reader(spec.name)
+    return reader.files()
+
+
+@contextlib.contextmanager
+def _tempfile(reader, suffix=''):
+    # Not using tempfile.NamedTemporaryFile as it leads to deeper 'try'
+    # blocks due to the need to close the temporary file to work on Windows
+    # properly.
+    fd, raw_path = tempfile.mkstemp(suffix=suffix)
+    try:
+        try:
+            os.write(fd, reader())
+        finally:
+            os.close(fd)
+        del reader
+        yield pathlib.Path(raw_path)
+    finally:
+        try:
+            os.remove(raw_path)
+        except FileNotFoundError:
+            pass
+
+
+@functools.singledispatch
+def as_file(path):
+    """
+    Given a Traversable object, return that object as a
+    path on the local file system in a context manager.
+    """
+    return _tempfile(path.read_bytes, suffix=path.name)
+
+
+@as_file.register(pathlib.Path)
+@contextlib.contextmanager
+def _(path):
+    """
+    Degenerate behavior for pathlib.Path objects.
+    """
+    yield path

lib/importlib_resources/_compat.py

@@ -0,0 +1,98 @@
+# flake8: noqa
+
+import abc
+import sys
+import pathlib
+from contextlib import suppress
+
+if sys.version_info >= (3, 10):
+    from zipfile import Path as ZipPath  # type: ignore
+else:
+    from zipp import Path as ZipPath  # type: ignore
+
+
+try:
+    from typing import runtime_checkable  # type: ignore
+except ImportError:
+
+    def runtime_checkable(cls):  # type: ignore
+        return cls
+
+
+try:
+    from typing import Protocol  # type: ignore
+except ImportError:
+    Protocol = abc.ABC  # type: ignore
+
+
+class TraversableResourcesLoader:
+    """
+    Adapt loaders to provide TraversableResources and other
+    compatibility.
+
+    Used primarily for Python 3.9 and earlier where the native
+    loaders do not yet implement TraversableResources.
+    """
+
+    def __init__(self, spec):
+        self.spec = spec
+
+    @property
+    def path(self):
+        return self.spec.origin
+
+    def get_resource_reader(self, name):
+        from . import readers, _adapters
+
+        def _zip_reader(spec):
+            with suppress(AttributeError):
+                return readers.ZipReader(spec.loader, spec.name)
+
+        def _namespace_reader(spec):
+            with suppress(AttributeError, ValueError):
+                return readers.NamespaceReader(spec.submodule_search_locations)
+
+        def _available_reader(spec):
+            with suppress(AttributeError):
+                return spec.loader.get_resource_reader(spec.name)
+
+        def _native_reader(spec):
+            reader = _available_reader(spec)
+            return reader if hasattr(reader, 'files') else None
+
+        def _file_reader(spec):
+            try:
+                path = pathlib.Path(self.path)
+            except TypeError:
+                return None
+            if path.exists():
+                return readers.FileReader(self)
+
+        return (
+            # native reader if it supplies 'files'
+            _native_reader(self.spec)
+            or
+            # local ZipReader if a zip module
+            _zip_reader(self.spec)
+            or
+            # local NamespaceReader if a namespace module
+            _namespace_reader(self.spec)
+            or
+            # local FileReader
+            _file_reader(self.spec)
+            # fallback - adapt the spec ResourceReader to TraversableReader
+            or _adapters.CompatibilityFiles(self.spec)
+        )
+
+
+def wrap_spec(package):
+    """
+    Construct a package spec with traversable compatibility
+    on the spec/loader/reader.
+
+    Supersedes _adapters.wrap_spec to use TraversableResourcesLoader
+    from above for older Python compatibility (<3.10).
+    """
+    from . import _adapters
+
+    return _adapters.SpecLoaderAdapter(package.__spec__, TraversableResourcesLoader)

lib/importlib_resources/_itertools.py

@@ -0,0 +1,35 @@
+from itertools import filterfalse
+
+from typing import (
+    Callable,
+    Iterable,
+    Iterator,
+    Optional,
+    Set,
+    TypeVar,
+    Union,
+)
+
+# Type and type variable definitions
+_T = TypeVar('_T')
+_U = TypeVar('_U')
+
+
+def unique_everseen(
+    iterable: Iterable[_T], key: Optional[Callable[[_T], _U]] = None
+) -> Iterator[_T]:
+    "List unique elements, preserving order. Remember all elements ever seen."
+    # unique_everseen('AAAABBBCCDAABBB') --> A B C D
+    # unique_everseen('ABBCcAD', str.lower) --> A B C D
+    seen: Set[Union[_T, _U]] = set()
+    seen_add = seen.add
+    if key is None:
+        for element in filterfalse(seen.__contains__, iterable):
+            seen_add(element)
+            yield element
+    else:
+        for element in iterable:
+            k = key(element)
+            if k not in seen:
+                seen_add(k)
+                yield element

lib/importlib_resources/_legacy.py

@@ -0,0 +1,121 @@
+import functools
+import os
+import pathlib
+import types
+import warnings
+
+from typing import Union, Iterable, ContextManager, BinaryIO, TextIO, Any
+
+from . import _common
+
+Package = Union[types.ModuleType, str]
+Resource = str
+
+
+def deprecated(func):
+    @functools.wraps(func)
+    def wrapper(*args, **kwargs):
+        warnings.warn(
+            f"{func.__name__} is deprecated. Use files() instead. "
+            "Refer to https://importlib-resources.readthedocs.io"
+            "/en/latest/using.html#migrating-from-legacy for migration advice.",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        return func(*args, **kwargs)
+
+    return wrapper
+
+
+def normalize_path(path):
+    # type: (Any) -> str
+    """Normalize a path by ensuring it is a string.
+
+    If the resulting string contains path separators, an exception is raised.
+    """
+    str_path = str(path)
+    parent, file_name = os.path.split(str_path)
+    if parent:
+        raise ValueError(f'{path!r} must be only a file name')
+    return file_name
+
+
+@deprecated
+def open_binary(package: Package, resource: Resource) -> BinaryIO:
+    """Return a file-like object opened for binary reading of the resource."""
+    return (_common.files(package) / normalize_path(resource)).open('rb')
+
+
+@deprecated
+def read_binary(package: Package, resource: Resource) -> bytes:
+    """Return the binary contents of the resource."""
+    return (_common.files(package) / normalize_path(resource)).read_bytes()
+
+
+@deprecated
+def open_text(
+    package: Package,
+    resource: Resource,
+    encoding: str = 'utf-8',
+    errors: str = 'strict',
+) -> TextIO:
+    """Return a file-like object opened for text reading of the resource."""
+    return (_common.files(package) / normalize_path(resource)).open(
+        'r', encoding=encoding, errors=errors
+    )
+
+
+@deprecated
+def read_text(
+    package: Package,
+    resource: Resource,
+    encoding: str = 'utf-8',
+    errors: str = 'strict',
+) -> str:
+    """Return the decoded string of the resource.
+
+    The decoding-related arguments have the same semantics as those of
+    bytes.decode().
+    """
+    with open_text(package, resource, encoding, errors) as fp:
+        return fp.read()
+
+
+@deprecated
+def contents(package: Package) -> Iterable[str]:
+    """Return an iterable of entries in `package`.
+
+    Note that not all entries are resources.  Specifically, directories are
+    not considered resources.  Use `is_resource()` on each entry returned here
+    to check if it is a resource or not.
+    """
+    return [path.name for path in _common.files(package).iterdir()]
+
+
+@deprecated
+def is_resource(package: Package, name: str) -> bool:
+    """True if `name` is a resource inside `package`.
+
+    Directories are *not* resources.
+    """
+    resource = normalize_path(name)
+    return any(
+        traversable.name == resource and traversable.is_file()
+        for traversable in _common.files(package).iterdir()
+    )
+
+
+@deprecated
+def path(
+    package: Package,
+    resource: Resource,
+) -> ContextManager[pathlib.Path]:
+    """A context manager providing a file path object to the resource.
+
+    If the resource does not already exist on its own on the file system,
+    a temporary file will be created. If the file was created, the file
+    will be deleted upon exiting the context manager (no exception is
+    raised if the file was deleted prior to the context manager
+    exiting).
+    """
+    return _common.as_file(_common.files(package) / normalize_path(resource))

lib/importlib_resources/abc.py

@@ -0,0 +1,137 @@
+import abc
+from typing import BinaryIO, Iterable, Text
+
+from ._compat import runtime_checkable, Protocol
+
+
+class ResourceReader(metaclass=abc.ABCMeta):
+    """Abstract base class for loaders to provide resource reading support."""
+
+    @abc.abstractmethod
+    def open_resource(self, resource: Text) -> BinaryIO:
+        """Return an opened, file-like object for binary reading.
+
+        The 'resource' argument is expected to represent only a file name.
+        If the resource cannot be found, FileNotFoundError is raised.
+        """
+        # This deliberately raises FileNotFoundError instead of
+        # NotImplementedError so that if this method is accidentally called,
+        # it'll still do the right thing.
+        raise FileNotFoundError
+
+    @abc.abstractmethod
+    def resource_path(self, resource: Text) -> Text:
+        """Return the file system path to the specified resource.
+
+        The 'resource' argument is expected to represent only a file name.
+        If the resource does not exist on the file system, raise
+        FileNotFoundError.
+        """
+        # This deliberately raises FileNotFoundError instead of
+        # NotImplementedError so that if this method is accidentally called,
+        # it'll still do the right thing.
+        raise FileNotFoundError
+
+    @abc.abstractmethod
+    def is_resource(self, path: Text) -> bool:
+        """Return True if the named 'path' is a resource.
+
+        Files are resources, directories are not.
+        """
+        raise FileNotFoundError
+
+    @abc.abstractmethod
+    def contents(self) -> Iterable[str]:
+        """Return an iterable of entries in `package`."""
+        raise FileNotFoundError
+
+
+@runtime_checkable
+class Traversable(Protocol):
+    """
+    An object with a subset of pathlib.Path methods suitable for
+    traversing directories and opening files.
+    """
+
+    @abc.abstractmethod
+    def iterdir(self):
+        """
+        Yield Traversable objects in self
+        """
+
+    def read_bytes(self):
+        """
+        Read contents of self as bytes
+        """
+        with self.open('rb') as strm:
+            return strm.read()
+
+    def read_text(self, encoding=None):
+        """
+        Read contents of self as text
+        """
+        with self.open(encoding=encoding) as strm:
+            return strm.read()
+
+    @abc.abstractmethod
+    def is_dir(self) -> bool:
+        """
+        Return True if self is a directory
+        """
+
+    @abc.abstractmethod
+    def is_file(self) -> bool:
+        """
+        Return True if self is a file
+        """
+
+    @abc.abstractmethod
+    def joinpath(self, child):
+        """
+        Return Traversable child in self
+        """
+
+    def __truediv__(self, child):
+        """
+        Return Traversable child in self
+        """
+        return self.joinpath(child)
+
+    @abc.abstractmethod
+    def open(self, mode='r', *args, **kwargs):
+        """
+        mode may be 'r' or 'rb' to open as text or binary. Return a handle
+        suitable for reading (same as pathlib.Path.open).
+
+        When opening as text, accepts encoding parameters such as those
+        accepted by io.TextIOWrapper.
+        """
+
+    @abc.abstractproperty
+    def name(self) -> str:
+        """
+        The base name of this object without any parent references.
+        """
+
+
+class TraversableResources(ResourceReader):
+    """
+    The required interface for providing traversable
+    resources.
+    """
+
+    @abc.abstractmethod
+    def files(self):
+        """Return a Traversable object for the loaded package."""
+
+    def open_resource(self, resource):
+        return self.files().joinpath(resource).open('rb')
+
+    def resource_path(self, resource):
+        raise FileNotFoundError(resource)
+
+    def is_resource(self, path):
+        return self.files().joinpath(path).is_file()
+
+    def contents(self):
+        return (item.name for item in self.files().iterdir())

lib/importlib_resources/readers.py

@@ -0,0 +1,122 @@
+import collections
+import pathlib
+import operator
+
+from . import abc
+
+from ._itertools import unique_everseen
+from ._compat import ZipPath
+
+
+def remove_duplicates(items):
+    return iter(collections.OrderedDict.fromkeys(items))
+
+
+class FileReader(abc.TraversableResources):
+    def __init__(self, loader):
+        self.path = pathlib.Path(loader.path).parent
+
+    def resource_path(self, resource):
+        """
+        Return the file system path to prevent
+        `resources.path()` from creating a temporary
+        copy.
+        """
+        return str(self.path.joinpath(resource))
+
+    def files(self):
+        return self.path
+
+
+class ZipReader(abc.TraversableResources):
+    def __init__(self, loader, module):
+        _, _, name = module.rpartition('.')
+        self.prefix = loader.prefix.replace('\\', '/') + name + '/'
+        self.archive = loader.archive
+
+    def open_resource(self, resource):
+        try:
+            return super().open_resource(resource)
+        except KeyError as exc:
+            raise FileNotFoundError(exc.args[0])
+
+    def is_resource(self, path):
+        # workaround for `zipfile.Path.is_file` returning true
+        # for non-existent paths.
+        target = self.files().joinpath(path)
+        return target.is_file() and target.exists()
+
+    def files(self):
+        return ZipPath(self.archive, self.prefix)
+
+
+class MultiplexedPath(abc.Traversable):
+    """
+    Given a series of Traversable objects, implement a merged
+    version of the interface across all objects. Useful for
+    namespace packages which may be multihomed at a single
+    name.
+    """
+
+    def __init__(self, *paths):
+        self._paths = list(map(pathlib.Path, remove_duplicates(paths)))
+        if not self._paths:
+            message = 'MultiplexedPath must contain at least one path'
+            raise FileNotFoundError(message)
+        if not all(path.is_dir() for path in self._paths):
+            raise NotADirectoryError('MultiplexedPath only supports directories')
+
+    def iterdir(self):
+        files = (file for path in self._paths for file in path.iterdir())
+        return unique_everseen(files, key=operator.attrgetter('name'))
+
+    def read_bytes(self):
+        raise FileNotFoundError(f'{self} is not a file')
+
+    def read_text(self, *args, **kwargs):
+        raise FileNotFoundError(f'{self} is not a file')
+
+    def is_dir(self):
+        return True
+
+    def is_file(self):
+        return False
+
+    def joinpath(self, child):
+        # first try to find child in current paths
+        for file in self.iterdir():
+            if file.name == child:
+                return file
+        # if it does not exist, construct it with the first path
+        return self._paths[0] / child
+
+    __truediv__ = joinpath
+
+    def open(self, *args, **kwargs):
+        raise FileNotFoundError(f'{self} is not a file')
+
+    @property
+    def name(self):
+        return self._paths[0].name
+
+    def __repr__(self):
+        paths = ', '.join(f"'{path}'" for path in self._paths)
+        return f'MultiplexedPath({paths})'
+
+
+class NamespaceReader(abc.TraversableResources):
+    def __init__(self, namespace_path):
+        if 'NamespacePath' not in str(namespace_path):
+            raise ValueError('Invalid path')
+        self.path = MultiplexedPath(*list(namespace_path))
+
+    def resource_path(self, resource):
+        """
+        Return the file system path to prevent
+        `resources.path()` from creating a temporary
+        copy.
+        """
+        return str(self.path.joinpath(resource))
+
+    def files(self):
+        return self.path

lib/importlib_resources/simple.py

@@ -0,0 +1,116 @@
+"""
+Interface adapters for low-level readers.
+"""
+
+import abc
+import io
+import itertools
+from typing import BinaryIO, List
+
+from .abc import Traversable, TraversableResources
+
+
+class SimpleReader(abc.ABC):
+    """
+    The minimum, low-level interface required from a resource
+    provider.
+    """
+
+    @abc.abstractproperty
+    def package(self):
+        # type: () -> str
+        """
+        The name of the package for which this reader loads resources.
+        """
+
+    @abc.abstractmethod
+    def children(self):
+        # type: () -> List['SimpleReader']
+        """
+        Obtain an iterable of SimpleReader for available
+        child containers (e.g. directories).
+        """
+
+    @abc.abstractmethod
+    def resources(self):
+        # type: () -> List[str]
+        """
+        Obtain available named resources for this virtual package.
+        """
+
+    @abc.abstractmethod
+    def open_binary(self, resource):
+        # type: (str) -> BinaryIO
+        """
+        Obtain a File-like for a named resource.
+        """
+
+    @property
+    def name(self):
+        return self.package.split('.')[-1]
+
+
+class ResourceHandle(Traversable):
+    """
+    Handle to a named resource in a ResourceReader.
+    """
+
+    def __init__(self, parent, name):
+        # type: (ResourceContainer, str) -> None
+        self.parent = parent
+        self.name = name  # type: ignore
+
+    def is_file(self):
+        return True
+
+    def is_dir(self):
+        return False
+
+    def open(self, mode='r', *args, **kwargs):
+        stream = self.parent.reader.open_binary(self.name)
+        if 'b' not in mode:
+            stream = io.TextIOWrapper(*args, **kwargs)
+        return stream
+
+    def joinpath(self, name):
+        raise RuntimeError("Cannot traverse into a resource")
+
+
+class ResourceContainer(Traversable):
+    """
+    Traversable container for a package's resources via its reader.
+    """
+
+    def __init__(self, reader):
+        # type: (SimpleReader) -> None
+        self.reader = reader
+
+    def is_dir(self):
+        return True
+
+    def is_file(self):
+        return False
+
+    def iterdir(self):
+        files = (ResourceHandle(self, name) for name in self.reader.resources)
+        dirs = map(ResourceContainer, self.reader.children())
+        return itertools.chain(files, dirs)
+
+    def open(self, *args, **kwargs):
+        raise IsADirectoryError()
+
+    def joinpath(self, name):
+        return next(
+            traversable for traversable in self.iterdir() if traversable.name == name
+        )
+
+
+class TraversableReader(TraversableResources, SimpleReader):
+    """
+    A TraversableResources based on SimpleReader. Resource providers
+    may derive from this class to provide the TraversableResources
+    interface by supplying the SimpleReader interface.
+    """
+
+    def files(self):
+        return ResourceContainer(self)

lib/ordereddict.py

@@ -1,127 +0,0 @@
-# Copyright (c) 2009 Raymond Hettinger
-#
-# Permission is hereby granted, free of charge, to any person
-# obtaining a copy of this software and associated documentation files
-# (the "Software"), to deal in the Software without restriction,
-# including without limitation the rights to use, copy, modify, merge,
-# publish, distribute, sublicense, and/or sell copies of the Software,
-# and to permit persons to whom the Software is furnished to do so,
-# subject to the following conditions:
-#
-#     The above copyright notice and this permission notice shall be
-#     included in all copies or substantial portions of the Software.
-#
-#     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-#     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
-#     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-#     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
-#     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
-#     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-#     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
-#     OTHER DEALINGS IN THE SOFTWARE.
-
-from UserDict import DictMixin
-
-class OrderedDict(dict, DictMixin):
-
-    def __init__(self, *args, **kwds):
-        if len(args) > 1:
-            raise TypeError('expected at most 1 arguments, got %d' % len(args))
-        try:
-            self.__end
-        except AttributeError:
-            self.clear()
-        self.update(*args, **kwds)
-
-    def clear(self):
-        self.__end = end = []
-        end += [None, end, end]         # sentinel node for doubly linked list
-        self.__map = {}                 # key --> [key, prev, next]
-        dict.clear(self)
-
-    def __setitem__(self, key, value):
-        if key not in self:
-            end = self.__end
-            curr = end[1]
-            curr[2] = end[1] = self.__map[key] = [key, curr, end]
-        dict.__setitem__(self, key, value)
-
-    def __delitem__(self, key):
-        dict.__delitem__(self, key)
-        key, prev, next = self.__map.pop(key)
-        prev[2] = next
-        next[1] = prev
-
-    def __iter__(self):
-        end = self.__end
-        curr = end[2]
-        while curr is not end:
-            yield curr[0]
-            curr = curr[2]
-
-    def __reversed__(self):
-        end = self.__end
-        curr = end[1]
-        while curr is not end:
-            yield curr[0]
-            curr = curr[1]
-
-    def popitem(self, last=True):
-        if not self:
-            raise KeyError('dictionary is empty')
-        if last:
-            key = next(reversed(self))
-        else:
-            key = next(iter(self))
-        value = self.pop(key)
-        return key, value
-
-    def __reduce__(self):
-        items = [[k, self[k]] for k in self]
-        tmp = self.__map, self.__end
-        del self.__map, self.__end
-        inst_dict = vars(self).copy()
-        self.__map, self.__end = tmp
-        if inst_dict:
-            return (self.__class__, (items,), inst_dict)
-        return self.__class__, (items,)
-
-    def keys(self):
-        return list(self)
-
-    setdefault = DictMixin.setdefault
-    update = DictMixin.update
-    pop = DictMixin.pop
-    values = DictMixin.values
-    items = DictMixin.items
-    iterkeys = DictMixin.iterkeys
-    itervalues = DictMixin.itervalues
-    iteritems = DictMixin.iteritems
-
-    def __repr__(self):
-        if not self:
-            return '%s()' % (self.__class__.__name__,)
-        return '%s(%r)' % (self.__class__.__name__, list(self.items()))
-
-    def copy(self):
-        return self.__class__(self)
-
-    @classmethod
-    def fromkeys(cls, iterable, value=None):
-        d = cls()
-        for key in iterable:
-            d[key] = value
-        return d
-
-    def __eq__(self, other):
-        if isinstance(other, OrderedDict):
-            if len(self) != len(other):
-                return False
-            for p, q in  zip(list(self.items()), list(other.items())):
-                if p != q:
-                    return False
-            return True
-        return dict.__eq__(self, other)
-
-    def __ne__(self, other):
-        return not self == other