Update changelog for v0.6.1

xld - use native plistlib instead of biplist

rutracker - use new Result object

seacher - small tweaks

transmission - str no longer required

.flake8

@@ -1,3 +0,0 @@
-[flake8]
-exclude = .git,data,init-scripts,lib
-ignore = E501

.github/workflows/check.yaml

@@ -1,29 +0,0 @@
-name: check 
-
-on: [push, pull_request]
-
-jobs:
-  build:
-
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python-version: [3.8, 3.9, 3.10]
-
-    steps:
-      - uses: actions/checkout@v2
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
-        with:
-          python-version: ${{ matrix.python-version }}
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install -r requirements-dev.txt
-      - name: Lint with flake8
-        run: |
-          # stop the build if there are Python syntax errors or undefined names
-          flake8 .
-      - name: Test with nosetests 
-        run: |
-          nosetests

.travis.yml

@@ -0,0 +1,25 @@
+# Travis CI configuration file
+# http://about.travis-ci.org/docs/
+
+language: python
+
+sudo: false
+
+cache:
+  pip: true
+  directories:
+    - lib
+
+python:
+  - "2.7"
+
+install:
+  - pip install -r requirements-dev.txt
+
+script:
+  - pep8 headphones
+  - pyflakes headphones
+  - nosetests
+
+after_success:
+  - if [[ $TRAVIS_PYTHON_VERSION == "2.7" ]]; then coveralls; fi

CHANGELOG.md

@@ -1,5 +1,21 @@
 # Changelog
 
+## v0.6.1
+Released 26 November 2023
+
+Highlights:
+* Dependency updates to work with > Python 3.11
+
+The full list of commits can be found [here](https://github.com/rembo10/headphones/compare/v0.6.0...v0.6.1).
+
+## v0.6.0
+Released 13 November 2022
+
+Highlights:
+* Updated to python 3
+
+The full list of commits can be found [here](https://github.com/rembo10/headphones/compare/v0.5.20...v0.6.0).
+
 ## v0.5.20
 Released 15 October 2021
 

Headphones.py

@@ -17,8 +17,8 @@
 import os
 import sys
 
-if sys.version_info <= (3, 5):
+if sys.version_info <= (3, 6):
-    sys.stdout.write("Headphones requires Python >= 3.5\n")
+    sys.stdout.write("Headphones requires Python >= 3.7\n")
     sys.exit(1)
 
 # Ensure lib added to path, before any other imports

data/interfaces/default/config.html

@@ -1370,17 +1370,20 @@
                             <div class="row">
                                 <label>File Format</label>
                                 <input type="text" name="file_format" value="${config['file_format']}" size="43">
-                                <small>Use: $Disc/$disc (disc #), $Track/$track (track #), $Title/$title, $Artist/$artist, $Album/$album and $Year/$year. Put optional variables in curly braces, use single-quote marks to escape curly braces literally ('{', '}').</small>
+                                <small>Use: In addition to the above, there is also $Title/$title (track title), $Track (track #), $Disc (disc #), $DiscTotal.</small>
                             </div>
-                            <div class="checkbox row clearfix">
+                            <div class="checkbox row left clearfix nopad">
                                 <input type="checkbox" name="file_underscores" id="file_underscores" value="1" ${config['file_underscores']}/><label>Use underscores instead of spaces</label>
                             </div>
+                            <div class="checkbox row left clearfix nopad">
+                                <input type="checkbox" name="rename_single_disc_ignore" id="rename_single_disc_ignore" value="1" ${config['rename_single_disc_ignore']}/><label>Don't include disc# for single disc albums</label>
+                            </div>
                         </fieldset>
 
                         <fieldset>
                             <legend>Re-Encoding Options</legend>
                             <small class="heading"><i class="fa fa-info-circle"></i> Note: this option requires the lame, ffmpeg or xld encoder</small>
-                            <div class="checkbox row clearfix">
+                            <div class="checkbox row left clearfix nopad">
                                 <input type="checkbox" name="music_encoder" id="music_encoder" value="1" ${config['music_encoder']}/><label>Re-encode downloads during postprocessing</label>
                             </div>
                             <div id="encoderoptions" class="row clearfix checkbox">
@@ -1651,6 +1654,16 @@
                                 </div>
                             </fieldset>
 
+                            <fieldset>
+                                <legend>Last.fm</legend>
+                                <div id="lastfmoptions">
+                                    <div class="row">
+                                        <label>API Key</label>
+                                        <input type="text" name="lastfm_apikey" value="${config['lastfm_apikey']}" size="40" />
+                                    </div>
+                                </div>
+                            </fieldset>
+
                             <fieldset>
                                 <legend>Songkick</legend>
                                 <div class="row checkbox">

headphones/api.py

@@ -474,12 +474,8 @@ class Api(object):
         # Handle situations where the torrent url contains arguments that are
         # parsed
         if kwargs:
-            import urllib.request
+            import urllib.request, urllib.parse, urllib.error
-            import urllib.parse
+            import urllib.request, urllib.error, urllib.parse
-            import urllib.error
-            import urllib.request
-            import urllib.error
-            import urllib.parse
             url = urllib.parse.quote(
                 url, safe=":?/=&") + '&' + urllib.parse.urlencode(kwargs)
 

headphones/cache.py

@@ -388,9 +388,9 @@ class Cache(object):
             else:
                 if dbalbum['Type'] != "part of":
                     data = lastfm.request_lastfm("album.getinfo",
-                                                 artist=helpers.clean_musicbrainz_name(dbalbum['ArtistName']),
+                                                artist=helpers.clean_musicbrainz_name(dbalbum['ArtistName']),
-                                                 album=helpers.clean_musicbrainz_name(dbalbum['AlbumTitle']),
+                                                album=helpers.clean_musicbrainz_name(dbalbum['AlbumTitle']),
-                                                 api_key=LASTFM_API_KEY)
+                                                api_key=LASTFM_API_KEY)
                 else:
 
                     # Series, use actual artist for the release-group
@@ -484,7 +484,7 @@ class Cache(object):
                                                 self.id + '_fanart_' + '.' + helpers.today() + ext)
                 else:
                     artwork_path = os.path.join(self.path_to_art_cache,
-                                                self.id + '.' + helpers.today() + ext)
+                                            self.id + '.' + helpers.today() + ext)
                 try:
                     with open(artwork_path, 'wb') as f:
                         f.write(artwork)

headphones/classes.py

@@ -18,9 +18,7 @@
 #######################################
 
 
-import urllib.request
+import urllib.request, urllib.parse, urllib.error
-import urllib.parse
-import urllib.error
 
 from .common import USER_AGENT
 

headphones/common.py

@@ -77,7 +77,7 @@ class Quality:
         toReturn = {}
         for x in list(Quality.qualityStrings.keys()):
             toReturn[Quality.compositeStatus(status, x)] = Quality.statusPrefixes[status] + " (" + \
-                Quality.qualityStrings[x] + ")"
+                                                           Quality.qualityStrings[x] + ")"
         return toReturn
 
     @staticmethod

headphones/config.py

@@ -31,7 +31,6 @@ class path(str):
     def __repr__(self):
         return 'headphones.config.path(%s)' % self
 
-
 _CONFIG_DEFINITIONS = {
     'ADD_ALBUM_ART': (int, 'General', 0),
     'ADVANCEDENCODER': (str, 'General', ''),
@@ -156,9 +155,10 @@ _CONFIG_DEFINITIONS = {
     'KEEP_TORRENT_FILES': (int, 'General', 0),
     'KEEP_TORRENT_FILES_DIR': (path, 'General', ''),
     'LASTFM_USERNAME': (str, 'General', ''),
+    'LASTFM_APIKEY': (str, 'General', ''),
     'LAUNCH_BROWSER': (int, 'General', 1),
     'LIBRARYSCAN': (int, 'General', 1),
-    'LIBRARYSCAN_INTERVAL': (int, 'General', 300),
+    'LIBRARYSCAN_INTERVAL': (int, 'General', 24),
     'LMS_ENABLED': (int, 'LMS', 0),
     'LMS_HOST': (str, 'LMS', ''),
     'LOG_DIR': (path, 'General', ''),
@@ -241,6 +241,7 @@ _CONFIG_DEFINITIONS = {
     'QBITTORRENT_PASSWORD': (str, 'QBitTorrent', ''),
     'QBITTORRENT_USERNAME': (str, 'QBitTorrent', ''),
     'RENAME_FILES': (int, 'General', 0),
+    'RENAME_SINGLE_DISC_IGNORE': (int, 'General', 0),
     'RENAME_UNPROCESSED': (bool_int, 'General', 1),
     'RENAME_FROZEN': (bool_int, 'General', 1),
     'REPLACE_EXISTING_FOLDERS': (int, 'General', 0),
@@ -328,7 +329,7 @@ class Config(object):
     def __init__(self, config_file):
         """ Initialize the config with values from a file """
         self._config_file = config_file
-        self._config = ConfigParser()
+        self._config = ConfigParser(interpolation=None)
         self._config.read(self._config_file)
         for key in list(_CONFIG_DEFINITIONS.keys()):
             self.check_setting(key)
@@ -364,12 +365,12 @@ class Config(object):
 
         try:
             my_val = definition_type(self._config[section][ini_key])
-            # ConfigParser interprets empty strings in the config
+            # ConfigParser interprets quotes in the config
             # literally, so we need to sanitize it. It's not really
             # a config upgrade, since a user can at any time put 
-            # some_key = ''
+            # some_key = 'some_val'
-            if my_val == '""' or my_val == "''":
+            if type(my_val) in [str, path]:
-                my_val = ''
+                my_val = my_val.strip('"').strip("'")
         except Exception:
             my_val = default
             self._config[section][ini_key] = str(my_val)
@@ -377,7 +378,7 @@ class Config(object):
 
     def write(self):
         """ Make a copy of the stored config and write it to the configured file """
-        new_config = ConfigParser()
+        new_config = ConfigParser(interpolation=None)
 
         # first copy over everything from the old config, even if it is not
         # correctly defined to keep from losing data
@@ -408,7 +409,7 @@ class Config(object):
         """ Return the extra newznab tuples """
         extra_newznabs = list(
             zip(*[itertools.islice(self.EXTRA_NEWZNABS, i, None, 3)
-                  for i in range(3)])
+                             for i in range(3)])
         )
         return extra_newznabs
 
@@ -427,7 +428,7 @@ class Config(object):
         """ Return the extra torznab tuples """
         extra_torznabs = list(
             zip(*[itertools.islice(self.EXTRA_TORZNABS, i, None, 4)
-                  for i in range(4)])
+                             for i in range(4)])
         )
         return extra_torznabs
 
@@ -504,7 +505,7 @@ class Config(object):
             if self.EXTRA_TORZNABS:
                 extra_torznabs = list(
                     zip(*[itertools.islice(self.EXTRA_TORZNABS, i, None, 3)
-                          for i in range(3)])
+                                     for i in range(3)])
                 )
                 new_torznabs = []
                 for torznab in extra_torznabs:

headphones/db.py

@@ -18,6 +18,7 @@
 ###################################
 
 
+
 import time
 
 import sqlite3
@@ -116,7 +117,7 @@ class DBConnection:
                     break
 
             except sqlite3.OperationalError as e:
-                if "unable to open database file" in e.message or "database is locked" in e.message:
+                if "unable to open database file" in str(e) or "database is locked" in str(e):
                     dberror = e
                     if args is None:
                         logger.debug('Database error: %s. Query: %s', e, query)

headphones/deluge.py

@@ -35,6 +35,7 @@
 # along with SickRage. If not, see <http://www.gnu.org/licenses/>.
 
 
+
 from headphones import logger
 
 import time
@@ -88,7 +89,7 @@ def addTorrent(link, data=None, name=None):
         if link.lower().startswith('magnet:'):
             logger.debug('Deluge: Got a magnet link: %s' % _scrubber(link))
             result = {'type': 'magnet',
-                      'url': link}
+                        'url': link}
             retid = _add_torrent_magnet(result)
 
         elif link.lower().startswith('http://') or link.lower().startswith('https://'):
@@ -142,8 +143,8 @@ def addTorrent(link, data=None, name=None):
             except:
                 logger.debug('Deluge: Sending Deluge torrent with problematic name and some content')
             result = {'type': 'torrent',
-                      'name': name,
+                        'name': name,
-                      'content': torrentfile}
+                        'content': torrentfile}
             retid = _add_torrent_file(result)
 
         # elif link.endswith('.torrent') or data:
@@ -174,8 +175,8 @@ def addTorrent(link, data=None, name=None):
             except UnicodeDecodeError:
                 logger.debug('Deluge: Sending Deluge torrent with name %s and content [%s...]' % (name.decode('utf-8'), str(torrentfile)[:40]))
             result = {'type': 'torrent',
-                      'name': name,
+                        'name': name,
-                      'content': torrentfile}
+                        'content': torrentfile}
             retid = _add_torrent_file(result)
 
         else:
@@ -207,7 +208,7 @@ def getTorrentFolder(result):
                                 ],
                                 "id": 21})
         response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth,
-                                 verify=deluge_verify_cert, headers=headers)
+            verify=deluge_verify_cert, headers=headers)
         result['total_done'] = json.loads(response.text)['result']['total_done']
 
         tries = 0
@@ -215,7 +216,7 @@ def getTorrentFolder(result):
             tries += 1
             time.sleep(5)
             response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth,
-                                     verify=deluge_verify_cert, headers=headers)
+                verify=deluge_verify_cert, headers=headers)
             result['total_done'] = json.loads(response.text)['result']['total_done']
 
         post_data = json.dumps({"method": "web.get_torrent_status",
@@ -234,7 +235,7 @@ def getTorrentFolder(result):
                                 "id": 23})
 
         response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth,
-                                 verify=deluge_verify_cert, headers=headers)
+            verify=deluge_verify_cert, headers=headers)
 
         result['save_path'] = json.loads(response.text)['result']['save_path']
         result['name'] = json.loads(response.text)['result']['name']
@@ -263,7 +264,7 @@ def removeTorrent(torrentid, remove_data=False):
                                 "id": 26})
 
         response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth,
-                                 verify=deluge_verify_cert, headers=headers)
+            verify=deluge_verify_cert, headers=headers)
 
         try:
             state = json.loads(response.text)['result']['state']
@@ -282,10 +283,10 @@ def removeTorrent(torrentid, remove_data=False):
                                     "params": [
                                         torrentid,
                                         remove_data
-                                    ],
+                                        ],
                                     "id": 25})
             response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth,
-                                     verify=deluge_verify_cert, headers=headers)
+                verify=deluge_verify_cert, headers=headers)
             result = json.loads(response.text)['result']
 
             return result
@@ -328,12 +329,12 @@ def _get_auth():
                             "id": 1})
     try:
         response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth,
-                                 verify=deluge_verify_cert, headers=headers)
+            verify=deluge_verify_cert, headers=headers)
     except requests.ConnectionError:
         try:
             logger.debug('Deluge: Connection failed, let\'s try HTTPS just in case')
             response = requests.post(delugeweb_url.replace('http:', 'https:'), data=post_data.encode('utf-8'), cookies=delugeweb_auth,
-                                     verify=deluge_verify_cert, headers=headers)
+                verify=deluge_verify_cert, headers=headers)
             # If the previous line didn't fail, change delugeweb_url for the rest of this session
             logger.error('Deluge: Switching to HTTPS, but certificate won\'t be verified because NO CERTIFICATE WAS CONFIGURED!')
             delugeweb_url = delugeweb_url.replace('http:', 'https:')
@@ -358,7 +359,7 @@ def _get_auth():
                             "id": 10})
     try:
         response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth,
-                                 verify=deluge_verify_cert, headers=headers)
+            verify=deluge_verify_cert, headers=headers)
     except Exception as e:
         logger.error('Deluge: Authentication failed: %s' % str(e))
         formatted_lines = traceback.format_exc().splitlines()
@@ -375,7 +376,7 @@ def _get_auth():
                                 "id": 11})
         try:
             response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth,
-                                     verify=deluge_verify_cert, headers=headers)
+                verify=deluge_verify_cert, headers=headers)
         except Exception as e:
             logger.error('Deluge: Authentication failed: %s' % str(e))
             formatted_lines = traceback.format_exc().splitlines()
@@ -394,7 +395,7 @@ def _get_auth():
 
         try:
             response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth,
-                                     verify=deluge_verify_cert, headers=headers)
+                verify=deluge_verify_cert, headers=headers)
         except Exception as e:
             logger.error('Deluge: Authentication failed: %s' % str(e))
             formatted_lines = traceback.format_exc().splitlines()
@@ -407,7 +408,7 @@ def _get_auth():
 
         try:
             response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth,
-                                     verify=deluge_verify_cert, headers=headers)
+                verify=deluge_verify_cert, headers=headers)
         except Exception as e:
             logger.error('Deluge: Authentication failed: %s' % str(e))
             formatted_lines = traceback.format_exc().splitlines()
@@ -432,7 +433,7 @@ def _add_torrent_magnet(result):
                                 "params": [result['url'], {}],
                                 "id": 2})
         response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth,
-                                 verify=deluge_verify_cert, headers=headers)
+            verify=deluge_verify_cert, headers=headers)
         result['hash'] = json.loads(response.text)['result']
         logger.debug('Deluge: Response was %s' % str(json.loads(response.text)))
         return json.loads(response.text)['result']
@@ -452,7 +453,7 @@ def _add_torrent_url(result):
                                 "params": [result['url'], {}],
                                 "id": 32})
         response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth,
-                                 verify=deluge_verify_cert, headers=headers)
+            verify=deluge_verify_cert, headers=headers)
         result['location'] = json.loads(response.text)['result']
         logger.debug('Deluge: Response was %s' % str(json.loads(response.text)))
         return json.loads(response.text)['result']
@@ -471,10 +472,10 @@ def _add_torrent_file(result):
         # content is torrent file contents that needs to be encoded to base64
         post_data = json.dumps({"method": "core.add_torrent_file",
                                 "params": [result['name'] + '.torrent',
-                                           b64encode(result['content']).decode(), {}],
+                                b64encode(result['content']).decode(), {}],
                                 "id": 2})
         response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth,
-                                 verify=deluge_verify_cert, headers=headers)
+            verify=deluge_verify_cert, headers=headers)
         result['hash'] = json.loads(response.text)['result']
         logger.debug('Deluge: Response was %s' % str(json.loads(response.text)))
         return json.loads(response.text)['result']
@@ -501,7 +502,7 @@ def setTorrentLabel(result):
                                 "params": [],
                                 "id": 3})
         response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth,
-                                 verify=deluge_verify_cert, headers=headers)
+            verify=deluge_verify_cert, headers=headers)
         labels = json.loads(response.text)['result']
 
         if labels is not None:
@@ -512,7 +513,7 @@ def setTorrentLabel(result):
                                             "params": [label],
                                             "id": 4})
                     response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth,
-                                             verify=deluge_verify_cert, headers=headers)
+                        verify=deluge_verify_cert, headers=headers)
                     logger.debug('Deluge: %s label added to Deluge' % label)
                 except Exception as e:
                     logger.error('Deluge: Setting label failed: %s' % str(e))
@@ -524,7 +525,7 @@ def setTorrentLabel(result):
                                     "params": [result['hash'], label],
                                     "id": 5})
             response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth,
-                                     verify=deluge_verify_cert, headers=headers)
+                verify=deluge_verify_cert, headers=headers)
             logger.debug('Deluge: %s label added to torrent' % label)
         else:
             logger.debug('Deluge: Label plugin not detected')
@@ -548,12 +549,12 @@ def setSeedRatio(result):
                                     "params": [result['hash'], True],
                                     "id": 5})
             response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth,
-                                     verify=deluge_verify_cert, headers=headers)
+                verify=deluge_verify_cert, headers=headers)
             post_data = json.dumps({"method": "core.set_torrent_stop_ratio",
                                     "params": [result['hash'], float(ratio)],
                                     "id": 6})
             response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth,
-                                     verify=deluge_verify_cert, headers=headers)
+                verify=deluge_verify_cert, headers=headers)
 
             return not json.loads(response.text)['error']
 
@@ -576,7 +577,7 @@ def setTorrentPath(result):
                                     "params": [result['hash'], True],
                                     "id": 7})
             response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth,
-                                     verify=deluge_verify_cert, headers=headers)
+                verify=deluge_verify_cert, headers=headers)
 
             if headphones.CONFIG.DELUGE_DONE_DIRECTORY:
                 move_to = headphones.CONFIG.DELUGE_DONE_DIRECTORY
@@ -590,7 +591,7 @@ def setTorrentPath(result):
                                     "params": [result['hash'], move_to],
                                     "id": 8})
             response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth,
-                                     verify=deluge_verify_cert, headers=headers)
+                verify=deluge_verify_cert, headers=headers)
 
             return not json.loads(response.text)['error']
 
@@ -613,7 +614,7 @@ def setTorrentPause(result):
                                     "params": [[result['hash']]],
                                     "id": 9})
             response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth,
-                                     verify=deluge_verify_cert, headers=headers)
+                verify=deluge_verify_cert, headers=headers)
 
             return not json.loads(response.text)['error']
 

headphones/getXldProfile.py

@@ -1,6 +1,6 @@
 import os.path
 
-import biplist
+import plistlib
 from headphones import logger
 
 
@@ -14,8 +14,9 @@ def getXldProfile(xldProfile):
 
     # Get xld preferences plist
     try:
-        preferences = biplist.readPlist(expanded)
+        with open(expanded, 'rb') as _f:
-    except (biplist.InvalidPlistException, biplist.NotBinaryPlistException) as e:
+            preferences = plistlib.load(_f)
+    except Exception as e:
         logger.error("Error reading xld preferences plist: %s", e)
         return (xldProfileNotFound, None, None)
 

headphones/helpers.py

@@ -14,25 +14,25 @@
 #  You should have received a copy of the GNU General Public License
 #  along with Headphones.  If not, see <http://www.gnu.org/licenses/>.
 
-from operator import itemgetter
+import os
-import unicodedata
+import re
-import datetime
 import shutil
-import time
 import sys
 import tempfile
-import glob
+import time
+import unicodedata
+from contextlib import contextmanager
+from datetime import datetime, date
+from fnmatch import fnmatch
+from functools import cmp_to_key
+from glob import glob
+from operator import itemgetter
 
 from beets import logging as beetslogging
-import six
-from contextlib import contextmanager
-
-import fnmatch
-import functools
-import re
-import os
 from mediafile import MediaFile, FileTypeError, UnreadableFileError
+from six import text_type
 from unidecode import unidecode
+
 import headphones
 
 
@@ -42,7 +42,6 @@ RE_FEATURING = re.compile(r"[fF]t\.|[fF]eaturing|[fF]eat\.|\b[wW]ith\b|&|vs\.")
 RE_CD_ALBUM = re.compile(r"\(?((CD|disc)\s*[0-9]+)\)?", re.I)
 RE_CD = re.compile(r"^(CD|dics)\s*[0-9]+$", re.I)
 
-
 def cmp(x, y):
     """
     Replacement for built-in function cmp that was removed in Python 3
@@ -53,8 +52,14 @@ def cmp(x, y):
 
     https://portingguide.readthedocs.io/en/latest/comparisons.html#the-cmp-function
     """
-    return (x > y) - (x < y)
+    if x is None and y is None:
-
+        return 0
+    elif x is None:
+        return -1
+    elif y is None:
+        return 1
+    else:
+        return (x > y) - (x < y)
 
 def multikeysort(items, columns):
     comparers = [
@@ -69,7 +74,7 @@ def multikeysort(items, columns):
         else:
             return 0
 
-    return sorted(items, key=functools.cmp_to_key(comparer))
+    return sorted(items, key=cmp_to_key(comparer))
 
 
 def checked(variable):
@@ -151,28 +156,25 @@ def convert_seconds(s):
 
 
 def today():
-    today = datetime.date.today()
+    return date.isoformat(date.today())
-    yyyymmdd = datetime.date.isoformat(today)
-    return yyyymmdd
 
 
 def now():
-    now = datetime.datetime.now()
+    now = datetime.now()
     return now.strftime("%Y-%m-%d %H:%M:%S")
 
 
-def get_age(date):
+def is_valid_date(d):
-    try:
+    if not d:
-        split_date = date.split('-')
-    except:
         return False
+    else:
+        return bool(re.match(r'\d{4}-\d{2}-\d{2}', d))
 
-    try:
-        days_old = int(split_date[0]) * 365 + int(split_date[1]) * 30 + int(split_date[2])
-    except (IndexError, ValueError):
-        days_old = False
 
-    return days_old
+def age(d):
+    '''Requires a valid date'''
+    delta = date.today() - date.fromisoformat(d)
+    return delta.days
 
 
 def bytes_to_mb(bytes):
@@ -234,7 +236,7 @@ def pattern_substitute(pattern, dic, normalize=False):
                         j = unicodedata.normalize('NFC', j)
                 except TypeError:
                     j = unicodedata.normalize('NFC',
-                                              j.decode(headphones.SYS_ENCODING, 'replace'))
+                        j.decode(headphones.SYS_ENCODING, 'replace'))
             new_dic[i] = j
         dic = new_dic
     return pathrender.render(pattern, dic)[0]
@@ -279,7 +281,7 @@ _XLATE_GRAPHICAL_AND_DIACRITICAL = {
     'Ǥ': 'G', 'ǥ': 'g', 'DZ': 'DZ', 'Dz': 'Dz', 'dz': 'dz',
     'Ȥ': 'Z', 'ȥ': 'z', '№': 'No.',
     'º': 'o.',        # normalize Nº abbrev (popular w/ classical music),
-    # this is 'masculine ordering indicator', not degree
+                       # this is 'masculine ordering indicator', not degree
 }
 
 _XLATE_SPECIAL = {
@@ -504,7 +506,7 @@ def path_match_patterns(path, patterns):
     """
 
     for pattern in patterns:
-        if fnmatch.fnmatch(path, pattern):
+        if fnmatch(path, pattern):
             return True
 
     # No match
@@ -710,7 +712,7 @@ def preserve_torrent_directory(albumpath, forced=False, single=False):
             workdir = os.path.join(tempdir, prefix)
             workdir = re.sub(r'\[', '[[]', workdir)
             workdir = re.sub(r'(?<!\[)\]', '[]]', workdir)
-            if len(glob.glob(workdir + '*/')) >= 3:
+            if len(glob(workdir + '*/')) >= 3:
                 logger.error(
                     "Looks like a temp directory has previously been created "
                     "for this albumpath, not continuing "
@@ -1029,7 +1031,7 @@ class BeetsLogCapture(beetslogging.Handler):
         self.messages = []
 
     def emit(self, record):
-        self.messages.append(six.text_type(record.msg))
+        self.messages.append(text_type(record.msg))
 
 
 @contextmanager
@@ -1041,3 +1043,10 @@ def capture_beets_log(logger='beets'):
         yield capture.messages
     finally:
         log.removeHandler(capture)
+
+def have_pct_have_total(db_artist):
+    have_tracks = db_artist['HaveTracks'] or 0
+    total_tracks = db_artist['TotalTracks'] or 0 
+    have_pct = have_tracks / total_tracks if total_tracks else 0
+    return (have_pct, total_tracks)
+

headphones/helpers_test.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 from .unittestcompat import TestCase
-from headphones.helpers import clean_name
+from headphones.helpers import clean_name, is_valid_date, age
 
 
 class HelpersTest(TestCase):
@@ -14,9 +14,9 @@ class HelpersTest(TestCase):
             'Symphonęy Nº9': 'Symphoney No.9',
             'ÆæßðÞIJij': 'AeaessdThIJıj',
             'Obsessió (Cerebral Apoplexy remix)': 'obsessio cerebral '
-            'apoplexy remix',
+                                                    'apoplexy remix',
             'Doktór Hałabała i siedmiu zbojów': 'doktor halabala i siedmiu '
-            'zbojow',
+                                                    'zbojow',
             'Arbetets Söner och Döttrar': 'arbetets soner och dottrar',
             'Björk Guðmundsdóttir': 'bjork gudmundsdottir',
             'L\'Arc~en~Ciel': 'larc en ciel',
@@ -46,3 +46,13 @@ class HelpersTest(TestCase):
         self.assertEqual(
             test, expected, "check clean_name() with narrow non-ascii input"
         )
+
+    def test_is_valid_date(date):
+        test_cases = [
+            ('2021-11-12', True, "check is_valid_date returns True for valid date"),
+            (None, False, "check is_valid_date returns False for None"),
+            ('2021-11', False, "check is_valid_date returns False for incomplete"),
+            ('2021', False, "check is_valid_date returns False for incomplete")
+        ]
+        for input, expected, desc in test_cases:
+            self.assertEqual(is_valid_date(input), expected, desc)

headphones/importer.py

@@ -39,7 +39,7 @@ def is_exists(artistid):
 
     if any(artistid in x for x in artistlist):
         logger.info(artistlist[0][
-            1] + " is already in the database. Updating 'have tracks', but not artist information")
+                        1] + " is already in the database. Updating 'have tracks', but not artist information")
         return True
     else:
         return False
@@ -102,12 +102,7 @@ def artistlist_to_mbids(artistlist, forced=False):
             myDB.action('DELETE from newartists WHERE ArtistName=?', [artist])
 
     # Update the similar artist tag cloud:
-    logger.info('Updating artist information from Last.fm')
+    lastfm.getSimilar()
-
-    try:
-        lastfm.getSimilar()
-    except Exception as e:
-        logger.warn('Failed to update artist information from Last.fm: %s' % e)
 
 
 def addArtistIDListToDB(artistidlist):
@@ -245,7 +240,7 @@ def addArtisttoDB(artistid, extrasonly=False, forcefull=False, type="artist"):
         rgid = rg['id']
         skip_log = 0
         # Make a user configurable variable to skip update of albums with release dates older than this date (in days)
-        pause_delta = headphones.CONFIG.MB_IGNORE_AGE
+        ignore_age = headphones.CONFIG.MB_IGNORE_AGE
 
         rg_exists = myDB.action("SELECT * from albums WHERE AlbumID=?", [rg['id']]).fetchone()
 
@@ -274,18 +269,18 @@ def addArtisttoDB(artistid, extrasonly=False, forcefull=False, type="artist"):
                     if len(check_release_date) == 10:
                         release_date = check_release_date
                     elif len(check_release_date) == 7:
-                        release_date = check_release_date + "-31"
+                        release_date = check_release_date + "-27"
                     elif len(check_release_date) == 4:
-                        release_date = check_release_date + "-12-31"
+                        release_date = check_release_date + "-12-27"
                     else:
                         release_date = today
-                    if helpers.get_age(today) - helpers.get_age(release_date) < pause_delta:
+                    if helpers.age(release_date) < ignore_age:
                         logger.info("[%s] Now updating: %s (Release Date <%s Days)",
-                                    artist['artist_name'], rg['title'], pause_delta)
+                                    artist['artist_name'], rg['title'], ignore_age)
                         new_releases = mb.get_new_releases(rgid, includeExtras, True)
                     else:
                         logger.info("[%s] Skipping: %s (Release Date >%s Days)",
-                                    artist['artist_name'], rg['title'], pause_delta)
+                                    artist['artist_name'], rg['title'], ignore_age)
                         skip_log = 1
                         new_releases = 0
 
@@ -450,14 +445,9 @@ def addArtisttoDB(artistid, extrasonly=False, forcefull=False, type="artist"):
 
                 if headphones.CONFIG.AUTOWANT_ALL:
                     newValueDict['Status'] = "Wanted"
-                elif album['ReleaseDate'] > today and headphones.CONFIG.AUTOWANT_UPCOMING:
+                elif headphones.CONFIG.AUTOWANT_UPCOMING:
-                    newValueDict['Status'] = "Wanted"
+                    if helpers.is_valid_date(album['ReleaseDate']) and helpers.age(album['ReleaseDate']) < 21:
-                # Sometimes "new" albums are added to musicbrainz after their release date, so let's try to catch these
+                        newValueDict['Status'] = "Wanted"
-                # The first test just makes sure we have year-month-day
-                elif helpers.get_age(album['ReleaseDate']) and helpers.get_age(
-                        today) - helpers.get_age(
-                        album['ReleaseDate']) < 21 and headphones.CONFIG.AUTOWANT_UPCOMING:
-                    newValueDict['Status'] = "Wanted"
                 else:
                     newValueDict['Status'] = "Skipped"
 

headphones/lastfm.py

@@ -23,7 +23,7 @@ from headphones import db, logger, request
 TIMEOUT = 60.0  # seconds
 REQUEST_LIMIT = 1.0 / 5  # seconds
 ENTRY_POINT = "https://ws.audioscrobbler.com/2.0/"
-API_KEY = "395e6ec6bb557382fc41fde867bce66f"
+APP_API_KEY = "395e6ec6bb557382fc41fde867bce66f"
 
 # Required for API request limit
 lastfm_lock = headphones.lock.TimedLock(REQUEST_LIMIT)
@@ -31,7 +31,7 @@ lastfm_lock = headphones.lock.TimedLock(REQUEST_LIMIT)
 
 def request_lastfm(method, **kwargs):
     """
-    Call a Last.FM API method. Automatically sets the method and API key. Method
+    Call a Last.fm API method. Automatically sets the method and API key. Method
     will return the result if no error occured.
 
     By default, this method will request the JSON format, since it is more
@@ -40,35 +40,42 @@ def request_lastfm(method, **kwargs):
 
     # Prepare request
     kwargs["method"] = method
-    kwargs.setdefault("api_key", API_KEY)
+    kwargs.setdefault("api_key", headphones.CONFIG.LASTFM_APIKEY or APP_API_KEY)
     kwargs.setdefault("format", "json")
 
     # Send request
-    logger.debug("Calling Last.FM method: %s", method)
+    logger.debug("Calling Last.fm method: %s", method)
-    logger.debug("Last.FM call parameters: %s", kwargs)
+    logger.debug("Last.fm call parameters: %s", kwargs)
 
     data = request.request_json(ENTRY_POINT, timeout=TIMEOUT, params=kwargs, lock=lastfm_lock)
 
     # Parse response and check for errors.
     if not data:
-        logger.error("Error calling Last.FM method: %s", method)
+        logger.error("Error calling Last.fm method: %s", method)
         return
 
     if "error" in data:
-        logger.debug("Last.FM returned an error: %s", data["message"])
+        logger.debug("Last.fm returned an error: %s", data["message"])
         return
 
     return data
 
 
 def getSimilar():
-    myDB = db.DBConnection()
+    if not headphones.CONFIG.LASTFM_APIKEY:
-    results = myDB.select("SELECT ArtistID from artists ORDER BY HaveTracks DESC")
+        logger.info(
+            'To update the Similar Artists cloud tag, create a Last.fm application api key '
+            'and add it under the Advanced config tab'
+        )
+        return
 
-    logger.info("Fetching similar artists from Last.FM for tag cloud")
+    myDB = db.DBConnection()
+    results = myDB.select("SELECT ArtistID from artists ORDER BY HaveTracks DESC LIMIT 10")
+
+    logger.info("Fetching similar artists from Last.fm for tag cloud")
     artistlist = []
 
-    for result in results[:12]:
+    for result in results:
         data = request_lastfm("artist.getsimilar", mbid=result["ArtistId"])
 
         if data and "similarartists" in data:
@@ -85,7 +92,7 @@ def getSimilar():
                     artistlist.append((artist_name, artist_mbid))
 
     # Add new artists to tag cloud
-    logger.debug("Fetched %d artists from Last.FM", len(artistlist))
+    logger.debug("Fetched %d artists from Last.fm", len(artistlist))
     count = defaultdict(int)
 
     for artist, mbid in artistlist:
@@ -103,7 +110,7 @@ def getSimilar():
 
         myDB.action("INSERT INTO lastfmcloud VALUES( ?, ?, ?)", [artist_name, artist_mbid, count])
 
-    logger.debug("Inserted %d artists into Last.FM tag cloud", len(top_list))
+    logger.debug("Inserted %d artists into Last.fm tag cloud", len(top_list))
 
 
 def getArtists():
@@ -111,16 +118,16 @@ def getArtists():
     results = myDB.select("SELECT ArtistID from artists")
 
     if not headphones.CONFIG.LASTFM_USERNAME:
-        logger.warn("Last.FM username not set, not importing artists.")
+        logger.warn("Last.fm username not set, not importing artists.")
         return
 
-    logger.info("Fetching artists from Last.FM for username: %s", headphones.CONFIG.LASTFM_USERNAME)
+    logger.info("Fetching artists from Last.fm for username: %s", headphones.CONFIG.LASTFM_USERNAME)
     data = request_lastfm("library.getartists", limit=1000, user=headphones.CONFIG.LASTFM_USERNAME)
 
     if data and "artists" in data:
         artistlist = []
         artists = data["artists"]["artist"]
-        logger.debug("Fetched %d artists from Last.FM", len(artists))
+        logger.debug("Fetched %d artists from Last.fm", len(artists))
 
         for artist in artists:
             artist_mbid = artist["mbid"]
@@ -133,20 +140,20 @@ def getArtists():
         for artistid in artistlist:
             importer.addArtisttoDB(artistid)
 
-        logger.info("Imported %d new artists from Last.FM", len(artistlist))
+        logger.info("Imported %d new artists from Last.fm", len(artistlist))
 
 
 def getTagTopArtists(tag, limit=50):
     myDB = db.DBConnection()
     results = myDB.select("SELECT ArtistID from artists")
 
-    logger.info("Fetching top artists from Last.FM for tag: %s", tag)
+    logger.info("Fetching top artists from Last.fm for tag: %s", tag)
     data = request_lastfm("tag.gettopartists", limit=limit, tag=tag)
 
     if data and "topartists" in data:
         artistlist = []
         artists = data["topartists"]["artist"]
-        logger.debug("Fetched %d artists from Last.FM", len(artists))
+        logger.debug("Fetched %d artists from Last.fm", len(artists))
 
         for artist in artists:
             try:
@@ -162,4 +169,4 @@ def getTagTopArtists(tag, limit=50):
         for artistid in artistlist:
             importer.addArtisttoDB(artistid)
 
-        logger.debug("Added %d new artists from Last.FM", len(artistlist))
+        logger.debug("Added %d new artists from Last.fm", len(artistlist))

headphones/librarysync.py

@@ -77,9 +77,9 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None,
                 if track['ArtistName']:
                     # Make sure deleted files get accounted for when updating artist track counts
                     new_artists.append(track['ArtistName'])
-                myDB.action('DELETE FROM have WHERE Location=?', [Track['Location']])
+                myDB.action('DELETE FROM have WHERE Location=?', [track['Location']])
                 logger.info(
-                    f"{Track['Location']} removed from Headphones, as it "
+                    f"{track['Location']} removed from Headphones, as it "
                     f"is no longer on disk"
                 )
 
@@ -152,7 +152,7 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None,
 
                 # track_list.append(track_dict)
                 check_exist_track = myDB.action("SELECT * FROM have WHERE Location=?",
-                                                [track_path]).fetchone()
+                                               [track_path]).fetchone()
                 # Only attempt to match tracks that are new, haven't yet been matched, or metadata has changed.
                 if not check_exist_track:
                     # This is a new track
@@ -167,7 +167,7 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None,
                         if f_artist and f_artist != check_exist_track['ArtistName']:
                             new_artists.append(f_artist)
                         elif f_artist and f_artist == check_exist_track['ArtistName'] and \
-                                check_exist_track['Matched'] != "Ignored":
+                                        check_exist_track['Matched'] != "Ignored":
                             new_artists.append(f_artist)
                         else:
                             continue
@@ -191,16 +191,18 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None,
     # Now we start track matching
     logger.info(f"{new_track_count} new/modified tracks found and added to the database")
     dbtracks = myDB.action(
-        "SELECT * FROM have WHERE Matched IS NULL AND LOCATION LIKE ?",
+            "SELECT * FROM have WHERE Matched IS NULL AND LOCATION LIKE ?",
-        [f"{dir}%"]
+            [f"{dir}%"]
-    )
+        )
     dbtracks_count = myDB.action(
-        "SELECT COUNT(*) FROM have WHERE Matched IS NULL AND LOCATION LIKE ?",
+            "SELECT COUNT(*) FROM have WHERE Matched IS NULL AND LOCATION LIKE ?",
-        [f"{dir}%"]
+            [f"{dir}%"]
-    ).fetchone()[0]
+        ).fetchone()[0]
     logger.info(f"Found {dbtracks_count} new/modified tracks in `{dir}`")
     logger.info("Matching tracks to the appropriate releases....")
 
+
+
     # Sort the track_list by most vague (e.g. no trackid or releaseid)
     # to most specific (both trackid & releaseid)
     # When we insert into the database, the tracks with the most 
@@ -208,6 +210,7 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None,
 
     sorted_dbtracks = helpers.multikeysort(dbtracks, ['ArtistName', 'AlbumTitle'])
 
+
     # We'll use this to give a % completion, just because the
     # track matching might take a while
     tracks_completed = 0
@@ -224,8 +227,8 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None,
 
         tracks_completed += 1
         completion_percentage = math.floor(
-            float(tracks_completed) / dbtracks_count * 1000
+                float(tracks_completed) / dbtracks_count * 1000
-        ) / 10
+            ) / 10
 
         if completion_percentage >= (last_completion_percentage + 10):
             logger.info("Track matching is " + str(completion_percentage) + "% complete")

headphones/mb.py

@@ -14,20 +14,14 @@
 #  along with Headphones.  If not, see <http://www.gnu.org/licenses/>.
 
 
-from headphones import logger, db, helpers
+from collections import OrderedDict
+
+import musicbrainzngs
 
 import headphones
-import musicbrainzngs
 import headphones.lock
+from headphones import logger, db, helpers
 
-try:
-    # pylint:disable=E0611
-    # ignore this error because we are catching the ImportError
-    from collections import OrderedDict
-    # pylint:enable=E0611
-except ImportError:
-    # Python 2.6.x fallback, from libs
-    from ordereddict import OrderedDict
 
 mb_lock = headphones.lock.TimedLock(0)
 
@@ -97,7 +91,7 @@ def findArtist(name, limit=1):
         try:
             artistResults = musicbrainzngs.search_artists(limit=limit, **criteria)['artist-list']
         except ValueError as e:
-            if "at least one query term is required" in e.message:
+            if "at least one query term is required" in str(e):
                 logger.error(
                     "Tried to search without a term, or an empty one. Provided artist (probably emtpy): %s",
                     name)

headphones/metadata.py

@@ -38,7 +38,6 @@ class MetadataDict(dict):
     lowercase) in member variable self._lower. If case-sensitive lookup
     fails, another case-insensitive attempt is made.
     """
-
     def __setitem__(self, key, value):
         super(MetadataDict, self).__setitem__(key, value)
         self._lower.__setitem__(key.lower(), value)
@@ -80,6 +79,7 @@ class Vars:
     Metadata $variable names (only ones set explicitly by headphones).
     """
     DISC = '$Disc'
+    DISC_TOTAL = '$DiscTotal'
     TRACK = '$Track'
     TITLE = '$Title'
     ARTIST = '$Artist'
@@ -172,7 +172,7 @@ def _lower(s):
     return None
 
 
-def file_metadata(path, release):
+def file_metadata(path, release, single_disc_ignore=False):
     # type: (str,sqlite3.Row)->Tuple[Mapping[str,str],bool]
     """
     Prepare metadata dictionary for path substitution, based on file name,
@@ -195,7 +195,13 @@ def file_metadata(path, release):
     _row_to_dict(release, res)
 
     date, year = _date_year(release)
-    if not f.disc:
+
+    if not f.disctotal or (f.disctotal == 1 and single_disc_ignore):
+        disc_total = ''
+    else:
+        disc_total = '%d' % f.disctotal
+
+    if not f.disc or (f.disctotal == 1 and single_disc_ignore):
         disc_number = ''
     else:
         disc_number = '%d' % f.disc
@@ -227,6 +233,7 @@ def file_metadata(path, release):
     album_title = release['AlbumTitle']
     override_values = {
         Vars.DISC: disc_number,
+        Vars.DISC_TOTAL: disc_total,
         Vars.TRACK: track_number,
         Vars.TITLE: title,
         Vars.ARTIST: artist_name,

headphones/music_encoder.py

@@ -30,7 +30,6 @@ from . import getXldProfile
 
 
 def encode(albumPath):
-    print(albumPath)
     use_xld = headphones.CONFIG.ENCODER == 'xld'
 
     # Return if xld details not found

headphones/notifiers.py

@@ -1,7 +1,5 @@
 from urllib.parse import urlencode, quote_plus
-import urllib.request
+import urllib.request, urllib.parse, urllib.error
-import urllib.parse
-import urllib.error
 import subprocess
 import json
 from email.mime.text import MIMEText
@@ -9,9 +7,7 @@ import smtplib
 import email.utils
 from http.client import HTTPSConnection
 from urllib.parse import parse_qsl
-import urllib.request
+import urllib.request, urllib.error, urllib.parse
-import urllib.error
-import urllib.parse
 import requests as requests
 
 import os.path
@@ -250,7 +246,7 @@ class XBMC(object):
 
                 if version < 12:  # Eden
                     notification = header + "," + message + "," + time + \
-                        "," + albumartpath
+                                   "," + albumartpath
                     notifycommand = {'command': 'ExecBuiltIn',
                                      'parameter': 'Notification(' +
                                                   notification + ')'}
@@ -444,7 +440,7 @@ class Plex(object):
 
                 if version < 12:  # Eden
                     notification = header + "," + message + "," + time + \
-                        "," + albumartpath
+                                   "," + albumartpath
                     notifycommand = {'command': 'ExecBuiltIn',
                                      'parameter': 'Notification(' +
                                                   notification + ')'}
@@ -608,12 +604,12 @@ class JOIN(object):
             self.url += '&deviceId={deviceid}'
 
         response = urllib.request.urlopen(self.url.format(apikey=self.apikey,
-                                                          title=quote_plus(event),
+                                                   title=quote_plus(event),
-                                                          text=quote_plus(
+                                                   text=quote_plus(
-                                                              message.encode(
+                                                       message.encode(
-                                                                  "utf-8")),
+                                                           "utf-8")),
-                                                          icon=icon,
+                                                   icon=icon,
-                                                          deviceid=self.deviceid))
+                                                   deviceid=self.deviceid))
 
         if response:
             logger.info("Join notifications sent.")
@@ -737,8 +733,8 @@ class TwitterNotifier(object):
     def notify_download(self, title):
         if headphones.CONFIG.TWITTER_ENABLED:
             self._notifyTwitter(common.notifyStrings[
-                common.NOTIFY_DOWNLOAD] + ': ' +
+                                    common.NOTIFY_DOWNLOAD] + ': ' +
-                title + ' at ' + helpers.now())
+                                title + ' at ' + helpers.now())
 
     def test_notify(self):
         return self._notifyTwitter(
@@ -802,7 +798,7 @@ class TwitterNotifier(object):
         if resp['status'] != '200':
             logger.info('The request for a token with did not succeed: ' + str(
                 resp['status']),
-                logger.ERROR)
+                        logger.ERROR)
             return False
         else:
             logger.info('Your Twitter Access Token key: %s' % access_token[
@@ -1024,7 +1020,7 @@ class TELEGRAM(object):
         # MusicBrainz link
         if rgid:
             message += '\n\n <a href="https://musicbrainz.org/' \
-                'release-group/%s">MusicBrainz</a>' % rgid
+                      'release-group/%s">MusicBrainz</a>' % rgid
 
         # Send image
         response = None

headphones/nzbget.py

@@ -70,7 +70,8 @@ def sendNZB(nzb):
     nzbcontent64 = None
     if nzb.resultType == "nzbdata":
         data = nzb.extraInfo[0]
-        nzbcontent64 = standard_b64encode(data)
+        # NZBGet needs a string, not bytes
+        nzbcontent64 = standard_b64encode(data).decode("utf-8")
 
     logger.info("Sending NZB to NZBget")
     logger.debug("URL: " + url)

headphones/pathrender.py

@@ -38,7 +38,6 @@ __author__ = "Andrzej Ciarkowski <andrzej.ciarkowski@gmail.com>"
 
 class _PatternElement(object):
     '''ABC for hierarchy of path name renderer pattern elements.'''
-
     def render(self, replacement):
         # type: (Mapping[str,str]) -> str
         '''Format this _PatternElement into string using provided substitution dictionary.'''
@@ -56,7 +55,6 @@ class _Generator(_PatternElement):
 
 class _Replacement(_Generator):
     '''Replacement variable, eg. $title.'''
-
     def __init__(self, pattern):
         # type: (str)
         self._pattern = pattern
@@ -83,7 +81,6 @@ class _Replacement(_Generator):
 
 class _LiteralText(_PatternElement):
     '''Just a plain piece of text to be rendered "as is".'''
-
     def __init__(self, text):
         # type: (str)
         self._text = text

headphones/postprocessor.py

@@ -65,7 +65,6 @@ def checkFolder():
                             folder_name = torrent_folder_name
 
                 if folder_name:
-                    print(folder_name)
                     album_path = os.path.join(download_dir, folder_name)
                     logger.debug("Checking if %s exists" % album_path)
 
@@ -80,7 +79,6 @@ def checkFolder():
 
 
 def verify(albumid, albumpath, Kind=None, forced=False, keep_original_folder=False, single=False):
-    print(albumpath)
     myDB = db.DBConnection()
     release = myDB.action('SELECT * from albums WHERE AlbumID=?', [albumid]).fetchone()
     tracks = myDB.select('SELECT * from tracks WHERE AlbumID=?', [albumid])
@@ -342,7 +340,6 @@ def verify(albumid, albumpath, Kind=None, forced=False, keep_original_folder=Fal
     logger.warn(f"Could not identify {albumpath}. It may not be the intended album")
     markAsUnprocessed(albumid, albumpath, keep_original_folder)
 
-
 def markAsUnprocessed(albumid, albumpath, keep_original_folder=False):
     myDB = db.DBConnection()
     myDB.action(
@@ -420,7 +417,7 @@ def doPostProcessing(albumid, albumpath, release, tracks, downloaded_track_list,
                 logger.debug("Write check exact error: %s", e)
                 logger.error(
                     f"`{downloaded_track}` is not writable. This is required "
-                    "for some post processing steps. Not continuing."
+                     "for some post processing steps. Not continuing."
                 )
                 if new_folder:
                     shutil.rmtree(new_folder)
@@ -596,7 +593,7 @@ def doPostProcessing(albumid, albumpath, release, tracks, downloaded_track_list,
         logger.info("Twitter notifications temporarily disabled")
         #logger.info("Sending Twitter notification")
         #twitter = notifiers.TwitterNotifier()
-        # twitter.notify_download(pushmessage)
+        #twitter.notify_download(pushmessage)
 
     if headphones.CONFIG.OSX_NOTIFY_ENABLED:
         from headphones import cache
@@ -1088,7 +1085,11 @@ def renameFiles(albumpath, downloaded_track_list, release):
     # Until tagging works better I'm going to rely on the already provided metadata
 
     for downloaded_track in downloaded_track_list:
-        md, from_metadata = metadata.file_metadata(downloaded_track, release)
+        md, from_metadata = metadata.file_metadata(
+            downloaded_track,
+            release,
+            headphones.CONFIG.RENAME_SINGLE_DISC_IGNORE
+        )
         if md is None:
             # unable to parse media file, skip file
             continue
@@ -1136,7 +1137,6 @@ def updateFilePermissions(albumpaths):
                     logger.error(f"Could not change permissions for `{full_path}`")
                     continue
 
-
 def renameUnprocessedFolder(path, tag):
     """
     Rename a unprocessed folder to a new unique name to indicate a certain

headphones/qbittorrent.py

@@ -13,12 +13,8 @@
 #  You should have received a copy of the GNU General Public License
 #  along with Headphones.  If not, see <http://www.gnu.org/licenses/>.
 
-import urllib.request
+import urllib.request, urllib.parse, urllib.error
-import urllib.parse
+import urllib.request, urllib.error, urllib.parse
-import urllib.error
-import urllib.request
-import urllib.error
-import urllib.parse
 import http.cookiejar
 import json
 import time
@@ -85,7 +81,7 @@ class qbittorrentclient(object):
             logger.debug('Error getting SID. qBittorrent responded with error: ' + str(err.reason))
             return
         for cookie in self.cookiejar:
-            logger.debug('login cookie: ' + cookie.name + ', value: ' + cookie.value)
+                logger.debug('login cookie: ' + cookie.name + ', value: ' + cookie.value)
         return
 
     def _command(self, command, args=None, content_type=None, files=None):

headphones/request.py

@@ -220,7 +220,7 @@ def server_message(response):
 
     # First attempt is to 'read' the response as HTML
     if response.headers.get("content-type") and \
-            "text/html" in response.headers.get("content-type"):
+                    "text/html" in response.headers.get("content-type"):
         try:
             soup = BeautifulSoup(response.content, "html.parser")
         except Exception:

headphones/rutracker.py

@@ -1,8 +1,6 @@
 #!/usr/bin/env python
 
-import urllib.request
+import urllib.request, urllib.parse, urllib.error
-import urllib.parse
-import urllib.error
 import time
 from urllib.parse import urlparse
 import re
@@ -13,6 +11,7 @@ from bs4 import BeautifulSoup
 
 import headphones
 from headphones import logger
+from headphones.types import Result
 
 
 class Rutracker(object):
@@ -162,7 +161,7 @@ class Rutracker(object):
                     torrent_id = dict([part.split('=') for part in urlparse(url)[4].split('&')])[
                         't']
                     topicurl = 'https://rutracker.org/forum/viewtopic.php?t=' + torrent_id
-                    rulist.append((title, size, topicurl, 'rutracker.org', 'torrent', True))
+                    rulist.append(Result(title, size, url, 'rutracker.org', 'torrent', True))
                 else:
                     logger.info("%s is larger than the maxsize or has too little seeders for this category, "
                                 "skipping. (Size: %i bytes, Seeders: %i)" % (title, size, int(seeds)))

headphones/sab.py

@@ -30,7 +30,7 @@ def sab_api_call(request_type=None, params={}, **kwargs):
 
     if headphones.CONFIG.SAB_HOST.endswith('/'):
         headphones.CONFIG.SAB_HOST = headphones.CONFIG.SAB_HOST[
-            0:len(headphones.CONFIG.SAB_HOST) - 1]
+                                     0:len(headphones.CONFIG.SAB_HOST) - 1]
 
     url = headphones.CONFIG.SAB_HOST + "/" + "api?"
 

headphones/searcher.py

@@ -15,30 +15,32 @@
 
 # NZBGet support added by CurlyMo <curlymoo1@gmail.com> as a part of XBian - XBMC on the Raspberry Pi
 
-from base64 import b16encode, b32decode
+import os
-from hashlib import sha1
+import re
 import string
 import random
-import urllib.request
+import urllib.request, urllib.parse, urllib.error
-import urllib.parse
-import urllib.error
 import datetime
 import subprocess
 import unicodedata
 import urllib.parse
+from base64 import b16encode, b32decode
+from hashlib import sha1
 
-import os
+from bencode import encode as bencode
-import re
+from bencode import decode as bdecode
 from pygazelle import api as gazelleapi
 from pygazelle import encoding as gazelleencoding
 from pygazelle import format as gazelleformat
 from pygazelle import release_type as gazellerelease_type
+from unidecode import unidecode
+
 import headphones
 from headphones.common import USER_AGENT
+from headphones.types import Result
 from headphones import logger, db, helpers, classes, sab, nzbget, request
 from headphones import utorrent, transmission, notifiers, rutracker, deluge, qbittorrent
-from bencode import encode as bencode
+
-from bencode import decode as bdecode
 
 # Magnet to torrent services, for Black hole. Stolen from CouchPotato.
 TORRENT_TO_MAGNET_SERVICES = [
@@ -54,6 +56,7 @@ ruobj = None
 redobj = None
 
 
+
 def fix_url(s, charset="utf-8"):
     """
     Fix the URL so it is proper formatted and encoded.
@@ -79,8 +82,8 @@ def torrent_to_file(target_file, data):
             fp.write(data)
     except IOError as e:
         logger.error(
-            "Could not write torrent file '%s': %s. Skipping.",
+            f"Could not write `{target_file}`: {str(e)}"
-            target_file, e.message)
+        )
         return
 
     # Try to change permissions
@@ -138,7 +141,7 @@ def calculate_torrent_hash(link, data=None):
         if len(torrent_hash) == 32:
             torrent_hash = b16encode(b32decode(torrent_hash)).lower()
     elif data:
-        info = bdecode(data)["info"]
+        info = bdecode(data)[b"info"]
         torrent_hash = sha1(bencode(info)).hexdigest()
     else:
         raise ValueError("Cannot calculate torrent hash without magnet link "
@@ -320,7 +323,7 @@ def do_sorted_search(album, new, losslessOnly, choose_specific_download=False):
         return results
 
     # Filter all results that do not comply
-    results = [result for result in results if result[5]]
+    results = [result for result in results if result.matches]
 
     # Sort the remaining results
     sorted_search_results = sort_search_results(results, album, new, albumlength)
@@ -328,11 +331,14 @@ def do_sorted_search(album, new, losslessOnly, choose_specific_download=False):
     if not sorted_search_results:
         return
 
-    logger.info("Making sure we can download the best result")
+    logger.info(
-    (data, bestqual) = preprocess(sorted_search_results)
+        "Making sure we can download the best result: "
+        f"{sorted_search_results[0].title} from {sorted_search_results[0].provider}"
+    )
+    (data, result) = preprocess(sorted_search_results)
 
-    if data and bestqual:
+    if data and result:
-        send_to_downloader(data, bestqual, album)
+        send_to_downloader(data, result, album)
 
 
 def more_filtering(results, album, albumlength, new):
@@ -368,36 +374,46 @@ def more_filtering(results, album, albumlength, new):
 
     for result in results:
 
-        if low_size_limit and (int(result[1]) < low_size_limit):
+        if low_size_limit and result.size < low_size_limit:
             logger.info(
-                "%s from %s is too small for this album - not considering it. (Size: %s, Minsize: %s)",
+                f"{result.title} from {result.provider} is too small for this album. "
-                result[0], result[3], helpers.bytes_to_mb(result[1]),
+                f"(Size: {result.size}, MinSize: {helpers.bytes_to_mb(low_size_limit)})"
-                helpers.bytes_to_mb(low_size_limit))
+            )
             continue
 
-        if high_size_limit and (int(result[1]) > high_size_limit):
+        if high_size_limit and result.size > high_size_limit:
             logger.info(
-                "%s from %s is too large for this album - not considering it. (Size: %s, Maxsize: %s)",
+                f"{result.title} from {result.provider} is too large for this album. "
-                result[0], result[3], helpers.bytes_to_mb(result[1]),
+                f"(Size: {result.size}, MaxSize: {helpers.bytes_to_mb(high_size_limit)})"
-                helpers.bytes_to_mb(high_size_limit))
+            )
-
             # Keep lossless results if there are no good lossy matches
-            if not (allow_lossless and 'flac' in result[0].lower()):
+            if not (allow_lossless and 'flac' in result.title.lower()):
                 continue
 
         if new:
-            alreadydownloaded = myDB.select('SELECT * from snatched WHERE URL=?', [result[2]])
+            alreadydownloaded = myDB.select(
-
+                "SELECT * from snatched WHERE URL=?", [result.url]
+            )
             if len(alreadydownloaded):
                 logger.info(
-                    '%s has already been downloaded from %s. Skipping.' % (result[0], result[3]))
+                    f"{result.title} has already been downloaded from "
+                    f"{result.provider}. Skipping."
+                )
                 continue
 
         newlist.append(result)
 
-    results = newlist
+    return newlist
 
-    return results
+
+def sort_by_priority_then_size(rs):
+    return list(map(lambda x: x[0],
+        sorted(
+            rs,
+            key=lambda x: (x[0].matches, x[1], x[0].size),
+            reverse=True
+        )
+    ))
 
 
 def sort_search_results(resultlist, album, new, albumlength):
@@ -407,83 +423,74 @@ def sort_search_results(resultlist, album, new, albumlength):
         return None
 
     # Add a priority if it has any of the preferred words
-    temp_list = []
+    results_with_priority = []
-    preferred_words = None
+    preferred_words = helpers.split_string(headphones.CONFIG.PREFERRED_WORDS)
-    if headphones.CONFIG.PREFERRED_WORDS:
-        preferred_words = helpers.split_string(headphones.CONFIG.PREFERRED_WORDS)
     for result in resultlist:
         priority = 0
-        if preferred_words:
+        for word in preferred_words:
-            if any(word.lower() in result[0].lower() for word in preferred_words):
+            if word.lower() in [result.title.lower(), result.provider.lower()]:
-                priority = 1
+                priority += len(preferred_words) - preferred_words.index(word)
-            # add a search provider priority (weighted based on position)
+        results_with_priority.append((result, priority))
-            i = next((i for i, word in enumerate(preferred_words) if word in result[3].lower()),
-                     None)
-            if i is not None:
-                priority += round((len(preferred_words) - i) / float(len(preferred_words)), 2)
 
-        temp_list.append((result[0], result[1], result[2], result[3], result[4], priority))
-
-    resultlist = temp_list
-
-    # if headphones.CONFIG.PREFERRED_QUALITY == 2 and headphones.CONFIG.PREFERRED_BITRATE and result[3] != 'Orpheus.network':
     if headphones.CONFIG.PREFERRED_QUALITY == 2 and headphones.CONFIG.PREFERRED_BITRATE:
 
         try:
             targetsize = albumlength / 1000 * int(headphones.CONFIG.PREFERRED_BITRATE) * 128
-
             if not targetsize:
-                logger.info('No track information for %s - %s. Defaulting to highest quality' % (
+                logger.info(
-                    album['ArtistName'], album['AlbumTitle']))
+                    f"No track information for {album['ArtistName']} - "
-                finallist = sorted(resultlist, key=lambda title: (title[5], int(title[1])),
+                    f"{album['AlbumTitle']}. Defaulting to highest quality"
-                                   reverse=True)
+                )
+                return sort_by_priority_then_size(results_with_priority)
 
             else:
-                newlist = []
+                lossy_results_with_delta = []
-                flac_list = []
+                lossless_results = []
 
-                for result in resultlist:
+                for result, priority in results_with_priority:
 
                     # Add lossless results to the "flac list" which we can use if there are no good lossy matches
-                    if 'flac' in result[0].lower():
+                    if 'flac' in result.title.lower():
-                        flac_list.append(
+                        lossless_results.append((result, priority))
-                            (result[0], result[1], result[2], result[3], result[4], result[5]))
+                    else:
-                        continue
+                        delta = abs(targetsize - result.size)
+                        lossy_results_with_delta.append((result, priority, delta))
 
-                    delta = abs(targetsize - int(result[1]))
+                return list(map(lambda x: x[0],
-                    newlist.append(
+                    sorted(
-                        (result[0], result[1], result[2], result[3], result[4], result[5], delta))
+                        lossy_results_with_delta,
+                        key=lambda x: (-x[0].matches, -x[1], x[2])
+                    )
+                ))
 
-                finallist = sorted(newlist, key=lambda title: (-title[5], title[6]))
+                if (
-
+                        not len(lossy_results_with_delta)
-                if not len(finallist) and len(
+                        and len(lossless_results)
-                        flac_list) and headphones.CONFIG.PREFERRED_BITRATE_ALLOW_LOSSLESS:
+                        and headphones.CONFIG.PREFERRED_BITRATE_ALLOW_LOSSLESS
+                    ):
                     logger.info(
-                        "Since there were no appropriate lossy matches (and at least one lossless match), going to use lossless instead")
+                        "Since there were no appropriate lossy matches "
-                    finallist = sorted(flac_list, key=lambda title: (title[5], int(title[1])),
+                        "(and at least one lossless match), going to use "
-                                       reverse=True)
+                        "lossless instead"
+                    )
+                    return sort_by_priority_then_size(results_with_priority)
+
         except Exception:
             logger.exception('Unhandled exception')
-            logger.info('No track information for %s - %s. Defaulting to highest quality',
+            logger.info(
-                        album['ArtistName'], album['AlbumTitle'])
+                f"No track information for {album['ArtistName']} - "
-
+                f"{album['AlbumTitle']}. Defaulting to highest quality"
-            finallist = sorted(resultlist, key=lambda title: (title[5], int(title[1])),
+            )
-                               reverse=True)
+            return sort_by_priority_then_size(results_with_priority)
 
     else:
+        return sort_by_priority_then_size(results_with_priority)
 
-        finallist = sorted(resultlist, key=lambda title: (title[5], int(title[1])), reverse=True)
+    logger.info(
-
+        f"No appropriate matches found for {album['ArtistName']} - "
-        # keep number of seeders order for Orpheus.network
+        f"{album['AlbumTitle']}"
-        # if result[3] == 'Orpheus.network':
+    )
-        #    finallist = resultlist
+    return None
-
-    if not len(finallist):
-        logger.info('No appropriate matches found for %s - %s', album['ArtistName'],
-                    album['AlbumTitle'])
-        return None
-
-    return finallist
 
 
 def get_year_from_release_date(release_date):
@@ -500,11 +507,22 @@ def searchNZB(album, new=False, losslessOnly=False, albumlength=None,
     reldate = album['ReleaseDate']
     year = get_year_from_release_date(reldate)
 
-    dic = {'...': '', ' & ': ' ', ' = ': ' ', '?': '', '$': 's', ' + ': ' ', '"': '', ',': '',
+    replacements = {
-           '*': '', '.': '', ':': ''}
+        '...': '',
+        ' & ': ' ',
+        ' = ': ' ',
+        '?': '',
+        '$': 's',
+        ' + ': ' ',
+        '"': '',
+        ',': '',
+        '*': '',
+        '.': '',
+        ':': ''
+    }
 
-    cleanalbum = helpers.latinToAscii(helpers.replace_all(album['AlbumTitle'], dic)).strip()
+    cleanalbum = unidecode(helpers.replace_all(album['AlbumTitle'], replacements)).strip()
-    cleanartist = helpers.latinToAscii(helpers.replace_all(album['ArtistName'], dic)).strip()
+    cleanartist = unidecode(helpers.replace_all(album['ArtistName'], replacements)).strip()
 
     # Use the provided search term if available, otherwise build a search term
     if album['SearchTerm']:
@@ -580,7 +598,7 @@ def searchNZB(album, new=False, losslessOnly=False, albumlength=None,
                         title = item.title
                         size = int(item.links[1]['length'])
 
-                        resultlist.append((title, size, url, provider, 'nzb', True))
+                        resultlist.append(Result(title, size, url, provider, 'nzb', True))
                         logger.info('Found %s. Size: %s' % (title, helpers.bytes_to_mb(size)))
                     except Exception as e:
                         logger.error("An unknown error occurred trying to parse the feed: %s" % e)
@@ -653,7 +671,7 @@ def searchNZB(album, new=False, losslessOnly=False, albumlength=None,
                             if all(word.lower() in title.lower() for word in term.split()):
                                 logger.info(
                                     'Found %s. Size: %s' % (title, helpers.bytes_to_mb(size)))
-                                resultlist.append((title, size, url, provider, 'nzb', True))
+                                resultlist.append(Result(title, size, url, provider, 'nzb', True))
                             else:
                                 logger.info('Skipping %s, not all search term words found' % title)
 
@@ -701,7 +719,7 @@ def searchNZB(album, new=False, losslessOnly=False, albumlength=None,
                         title = item.title
                         size = int(item.links[1]['length'])
 
-                        resultlist.append((title, size, url, provider, 'nzb', True))
+                        resultlist.append(Result(title, size, url, provider, 'nzb', True))
                         logger.info('Found %s. Size: %s' % (title, helpers.bytes_to_mb(size)))
                     except Exception as e:
                         logger.exception("Unhandled exception while parsing feed")
@@ -748,7 +766,7 @@ def searchNZB(album, new=False, losslessOnly=False, albumlength=None,
                         title = item['release']
                         size = int(item['sizebytes'])
 
-                        resultlist.append((title, size, url, provider, 'nzb', True))
+                        resultlist.append(Result(title, size, url, provider, 'nzb', True))
                         logger.info('Found %s. Size: %s', title, helpers.bytes_to_mb(size))
                     except Exception as e:
                         logger.exception("Unhandled exception")
@@ -760,7 +778,7 @@ def searchNZB(album, new=False, losslessOnly=False, albumlength=None,
     # Also will filter flac & remix albums if not specifically looking for it
     # This code also checks the ignored words and required words
     results = [result for result in resultlist if
-               verifyresult(result[0], artistterm, term, losslessOnly)]
+               verifyresult(result.title, artistterm, term, losslessOnly)]
 
     # Additional filtering for size etc
     if results and not choose_specific_download:
@@ -769,16 +787,18 @@ def searchNZB(album, new=False, losslessOnly=False, albumlength=None,
     return results
 
 
-def send_to_downloader(data, bestqual, album):
+def send_to_downloader(data, result, album):
-    logger.info('Found best result from %s: <a href="%s">%s</a> - %s', bestqual[3], bestqual[2],
+    logger.info(
-                bestqual[0], helpers.bytes_to_mb(bestqual[1]))
+        f"Found best result from {result.provider}: <a href=\"{result.url}\">"
+        f"{result.title}</a> - {helpers.bytes_to_mb(result.size)}"
+    )
     # Get rid of any dodgy chars here so we can prevent sab from renaming our downloads
-    kind = bestqual[4]
+    kind = result.kind
     seed_ratio = None
     torrentid = None
 
     if kind == 'nzb':
-        folder_name = helpers.sab_sanitize_foldername(bestqual[0])
+        folder_name = helpers.sab_sanitize_foldername(result.title)
 
         if headphones.CONFIG.NZB_DOWNLOADER == 1:
 
@@ -811,7 +831,7 @@ def send_to_downloader(data, bestqual, album):
             try:
                 prev = os.umask(headphones.UMASK)
 
-                with open(download_path, 'w') as fp:
+                with open(download_path, 'wb') as fp:
                     fp.write(data)
 
                 os.umask(prev)
@@ -821,8 +841,8 @@ def send_to_downloader(data, bestqual, album):
                 return
     else:
         folder_name = '%s - %s [%s]' % (
-            helpers.latinToAscii(album['ArtistName']).replace('/', '_'),
+            unidecode(album['ArtistName']).replace('/', '_'),
-            helpers.latinToAscii(album['AlbumTitle']).replace('/', '_'),
+            unidecode(album['AlbumTitle']).replace('/', '_'),
             get_year_from_release_date(album['ReleaseDate']))
 
         # Blackhole
@@ -832,26 +852,26 @@ def send_to_downloader(data, bestqual, album):
             torrent_name = helpers.replace_illegal_chars(folder_name) + '.torrent'
             download_path = os.path.join(headphones.CONFIG.TORRENTBLACKHOLE_DIR, torrent_name)
 
-            if bestqual[2].lower().startswith("magnet:"):
+            if result.url.lower().startswith("magnet:"):
                 if headphones.CONFIG.MAGNET_LINKS == 1:
                     try:
                         if headphones.SYS_PLATFORM == 'win32':
-                            os.startfile(bestqual[2])
+                            os.startfile(result.url)
                         elif headphones.SYS_PLATFORM == 'darwin':
-                            subprocess.Popen(["open", bestqual[2]], stdout=subprocess.PIPE,
+                            subprocess.Popen(["open", result.url], stdout=subprocess.PIPE,
                                              stderr=subprocess.PIPE)
                         else:
-                            subprocess.Popen(["xdg-open", bestqual[2]], stdout=subprocess.PIPE,
+                            subprocess.Popen(["xdg-open", result.url], stdout=subprocess.PIPE,
                                              stderr=subprocess.PIPE)
 
                         # Gonna just take a guess at this..... Is there a better way to find this out?
-                        folder_name = bestqual[0]
+                        folder_name = result.title
                     except Exception as e:
                         logger.error("Error opening magnet link: %s" % str(e))
                         return
                 elif headphones.CONFIG.MAGNET_LINKS == 2:
                     # Procedure adapted from CouchPotato
-                    torrent_hash = calculate_torrent_hash(bestqual[2])
+                    torrent_hash = calculate_torrent_hash(result.url)
 
                     # Randomize list of services
                     services = TORRENT_TO_MAGNET_SERVICES[:]
@@ -865,8 +885,9 @@ def send_to_downloader(data, bestqual, album):
                             if not torrent_to_file(download_path, data):
                                 return
                             # Extract folder name from torrent
-                            folder_name = read_torrent_name(download_path,
+                            folder_name = read_torrent_name(
-                                                            bestqual[0])
+                                    download_path,
+                                    result.title)
 
                             # Break for loop
                             break
@@ -890,7 +911,7 @@ def send_to_downloader(data, bestqual, album):
                     return
 
                 # Extract folder name from torrent
-                folder_name = read_torrent_name(download_path, bestqual[0])
+                folder_name = read_torrent_name(download_path, result.title)
                 if folder_name:
                     logger.info('Torrent folder name: %s' % folder_name)
 
@@ -898,10 +919,10 @@ def send_to_downloader(data, bestqual, album):
             logger.info("Sending torrent to Transmission")
 
             # Add torrent
-            if bestqual[3] == 'rutracker.org':
+            if result.provider == 'rutracker.org':
                 torrentid = transmission.addTorrent('', data)
             else:
-                torrentid = transmission.addTorrent(bestqual[2])
+                torrentid = transmission.addTorrent(result.url)
 
             if not torrentid:
                 logger.error("Error sending torrent to Transmission. Are you sure it's running?")
@@ -915,7 +936,7 @@ def send_to_downloader(data, bestqual, album):
                 return
 
             # Set Seed Ratio
-            seed_ratio = get_seed_ratio(bestqual[3])
+            seed_ratio = get_seed_ratio(result.provider)
             if seed_ratio is not None:
                 transmission.setSeedRatio(torrentid, seed_ratio)
 
@@ -924,10 +945,10 @@ def send_to_downloader(data, bestqual, album):
 
             try:
                 # Add torrent
-                if bestqual[3] == 'rutracker.org':
+                if result.provider == 'rutracker.org':
                     torrentid = deluge.addTorrent('', data)
                 else:
-                    torrentid = deluge.addTorrent(bestqual[2])
+                    torrentid = deluge.addTorrent(result.url)
 
                 if not torrentid:
                     logger.error("Error sending torrent to Deluge. Are you sure it's running? Maybe the torrent already exists?")
@@ -942,7 +963,7 @@ def send_to_downloader(data, bestqual, album):
                     deluge.setTorrentLabel({'hash': torrentid})
 
                 # Set Seed Ratio
-                seed_ratio = get_seed_ratio(bestqual[3])
+                seed_ratio = get_seed_ratio(result.provider)
                 if seed_ratio is not None:
                     deluge.setSeedRatio({'hash': torrentid, 'ratio': seed_ratio})
 
@@ -965,13 +986,13 @@ def send_to_downloader(data, bestqual, album):
             logger.info("Sending torrent to uTorrent")
 
             # Add torrent
-            if bestqual[3] == 'rutracker.org':
+            if result.provider == 'rutracker.org':
                 ruobj.utorrent_add_file(data)
             else:
-                utorrent.addTorrent(bestqual[2])
+                utorrent.addTorrent(result.url)
 
             # Get hash
-            torrentid = calculate_torrent_hash(bestqual[2], data)
+            torrentid = calculate_torrent_hash(result.url, data)
             if not torrentid:
                 logger.error('Torrent id could not be determined')
                 return
@@ -989,23 +1010,23 @@ def send_to_downloader(data, bestqual, album):
                 utorrent.labelTorrent(torrentid)
 
             # Set Seed Ratio
-            seed_ratio = get_seed_ratio(bestqual[3])
+            seed_ratio = get_seed_ratio(result.provider)
             if seed_ratio is not None:
                 utorrent.setSeedRatio(torrentid, seed_ratio)
         else:  # if headphones.CONFIG.TORRENT_DOWNLOADER == 4:
             logger.info("Sending torrent to QBiTorrent")
 
             # Add torrent
-            if bestqual[3] == 'rutracker.org':
+            if result.provider == 'rutracker.org':
                 if qbittorrent.apiVersion2:
                     qbittorrent.addFile(data)
                 else:
                     ruobj.qbittorrent_add_file(data)
             else:
-                qbittorrent.addTorrent(bestqual[2])
+                qbittorrent.addTorrent(result.url)
 
             # Get hash
-            torrentid = calculate_torrent_hash(bestqual[2], data)
+            torrentid = calculate_torrent_hash(result.url, data)
             torrentid = torrentid.lower()
             if not torrentid:
                 logger.error('Torrent id could not be determined')
@@ -1020,29 +1041,49 @@ def send_to_downloader(data, bestqual, album):
                 return
 
             # Set Seed Ratio
-            seed_ratio = get_seed_ratio(bestqual[3])
+            # Oh my god why is this repeated again for the 100th time
+            seed_ratio = get_seed_ratio(result.provider)
             if seed_ratio is not None:
                 qbittorrent.setSeedRatio(torrentid, seed_ratio)
 
     myDB = db.DBConnection()
     myDB.action('UPDATE albums SET status = "Snatched" WHERE AlbumID=?', [album['AlbumID']])
-    myDB.action('INSERT INTO snatched VALUES( ?, ?, ?, ?, DATETIME("NOW", "localtime"), ?, ?, ?, ?)',
+    myDB.action(
-                [album['AlbumID'], bestqual[0], bestqual[1], bestqual[2], "Snatched", folder_name,
+        "INSERT INTO snatched VALUES (?, ?, ?, ?, DATETIME('NOW', 'localtime'), "
-                 kind, torrentid])
+        "?, ?, ?, ?)", [
+            album['AlbumID'],
+            result.title,
+            result.size,
+            result.url,
+            "Snatched",
+            folder_name,
+            kind,
+            torrentid
+        ]
+    )
 
-    # Store the torrent id so we can check later if it's finished seeding and can be removed
+    # Additional record for post processing or scheduled job to remove the torrent when finished seeding
     if seed_ratio is not None and seed_ratio != 0 and torrentid:
         myDB.action(
-            'INSERT INTO snatched VALUES( ?, ?, ?, ?, DATETIME("NOW", "localtime"), ?, ?, ?, ?)',
+            "INSERT INTO snatched VALUES (?, ?, ?, ?, DATETIME('NOW', 'localtime'), "
-            [album['AlbumID'], bestqual[0], bestqual[1], bestqual[2], "Seed_Snatched", folder_name,
+            "?, ?, ?, ?)", [
-             kind, torrentid])
+                album['AlbumID'],
+                result.title,
+                result.size,
+                result.url,
+                "Seed_Snatched",
+                folder_name,
+                kind,
+                torrentid
+            ]
+        )
 
     # notify
     artist = album[1]
     albumname = album[2]
     rgid = album[6]
     title = artist + ' - ' + albumname
-    provider = bestqual[3]
+    provider = result.provider
     if provider.startswith(("http://", "https://")):
         provider = provider.split("//")[1]
     name = folder_name if folder_name else None
@@ -1083,7 +1124,7 @@ def send_to_downloader(data, bestqual, album):
         logger.info("Twitter notifications temporarily disabled")
         #logger.info("Sending Twitter notification")
         #twitter = notifiers.TwitterNotifier()
-        # twitter.notify_snatch(name)
+        #twitter.notify_snatch(name)
     if headphones.CONFIG.NMA_ENABLED and headphones.CONFIG.NMA_ONSNATCH:
         logger.info("Sending NMA notification")
         nma = notifiers.NMA()
@@ -1211,13 +1252,22 @@ def searchTorrent(album, new=False, losslessOnly=False, albumlength=None,
     year = get_year_from_release_date(reldate)
 
     # MERGE THIS WITH THE TERM CLEANUP FROM searchNZB
-    dic = {'...': '', ' & ': ' ', ' = ': ' ', '?': '', '$': 's', ' + ': ' ', '"': '', ',': ' ',
+    replacements = {
-           '*': ''}
+        '...': '',
+        ' & ': ' ',
+        ' = ': ' ',
+        '?': '',
+        '$': 's',
+        ' + ': ' ',
+        '"': '',
+        ',': ' ',
+        '*': ''
+    }
 
-    semi_cleanalbum = helpers.replace_all(album['AlbumTitle'], dic)
+    semi_cleanalbum = helpers.replace_all(album['AlbumTitle'], replacements)
-    cleanalbum = helpers.latinToAscii(semi_cleanalbum)
+    cleanalbum = unidecode(semi_cleanalbum)
-    semi_cleanartist = helpers.replace_all(album['ArtistName'], dic)
+    semi_cleanartist = helpers.replace_all(album['ArtistName'], replacements)
-    cleanartist = helpers.latinToAscii(semi_cleanartist)
+    cleanartist = unidecode(semi_cleanartist)
 
     # Use provided term if available, otherwise build our own (this code needs to be cleaned up since a lot
     # of these torrent providers are just using cleanartist/cleanalbum terms
@@ -1352,7 +1402,7 @@ def searchTorrent(album, new=False, losslessOnly=False, albumlength=None,
                             if all(word.lower() in title.lower() for word in term.split()):
                                 if size < maxsize and minimumseeders < seeders:
                                     logger.info('Found %s. Size: %s' % (title, helpers.bytes_to_mb(size)))
-                                    resultlist.append((title, size, url, provider, 'torrent', True))
+                                    resultlist.append(Result(title, size, url, provider, 'torrent', True))
                                 else:
                                     logger.info(
                                         '%s is larger than the maxsize or has too little seeders for this category, '
@@ -1426,7 +1476,7 @@ def searchTorrent(album, new=False, losslessOnly=False, albumlength=None,
                         desc_match = re.search(r"Size: (\d+)<", item.description)
                         size = int(desc_match.group(1))
                         url = item.link
-                        resultlist.append((title, size, url, provider, 'torrent', True))
+                        resultlist.append(Result(title, size, url, provider, 'torrent', True))
                         logger.info('Found %s. Size: %s', title, helpers.bytes_to_mb(size))
                     except Exception as e:
                         logger.error(
@@ -1505,8 +1555,8 @@ def searchTorrent(album, new=False, losslessOnly=False, albumlength=None,
             try:
                 logger.info("Attempting to log in to Orpheus.network...")
                 orpheusobj = gazelleapi.GazelleAPI(headphones.CONFIG.ORPHEUS_USERNAME,
-                                                   headphones.CONFIG.ORPHEUS_PASSWORD,
+                                                headphones.CONFIG.ORPHEUS_PASSWORD,
-                                                   headphones.CONFIG.ORPHEUS_URL)
+                                                headphones.CONFIG.ORPHEUS_URL)
                 orpheusobj._login()
             except Exception as e:
                 orpheusobj = None
@@ -1552,13 +1602,13 @@ def searchTorrent(album, new=False, losslessOnly=False, albumlength=None,
                 if usersearchterm:
                     all_torrents.extend(
                         orpheusobj.search_torrents(searchstr=usersearchterm, format=search_format,
-                                                   encoding=bitrate_string, releasetype=album_type)['results'])
+                                                encoding=bitrate_string, releasetype=album_type)['results'])
                 else:
                     all_torrents.extend(orpheusobj.search_torrents(artistname=semi_clean_artist_term,
-                                                                   groupname=semi_clean_album_term,
+                                                                groupname=semi_clean_album_term,
-                                                                   format=search_format,
+                                                                format=search_format,
-                                                                   encoding=bitrate_string,
+                                                                encoding=bitrate_string,
-                                                                   releasetype=album_type)['results'])
+                                                                releasetype=album_type)['results'])
 
             # filter on format, size, and num seeders
             logger.info("Filtering torrents by format, maximum size, and minimum seeders...")
@@ -1591,11 +1641,16 @@ def searchTorrent(album, new=False, losslessOnly=False, albumlength=None,
             for torrent in match_torrents:
                 if not torrent.file_path:
                     torrent.group.update_group_data()  # will load the file_path for the individual torrents
-                resultlist.append((torrent.file_path,
+                resultlist.append(
-                                   torrent.size,
+                    Result(
-                                   orpheusobj.generate_torrent_link(torrent.id),
+                        torrent.file_path,
-                                   provider,
+                        torrent.size,
-                                   'torrent', True))
+                        orpheusobj.generate_torrent_link(torrent.id),
+                        provider,
+                        'torrent',
+                        True
+                    )
+                )
 
     # Redacted - Using same logic as What.CD as it's also Gazelle, so should really make this into something reusable
     if headphones.CONFIG.REDACTED:
@@ -1636,8 +1691,8 @@ def searchTorrent(album, new=False, losslessOnly=False, albumlength=None,
             try:
                 logger.info("Attempting to log in to Redacted...")
                 redobj = gazelleapi.GazelleAPI(headphones.CONFIG.REDACTED_USERNAME,
-                                               headphones.CONFIG.REDACTED_PASSWORD,
+                                                headphones.CONFIG.REDACTED_PASSWORD,
-                                               providerurl)
+                                                providerurl)
                 redobj._login()
             except Exception as e:
                 redobj = None
@@ -1651,12 +1706,12 @@ def searchTorrent(album, new=False, losslessOnly=False, albumlength=None,
                 if usersearchterm:
                     all_torrents.extend(
                         redobj.search_torrents(searchstr=usersearchterm, format=search_format,
-                                               encoding=bitrate_string)['results'])
+                                                encoding=bitrate_string)['results'])
                 else:
                     all_torrents.extend(redobj.search_torrents(artistname=semi_clean_artist_term,
-                                                               groupname=semi_clean_album_term,
+                                                                groupname=semi_clean_album_term,
-                                                               format=search_format,
+                                                                format=search_format,
-                                                               encoding=bitrate_string)['results'])
+                                                                encoding=bitrate_string)['results'])
 
             # filter on format, size, and num seeders
             logger.info("Filtering torrents by format, maximum size, and minimum seeders...")
@@ -1692,11 +1747,16 @@ def searchTorrent(album, new=False, losslessOnly=False, albumlength=None,
                 if not torrent.file_path:
                     torrent.group.update_group_data()  # will load the file_path for the individual torrents
                 use_token = headphones.CONFIG.REDACTED_USE_FLTOKEN and torrent.can_use_token
-                resultlist.append((torrent.file_path,
+                resultlist.append(
-                                   torrent.size,
+                    Result(
-                                   redobj.generate_torrent_link(torrent.id, use_token),
+                        torrent.file_path,
-                                   provider,
+                        torrent.size,
-                                   'torrent', True))
+                        redobj.generate_torrent_link(torrent.id, use_token),
+                        provider,
+                        'torrent',
+                        True
+                    )
+                )
 
     # Pirate Bay
     if headphones.CONFIG.PIRATEBAY:
@@ -1770,7 +1830,7 @@ def searchTorrent(album, new=False, losslessOnly=False, albumlength=None,
                             logger.info('%s is larger than the maxsize or has too little seeders for this category, '
                                         'skipping. (Size: %i bytes, Seeders: %i)' % (title, size, int(seeds)))
 
-                        resultlist.append((title, size, url, provider, "torrent", match))
+                        resultlist.append(Result(title, size, url, provider, "torrent", match))
                     except Exception as e:
                         logger.error("An unknown error occurred in the Pirate Bay parser: %s" % e)
 
@@ -1793,7 +1853,7 @@ def searchTorrent(album, new=False, losslessOnly=False, albumlength=None,
         headers = {
             'User-Agent': 'Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2243.2 Safari/537.36'}
         provider_url = fix_url(headphones.CONFIG.OLDPIRATEBAY_URL) + \
-            "/search.php?" + urllib.parse.urlencode({"q": tpb_term, "iht": 6})
+                       "/search.php?" + urllib.parse.urlencode({"q": tpb_term, "iht": 6})
 
         data = request.request_soup(url=provider_url, headers=headers)
 
@@ -1824,7 +1884,7 @@ def searchTorrent(album, new=False, losslessOnly=False, albumlength=None,
                             logger.info('%s is larger than the maxsize or has too little seeders for this category, '
                                         'skipping. (Size: %i bytes, Seeders: %i)' % (title, size, int(seeds)))
 
-                        resultlist.append((title, size, url, provider, "torrent", match))
+                        resultlist.append(Result(title, size, url, provider, "torrent", match))
                     except Exception as e:
                         logger.error(
                             "An unknown error occurred in the Old Pirate Bay parser: %s" % e)
@@ -1832,10 +1892,9 @@ def searchTorrent(album, new=False, losslessOnly=False, albumlength=None,
     # attempt to verify that this isn't a substring result
     # when looking for "Foo - Foo" we don't want "Foobar"
     # this should be less of an issue when it isn't a self-titled album so we'll only check vs artist
-    results = [result for result in resultlist if verifyresult(result[0], artistterm, term, losslessOnly)]
+    results = [result for result in resultlist if verifyresult(result.title, artistterm, term, losslessOnly)]
 
     # Additional filtering for size etc
-    # if results and not choose_specific_download and result[3] != 'Orpheus.network':
     if results and not choose_specific_download:
         results = more_filtering(results, album, albumlength, new)
 
@@ -1847,28 +1906,51 @@ def searchTorrent(album, new=False, losslessOnly=False, albumlength=None,
 
 def preprocess(resultlist):
     for result in resultlist:
-        if result[4] == 'torrent':
 
-            headers = {}
+        if result.provider in ["The Pirate Bay", "Old Pirate Bay"]:
+            headers = {
+                'User-Agent':
+                    'Mozilla/5.0 (Windows NT 6.3; Win64; x64) \
+                    AppleWebKit/537.36 (KHTML, like Gecko) \
+                    Chrome/41.0.2243.2 Safari/537.36'
+            }
+        else:
+            headers = {'User-Agent': USER_AGENT}
+
+        if result.kind == 'torrent':
 
             # rutracker always needs the torrent data
-            if result[3] == 'rutracker.org':
+            if result.provider == 'rutracker.org':
-                return ruobj.get_torrent_data(result[2]), result
+                return ruobj.get_torrent_data(result.url), result
 
             # Jackett sometimes redirects
-            jackett_content = None
+            if result.provider.startswith('Jackett_') or 'torznab' in result.provider.lower():
-            if result[3].startswith('Jackett_') or 'torznab' in result[3].lower():
+                r = request.request_response(url=result.url, headers=headers, allow_redirects=False)
-                r = request.request_response(url=result[2], headers=headers, allow_redirects=False)
                 if r:
-                    jackett_content = r.content
                     link = r.headers.get('Location')
-                    if link and link != result[2]:
+                    if link and link != result.url:
                         if link.startswith('magnet:'):
-                            result = (result[0], result[1], link, result[3], "magnet", result[5])
+                            result = Result(
+                                result.url,
+                                result.size,
+                                link,
+                                result.provider,
+                                "magnet",
+                                result.matches
+                            )
                             return "d10:magnet-uri%d:%se" % (len(link), link), result
                         else:
-                            result = (result[0], result[1], link, result[3], result[4], result[5])
+                            result = Result(
+                                result.url,
+                                result.size,
+                                link,
+                                result.provider,
+                                result.kind,
+                                result.matches
+                            )
                             return True, result
+                    else:
+                        return r.content, result
 
             # Get out of here if we're using Transmission or Deluge
             # if not a magnet link still need the .torrent to generate hash... uTorrent support labeling
@@ -1876,31 +1958,22 @@ def preprocess(resultlist):
                 return True, result
 
             # Get out of here if it's a magnet link
-            if result[2].lower().startswith("magnet:"):
+            if result.url.lower().startswith("magnet:"):
                 return True, result
 
             # Download the torrent file
+            return request.request_content(url=result.url, headers=headers), result
 
-            if result[3] == 'Orpheus.network':
+        if result.kind == 'magnet':
-                headers['User-Agent'] = 'Headphones'
+            magnet_link = result.url
-            elif result[3] == 'Redacted':
-                headers['User-Agent'] = 'Headphones'
-            elif result[3] == "The Pirate Bay" or result[3] == "Old Pirate Bay":
-                headers['User-Agent'] = 'Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2243.2 Safari/537.36'
-            elif jackett_content:
-                return jackett_content, result
-
-            return request.request_content(url=result[2], headers=headers), result
-
-        if result[4] == 'magnet':
-            magnet_link = result[2]
             return "d10:magnet-uri%d:%se" % (len(magnet_link), magnet_link), result
 
         else:
-            headers = {'User-Agent': USER_AGENT}
+            if result.provider == 'headphones':
-
+                return request.request_content(
-            if result[3] == 'headphones':
+                    url=result.url,
-                return request.request_content(url=result[2], headers=headers,
+                    headers=headers,
-                                               auth=(headphones.CONFIG.HPUSER, headphones.CONFIG.HPPASS)), result
+                    auth=(headphones.CONFIG.HPUSER, headphones.CONFIG.HPPASS)
+                    ), result
             else:
-                return request.request_content(url=result[2], headers=headers), result
+                return request.request_content(url=result.url, headers=headers), result

headphones/transmission.py

@@ -15,7 +15,7 @@
 
 import time
 import json
-import base64
+from base64 import b64encode
 import urllib.parse
 import os
 
@@ -36,10 +36,10 @@ def addTorrent(link, data=None):
 
     if link.endswith('.torrent') and not link.startswith(('http', 'magnet')) or data:
         if data:
-            metainfo = str(base64.b64encode(data))
+            metainfo = b64encode(data).decode("utf-8")
         else:
             with open(link, 'rb') as f:
-                metainfo = str(base64.b64encode(f.read()))
+                metainfo = b64encode(f.read()).decode("utf-8")
         arguments = {'metainfo': metainfo, 'download-dir': headphones.CONFIG.DOWNLOAD_TORRENT_DIR}
     else:
         arguments = {'filename': link, 'download-dir': headphones.CONFIG.DOWNLOAD_TORRENT_DIR}
@@ -183,15 +183,15 @@ def torrentAction(method, arguments):
         if _session_id is not None:
             headers = {'x-transmission-session-id': _session_id}
             response = request.request_response(host, method="POST",
-                                                data=data_json, headers=headers, auth=auth,
+                data=data_json, headers=headers, auth=auth,
-                                                whitelist_status_code=[200, 401, 409])
+                whitelist_status_code=[200, 401, 409])
         else:
             response = request.request_response(host, auth=auth,
-                                                whitelist_status_code=[401, 409])
+                                            whitelist_status_code=[401, 409])
         if response.status_code == 401:
             if auth:
                 logger.error("Username and/or password not accepted by "
-                             "Transmission")
+                            "Transmission")
             else:
                 logger.error("Transmission authorization required")
             return
@@ -205,5 +205,4 @@ def torrentAction(method, arguments):
             continue
 
         resp_json = response.json()
-        print(resp_json)
         return resp_json

headphones/types.py

@@ -0,0 +1,10 @@
+from dataclasses import dataclass
+
+@dataclass(frozen=True)
+class Result:
+    title: str
+    size: int
+    url: str
+    provider: str
+    kind: str
+    matches: bool

headphones/utorrent.py

@@ -13,15 +13,11 @@
 #  You should have received a copy of the GNU General Public License
 #  along with Headphones.  If not, see <http://www.gnu.org/licenses/>.
 
-import urllib.request
+import urllib.request, urllib.parse, urllib.error
-import urllib.parse
-import urllib.error
 import json
 import time
 from collections import namedtuple
-import urllib.request
+import urllib.request, urllib.error, urllib.parse
-import urllib.error
-import urllib.parse
 import urllib.parse
 import http.cookiejar
 

headphones/webserve.py

@@ -15,38 +15,46 @@
 
 # NZBGet support added by CurlyMo <curlymoo1@gmail.com> as a part of XBian - XBMC on the Raspberry Pi
 
-from operator import itemgetter
-import threading
-import secrets
-import random
-import urllib.request
-import urllib.parse
-import urllib.error
 import json
-import time
-import sys
-from html import escape as html_escape
-import urllib.request
-import urllib.error
-import urllib.parse
-
 import os
+import random
 import re
-from headphones import logger, searcher, db, importer, mb, lastfm, librarysync, helpers, notifiers, crier
+import secrets
-from headphones.helpers import checked, radio, today, clean_name
+import sys
-from mako.lookup import TemplateLookup
+import threading
-from mako import exceptions
+import time
-import headphones
+from collections import OrderedDict
-import cherrypy
+from dataclasses import asdict
+from html import escape as html_escape
+from operator import itemgetter
+from urllib import parse
 
-try:
+import cherrypy
-    # pylint:disable=E0611
+from mako import exceptions
-    # ignore this error because we are catching the ImportError
+from mako.lookup import TemplateLookup
-    from collections import OrderedDict
+
-    # pylint:enable=E0611
+import headphones
-except ImportError:
+from headphones import (
-    # Python 2.6.x fallback, from libs
+    crier,
-    from ordereddict import OrderedDict
+    db,
+    importer,
+    lastfm,
+    librarysync,
+    logger,
+    mb,
+    notifiers,
+    searcher,
+)
+from headphones.helpers import (
+    checked,
+    clean_name,
+    have_pct_have_total,
+    pattern_substitute,
+    radio,
+    replace_illegal_chars,
+    today,
+)
+from headphones.types import Result
 
 
 def serve_template(templatename, **kwargs):
@@ -330,9 +338,9 @@ class WebInterface(object):
                   '$first': firstchar.lower(),
                   }
 
-        folder = helpers.pattern_substitute(folder_format.strip(), values, normalize=True)
+        folder = pattern_substitute(folder_format.strip(), values, normalize=True)
 
-        folder = helpers.replace_illegal_chars(folder, type="folder")
+        folder = replace_illegal_chars(folder, type="folder")
         folder = folder.replace('./', '_/').replace('/.', '/_')
 
         if folder.endswith('.'):
@@ -444,40 +452,27 @@ class WebInterface(object):
     @cherrypy.expose
     @cherrypy.tools.json_out()
     def choose_specific_download(self, AlbumID):
-        results = searcher.searchforalbum(AlbumID, choose_specific_download=True)
+        results = searcher.searchforalbum(AlbumID, choose_specific_download=True) or []
-
+        return list(map(asdict, results))
-        data = []
-
-        for result in results:
-            result_dict = {
-                'title': result[0],
-                'size': result[1],
-                'url': result[2],
-                'provider': result[3],
-                'kind': result[4],
-                'matches': result[5]
-            }
-            data.append(result_dict)
-        return data
 
     @cherrypy.expose
     @cherrypy.tools.json_out()
     def download_specific_release(self, AlbumID, title, size, url, provider, kind, **kwargs):
         # Handle situations where the torrent url contains arguments that are parsed
         if kwargs:
-            url = urllib.parse.quote(url, safe=":?/=&") + '&' + urllib.parse.urlencode(kwargs)
+            url = parse.quote(url, safe=":?/=&") + '&' + parse.urlencode(kwargs)
         try:
-            result = [(title, int(size), url, provider, kind)]
+            result = [Result(title, int(size), url, provider, kind, True)]
         except ValueError:
-            result = [(title, float(size), url, provider, kind)]
+            result = [Result(title, float(size), url, provider, kind, True)]
 
         logger.info("Making sure we can download the chosen result")
-        (data, bestqual) = searcher.preprocess(result)
+        data, result = searcher.preprocess(result)
 
-        if data and bestqual:
+        if data and result:
             myDB = db.DBConnection()
             album = myDB.action('SELECT * from albums WHERE AlbumID=?', [AlbumID]).fetchone()
-            searcher.send_to_downloader(data, bestqual, album)
+            searcher.send_to_downloader(data, result, album)
             return {'result': 'success'}
         else:
             return {'result': 'failure'}
@@ -590,7 +585,7 @@ class WebInterface(object):
         for albums in have_albums:
             # Have to skip over manually matched tracks
             if albums['ArtistName'] and albums['AlbumTitle'] and albums['TrackTitle']:
-                original_clean = helpers.clean_name(
+                original_clean = clean_name(
                     albums['ArtistName'] + " " + albums['AlbumTitle'] + " " + albums['TrackTitle'])
                 # else:
                 #     original_clean = None
@@ -637,8 +632,8 @@ class WebInterface(object):
                 (artist, album))
 
         elif action == "matchArtist":
-            existing_artist_clean = helpers.clean_name(existing_artist).lower()
+            existing_artist_clean = clean_name(existing_artist).lower()
-            new_artist_clean = helpers.clean_name(new_artist).lower()
+            new_artist_clean = clean_name(new_artist).lower()
             if new_artist_clean != existing_artist_clean:
                 have_tracks = myDB.action(
                     'SELECT Matched, CleanName, Location, BitRate, Format FROM have WHERE ArtistName=?',
@@ -682,10 +677,10 @@ class WebInterface(object):
                     "Artist %s already named appropriately; nothing to modify" % existing_artist)
 
         elif action == "matchAlbum":
-            existing_artist_clean = helpers.clean_name(existing_artist).lower()
+            existing_artist_clean = clean_name(existing_artist).lower()
-            new_artist_clean = helpers.clean_name(new_artist).lower()
+            new_artist_clean = clean_name(new_artist).lower()
-            existing_album_clean = helpers.clean_name(existing_album).lower()
+            existing_album_clean = clean_name(existing_album).lower()
-            new_album_clean = helpers.clean_name(new_album).lower()
+            new_album_clean = clean_name(new_album).lower()
             existing_clean_string = existing_artist_clean + " " + existing_album_clean
             new_clean_string = new_artist_clean + " " + new_album_clean
             if existing_clean_string != new_clean_string:
@@ -741,7 +736,7 @@ class WebInterface(object):
             'SELECT ArtistName, AlbumTitle, TrackTitle, CleanName, Matched from have')
         for albums in manualalbums:
             if albums['ArtistName'] and albums['AlbumTitle'] and albums['TrackTitle']:
-                original_clean = helpers.clean_name(
+                original_clean = clean_name(
                     albums['ArtistName'] + " " + albums['AlbumTitle'] + " " + albums['TrackTitle'])
                 if albums['Matched'] == "Ignored" or albums['Matched'] == "Manual" or albums[
                         'CleanName'] != original_clean:
@@ -782,14 +777,14 @@ class WebInterface(object):
                 [artist])
             update_count = 0
             for tracks in update_clean:
-                original_clean = helpers.clean_name(
+                original_clean = clean_name(
                     tracks['ArtistName'] + " " + tracks['AlbumTitle'] + " " + tracks[
                         'TrackTitle']).lower()
                 album = tracks['AlbumTitle']
                 track_title = tracks['TrackTitle']
                 if tracks['CleanName'] != original_clean:
                     artist_id_check = myDB.action('SELECT ArtistID FROM tracks WHERE CleanName = ?',
-                                                  [tracks['CleanName']]).fetchone()
+                                                 [tracks['CleanName']]).fetchone()
                     if artist_id_check:
                         artist_id = artist_id_check[0]
                     myDB.action(
@@ -814,7 +809,7 @@ class WebInterface(object):
                 (artist, album))
             update_count = 0
             for tracks in update_clean:
-                original_clean = helpers.clean_name(
+                original_clean = clean_name(
                     tracks['ArtistName'] + " " + tracks['AlbumTitle'] + " " + tracks[
                         'TrackTitle']).lower()
                 track_title = tracks['TrackTitle']
@@ -1022,9 +1017,7 @@ class WebInterface(object):
             totalcount = myDB.select('SELECT COUNT(*) from artists')[0][0]
 
         if sortbyhavepercent:
-            filtered.sort(key=lambda x: (
+            filtered.sort(key=have_pct_have_total, reverse=sSortDir_0 == "asc")
-                float(x['HaveTracks']) / x['TotalTracks'] if x['TotalTracks'] > 0 else 0.0,
-                x['HaveTracks'] if x['HaveTracks'] else 0.0), reverse=sSortDir_0 == "asc")
 
         # can't figure out how to change the datatables default sorting order when its using an ajax datasource so ill
         # just reverse it here and the first click on the "Latest Album" header will sort by descending release date
@@ -1275,6 +1268,7 @@ class WebInterface(object):
             "cue_split_shntool_path": headphones.CONFIG.CUE_SPLIT_SHNTOOL_PATH,
             "move_files": checked(headphones.CONFIG.MOVE_FILES),
             "rename_files": checked(headphones.CONFIG.RENAME_FILES),
+            "rename_single_disc_ignore": checked(headphones.CONFIG.RENAME_SINGLE_DISC_IGNORE),
             "correct_metadata": checked(headphones.CONFIG.CORRECT_METADATA),
             "cleanup_files": checked(headphones.CONFIG.CLEANUP_FILES),
             "keep_nfo": checked(headphones.CONFIG.KEEP_NFO),
@@ -1391,6 +1385,7 @@ class WebInterface(object):
             "custompass": headphones.CONFIG.CUSTOMPASS,
             "hpuser": headphones.CONFIG.HPUSER,
             "hppass": headphones.CONFIG.HPPASS,
+            "lastfm_apikey": headphones.CONFIG.LASTFM_APIKEY,
             "songkick_enabled": checked(headphones.CONFIG.SONGKICK_ENABLED),
             "songkick_apikey": headphones.CONFIG.SONGKICK_APIKEY,
             "songkick_location": headphones.CONFIG.SONGKICK_LOCATION,
@@ -1467,8 +1462,8 @@ class WebInterface(object):
             "use_waffles", "use_rutracker",
             "use_orpheus", "use_redacted", "redacted_use_fltoken", "preferred_bitrate_allow_lossless",
             "detect_bitrate", "ignore_clean_releases", "freeze_db", "cue_split", "move_files",
-            "rename_files", "correct_metadata", "cleanup_files", "keep_nfo", "add_album_art",
+            "rename_files", "rename_single_disc_ignore", "correct_metadata", "cleanup_files",
-            "embed_album_art", "embed_lyrics",
+            "keep_nfo", "add_album_art", "embed_album_art", "embed_lyrics",
             "replace_existing_folders", "keep_original_folder", "file_underscores",
             "include_extras", "official_releases_only",
             "wait_until_release_date", "autowant_upcoming", "autowant_all",

lib/apscheduler/__init__.py

@@ -1,5 +1,10 @@
-version_info = (3, 0, 1)
+from pkg_resources import get_distribution, DistributionNotFound
-version = '3.0.1'
-release = '3.0.1'
 
-__version__ = release  # PEP 396
+try:
+    release = get_distribution('APScheduler').version.split('-')[0]
+except DistributionNotFound:
+    release = '3.5.0'
+
+version_info = tuple(int(x) if x.isdigit() else x for x in release.split('.'))
+version = __version__ = '.'.join(str(x) for x in version_info[:3])
+del get_distribution, DistributionNotFound

lib/apscheduler/events.py

@@ -1,25 +1,33 @@
-__all__ = ('EVENT_SCHEDULER_START', 'EVENT_SCHEDULER_SHUTDOWN', 'EVENT_EXECUTOR_ADDED', 'EVENT_EXECUTOR_REMOVED',
+__all__ = ('EVENT_SCHEDULER_STARTED', 'EVENT_SCHEDULER_SHUTDOWN', 'EVENT_SCHEDULER_PAUSED',
-           'EVENT_JOBSTORE_ADDED', 'EVENT_JOBSTORE_REMOVED', 'EVENT_ALL_JOBS_REMOVED', 'EVENT_JOB_ADDED',
+           'EVENT_SCHEDULER_RESUMED', 'EVENT_EXECUTOR_ADDED', 'EVENT_EXECUTOR_REMOVED',
-           'EVENT_JOB_REMOVED', 'EVENT_JOB_MODIFIED', 'EVENT_JOB_EXECUTED', 'EVENT_JOB_ERROR', 'EVENT_JOB_MISSED',
+           'EVENT_JOBSTORE_ADDED', 'EVENT_JOBSTORE_REMOVED', 'EVENT_ALL_JOBS_REMOVED',
-           'SchedulerEvent', 'JobEvent', 'JobExecutionEvent')
+           'EVENT_JOB_ADDED', 'EVENT_JOB_REMOVED', 'EVENT_JOB_MODIFIED', 'EVENT_JOB_EXECUTED',
+           'EVENT_JOB_ERROR', 'EVENT_JOB_MISSED', 'EVENT_JOB_SUBMITTED', 'EVENT_JOB_MAX_INSTANCES',
+           'SchedulerEvent', 'JobEvent', 'JobExecutionEvent', 'JobSubmissionEvent')
 
 
-EVENT_SCHEDULER_START = 1
+EVENT_SCHEDULER_STARTED = EVENT_SCHEDULER_START = 2 ** 0
-EVENT_SCHEDULER_SHUTDOWN = 2
+EVENT_SCHEDULER_SHUTDOWN = 2 ** 1
-EVENT_EXECUTOR_ADDED = 4
+EVENT_SCHEDULER_PAUSED = 2 ** 2
-EVENT_EXECUTOR_REMOVED = 8
+EVENT_SCHEDULER_RESUMED = 2 ** 3
-EVENT_JOBSTORE_ADDED = 16
+EVENT_EXECUTOR_ADDED = 2 ** 4
-EVENT_JOBSTORE_REMOVED = 32
+EVENT_EXECUTOR_REMOVED = 2 ** 5
-EVENT_ALL_JOBS_REMOVED = 64
+EVENT_JOBSTORE_ADDED = 2 ** 6
-EVENT_JOB_ADDED = 128
+EVENT_JOBSTORE_REMOVED = 2 ** 7
-EVENT_JOB_REMOVED = 256
+EVENT_ALL_JOBS_REMOVED = 2 ** 8
-EVENT_JOB_MODIFIED = 512
+EVENT_JOB_ADDED = 2 ** 9
-EVENT_JOB_EXECUTED = 1024
+EVENT_JOB_REMOVED = 2 ** 10
-EVENT_JOB_ERROR = 2048
+EVENT_JOB_MODIFIED = 2 ** 11
-EVENT_JOB_MISSED = 4096
+EVENT_JOB_EXECUTED = 2 ** 12
-EVENT_ALL = (EVENT_SCHEDULER_START | EVENT_SCHEDULER_SHUTDOWN | EVENT_JOBSTORE_ADDED | EVENT_JOBSTORE_REMOVED |
+EVENT_JOB_ERROR = 2 ** 13
+EVENT_JOB_MISSED = 2 ** 14
+EVENT_JOB_SUBMITTED = 2 ** 15
+EVENT_JOB_MAX_INSTANCES = 2 ** 16
+EVENT_ALL = (EVENT_SCHEDULER_STARTED | EVENT_SCHEDULER_SHUTDOWN | EVENT_SCHEDULER_PAUSED |
+             EVENT_SCHEDULER_RESUMED | EVENT_EXECUTOR_ADDED | EVENT_EXECUTOR_REMOVED |
+             EVENT_JOBSTORE_ADDED | EVENT_JOBSTORE_REMOVED | EVENT_ALL_JOBS_REMOVED |
              EVENT_JOB_ADDED | EVENT_JOB_REMOVED | EVENT_JOB_MODIFIED | EVENT_JOB_EXECUTED |
-             EVENT_JOB_ERROR | EVENT_JOB_MISSED)
+             EVENT_JOB_ERROR | EVENT_JOB_MISSED | EVENT_JOB_SUBMITTED | EVENT_JOB_MAX_INSTANCES)
 
 
 class SchedulerEvent(object):
@@ -55,9 +63,21 @@ class JobEvent(SchedulerEvent):
         self.jobstore = jobstore
 
 
+class JobSubmissionEvent(JobEvent):
+    """
+    An event that concerns the submission of a job to its executor.
+
+    :ivar scheduled_run_times: a list of datetimes when the job was intended to run
+    """
+
+    def __init__(self, code, job_id, jobstore, scheduled_run_times):
+        super(JobSubmissionEvent, self).__init__(code, job_id, jobstore)
+        self.scheduled_run_times = scheduled_run_times
+
+
 class JobExecutionEvent(JobEvent):
     """
-    An event that concerns the execution of individual jobs.
+    An event that concerns the running of a job within its executor.
 
     :ivar scheduled_run_time: the time when the job was scheduled to be run
     :ivar retval: the return value of the successfully executed job
@@ -65,7 +85,8 @@ class JobExecutionEvent(JobEvent):
     :ivar traceback: a formatted traceback for the exception
     """
 
-    def __init__(self, code, job_id, jobstore, scheduled_run_time, retval=None, exception=None, traceback=None):
+    def __init__(self, code, job_id, jobstore, scheduled_run_time, retval=None, exception=None,
+                 traceback=None):
         super(JobExecutionEvent, self).__init__(code, job_id, jobstore)
         self.scheduled_run_time = scheduled_run_time
         self.retval = retval

lib/apscheduler/executors/asyncio.py

@@ -1,28 +1,52 @@
+from __future__ import absolute_import
 
 import sys
 
 from apscheduler.executors.base import BaseExecutor, run_job
+from apscheduler.executors.base_py3 import run_coroutine_job
+from apscheduler.util import iscoroutinefunction_partial
 
 
 class AsyncIOExecutor(BaseExecutor):
     """
     Runs jobs in the default executor of the event loop.
 
+    If the job function is a native coroutine function, it is scheduled to be run directly in the
+    event loop as soon as possible. All other functions are run in the event loop's default
+    executor which is usually a thread pool.
+
     Plugin alias: ``asyncio``
     """
 
     def start(self, scheduler, alias):
         super(AsyncIOExecutor, self).start(scheduler, alias)
         self._eventloop = scheduler._eventloop
+        self._pending_futures = set()
+
+    def shutdown(self, wait=True):
+        # There is no way to honor wait=True without converting this method into a coroutine method
+        for f in self._pending_futures:
+            if not f.done():
+                f.cancel()
+
+        self._pending_futures.clear()
 
     def _do_submit_job(self, job, run_times):
         def callback(f):
+            self._pending_futures.discard(f)
             try:
                 events = f.result()
-            except:
+            except BaseException:
                 self._run_job_error(job.id, *sys.exc_info()[1:])
             else:
                 self._run_job_success(job.id, events)
 
-        f = self._eventloop.run_in_executor(None, run_job, job, job._jobstore_alias, run_times, self._logger.name)
+        if iscoroutinefunction_partial(job.func):
+            coro = run_coroutine_job(job, job._jobstore_alias, run_times, self._logger.name)
+            f = self._eventloop.create_task(coro)
+        else:
+            f = self._eventloop.run_in_executor(None, run_job, job, job._jobstore_alias, run_times,
+                                                self._logger.name)
+
         f.add_done_callback(callback)
+        self._pending_futures.add(f)

lib/apscheduler/executors/base.py

@@ -8,13 +8,15 @@ import sys
 from pytz import utc
 import six
 
-from apscheduler.events import JobExecutionEvent, EVENT_JOB_MISSED, EVENT_JOB_ERROR, EVENT_JOB_EXECUTED
+from apscheduler.events import (
+    JobExecutionEvent, EVENT_JOB_MISSED, EVENT_JOB_ERROR, EVENT_JOB_EXECUTED)
 
 
 class MaxInstancesReachedError(Exception):
     def __init__(self, job):
         super(MaxInstancesReachedError, self).__init__(
-            'Job "%s" has already reached its maximum number of instances (%d)' % (job.id, job.max_instances))
+            'Job "%s" has already reached its maximum number of instances (%d)' %
+            (job.id, job.max_instances))
 
 
 class BaseExecutor(six.with_metaclass(ABCMeta, object)):
@@ -30,13 +32,14 @@ class BaseExecutor(six.with_metaclass(ABCMeta, object)):
 
     def start(self, scheduler, alias):
         """
-        Called by the scheduler when the scheduler is being started or when the executor is being added to an already
+        Called by the scheduler when the scheduler is being started or when the executor is being
-        running scheduler.
+        added to an already running scheduler.
 
-        :param apscheduler.schedulers.base.BaseScheduler scheduler: the scheduler that is starting this executor
+        :param apscheduler.schedulers.base.BaseScheduler scheduler: the scheduler that is starting
+            this executor
         :param str|unicode alias: alias of this executor as it was assigned to the scheduler
-        """
 
+        """
         self._scheduler = scheduler
         self._lock = scheduler._create_lock()
         self._logger = logging.getLogger('apscheduler.executors.%s' % alias)
@@ -45,7 +48,8 @@ class BaseExecutor(six.with_metaclass(ABCMeta, object)):
         """
         Shuts down this executor.
 
-        :param bool wait: ``True`` to wait until all submitted jobs have been executed
+        :param bool wait: ``True`` to wait until all submitted jobs
+            have been executed
         """
 
     def submit_job(self, job, run_times):
@@ -53,10 +57,12 @@ class BaseExecutor(six.with_metaclass(ABCMeta, object)):
         Submits job for execution.
 
         :param Job job: job to execute
-        :param list[datetime] run_times: list of datetimes specifying when the job should have been run
+        :param list[datetime] run_times: list of datetimes specifying
-        :raises MaxInstancesReachedError: if the maximum number of allowed instances for this job has been reached
+            when the job should have been run
-        """
+        :raises MaxInstancesReachedError: if the maximum number of
+            allowed instances for this job has been reached
 
+        """
         assert self._lock is not None, 'This executor has not been started yet'
         with self._lock:
             if self._instances[job.id] >= job.max_instances:
@@ -70,50 +76,71 @@ class BaseExecutor(six.with_metaclass(ABCMeta, object)):
         """Performs the actual task of scheduling `run_job` to be called."""
 
     def _run_job_success(self, job_id, events):
-        """Called by the executor with the list of generated events when `run_job` has been successfully called."""
+        """
+        Called by the executor with the list of generated events when :func:`run_job` has been
+        successfully called.
 
+        """
         with self._lock:
             self._instances[job_id] -= 1
+            if self._instances[job_id] == 0:
+                del self._instances[job_id]
 
         for event in events:
             self._scheduler._dispatch_event(event)
 
     def _run_job_error(self, job_id, exc, traceback=None):
-        """Called by the executor with the exception if there is an error calling `run_job`."""
+        """Called by the executor with the exception if there is an error  calling `run_job`."""
-
         with self._lock:
             self._instances[job_id] -= 1
+            if self._instances[job_id] == 0:
+                del self._instances[job_id]
 
         exc_info = (exc.__class__, exc, traceback)
         self._logger.error('Error running job %s', job_id, exc_info=exc_info)
 
 
 def run_job(job, jobstore_alias, run_times, logger_name):
-    """Called by executors to run the job. Returns a list of scheduler events to be dispatched by the scheduler."""
+    """
+    Called by executors to run the job. Returns a list of scheduler events to be dispatched by the
+    scheduler.
 
+    """
     events = []
     logger = logging.getLogger(logger_name)
     for run_time in run_times:
-        # See if the job missed its run time window, and handle possible misfires accordingly
+        # See if the job missed its run time window, and handle
+        # possible misfires accordingly
         if job.misfire_grace_time is not None:
             difference = datetime.now(utc) - run_time
             grace_time = timedelta(seconds=job.misfire_grace_time)
             if difference > grace_time:
-                events.append(JobExecutionEvent(EVENT_JOB_MISSED, job.id, jobstore_alias, run_time))
+                events.append(JobExecutionEvent(EVENT_JOB_MISSED, job.id, jobstore_alias,
+                                                run_time))
                 logger.warning('Run time of job "%s" was missed by %s', job, difference)
                 continue
 
         logger.info('Running job "%s" (scheduled at %s)', job, run_time)
         try:
             retval = job.func(*job.args, **job.kwargs)
-        except:
+        except BaseException:
             exc, tb = sys.exc_info()[1:]
             formatted_tb = ''.join(format_tb(tb))
-            events.append(JobExecutionEvent(EVENT_JOB_ERROR, job.id, jobstore_alias, run_time, exception=exc,
+            events.append(JobExecutionEvent(EVENT_JOB_ERROR, job.id, jobstore_alias, run_time,
-                                            traceback=formatted_tb))
+                                            exception=exc, traceback=formatted_tb))
             logger.exception('Job "%s" raised an exception', job)
+
+            # This is to prevent cyclic references that would lead to memory leaks
+            if six.PY2:
+                sys.exc_clear()
+                del tb
+            else:
+                import traceback
+                traceback.clear_frames(tb)
+                del tb
         else:
-            events.append(JobExecutionEvent(EVENT_JOB_EXECUTED, job.id, jobstore_alias, run_time, retval=retval))
+            events.append(JobExecutionEvent(EVENT_JOB_EXECUTED, job.id, jobstore_alias, run_time,
+                                            retval=retval))
             logger.info('Job "%s" executed successfully', job)
 
     return events

lib/apscheduler/executors/base_py3.py

@@ -0,0 +1,43 @@
+import logging
+import sys
+import traceback
+from datetime import datetime, timedelta
+from traceback import format_tb
+
+from pytz import utc
+
+from apscheduler.events import (
+    JobExecutionEvent, EVENT_JOB_MISSED, EVENT_JOB_ERROR, EVENT_JOB_EXECUTED)
+
+
+async def run_coroutine_job(job, jobstore_alias, run_times, logger_name):
+    """Coroutine version of run_job()."""
+    events = []
+    logger = logging.getLogger(logger_name)
+    for run_time in run_times:
+        # See if the job missed its run time window, and handle possible misfires accordingly
+        if job.misfire_grace_time is not None:
+            difference = datetime.now(utc) - run_time
+            grace_time = timedelta(seconds=job.misfire_grace_time)
+            if difference > grace_time:
+                events.append(JobExecutionEvent(EVENT_JOB_MISSED, job.id, jobstore_alias,
+                                                run_time))
+                logger.warning('Run time of job "%s" was missed by %s', job, difference)
+                continue
+
+        logger.info('Running job "%s" (scheduled at %s)', job, run_time)
+        try:
+            retval = await job.func(*job.args, **job.kwargs)
+        except BaseException:
+            exc, tb = sys.exc_info()[1:]
+            formatted_tb = ''.join(format_tb(tb))
+            events.append(JobExecutionEvent(EVENT_JOB_ERROR, job.id, jobstore_alias, run_time,
+                                            exception=exc, traceback=formatted_tb))
+            logger.exception('Job "%s" raised an exception', job)
+            traceback.clear_frames(tb)
+        else:
+            events.append(JobExecutionEvent(EVENT_JOB_EXECUTED, job.id, jobstore_alias, run_time,
+                                            retval=retval))
+            logger.info('Job "%s" executed successfully', job)
+
+    return events

lib/apscheduler/executors/debug.py

@@ -5,7 +5,8 @@ from apscheduler.executors.base import BaseExecutor, run_job
 
 class DebugExecutor(BaseExecutor):
     """
-    A special executor that executes the target callable directly instead of deferring it to a thread or process.
+    A special executor that executes the target callable directly instead of deferring it to a
+    thread or process.
 
     Plugin alias: ``debug``
     """
@@ -13,7 +14,7 @@ class DebugExecutor(BaseExecutor):
     def _do_submit_job(self, job, run_times):
         try:
             events = run_job(job, job._jobstore_alias, run_times, self._logger.name)
-        except:
+        except BaseException:
             self._run_job_error(job.id, *sys.exc_info()[1:])
         else:
             self._run_job_success(job.id, events)

lib/apscheduler/executors/gevent.py

@@ -1,4 +1,4 @@
-
+from __future__ import absolute_import
 import sys
 
 from apscheduler.executors.base import BaseExecutor, run_job
@@ -21,9 +21,10 @@ class GeventExecutor(BaseExecutor):
         def callback(greenlet):
             try:
                 events = greenlet.get()
-            except:
+            except BaseException:
                 self._run_job_error(job.id, *sys.exc_info()[1:])
             else:
                 self._run_job_success(job.id, events)
 
-        gevent.spawn(run_job, job, job._jobstore_alias, run_times, self._logger.name).link(callback)
+        gevent.spawn(run_job, job, job._jobstore_alias, run_times, self._logger.name).\
+            link(callback)

lib/apscheduler/executors/pool.py

@@ -3,6 +3,11 @@ import concurrent.futures
 
 from apscheduler.executors.base import BaseExecutor, run_job
 
+try:
+    from concurrent.futures.process import BrokenProcessPool
+except ImportError:
+    BrokenProcessPool = None
+
 
 class BasePoolExecutor(BaseExecutor):
     @abstractmethod
@@ -19,7 +24,13 @@ class BasePoolExecutor(BaseExecutor):
             else:
                 self._run_job_success(job.id, f.result())
 
-        f = self._pool.submit(run_job, job, job._jobstore_alias, run_times, self._logger.name)
+        try:
+            f = self._pool.submit(run_job, job, job._jobstore_alias, run_times, self._logger.name)
+        except BrokenProcessPool:
+            self._logger.warning('Process pool is broken; replacing pool with a fresh instance')
+            self._pool = self._pool.__class__(self._pool._max_workers)
+            f = self._pool.submit(run_job, job, job._jobstore_alias, run_times, self._logger.name)
+
         f.add_done_callback(callback)
 
     def shutdown(self, wait=True):
@@ -33,10 +44,13 @@ class ThreadPoolExecutor(BasePoolExecutor):
     Plugin alias: ``threadpool``
 
     :param max_workers: the maximum number of spawned threads.
+    :param pool_kwargs: dict of keyword arguments to pass to the underlying
+        ThreadPoolExecutor constructor
     """
 
-    def __init__(self, max_workers=10):
+    def __init__(self, max_workers=10, pool_kwargs=None):
-        pool = concurrent.futures.ThreadPoolExecutor(int(max_workers))
+        pool_kwargs = pool_kwargs or {}
+        pool = concurrent.futures.ThreadPoolExecutor(int(max_workers), **pool_kwargs)
         super(ThreadPoolExecutor, self).__init__(pool)
 
 
@@ -47,8 +61,11 @@ class ProcessPoolExecutor(BasePoolExecutor):
     Plugin alias: ``processpool``
 
     :param max_workers: the maximum number of spawned processes.
+    :param pool_kwargs: dict of keyword arguments to pass to the underlying
+        ProcessPoolExecutor constructor
     """
 
-    def __init__(self, max_workers=10):
+    def __init__(self, max_workers=10, pool_kwargs=None):
-        pool = concurrent.futures.ProcessPoolExecutor(int(max_workers))
+        pool_kwargs = pool_kwargs or {}
+        pool = concurrent.futures.ProcessPoolExecutor(int(max_workers), **pool_kwargs)
         super(ProcessPoolExecutor, self).__init__(pool)

lib/apscheduler/executors/tornado.py

@@ -0,0 +1,54 @@
+from __future__ import absolute_import
+
+import sys
+from concurrent.futures import ThreadPoolExecutor
+
+from tornado.gen import convert_yielded
+
+from apscheduler.executors.base import BaseExecutor, run_job
+
+try:
+    from apscheduler.executors.base_py3 import run_coroutine_job
+    from apscheduler.util import iscoroutinefunction_partial
+except ImportError:
+    def iscoroutinefunction_partial(func):
+        return False
+
+
+class TornadoExecutor(BaseExecutor):
+    """
+    Runs jobs either in a thread pool or directly on the I/O loop.
+
+    If the job function is a native coroutine function, it is scheduled to be run directly in the
+    I/O loop as soon as possible. All other functions are run in a thread pool.
+
+    Plugin alias: ``tornado``
+
+    :param int max_workers: maximum number of worker threads in the thread pool
+    """
+
+    def __init__(self, max_workers=10):
+        super(TornadoExecutor, self).__init__()
+        self.executor = ThreadPoolExecutor(max_workers)
+
+    def start(self, scheduler, alias):
+        super(TornadoExecutor, self).start(scheduler, alias)
+        self._ioloop = scheduler._ioloop
+
+    def _do_submit_job(self, job, run_times):
+        def callback(f):
+            try:
+                events = f.result()
+            except BaseException:
+                self._run_job_error(job.id, *sys.exc_info()[1:])
+            else:
+                self._run_job_success(job.id, events)
+
+        if iscoroutinefunction_partial(job.func):
+            f = run_coroutine_job(job, job._jobstore_alias, run_times, self._logger.name)
+        else:
+            f = self.executor.submit(run_job, job, job._jobstore_alias, run_times,
+                                     self._logger.name)
+
+        f = convert_yielded(f)
+        f.add_done_callback(callback)

lib/apscheduler/executors/twisted.py

@@ -1,4 +1,4 @@
-
+from __future__ import absolute_import
 
 from apscheduler.executors.base import BaseExecutor, run_job
 
@@ -21,5 +21,5 @@ class TwistedExecutor(BaseExecutor):
             else:
                 self._run_job_error(job.id, result.value, result.tb)
 
-        self._reactor.getThreadPool().callInThreadWithCallback(callback, run_job, job, job._jobstore_alias, run_times,
+        self._reactor.getThreadPool().callInThreadWithCallback(
-                                                               self._logger.name)
+            callback, run_job, job, job._jobstore_alias, run_times, self._logger.name)

lib/apscheduler/job.py

@@ -1,11 +1,17 @@
-from collections.abc import Iterable, Mapping
+from inspect import ismethod, isclass
 from uuid import uuid4
 
 import six
 
 from apscheduler.triggers.base import BaseTrigger
-from apscheduler.util import ref_to_obj, obj_to_ref, datetime_repr, repr_escape, get_callable_name, check_callable_args, \
+from apscheduler.util import (
-    convert_to_datetime
+    ref_to_obj, obj_to_ref, datetime_repr, repr_escape, get_callable_name, check_callable_args,
+    convert_to_datetime)
+
+try:
+    from collections.abc import Iterable, Mapping
+except ImportError:
+    from collections import Iterable, Mapping
 
 
 class Job(object):
@@ -21,13 +27,20 @@ class Job(object):
     :var bool coalesce: whether to only run the job once when several run times are due
     :var trigger: the trigger object that controls the schedule of this job
     :var str executor: the name of the executor that will run this job
-    :var int misfire_grace_time: the time (in seconds) how much this job's execution is allowed to be late
+    :var int misfire_grace_time: the time (in seconds) how much this job's execution is allowed to
-    :var int max_instances: the maximum number of concurrently executing instances allowed for this job
+        be late (``None`` means "allow the job to run no matter how late it is")
+    :var int max_instances: the maximum number of concurrently executing instances allowed for this
+        job
     :var datetime.datetime next_run_time: the next scheduled run time of this job
+
+    .. note::
+        The ``misfire_grace_time`` has some non-obvious effects on job execution. See the
+        :ref:`missed-job-executions` section in the documentation for an in-depth explanation.
     """
 
-    __slots__ = ('_scheduler', '_jobstore_alias', 'id', 'trigger', 'executor', 'func', 'func_ref', 'args', 'kwargs',
+    __slots__ = ('_scheduler', '_jobstore_alias', 'id', 'trigger', 'executor', 'func', 'func_ref',
-                 'name', 'misfire_grace_time', 'coalesce', 'max_instances', 'next_run_time')
+                 'args', 'kwargs', 'name', 'misfire_grace_time', 'coalesce', 'max_instances',
+                 'next_run_time', '__weakref__')
 
     def __init__(self, scheduler, id=None, **kwargs):
         super(Job, self).__init__()
@@ -38,53 +51,69 @@ class Job(object):
     def modify(self, **changes):
         """
         Makes the given changes to this job and saves it in the associated job store.
+
         Accepted keyword arguments are the same as the variables on this class.
 
         .. seealso:: :meth:`~apscheduler.schedulers.base.BaseScheduler.modify_job`
-        """
 
+        :return Job: this job instance
+
+        """
         self._scheduler.modify_job(self.id, self._jobstore_alias, **changes)
+        return self
 
     def reschedule(self, trigger, **trigger_args):
         """
         Shortcut for switching the trigger on this job.
 
         .. seealso:: :meth:`~apscheduler.schedulers.base.BaseScheduler.reschedule_job`
-        """
 
+        :return Job: this job instance
+
+        """
         self._scheduler.reschedule_job(self.id, self._jobstore_alias, trigger, **trigger_args)
+        return self
 
     def pause(self):
         """
         Temporarily suspend the execution of this job.
 
         .. seealso:: :meth:`~apscheduler.schedulers.base.BaseScheduler.pause_job`
-        """
 
+        :return Job: this job instance
+
+        """
         self._scheduler.pause_job(self.id, self._jobstore_alias)
+        return self
 
     def resume(self):
         """
         Resume the schedule of this job if previously paused.
 
         .. seealso:: :meth:`~apscheduler.schedulers.base.BaseScheduler.resume_job`
-        """
 
+        :return Job: this job instance
+
+        """
         self._scheduler.resume_job(self.id, self._jobstore_alias)
+        return self
 
     def remove(self):
         """
         Unschedules this job and removes it from its associated job store.
 
         .. seealso:: :meth:`~apscheduler.schedulers.base.BaseScheduler.remove_job`
-        """
 
+        """
         self._scheduler.remove_job(self.id, self._jobstore_alias)
 
     @property
     def pending(self):
-        """Returns ``True`` if the referenced job is still waiting to be added to its designated job store."""
+        """
+        Returns ``True`` if the referenced job is still waiting to be added to its designated job
+        store.
 
+        """
         return self._jobstore_alias is None
 
     #
@@ -97,8 +126,8 @@ class Job(object):
 
         :type now: datetime.datetime
         :rtype: list[datetime.datetime]
-        """
 
+        """
         run_times = []
         next_run_time = self.next_run_time
         while next_run_time and next_run_time <= now:
@@ -108,8 +137,11 @@ class Job(object):
         return run_times
 
     def _modify(self, **changes):
-        """Validates the changes to the Job and makes the modifications if and only if all of them validate."""
+        """
+        Validates the changes to the Job and makes the modifications if and only if all of them
+        validate.
 
+        """
         approved = {}
 
         if 'id' in changes:
@@ -125,7 +157,7 @@ class Job(object):
             args = changes.pop('args') if 'args' in changes else self.args
             kwargs = changes.pop('kwargs') if 'kwargs' in changes else self.kwargs
 
-            if isinstance(func, str):
+            if isinstance(func, six.string_types):
                 func_ref = func
                 func = ref_to_obj(func)
             elif callable(func):
@@ -177,7 +209,8 @@ class Job(object):
         if 'trigger' in changes:
             trigger = changes.pop('trigger')
             if not isinstance(trigger, BaseTrigger):
-                raise TypeError('Expected a trigger instance, got %s instead' % trigger.__class__.__name__)
+                raise TypeError('Expected a trigger instance, got %s instead' %
+                                trigger.__class__.__name__)
 
             approved['trigger'] = trigger
 
@@ -189,10 +222,12 @@ class Job(object):
 
         if 'next_run_time' in changes:
             value = changes.pop('next_run_time')
-            approved['next_run_time'] = convert_to_datetime(value, self._scheduler.timezone, 'next_run_time')
+            approved['next_run_time'] = convert_to_datetime(value, self._scheduler.timezone,
+                                                            'next_run_time')
 
         if changes:
-            raise AttributeError('The following are not modifiable attributes of Job: %s' % ', '.join(changes))
+            raise AttributeError('The following are not modifiable attributes of Job: %s' %
+                                 ', '.join(changes))
 
         for key, value in six.iteritems(approved):
             setattr(self, key, value)
@@ -200,9 +235,18 @@ class Job(object):
     def __getstate__(self):
         # Don't allow this Job to be serialized if the function reference could not be determined
         if not self.func_ref:
-            raise ValueError('This Job cannot be serialized since the reference to its callable (%r) could not be '
+            raise ValueError(
-                             'determined. Consider giving a textual reference (module:function name) instead.' %
+                'This Job cannot be serialized since the reference to its callable (%r) could not '
-                             (self.func,))
+                'be determined. Consider giving a textual reference (module:function name) '
+                'instead.' % (self.func,))
+
+        # Instance methods cannot survive serialization as-is, so store the "self" argument
+        # explicitly
+        func = self.func
+        if ismethod(func) and not isclass(func.__self__) and obj_to_ref(func) == self.func_ref:
+            args = (func.__self__,) + tuple(self.args)
+        else:
+            args = self.args
 
         return {
             'version': 1,
@@ -210,7 +254,7 @@ class Job(object):
             'func': self.func_ref,
             'trigger': self.trigger,
             'executor': self.executor,
-            'args': self.args,
+            'args': args,
             'kwargs': self.kwargs,
             'name': self.name,
             'misfire_grace_time': self.misfire_grace_time,
@@ -221,7 +265,8 @@ class Job(object):
 
     def __setstate__(self, state):
         if state.get('version', 1) > 1:
-            raise ValueError('Job has version %s, but only version 1 can be handled' % state['version'])
+            raise ValueError('Job has version %s, but only version 1 can be handled' %
+                             state['version'])
 
         self.id = state['id']
         self.func_ref = state['func']
@@ -245,8 +290,13 @@ class Job(object):
         return '<Job (id=%s name=%s)>' % (repr_escape(self.id), repr_escape(self.name))
 
     def __str__(self):
-        return '%s (trigger: %s, next run at: %s)' % (repr_escape(self.name), repr_escape(str(self.trigger)),
+        return repr_escape(self.__unicode__())
-                                                      datetime_repr(self.next_run_time))
 
     def __unicode__(self):
-        return six.u('%s (trigger: %s, next run at: %s)') % (self.name, self.trigger, datetime_repr(self.next_run_time))
+        if hasattr(self, 'next_run_time'):
+            status = ('next run at: ' + datetime_repr(self.next_run_time) if
+                      self.next_run_time else 'paused')
+        else:
+            status = 'pending'
+
+        return u'%s (trigger: %s, %s)' % (self.name, self.trigger, status)

lib/apscheduler/jobstores/base.py

@@ -8,23 +8,27 @@ class JobLookupError(KeyError):
     """Raised when the job store cannot find a job for update or removal."""
 
     def __init__(self, job_id):
-        super(JobLookupError, self).__init__(six.u('No job by the id of %s was found') % job_id)
+        super(JobLookupError, self).__init__(u'No job by the id of %s was found' % job_id)
 
 
 class ConflictingIdError(KeyError):
     """Raised when the uniqueness of job IDs is being violated."""
 
     def __init__(self, job_id):
-        super(ConflictingIdError, self).__init__(six.u('Job identifier (%s) conflicts with an existing job') % job_id)
+        super(ConflictingIdError, self).__init__(
+            u'Job identifier (%s) conflicts with an existing job' % job_id)
 
 
 class TransientJobError(ValueError):
-    """Raised when an attempt to add transient (with no func_ref) job to a persistent job store is detected."""
+    """
+    Raised when an attempt to add transient (with no func_ref) job to a persistent job store is
+    detected.
+    """
 
     def __init__(self, job_id):
         super(TransientJobError, self).__init__(
-            six.u('Job (%s) cannot be added to this job store because a reference to the callable could not be '
+            u'Job (%s) cannot be added to this job store because a reference to the callable '
-                  'determined.') % job_id)
+            u'could not be determined.' % job_id)
 
 
 class BaseJobStore(six.with_metaclass(ABCMeta)):
@@ -36,10 +40,11 @@ class BaseJobStore(six.with_metaclass(ABCMeta)):
 
     def start(self, scheduler, alias):
         """
-        Called by the scheduler when the scheduler is being started or when the job store is being added to an already
+        Called by the scheduler when the scheduler is being started or when the job store is being
-        running scheduler.
+        added to an already running scheduler.
 
-        :param apscheduler.schedulers.base.BaseScheduler scheduler: the scheduler that is starting this job store
+        :param apscheduler.schedulers.base.BaseScheduler scheduler: the scheduler that is starting
+            this job store
         :param str|unicode alias: alias of this job store as it was assigned to the scheduler
         """
 
@@ -50,13 +55,22 @@ class BaseJobStore(six.with_metaclass(ABCMeta)):
     def shutdown(self):
         """Frees any resources still bound to this job store."""
 
+    def _fix_paused_jobs_sorting(self, jobs):
+        for i, job in enumerate(jobs):
+            if job.next_run_time is not None:
+                if i > 0:
+                    paused_jobs = jobs[:i]
+                    del jobs[:i]
+                    jobs.extend(paused_jobs)
+                break
+
     @abstractmethod
     def lookup_job(self, job_id):
         """
         Returns a specific job, or ``None`` if it isn't found..
 
-        The job store is responsible for setting the ``scheduler`` and ``jobstore`` attributes of the returned job to
+        The job store is responsible for setting the ``scheduler`` and ``jobstore`` attributes of
-        point to the scheduler and itself, respectively.
+        the returned job to point to the scheduler and itself, respectively.
 
         :param str|unicode job_id: identifier of the job
         :rtype: Job
@@ -75,7 +89,8 @@ class BaseJobStore(six.with_metaclass(ABCMeta)):
     @abstractmethod
     def get_next_run_time(self):
         """
-        Returns the earliest run time of all the jobs stored in this job store, or ``None`` if there are no active jobs.
+        Returns the earliest run time of all the jobs stored in this job store, or ``None`` if
+        there are no active jobs.
 
         :rtype: datetime.datetime
         """
@@ -83,11 +98,12 @@ class BaseJobStore(six.with_metaclass(ABCMeta)):
     @abstractmethod
     def get_all_jobs(self):
         """
-        Returns a list of all jobs in this job store. The returned jobs should be sorted by next run time (ascending).
+        Returns a list of all jobs in this job store.
-        Paused jobs (next_run_time is None) should be sorted last.
+        The returned jobs should be sorted by next run time (ascending).
+        Paused jobs (next_run_time == None) should be sorted last.
 
-        The job store is responsible for setting the ``scheduler`` and ``jobstore`` attributes of the returned jobs to
+        The job store is responsible for setting the ``scheduler`` and ``jobstore`` attributes of
-        point to the scheduler and itself, respectively.
+        the returned jobs to point to the scheduler and itself, respectively.
 
         :rtype: list[Job]
         """

lib/apscheduler/jobstores/memory.py

@@ -1,4 +1,4 @@
-
+from __future__ import absolute_import
 
 from apscheduler.jobstores.base import BaseJobStore, JobLookupError, ConflictingIdError
 from apscheduler.util import datetime_to_utc_timestamp
@@ -13,7 +13,8 @@ class MemoryJobStore(BaseJobStore):
 
     def __init__(self):
         super(MemoryJobStore, self).__init__()
-        self._jobs = []  # list of (job, timestamp), sorted by next_run_time and job id (ascending)
+        # list of (job, timestamp), sorted by next_run_time and job id (ascending)
+        self._jobs = []
         self._jobs_index = {}  # id -> (job, timestamp) lookup table
 
     def lookup_job(self, job_id):
@@ -80,13 +81,13 @@ class MemoryJobStore(BaseJobStore):
 
     def _get_job_index(self, timestamp, job_id):
         """
-        Returns the index of the given job, or if it's not found, the index where the job should be inserted based on
+        Returns the index of the given job, or if it's not found, the index where the job should be
-        the given timestamp.
+        inserted based on the given timestamp.
 
         :type timestamp: int
         :type job_id: str
-        """
 
+        """
         lo, hi = 0, len(self._jobs)
         timestamp = float('inf') if timestamp is None else timestamp
         while lo < hi:

lib/apscheduler/jobstores/mongodb.py

@@ -1,11 +1,12 @@
-
+from __future__ import absolute_import
+import warnings
 
 from apscheduler.jobstores.base import BaseJobStore, JobLookupError, ConflictingIdError
 from apscheduler.util import maybe_ref, datetime_to_utc_timestamp, utc_timestamp_to_datetime
 from apscheduler.job import Job
 
 try:
-    import pickle as pickle
+    import cPickle as pickle
 except ImportError:  # pragma: nocover
     import pickle
 
@@ -19,16 +20,18 @@ except ImportError:  # pragma: nocover
 
 class MongoDBJobStore(BaseJobStore):
     """
-    Stores jobs in a MongoDB database. Any leftover keyword arguments are directly passed to pymongo's `MongoClient
+    Stores jobs in a MongoDB database. Any leftover keyword arguments are directly passed to
+    pymongo's `MongoClient
     <http://api.mongodb.org/python/current/api/pymongo/mongo_client.html#pymongo.mongo_client.MongoClient>`_.
 
     Plugin alias: ``mongodb``
 
     :param str database: database to store jobs in
     :param str collection: collection to store jobs in
-    :param client: a :class:`~pymongo.mongo_client.MongoClient` instance to use instead of providing connection
+    :param client: a :class:`~pymongo.mongo_client.MongoClient` instance to use instead of
-                   arguments
+        providing connection arguments
-    :param int pickle_protocol: pickle protocol level to use (for serialization), defaults to the highest available
+    :param int pickle_protocol: pickle protocol level to use (for serialization), defaults to the
+        highest available
     """
 
     def __init__(self, database='apscheduler', collection='jobs', client=None,
@@ -42,13 +45,22 @@ class MongoDBJobStore(BaseJobStore):
             raise ValueError('The "collection" parameter must not be empty')
 
         if client:
-            self.connection = maybe_ref(client)
+            self.client = maybe_ref(client)
         else:
             connect_args.setdefault('w', 1)
-            self.connection = MongoClient(**connect_args)
+            self.client = MongoClient(**connect_args)
 
-        self.collection = self.connection[database][collection]
+        self.collection = self.client[database][collection]
-        self.collection.ensure_index('next_run_time', sparse=True)
+
+    def start(self, scheduler, alias):
+        super(MongoDBJobStore, self).start(scheduler, alias)
+        self.collection.create_index('next_run_time', sparse=True)
+
+    @property
+    def connection(self):
+        warnings.warn('The "connection" member is deprecated -- use "client" instead',
+                      DeprecationWarning)
+        return self.client
 
     def lookup_job(self, job_id):
         document = self.collection.find_one(job_id, ['job_state'])
@@ -59,16 +71,19 @@ class MongoDBJobStore(BaseJobStore):
         return self._get_jobs({'next_run_time': {'$lte': timestamp}})
 
     def get_next_run_time(self):
-        document = self.collection.find_one({'next_run_time': {'$ne': None}}, fields=['next_run_time'],
+        document = self.collection.find_one({'next_run_time': {'$ne': None}},
+                                            projection=['next_run_time'],
                                             sort=[('next_run_time', ASCENDING)])
         return utc_timestamp_to_datetime(document['next_run_time']) if document else None
 
     def get_all_jobs(self):
-        return self._get_jobs({})
+        jobs = self._get_jobs({})
+        self._fix_paused_jobs_sorting(jobs)
+        return jobs
 
     def add_job(self, job):
         try:
-            self.collection.insert({
+            self.collection.insert_one({
                 '_id': job.id,
                 'next_run_time': datetime_to_utc_timestamp(job.next_run_time),
                 'job_state': Binary(pickle.dumps(job.__getstate__(), self.pickle_protocol))
@@ -81,20 +96,20 @@ class MongoDBJobStore(BaseJobStore):
             'next_run_time': datetime_to_utc_timestamp(job.next_run_time),
             'job_state': Binary(pickle.dumps(job.__getstate__(), self.pickle_protocol))
         }
-        result = self.collection.update({'_id': job.id}, {'$set': changes})
+        result = self.collection.update_one({'_id': job.id}, {'$set': changes})
-        if result and result['n'] == 0:
+        if result and result.matched_count == 0:
-            raise JobLookupError(id)
+            raise JobLookupError(job.id)
 
     def remove_job(self, job_id):
-        result = self.collection.remove(job_id)
+        result = self.collection.delete_one({'_id': job_id})
-        if result and result['n'] == 0:
+        if result and result.deleted_count == 0:
             raise JobLookupError(job_id)
 
     def remove_all_jobs(self):
-        self.collection.remove()
+        self.collection.delete_many({})
 
     def shutdown(self):
-        self.connection.disconnect()
+        self.client.close()
 
     def _reconstitute_job(self, job_state):
         job_state = pickle.loads(job_state)
@@ -107,18 +122,20 @@ class MongoDBJobStore(BaseJobStore):
     def _get_jobs(self, conditions):
         jobs = []
         failed_job_ids = []
-        for document in self.collection.find(conditions, ['_id', 'job_state'], sort=[('next_run_time', ASCENDING)]):
+        for document in self.collection.find(conditions, ['_id', 'job_state'],
+                                             sort=[('next_run_time', ASCENDING)]):
             try:
                 jobs.append(self._reconstitute_job(document['job_state']))
-            except:
+            except BaseException:
-                self._logger.exception('Unable to restore job "%s" -- removing it', document['_id'])
+                self._logger.exception('Unable to restore job "%s" -- removing it',
+                                       document['_id'])
                 failed_job_ids.append(document['_id'])
 
         # Remove all the jobs we failed to restore
         if failed_job_ids:
-            self.collection.remove({'_id': {'$in': failed_job_ids}})
+            self.collection.delete_many({'_id': {'$in': failed_job_ids}})
 
         return jobs
 
     def __repr__(self):
-        return '<%s (client=%s)>' % (self.__class__.__name__, self.connection)
+        return '<%s (client=%s)>' % (self.__class__.__name__, self.client)

lib/apscheduler/jobstores/redis.py

@@ -1,5 +1,7 @@
+from __future__ import absolute_import
+from datetime import datetime
 
-
+from pytz import utc
 import six
 
 from apscheduler.jobstores.base import BaseJobStore, JobLookupError, ConflictingIdError
@@ -7,26 +9,28 @@ from apscheduler.util import datetime_to_utc_timestamp, utc_timestamp_to_datetim
 from apscheduler.job import Job
 
 try:
-    import pickle as pickle
+    import cPickle as pickle
 except ImportError:  # pragma: nocover
     import pickle
 
 try:
-    from redis import StrictRedis
+    from redis import Redis
 except ImportError:  # pragma: nocover
     raise ImportError('RedisJobStore requires redis installed')
 
 
 class RedisJobStore(BaseJobStore):
     """
-    Stores jobs in a Redis database. Any leftover keyword arguments are directly passed to redis's StrictRedis.
+    Stores jobs in a Redis database. Any leftover keyword arguments are directly passed to redis's
+    :class:`~redis.StrictRedis`.
 
     Plugin alias: ``redis``
 
     :param int db: the database number to store jobs in
     :param str jobs_key: key to store jobs in
     :param str run_times_key: key to store the jobs' run times in
-    :param int pickle_protocol: pickle protocol level to use (for serialization), defaults to the highest available
+    :param int pickle_protocol: pickle protocol level to use (for serialization), defaults to the
+        highest available
     """
 
     def __init__(self, db=0, jobs_key='apscheduler.jobs', run_times_key='apscheduler.run_times',
@@ -43,7 +47,7 @@ class RedisJobStore(BaseJobStore):
         self.pickle_protocol = pickle_protocol
         self.jobs_key = jobs_key
         self.run_times_key = run_times_key
-        self.redis = StrictRedis(db=int(db), **connect_args)
+        self.redis = Redis(db=int(db), **connect_args)
 
     def lookup_job(self, job_id):
         job_state = self.redis.hget(self.jobs_key, job_id)
@@ -65,7 +69,8 @@ class RedisJobStore(BaseJobStore):
     def get_all_jobs(self):
         job_states = self.redis.hgetall(self.jobs_key)
         jobs = self._reconstitute_jobs(six.iteritems(job_states))
-        return sorted(jobs, key=lambda job: job.next_run_time)
+        paused_sort_key = datetime(9999, 12, 31, tzinfo=utc)
+        return sorted(jobs, key=lambda job: job.next_run_time or paused_sort_key)
 
     def add_job(self, job):
         if self.redis.hexists(self.jobs_key, job.id):
@@ -73,8 +78,12 @@ class RedisJobStore(BaseJobStore):
 
         with self.redis.pipeline() as pipe:
             pipe.multi()
-            pipe.hset(self.jobs_key, job.id, pickle.dumps(job.__getstate__(), self.pickle_protocol))
+            pipe.hset(self.jobs_key, job.id, pickle.dumps(job.__getstate__(),
-            pipe.zadd(self.run_times_key, datetime_to_utc_timestamp(job.next_run_time), job.id)
+                                                          self.pickle_protocol))
+            if job.next_run_time:
+                pipe.zadd(self.run_times_key,
+                          {job.id: datetime_to_utc_timestamp(job.next_run_time)})
+
             pipe.execute()
 
     def update_job(self, job):
@@ -82,11 +91,14 @@ class RedisJobStore(BaseJobStore):
             raise JobLookupError(job.id)
 
         with self.redis.pipeline() as pipe:
-            pipe.hset(self.jobs_key, job.id, pickle.dumps(job.__getstate__(), self.pickle_protocol))
+            pipe.hset(self.jobs_key, job.id, pickle.dumps(job.__getstate__(),
+                                                          self.pickle_protocol))
             if job.next_run_time:
-                pipe.zadd(self.run_times_key, datetime_to_utc_timestamp(job.next_run_time), job.id)
+                pipe.zadd(self.run_times_key,
+                          {job.id: datetime_to_utc_timestamp(job.next_run_time)})
             else:
                 pipe.zrem(self.run_times_key, job.id)
+
             pipe.execute()
 
     def remove_job(self, job_id):
@@ -121,7 +133,7 @@ class RedisJobStore(BaseJobStore):
         for job_id, job_state in job_states:
             try:
                 jobs.append(self._reconstitute_job(job_state))
-            except:
+            except BaseException:
                 self._logger.exception('Unable to restore job "%s" -- removing it', job_id)
                 failed_job_ids.append(job_id)
 

lib/apscheduler/jobstores/rethinkdb.py

@@ -0,0 +1,155 @@
+from __future__ import absolute_import
+
+from apscheduler.jobstores.base import BaseJobStore, JobLookupError, ConflictingIdError
+from apscheduler.util import maybe_ref, datetime_to_utc_timestamp, utc_timestamp_to_datetime
+from apscheduler.job import Job
+
+try:
+    import cPickle as pickle
+except ImportError:  # pragma: nocover
+    import pickle
+
+try:
+    from rethinkdb import RethinkDB
+except ImportError:  # pragma: nocover
+    raise ImportError('RethinkDBJobStore requires rethinkdb installed')
+
+
+class RethinkDBJobStore(BaseJobStore):
+    """
+    Stores jobs in a RethinkDB database. Any leftover keyword arguments are directly passed to
+    rethinkdb's `RethinkdbClient <http://www.rethinkdb.com/api/#connect>`_.
+
+    Plugin alias: ``rethinkdb``
+
+    :param str database: database to store jobs in
+    :param str collection: collection to store jobs in
+    :param client: a :class:`rethinkdb.net.Connection` instance to use instead of providing
+        connection arguments
+    :param int pickle_protocol: pickle protocol level to use (for serialization), defaults to the
+        highest available
+    """
+
+    def __init__(self, database='apscheduler', table='jobs', client=None,
+                 pickle_protocol=pickle.HIGHEST_PROTOCOL, **connect_args):
+        super(RethinkDBJobStore, self).__init__()
+
+        if not database:
+            raise ValueError('The "database" parameter must not be empty')
+        if not table:
+            raise ValueError('The "table" parameter must not be empty')
+
+        self.database = database
+        self.table_name = table
+        self.table = None
+        self.client = client
+        self.pickle_protocol = pickle_protocol
+        self.connect_args = connect_args
+        self.r = RethinkDB()
+        self.conn = None
+
+    def start(self, scheduler, alias):
+        super(RethinkDBJobStore, self).start(scheduler, alias)
+
+        if self.client:
+            self.conn = maybe_ref(self.client)
+        else:
+            self.conn = self.r.connect(db=self.database, **self.connect_args)
+
+        if self.database not in self.r.db_list().run(self.conn):
+            self.r.db_create(self.database).run(self.conn)
+
+        if self.table_name not in self.r.table_list().run(self.conn):
+            self.r.table_create(self.table_name).run(self.conn)
+
+        if 'next_run_time' not in self.r.table(self.table_name).index_list().run(self.conn):
+            self.r.table(self.table_name).index_create('next_run_time').run(self.conn)
+
+        self.table = self.r.db(self.database).table(self.table_name)
+
+    def lookup_job(self, job_id):
+        results = list(self.table.get_all(job_id).pluck('job_state').run(self.conn))
+        return self._reconstitute_job(results[0]['job_state']) if results else None
+
+    def get_due_jobs(self, now):
+        return self._get_jobs(self.r.row['next_run_time'] <= datetime_to_utc_timestamp(now))
+
+    def get_next_run_time(self):
+        results = list(
+            self.table
+            .filter(self.r.row['next_run_time'] != None)  # noqa
+            .order_by(self.r.asc('next_run_time'))
+            .map(lambda x: x['next_run_time'])
+            .limit(1)
+            .run(self.conn)
+        )
+        return utc_timestamp_to_datetime(results[0]) if results else None
+
+    def get_all_jobs(self):
+        jobs = self._get_jobs()
+        self._fix_paused_jobs_sorting(jobs)
+        return jobs
+
+    def add_job(self, job):
+        job_dict = {
+            'id': job.id,
+            'next_run_time': datetime_to_utc_timestamp(job.next_run_time),
+            'job_state': self.r.binary(pickle.dumps(job.__getstate__(), self.pickle_protocol))
+        }
+        results = self.table.insert(job_dict).run(self.conn)
+        if results['errors'] > 0:
+            raise ConflictingIdError(job.id)
+
+    def update_job(self, job):
+        changes = {
+            'next_run_time': datetime_to_utc_timestamp(job.next_run_time),
+            'job_state': self.r.binary(pickle.dumps(job.__getstate__(), self.pickle_protocol))
+        }
+        results = self.table.get_all(job.id).update(changes).run(self.conn)
+        skipped = False in map(lambda x: results[x] == 0, results.keys())
+        if results['skipped'] > 0 or results['errors'] > 0 or not skipped:
+            raise JobLookupError(job.id)
+
+    def remove_job(self, job_id):
+        results = self.table.get_all(job_id).delete().run(self.conn)
+        if results['deleted'] + results['skipped'] != 1:
+            raise JobLookupError(job_id)
+
+    def remove_all_jobs(self):
+        self.table.delete().run(self.conn)
+
+    def shutdown(self):
+        self.conn.close()
+
+    def _reconstitute_job(self, job_state):
+        job_state = pickle.loads(job_state)
+        job = Job.__new__(Job)
+        job.__setstate__(job_state)
+        job._scheduler = self._scheduler
+        job._jobstore_alias = self._alias
+        return job
+
+    def _get_jobs(self, predicate=None):
+        jobs = []
+        failed_job_ids = []
+        query = (self.table.filter(self.r.row['next_run_time'] != None).filter(predicate)  # noqa
+                 if predicate else self.table)
+        query = query.order_by('next_run_time', 'id').pluck('id', 'job_state')
+
+        for document in query.run(self.conn):
+            try:
+                jobs.append(self._reconstitute_job(document['job_state']))
+            except Exception:
+                self._logger.exception('Unable to restore job "%s" -- removing it', document['id'])
+                failed_job_ids.append(document['id'])
+
+        # Remove all the jobs we failed to restore
+        if failed_job_ids:
+            self.r.expr(failed_job_ids).for_each(
+                lambda job_id: self.table.get_all(job_id).delete()).run(self.conn)
+
+        return jobs
+
+    def __repr__(self):
+        connection = self.conn
+        return '<%s (connection=%s)>' % (self.__class__.__name__, connection)

lib/apscheduler/jobstores/sqlalchemy.py

@@ -1,38 +1,47 @@
-
+from __future__ import absolute_import
 
 from apscheduler.jobstores.base import BaseJobStore, JobLookupError, ConflictingIdError
 from apscheduler.util import maybe_ref, datetime_to_utc_timestamp, utc_timestamp_to_datetime
 from apscheduler.job import Job
 
 try:
-    import pickle as pickle
+    import cPickle as pickle
 except ImportError:  # pragma: nocover
     import pickle
 
 try:
-    from sqlalchemy import create_engine, Table, Column, MetaData, Unicode, Float, LargeBinary, select
+    from sqlalchemy import (
+        create_engine, Table, Column, MetaData, Unicode, Float, LargeBinary, select, and_)
     from sqlalchemy.exc import IntegrityError
+    from sqlalchemy.sql.expression import null
 except ImportError:  # pragma: nocover
     raise ImportError('SQLAlchemyJobStore requires SQLAlchemy installed')
 
 
 class SQLAlchemyJobStore(BaseJobStore):
     """
-    Stores jobs in a database table using SQLAlchemy. The table will be created if it doesn't exist in the database.
+    Stores jobs in a database table using SQLAlchemy.
+    The table will be created if it doesn't exist in the database.
 
     Plugin alias: ``sqlalchemy``
 
-    :param str url: connection string (see `SQLAlchemy documentation
+    :param str url: connection string (see
-                    <http://docs.sqlalchemy.org/en/latest/core/engines.html?highlight=create_engine#database-urls>`_
+        :ref:`SQLAlchemy documentation <sqlalchemy:database_urls>` on this)
-                    on this)
+    :param engine: an SQLAlchemy :class:`~sqlalchemy.engine.Engine` to use instead of creating a
-    :param engine: an SQLAlchemy Engine to use instead of creating a new one based on ``url``
+        new one based on ``url``
     :param str tablename: name of the table to store jobs in
-    :param metadata: a :class:`~sqlalchemy.MetaData` instance to use instead of creating a new one
+    :param metadata: a :class:`~sqlalchemy.schema.MetaData` instance to use instead of creating a
-    :param int pickle_protocol: pickle protocol level to use (for serialization), defaults to the highest available
+        new one
+    :param int pickle_protocol: pickle protocol level to use (for serialization), defaults to the
+        highest available
+    :param str tableschema: name of the (existing) schema in the target database where the table
+        should be
+    :param dict engine_options: keyword arguments to :func:`~sqlalchemy.create_engine`
+        (ignored if ``engine`` is given)
     """
 
     def __init__(self, url=None, engine=None, tablename='apscheduler_jobs', metadata=None,
-                 pickle_protocol=pickle.HIGHEST_PROTOCOL):
+                 pickle_protocol=pickle.HIGHEST_PROTOCOL, tableschema=None, engine_options=None):
         super(SQLAlchemyJobStore, self).__init__()
         self.pickle_protocol = pickle_protocol
         metadata = maybe_ref(metadata) or MetaData()
@@ -40,37 +49,46 @@ class SQLAlchemyJobStore(BaseJobStore):
         if engine:
             self.engine = maybe_ref(engine)
         elif url:
-            self.engine = create_engine(url)
+            self.engine = create_engine(url, **(engine_options or {}))
         else:
             raise ValueError('Need either "engine" or "url" defined')
 
-        # 191 = max key length in MySQL for InnoDB/utf8mb4 tables, 25 = precision that translates to an 8-byte float
+        # 191 = max key length in MySQL for InnoDB/utf8mb4 tables,
+        # 25 = precision that translates to an 8-byte float
         self.jobs_t = Table(
             tablename, metadata,
-            Column('id', Unicode(191, _warn_on_bytestring=False), primary_key=True),
+            Column('id', Unicode(191), primary_key=True),
             Column('next_run_time', Float(25), index=True),
-            Column('job_state', LargeBinary, nullable=False)
+            Column('job_state', LargeBinary, nullable=False),
+            schema=tableschema
         )
 
+    def start(self, scheduler, alias):
+        super(SQLAlchemyJobStore, self).start(scheduler, alias)
         self.jobs_t.create(self.engine, True)
 
     def lookup_job(self, job_id):
-        selectable = select([self.jobs_t.c.job_state]).where(self.jobs_t.c.id == job_id)
+        selectable = select(self.jobs_t.c.job_state).where(self.jobs_t.c.id == job_id)
-        job_state = self.engine.execute(selectable).scalar()
+        with self.engine.begin() as connection:
-        return self._reconstitute_job(job_state) if job_state else None
+            job_state = connection.execute(selectable).scalar()
+            return self._reconstitute_job(job_state) if job_state else None
 
     def get_due_jobs(self, now):
         timestamp = datetime_to_utc_timestamp(now)
         return self._get_jobs(self.jobs_t.c.next_run_time <= timestamp)
 
     def get_next_run_time(self):
-        selectable = select([self.jobs_t.c.next_run_time]).where(self.jobs_t.c.next_run_time != None).\
+        selectable = select(self.jobs_t.c.next_run_time).\
+            where(self.jobs_t.c.next_run_time != null()).\
             order_by(self.jobs_t.c.next_run_time).limit(1)
-        next_run_time = self.engine.execute(selectable).scalar()
+        with self.engine.begin() as connection:
-        return utc_timestamp_to_datetime(next_run_time)
+            next_run_time = connection.execute(selectable).scalar()
+            return utc_timestamp_to_datetime(next_run_time)
 
     def get_all_jobs(self):
-        return self._get_jobs()
+        jobs = self._get_jobs()
+        self._fix_paused_jobs_sorting(jobs)
+        return jobs
 
     def add_job(self, job):
         insert = self.jobs_t.insert().values(**{
@@ -78,29 +96,33 @@ class SQLAlchemyJobStore(BaseJobStore):
             'next_run_time': datetime_to_utc_timestamp(job.next_run_time),
             'job_state': pickle.dumps(job.__getstate__(), self.pickle_protocol)
         })
-        try:
+        with self.engine.begin() as connection:
-            self.engine.execute(insert)
+            try:
-        except IntegrityError:
+                connection.execute(insert)
-            raise ConflictingIdError(job.id)
+            except IntegrityError:
+                raise ConflictingIdError(job.id)
 
     def update_job(self, job):
         update = self.jobs_t.update().values(**{
             'next_run_time': datetime_to_utc_timestamp(job.next_run_time),
             'job_state': pickle.dumps(job.__getstate__(), self.pickle_protocol)
         }).where(self.jobs_t.c.id == job.id)
-        result = self.engine.execute(update)
+        with self.engine.begin() as connection:
-        if result.rowcount == 0:
+            result = connection.execute(update)
-            raise JobLookupError(id)
+            if result.rowcount == 0:
+                raise JobLookupError(job.id)
 
     def remove_job(self, job_id):
         delete = self.jobs_t.delete().where(self.jobs_t.c.id == job_id)
-        result = self.engine.execute(delete)
+        with self.engine.begin() as connection:
-        if result.rowcount == 0:
+            result = connection.execute(delete)
-            raise JobLookupError(job_id)
+            if result.rowcount == 0:
+                raise JobLookupError(job_id)
 
     def remove_all_jobs(self):
         delete = self.jobs_t.delete()
-        self.engine.execute(delete)
+        with self.engine.begin() as connection:
+            connection.execute(delete)
 
     def shutdown(self):
         self.engine.dispose()
@@ -116,20 +138,22 @@ class SQLAlchemyJobStore(BaseJobStore):
 
     def _get_jobs(self, *conditions):
         jobs = []
-        selectable = select([self.jobs_t.c.id, self.jobs_t.c.job_state]).order_by(self.jobs_t.c.next_run_time)
+        selectable = select(self.jobs_t.c.id, self.jobs_t.c.job_state).\
-        selectable = selectable.where(*conditions) if conditions else selectable
+            order_by(self.jobs_t.c.next_run_time)
+        selectable = selectable.where(and_(*conditions)) if conditions else selectable
         failed_job_ids = set()
-        for row in self.engine.execute(selectable):
+        with self.engine.begin() as connection:
-            try:
+            for row in connection.execute(selectable):
-                jobs.append(self._reconstitute_job(row.job_state))
+                try:
-            except:
+                    jobs.append(self._reconstitute_job(row.job_state))
-                self._logger.exception('Unable to restore job "%s" -- removing it', row.id)
+                except BaseException:
-                failed_job_ids.add(row.id)
+                    self._logger.exception('Unable to restore job "%s" -- removing it', row.id)
+                    failed_job_ids.add(row.id)
 
-        # Remove all the jobs we failed to restore
+            # Remove all the jobs we failed to restore
-        if failed_job_ids:
+            if failed_job_ids:
-            delete = self.jobs_t.delete().where(self.jobs_t.c.id.in_(failed_job_ids))
+                delete = self.jobs_t.delete().where(self.jobs_t.c.id.in_(failed_job_ids))
-            self.engine.execute(delete)
+                connection.execute(delete)
 
         return jobs
 

lib/apscheduler/jobstores/zookeeper.py

@@ -0,0 +1,178 @@
+from __future__ import absolute_import
+
+from datetime import datetime
+
+from pytz import utc
+from kazoo.exceptions import NoNodeError, NodeExistsError
+
+from apscheduler.jobstores.base import BaseJobStore, JobLookupError, ConflictingIdError
+from apscheduler.util import maybe_ref, datetime_to_utc_timestamp, utc_timestamp_to_datetime
+from apscheduler.job import Job
+
+try:
+    import cPickle as pickle
+except ImportError:  # pragma: nocover
+    import pickle
+
+try:
+    from kazoo.client import KazooClient
+except ImportError:  # pragma: nocover
+    raise ImportError('ZooKeeperJobStore requires Kazoo installed')
+
+
+class ZooKeeperJobStore(BaseJobStore):
+    """
+    Stores jobs in a ZooKeeper tree. Any leftover keyword arguments are directly passed to
+    kazoo's `KazooClient
+    <http://kazoo.readthedocs.io/en/latest/api/client.html>`_.
+
+    Plugin alias: ``zookeeper``
+
+    :param str path: path to store jobs in
+    :param client: a :class:`~kazoo.client.KazooClient` instance to use instead of
+        providing connection arguments
+    :param int pickle_protocol: pickle protocol level to use (for serialization), defaults to the
+        highest available
+    """
+
+    def __init__(self, path='/apscheduler', client=None, close_connection_on_exit=False,
+                 pickle_protocol=pickle.HIGHEST_PROTOCOL, **connect_args):
+        super(ZooKeeperJobStore, self).__init__()
+        self.pickle_protocol = pickle_protocol
+        self.close_connection_on_exit = close_connection_on_exit
+
+        if not path:
+            raise ValueError('The "path" parameter must not be empty')
+
+        self.path = path
+
+        if client:
+            self.client = maybe_ref(client)
+        else:
+            self.client = KazooClient(**connect_args)
+        self._ensured_path = False
+
+    def _ensure_paths(self):
+        if not self._ensured_path:
+            self.client.ensure_path(self.path)
+        self._ensured_path = True
+
+    def start(self, scheduler, alias):
+        super(ZooKeeperJobStore, self).start(scheduler, alias)
+        if not self.client.connected:
+            self.client.start()
+
+    def lookup_job(self, job_id):
+        self._ensure_paths()
+        node_path = self.path + "/" + str(job_id)
+        try:
+            content, _ = self.client.get(node_path)
+            doc = pickle.loads(content)
+            job = self._reconstitute_job(doc['job_state'])
+            return job
+        except BaseException:
+            return None
+
+    def get_due_jobs(self, now):
+        timestamp = datetime_to_utc_timestamp(now)
+        jobs = [job_def['job'] for job_def in self._get_jobs()
+                if job_def['next_run_time'] is not None and job_def['next_run_time'] <= timestamp]
+        return jobs
+
+    def get_next_run_time(self):
+        next_runs = [job_def['next_run_time'] for job_def in self._get_jobs()
+                     if job_def['next_run_time'] is not None]
+        return utc_timestamp_to_datetime(min(next_runs)) if len(next_runs) > 0 else None
+
+    def get_all_jobs(self):
+        jobs = [job_def['job'] for job_def in self._get_jobs()]
+        self._fix_paused_jobs_sorting(jobs)
+        return jobs
+
+    def add_job(self, job):
+        self._ensure_paths()
+        node_path = self.path + "/" + str(job.id)
+        value = {
+            'next_run_time': datetime_to_utc_timestamp(job.next_run_time),
+            'job_state': job.__getstate__()
+        }
+        data = pickle.dumps(value, self.pickle_protocol)
+        try:
+            self.client.create(node_path, value=data)
+        except NodeExistsError:
+            raise ConflictingIdError(job.id)
+
+    def update_job(self, job):
+        self._ensure_paths()
+        node_path = self.path + "/" + str(job.id)
+        changes = {
+            'next_run_time': datetime_to_utc_timestamp(job.next_run_time),
+            'job_state': job.__getstate__()
+        }
+        data = pickle.dumps(changes, self.pickle_protocol)
+        try:
+            self.client.set(node_path, value=data)
+        except NoNodeError:
+            raise JobLookupError(job.id)
+
+    def remove_job(self, job_id):
+        self._ensure_paths()
+        node_path = self.path + "/" + str(job_id)
+        try:
+            self.client.delete(node_path)
+        except NoNodeError:
+            raise JobLookupError(job_id)
+
+    def remove_all_jobs(self):
+        try:
+            self.client.delete(self.path, recursive=True)
+        except NoNodeError:
+            pass
+        self._ensured_path = False
+
+    def shutdown(self):
+        if self.close_connection_on_exit:
+            self.client.stop()
+            self.client.close()
+
+    def _reconstitute_job(self, job_state):
+        job_state = job_state
+        job = Job.__new__(Job)
+        job.__setstate__(job_state)
+        job._scheduler = self._scheduler
+        job._jobstore_alias = self._alias
+        return job
+
+    def _get_jobs(self):
+        self._ensure_paths()
+        jobs = []
+        failed_job_ids = []
+        all_ids = self.client.get_children(self.path)
+        for node_name in all_ids:
+            try:
+                node_path = self.path + "/" + node_name
+                content, _ = self.client.get(node_path)
+                doc = pickle.loads(content)
+                job_def = {
+                    'job_id': node_name,
+                    'next_run_time': doc['next_run_time'] if doc['next_run_time'] else None,
+                    'job_state': doc['job_state'],
+                    'job': self._reconstitute_job(doc['job_state']),
+                    'creation_time': _.ctime
+                }
+                jobs.append(job_def)
+            except BaseException:
+                self._logger.exception('Unable to restore job "%s" -- removing it' % node_name)
+                failed_job_ids.append(node_name)
+
+        # Remove all the jobs we failed to restore
+        if failed_job_ids:
+            for failed_id in failed_job_ids:
+                self.remove_job(failed_id)
+        paused_sort_key = datetime(9999, 12, 31, tzinfo=utc)
+        return sorted(jobs, key=lambda job_def: (job_def['job'].next_run_time or paused_sort_key,
+                                                 job_def['creation_time']))
+
+    def __repr__(self):
+        self._logger.exception('<%s (client=%s)>' % (self.__class__.__name__, self.client))
+        return '<%s (client=%s)>' % (self.__class__.__name__, self.client)

lib/apscheduler/schedulers/asyncio.py

@@ -1,22 +1,16 @@
-
+from __future__ import absolute_import
-from functools import wraps
+import asyncio
+from functools import wraps, partial
 
 from apscheduler.schedulers.base import BaseScheduler
 from apscheduler.util import maybe_ref
 
-try:
-    import asyncio
-except ImportError:  # pragma: nocover
-    try:
-        import trollius as asyncio
-    except ImportError:
-        raise ImportError('AsyncIOScheduler requires either Python 3.4 or the asyncio package installed')
-
 
 def run_in_event_loop(func):
     @wraps(func)
     def wrapper(self, *args, **kwargs):
-        self._eventloop.call_soon_threadsafe(func, self, *args, **kwargs)
+        wrapped = partial(func, self, *args, **kwargs)
+        self._eventloop.call_soon_threadsafe(wrapped)
     return wrapper
 
 
@@ -24,6 +18,8 @@ class AsyncIOScheduler(BaseScheduler):
     """
     A scheduler that runs on an asyncio (:pep:`3156`) event loop.
 
+    The default executor can run jobs based on native coroutines (``async def``).
+
     Extra options:
 
     ============== =============================================================
@@ -34,9 +30,11 @@ class AsyncIOScheduler(BaseScheduler):
     _eventloop = None
     _timeout = None
 
-    def start(self):
+    def start(self, paused=False):
-        super(AsyncIOScheduler, self).start()
+        if not self._eventloop:
-        self.wakeup()
+            self._eventloop = asyncio.get_event_loop()
+
+        super(AsyncIOScheduler, self).start(paused)
 
     @run_in_event_loop
     def shutdown(self, wait=True):
@@ -44,7 +42,7 @@ class AsyncIOScheduler(BaseScheduler):
         self._stop_timer()
 
     def _configure(self, config):
-        self._eventloop = maybe_ref(config.pop('event_loop', None)) or asyncio.get_event_loop()
+        self._eventloop = maybe_ref(config.pop('event_loop', None))
         super(AsyncIOScheduler, self)._configure(config)
 
     def _start_timer(self, wait_seconds):

lib/apscheduler/schedulers/background.py

@@ -1,3 +1,4 @@
+from __future__ import absolute_import
 
 from threading import Thread, Event
 
@@ -13,11 +14,12 @@ class BackgroundScheduler(BlockingScheduler):
 
     Extra options:
 
-    ========== ============================================================================================
+    ========== =============================================================================
-    ``daemon`` Set the ``daemon`` option in the background thread (defaults to ``True``,
+    ``daemon`` Set the ``daemon`` option in the background thread (defaults to ``True``, see
-               see `the documentation <https://docs.python.org/3.4/library/threading.html#thread-objects>`_
+               `the documentation
+               <https://docs.python.org/3.4/library/threading.html#thread-objects>`_
                for further details)
-    ========== ============================================================================================
+    ========== =============================================================================
     """
 
     _thread = None
@@ -26,14 +28,16 @@ class BackgroundScheduler(BlockingScheduler):
         self._daemon = asbool(config.pop('daemon', True))
         super(BackgroundScheduler, self)._configure(config)
 
-    def start(self):
+    def start(self, *args, **kwargs):
-        BaseScheduler.start(self)
+        if self._event is None or self._event.is_set():
-        self._event = Event()
+            self._event = Event()
+
+        BaseScheduler.start(self, *args, **kwargs)
         self._thread = Thread(target=self._main_loop, name='APScheduler')
         self._thread.daemon = self._daemon
         self._thread.start()
 
-    def shutdown(self, wait=True):
+    def shutdown(self, *args, **kwargs):
-        super(BackgroundScheduler, self).shutdown(wait)
+        super(BackgroundScheduler, self).shutdown(*args, **kwargs)
         self._thread.join()
         del self._thread

lib/apscheduler/schedulers/blocking.py

@@ -1,21 +1,23 @@
+from __future__ import absolute_import
 
 from threading import Event
 
-from apscheduler.schedulers.base import BaseScheduler
+from apscheduler.schedulers.base import BaseScheduler, STATE_STOPPED
+from apscheduler.util import TIMEOUT_MAX
 
 
 class BlockingScheduler(BaseScheduler):
     """
-    A scheduler that runs in the foreground (:meth:`~apscheduler.schedulers.base.BaseScheduler.start` will block).
+    A scheduler that runs in the foreground
+    (:meth:`~apscheduler.schedulers.base.BaseScheduler.start` will block).
     """
-
-    MAX_WAIT_TIME = 4294967  # Maximum value accepted by Event.wait() on Windows
-
     _event = None
 
-    def start(self):
+    def start(self, *args, **kwargs):
-        super(BlockingScheduler, self).start()
+        if self._event is None or self._event.is_set():
-        self._event = Event()
+            self._event = Event()
+
+        super(BlockingScheduler, self).start(*args, **kwargs)
         self._main_loop()
 
     def shutdown(self, wait=True):
@@ -23,10 +25,11 @@ class BlockingScheduler(BaseScheduler):
         self._event.set()
 
     def _main_loop(self):
-        while self.running:
+        wait_seconds = TIMEOUT_MAX
-            wait_seconds = self._process_jobs()
+        while self.state != STATE_STOPPED:
-            self._event.wait(wait_seconds if wait_seconds is not None else self.MAX_WAIT_TIME)
+            self._event.wait(wait_seconds)
             self._event.clear()
+            wait_seconds = self._process_jobs()
 
     def wakeup(self):
         self._event.set()

lib/apscheduler/schedulers/gevent.py

@@ -1,4 +1,4 @@
-
+from __future__ import absolute_import
 
 from apscheduler.schedulers.blocking import BlockingScheduler
 from apscheduler.schedulers.base import BaseScheduler
@@ -16,14 +16,14 @@ class GeventScheduler(BlockingScheduler):
 
     _greenlet = None
 
-    def start(self):
+    def start(self, *args, **kwargs):
-        BaseScheduler.start(self)
         self._event = Event()
+        BaseScheduler.start(self, *args, **kwargs)
         self._greenlet = gevent.spawn(self._main_loop)
         return self._greenlet
 
-    def shutdown(self, wait=True):
+    def shutdown(self, *args, **kwargs):
-        super(GeventScheduler, self).shutdown(wait)
+        super(GeventScheduler, self).shutdown(*args, **kwargs)
         self._greenlet.join()
         del self._greenlet
 

lib/apscheduler/schedulers/qt.py

@@ -1,17 +1,24 @@
-
+from __future__ import absolute_import
 
 from apscheduler.schedulers.base import BaseScheduler
 
 try:
     from PyQt5.QtCore import QObject, QTimer
-except ImportError:  # pragma: nocover
+except (ImportError, RuntimeError):  # pragma: nocover
     try:
         from PyQt4.QtCore import QObject, QTimer
     except ImportError:
         try:
-            from PySide.QtCore import QObject, QTimer  # flake8: noqa
+            from PySide6.QtCore import QObject, QTimer  # noqa
         except ImportError:
-            raise ImportError('QtScheduler requires either PyQt5, PyQt4 or PySide installed')
+            try:
+                from PySide2.QtCore import QObject, QTimer  # noqa
+            except ImportError:
+                try:
+                    from PySide.QtCore import QObject, QTimer  # noqa
+                except ImportError:
+                    raise ImportError('QtScheduler requires either PyQt5, PyQt4, PySide6, PySide2 '
+                                      'or PySide installed')
 
 
 class QtScheduler(BaseScheduler):
@@ -19,18 +26,15 @@ class QtScheduler(BaseScheduler):
 
     _timer = None
 
-    def start(self):
+    def shutdown(self, *args, **kwargs):
-        super(QtScheduler, self).start()
+        super(QtScheduler, self).shutdown(*args, **kwargs)
-        self.wakeup()
-
-    def shutdown(self, wait=True):
-        super(QtScheduler, self).shutdown(wait)
         self._stop_timer()
 
     def _start_timer(self, wait_seconds):
         self._stop_timer()
         if wait_seconds is not None:
-            self._timer = QTimer.singleShot(wait_seconds * 1000, self._process_jobs)
+            wait_time = min(int(wait_seconds * 1000), 2147483647)
+            self._timer = QTimer.singleShot(wait_time, self._process_jobs)
 
     def _stop_timer(self):
         if self._timer:

lib/apscheduler/schedulers/tornado.py

@@ -1,3 +1,4 @@
+from __future__ import absolute_import
 
 from datetime import timedelta
 from functools import wraps
@@ -22,6 +23,8 @@ class TornadoScheduler(BaseScheduler):
     """
     A scheduler that runs on a Tornado IOLoop.
 
+    The default executor can run jobs based on native coroutines (``async def``).
+
     =========== ===============================================================
     ``io_loop`` Tornado IOLoop instance to use (defaults to the global IO loop)
     =========== ===============================================================
@@ -30,10 +33,6 @@ class TornadoScheduler(BaseScheduler):
     _ioloop = None
     _timeout = None
 
-    def start(self):
-        super(TornadoScheduler, self).start()
-        self.wakeup()
-
     @run_in_ioloop
     def shutdown(self, wait=True):
         super(TornadoScheduler, self).shutdown(wait)
@@ -53,6 +52,10 @@ class TornadoScheduler(BaseScheduler):
             self._ioloop.remove_timeout(self._timeout)
             del self._timeout
 
+    def _create_default_executor(self):
+        from apscheduler.executors.tornado import TornadoExecutor
+        return TornadoExecutor()
+
     @run_in_ioloop
     def wakeup(self):
         self._stop_timer()

lib/apscheduler/schedulers/twisted.py

@@ -1,3 +1,4 @@
+from __future__ import absolute_import
 
 from functools import wraps
 
@@ -35,10 +36,6 @@ class TwistedScheduler(BaseScheduler):
         self._reactor = maybe_ref(config.pop('reactor', default_reactor))
         super(TwistedScheduler, self)._configure(config)
 
-    def start(self):
-        super(TwistedScheduler, self).start()
-        self.wakeup()
-
     @run_in_reactor
     def shutdown(self, wait=True):
         super(TwistedScheduler, self).shutdown(wait)

lib/apscheduler/triggers/base.py

@@ -1,4 +1,6 @@
 from abc import ABCMeta, abstractmethod
+from datetime import timedelta
+import random
 
 import six
 
@@ -6,11 +8,30 @@ import six
 class BaseTrigger(six.with_metaclass(ABCMeta)):
     """Abstract base class that defines the interface that every trigger must implement."""
 
+    __slots__ = ()
+
     @abstractmethod
     def get_next_fire_time(self, previous_fire_time, now):
         """
-        Returns the next datetime to fire on, If no such datetime can be calculated, returns ``None``.
+        Returns the next datetime to fire on, If no such datetime can be calculated, returns
+        ``None``.
 
         :param datetime.datetime previous_fire_time: the previous time the trigger was fired
         :param datetime.datetime now: current datetime
         """
+
+    def _apply_jitter(self, next_fire_time, jitter, now):
+        """
+        Randomize ``next_fire_time`` by adding a random value (the jitter).
+
+        :param datetime.datetime|None next_fire_time: next fire time without jitter applied. If
+            ``None``, returns ``None``.
+        :param int|None jitter: maximum number of seconds to add to ``next_fire_time``
+            (if ``None`` or ``0``, returns ``next_fire_time``)
+        :param datetime.datetime now: current datetime
+        :return datetime.datetime|None: next fire time with a jitter.
+        """
+        if next_fire_time is None or not jitter:
+            return next_fire_time
+
+        return next_fire_time + timedelta(seconds=random.uniform(0, jitter))

lib/apscheduler/triggers/combining.py

@@ -0,0 +1,95 @@
+from apscheduler.triggers.base import BaseTrigger
+from apscheduler.util import obj_to_ref, ref_to_obj
+
+
+class BaseCombiningTrigger(BaseTrigger):
+    __slots__ = ('triggers', 'jitter')
+
+    def __init__(self, triggers, jitter=None):
+        self.triggers = triggers
+        self.jitter = jitter
+
+    def __getstate__(self):
+        return {
+            'version': 1,
+            'triggers': [(obj_to_ref(trigger.__class__), trigger.__getstate__())
+                         for trigger in self.triggers],
+            'jitter': self.jitter
+        }
+
+    def __setstate__(self, state):
+        if state.get('version', 1) > 1:
+            raise ValueError(
+                'Got serialized data for version %s of %s, but only versions up to 1 can be '
+                'handled' % (state['version'], self.__class__.__name__))
+
+        self.jitter = state['jitter']
+        self.triggers = []
+        for clsref, state in state['triggers']:
+            cls = ref_to_obj(clsref)
+            trigger = cls.__new__(cls)
+            trigger.__setstate__(state)
+            self.triggers.append(trigger)
+
+    def __repr__(self):
+        return '<{}({}{})>'.format(self.__class__.__name__, self.triggers,
+                                   ', jitter={}'.format(self.jitter) if self.jitter else '')
+
+
+class AndTrigger(BaseCombiningTrigger):
+    """
+    Always returns the earliest next fire time that all the given triggers can agree on.
+    The trigger is considered to be finished when any of the given triggers has finished its
+    schedule.
+
+    Trigger alias: ``and``
+
+    :param list triggers: triggers to combine
+    :param int|None jitter: delay the job execution by ``jitter`` seconds at most
+    """
+
+    __slots__ = ()
+
+    def get_next_fire_time(self, previous_fire_time, now):
+        while True:
+            fire_times = [trigger.get_next_fire_time(previous_fire_time, now)
+                          for trigger in self.triggers]
+            if None in fire_times:
+                return None
+            elif min(fire_times) == max(fire_times):
+                return self._apply_jitter(fire_times[0], self.jitter, now)
+            else:
+                now = max(fire_times)
+
+    def __str__(self):
+        return 'and[{}]'.format(', '.join(str(trigger) for trigger in self.triggers))
+
+
+class OrTrigger(BaseCombiningTrigger):
+    """
+    Always returns the earliest next fire time produced by any of the given triggers.
+    The trigger is considered finished when all the given triggers have finished their schedules.
+
+    Trigger alias: ``or``
+
+    :param list triggers: triggers to combine
+    :param int|None jitter: delay the job execution by ``jitter`` seconds at most
+
+    .. note:: Triggers that depends on the previous fire time, such as the interval trigger, may
+        seem to behave strangely since they are always passed the previous fire time produced by
+        any of the given triggers.
+    """
+
+    __slots__ = ()
+
+    def get_next_fire_time(self, previous_fire_time, now):
+        fire_times = [trigger.get_next_fire_time(previous_fire_time, now)
+                      for trigger in self.triggers]
+        fire_times = [fire_time for fire_time in fire_times if fire_time is not None]
+        if fire_times:
+            return self._apply_jitter(min(fire_times), self.jitter, now)
+        else:
+            return None
+
+    def __str__(self):
+        return 'or[{}]'.format(', '.join(str(trigger) for trigger in self.triggers))

lib/apscheduler/triggers/cron/__init__.py

@@ -4,17 +4,20 @@ from tzlocal import get_localzone
 import six
 
 from apscheduler.triggers.base import BaseTrigger
-from apscheduler.triggers.cron.fields import BaseField, WeekField, DayOfMonthField, DayOfWeekField, DEFAULT_VALUES
+from apscheduler.triggers.cron.fields import (
-from apscheduler.util import datetime_ceil, convert_to_datetime, datetime_repr, astimezone
+    BaseField, MonthField, WeekField, DayOfMonthField, DayOfWeekField, DEFAULT_VALUES)
+from apscheduler.util import (
+    datetime_ceil, convert_to_datetime, datetime_repr, astimezone, localize, normalize)
 
 
 class CronTrigger(BaseTrigger):
     """
-    Triggers when current time matches all specified time constraints, similarly to how the UNIX cron scheduler works.
+    Triggers when current time matches all specified time constraints,
+    similarly to how the UNIX cron scheduler works.
 
     :param int|str year: 4-digit year
     :param int|str month: month (1-12)
-    :param int|str day: day of the (1-31)
+    :param int|str day: day of month (1-31)
     :param int|str week: ISO week (1-53)
     :param int|str day_of_week: number or name of weekday (0-6 or mon,tue,wed,thu,fri,sat,sun)
     :param int|str hour: hour (0-23)
@@ -22,8 +25,9 @@ class CronTrigger(BaseTrigger):
     :param int|str second: second (0-59)
     :param datetime|str start_date: earliest possible date/time to trigger on (inclusive)
     :param datetime|str end_date: latest possible date/time to trigger on (inclusive)
-    :param datetime.tzinfo|str timezone: time zone to use for the date/time calculations
+    :param datetime.tzinfo|str timezone: time zone to use for the date/time calculations (defaults
-                                         (defaults to scheduler timezone)
+        to scheduler timezone)
+    :param int|None jitter: delay the job execution by ``jitter`` seconds at most
 
     .. note:: The first weekday is always **monday**.
     """
@@ -31,7 +35,7 @@ class CronTrigger(BaseTrigger):
     FIELD_NAMES = ('year', 'month', 'day', 'week', 'day_of_week', 'hour', 'minute', 'second')
     FIELDS_MAP = {
         'year': BaseField,
-        'month': BaseField,
+        'month': MonthField,
         'week': WeekField,
         'day': DayOfMonthField,
         'day_of_week': DayOfWeekField,
@@ -40,15 +44,16 @@ class CronTrigger(BaseTrigger):
         'second': BaseField
     }
 
-    __slots__ = 'timezone', 'start_date', 'end_date', 'fields'
+    __slots__ = 'timezone', 'start_date', 'end_date', 'fields', 'jitter'
 
-    def __init__(self, year=None, month=None, day=None, week=None, day_of_week=None, hour=None, minute=None,
+    def __init__(self, year=None, month=None, day=None, week=None, day_of_week=None, hour=None,
-                 second=None, start_date=None, end_date=None, timezone=None):
+                 minute=None, second=None, start_date=None, end_date=None, timezone=None,
+                 jitter=None):
         if timezone:
             self.timezone = astimezone(timezone)
-        elif start_date and start_date.tzinfo:
+        elif isinstance(start_date, datetime) and start_date.tzinfo:
             self.timezone = start_date.tzinfo
-        elif end_date and end_date.tzinfo:
+        elif isinstance(end_date, datetime) and end_date.tzinfo:
             self.timezone = end_date.tzinfo
         else:
             self.timezone = get_localzone()
@@ -56,6 +61,8 @@ class CronTrigger(BaseTrigger):
         self.start_date = convert_to_datetime(start_date, self.timezone, 'start_date')
         self.end_date = convert_to_datetime(end_date, self.timezone, 'end_date')
 
+        self.jitter = jitter
+
         values = dict((key, value) for (key, value) in six.iteritems(locals())
                       if key in self.FIELD_NAMES and value is not None)
         self.fields = []
@@ -76,13 +83,35 @@ class CronTrigger(BaseTrigger):
             field = field_class(field_name, exprs, is_default)
             self.fields.append(field)
 
+    @classmethod
+    def from_crontab(cls, expr, timezone=None):
+        """
+        Create a :class:`~CronTrigger` from a standard crontab expression.
+
+        See https://en.wikipedia.org/wiki/Cron for more information on the format accepted here.
+
+        :param expr: minute, hour, day of month, month, day of week
+        :param datetime.tzinfo|str timezone: time zone to use for the date/time calculations (
+            defaults to scheduler timezone)
+        :return: a :class:`~CronTrigger` instance
+
+        """
+        values = expr.split()
+        if len(values) != 5:
+            raise ValueError('Wrong number of fields; got {}, expected 5'.format(len(values)))
+
+        return cls(minute=values[0], hour=values[1], day=values[2], month=values[3],
+                   day_of_week=values[4], timezone=timezone)
+
     def _increment_field_value(self, dateval, fieldnum):
         """
-        Increments the designated field and resets all less significant fields to their minimum values.
+        Increments the designated field and resets all less significant fields to their minimum
+        values.
 
         :type dateval: datetime
         :type fieldnum: int
-        :return: a tuple containing the new date, and the number of the field that was actually incremented
+        :return: a tuple containing the new date, and the number of the field that was actually
+            incremented
         :rtype: tuple
         """
 
@@ -115,7 +144,7 @@ class CronTrigger(BaseTrigger):
                     i += 1
 
         difference = datetime(**values) - dateval.replace(tzinfo=None)
-        return self.timezone.normalize(dateval + difference), fieldnum
+        return normalize(dateval + difference), fieldnum
 
     def _set_field_value(self, dateval, fieldnum, new_value):
         values = {}
@@ -128,12 +157,13 @@ class CronTrigger(BaseTrigger):
                 else:
                     values[field.name] = new_value
 
-        difference = datetime(**values) - dateval.replace(tzinfo=None)
+        return localize(datetime(**values), self.timezone)
-        return self.timezone.normalize(dateval + difference)
 
     def get_next_fire_time(self, previous_fire_time, now):
         if previous_fire_time:
-            start_date = max(now, previous_fire_time + timedelta(microseconds=1))
+            start_date = min(now, previous_fire_time + timedelta(microseconds=1))
+            if start_date == previous_fire_time:
+                start_date += timedelta(microseconds=1)
         else:
             start_date = max(now, self.start_date) if self.start_date else now
 
@@ -163,7 +193,34 @@ class CronTrigger(BaseTrigger):
                 return None
 
         if fieldnum >= 0:
-            return next_date
+            next_date = self._apply_jitter(next_date, self.jitter, now)
+            return min(next_date, self.end_date) if self.end_date else next_date
+
+    def __getstate__(self):
+        return {
+            'version': 2,
+            'timezone': self.timezone,
+            'start_date': self.start_date,
+            'end_date': self.end_date,
+            'fields': self.fields,
+            'jitter': self.jitter,
+        }
+
+    def __setstate__(self, state):
+        # This is for compatibility with APScheduler 3.0.x
+        if isinstance(state, tuple):
+            state = state[1]
+
+        if state.get('version', 1) > 2:
+            raise ValueError(
+                'Got serialized data for version %s of %s, but only versions up to 2 can be '
+                'handled' % (state['version'], self.__class__.__name__))
+
+        self.timezone = state['timezone']
+        self.start_date = state['start_date']
+        self.end_date = state['end_date']
+        self.fields = state['fields']
+        self.jitter = state.get('jitter')
 
     def __str__(self):
         options = ["%s='%s'" % (f.name, f) for f in self.fields if not f.is_default]
@@ -172,5 +229,11 @@ class CronTrigger(BaseTrigger):
     def __repr__(self):
         options = ["%s='%s'" % (f.name, f) for f in self.fields if not f.is_default]
         if self.start_date:
-            options.append("start_date='%s'" % datetime_repr(self.start_date))
+            options.append("start_date=%r" % datetime_repr(self.start_date))
-        return '<%s (%s)>' % (self.__class__.__name__, ', '.join(options))
+        if self.end_date:
+            options.append("end_date=%r" % datetime_repr(self.end_date))
+        if self.jitter:
+            options.append('jitter=%s' % self.jitter)
+
+        return "<%s (%s, timezone='%s')>" % (
+            self.__class__.__name__, ', '.join(options), self.timezone)

lib/apscheduler/triggers/cron/expressions.py

@@ -1,17 +1,16 @@
-"""
+"""This module contains the expressions applicable for CronTrigger's fields."""
-This module contains the expressions applicable for CronTrigger's fields.
-"""
 
 from calendar import monthrange
 import re
 
 from apscheduler.util import asint
 
-__all__ = ('AllExpression', 'RangeExpression', 'WeekdayRangeExpression', 'WeekdayPositionExpression',
+__all__ = ('AllExpression', 'RangeExpression', 'WeekdayRangeExpression',
-           'LastDayOfMonthExpression')
+           'WeekdayPositionExpression', 'LastDayOfMonthExpression')
 
 
 WEEKDAYS = ['mon', 'tue', 'wed', 'thu', 'fri', 'sat', 'sun']
+MONTHS = ['jan', 'feb', 'mar', 'apr', 'may', 'jun', 'jul', 'aug', 'sep', 'oct', 'nov', 'dec']
 
 
 class AllExpression(object):
@@ -22,6 +21,14 @@ class AllExpression(object):
         if self.step == 0:
             raise ValueError('Increment must be higher than 0')
 
+    def validate_range(self, field_name):
+        from apscheduler.triggers.cron.fields import MIN_VALUES, MAX_VALUES
+
+        value_range = MAX_VALUES[field_name] - MIN_VALUES[field_name]
+        if self.step and self.step > value_range:
+            raise ValueError('the step value ({}) is higher than the total range of the '
+                             'expression ({})'.format(self.step, value_range))
+
     def get_next_value(self, date, field):
         start = field.get_value(date)
         minval = field.get_min(date)
@@ -37,6 +44,9 @@ class AllExpression(object):
         if next <= maxval:
             return next
 
+    def __eq__(self, other):
+        return isinstance(other, self.__class__) and self.step == other.step
+
     def __str__(self):
         if self.step:
             return '*/%d' % self.step
@@ -51,7 +61,7 @@ class RangeExpression(AllExpression):
         r'(?P<first>\d+)(?:-(?P<last>\d+))?(?:/(?P<step>\d+))?$')
 
     def __init__(self, first, last=None, step=None):
-        AllExpression.__init__(self, step)
+        super(RangeExpression, self).__init__(step)
         first = asint(first)
         last = asint(last)
         if last is None and step is None:
@@ -61,25 +71,41 @@ class RangeExpression(AllExpression):
         self.first = first
         self.last = last
 
+    def validate_range(self, field_name):
+        from apscheduler.triggers.cron.fields import MIN_VALUES, MAX_VALUES
+
+        super(RangeExpression, self).validate_range(field_name)
+        if self.first < MIN_VALUES[field_name]:
+            raise ValueError('the first value ({}) is lower than the minimum value ({})'
+                             .format(self.first, MIN_VALUES[field_name]))
+        if self.last is not None and self.last > MAX_VALUES[field_name]:
+            raise ValueError('the last value ({}) is higher than the maximum value ({})'
+                             .format(self.last, MAX_VALUES[field_name]))
+        value_range = (self.last or MAX_VALUES[field_name]) - self.first
+        if self.step and self.step > value_range:
+            raise ValueError('the step value ({}) is higher than the total range of the '
+                             'expression ({})'.format(self.step, value_range))
+
     def get_next_value(self, date, field):
-        start = field.get_value(date)
+        startval = field.get_value(date)
         minval = field.get_min(date)
         maxval = field.get_max(date)
 
         # Apply range limits
         minval = max(minval, self.first)
-        if self.last is not None:
+        maxval = min(maxval, self.last) if self.last is not None else maxval
-            maxval = min(maxval, self.last)
+        nextval = max(minval, startval)
-        start = max(start, minval)
 
-        if not self.step:
+        # Apply the step if defined
-            next = start
+        if self.step:
-        else:
+            distance_to_next = (self.step - (nextval - minval)) % self.step
-            distance_to_next = (self.step - (start - minval)) % self.step
+            nextval += distance_to_next
-            next = start + distance_to_next
 
-        if next <= maxval:
+        return nextval if nextval <= maxval else None
-            return next
+
+    def __eq__(self, other):
+        return (isinstance(other, self.__class__) and self.first == other.first and
+                self.last == other.last)
 
     def __str__(self):
         if self.last != self.first and self.last is not None:
@@ -100,6 +126,37 @@ class RangeExpression(AllExpression):
         return "%s(%s)" % (self.__class__.__name__, ', '.join(args))
 
 
+class MonthRangeExpression(RangeExpression):
+    value_re = re.compile(r'(?P<first>[a-z]+)(?:-(?P<last>[a-z]+))?', re.IGNORECASE)
+
+    def __init__(self, first, last=None):
+        try:
+            first_num = MONTHS.index(first.lower()) + 1
+        except ValueError:
+            raise ValueError('Invalid month name "%s"' % first)
+
+        if last:
+            try:
+                last_num = MONTHS.index(last.lower()) + 1
+            except ValueError:
+                raise ValueError('Invalid month name "%s"' % last)
+        else:
+            last_num = None
+
+        super(MonthRangeExpression, self).__init__(first_num, last_num)
+
+    def __str__(self):
+        if self.last != self.first and self.last is not None:
+            return '%s-%s' % (MONTHS[self.first - 1], MONTHS[self.last - 1])
+        return MONTHS[self.first - 1]
+
+    def __repr__(self):
+        args = ["'%s'" % MONTHS[self.first]]
+        if self.last != self.first and self.last is not None:
+            args.append("'%s'" % MONTHS[self.last - 1])
+        return "%s(%s)" % (self.__class__.__name__, ', '.join(args))
+
+
 class WeekdayRangeExpression(RangeExpression):
     value_re = re.compile(r'(?P<first>[a-z]+)(?:-(?P<last>[a-z]+))?', re.IGNORECASE)
 
@@ -117,7 +174,7 @@ class WeekdayRangeExpression(RangeExpression):
         else:
             last_num = None
 
-        RangeExpression.__init__(self, first_num, last_num)
+        super(WeekdayRangeExpression, self).__init__(first_num, last_num)
 
     def __str__(self):
         if self.last != self.first and self.last is not None:
@@ -133,9 +190,11 @@ class WeekdayRangeExpression(RangeExpression):
 
 class WeekdayPositionExpression(AllExpression):
     options = ['1st', '2nd', '3rd', '4th', '5th', 'last']
-    value_re = re.compile(r'(?P<option_name>%s) +(?P<weekday_name>(?:\d+|\w+))' % '|'.join(options), re.IGNORECASE)
+    value_re = re.compile(r'(?P<option_name>%s) +(?P<weekday_name>(?:\d+|\w+))' %
+                          '|'.join(options), re.IGNORECASE)
 
     def __init__(self, option_name, weekday_name):
+        super(WeekdayPositionExpression, self).__init__(None)
         try:
             self.option_num = self.options.index(option_name.lower())
         except ValueError:
@@ -147,8 +206,7 @@ class WeekdayPositionExpression(AllExpression):
             raise ValueError('Invalid weekday name "%s"' % weekday_name)
 
     def get_next_value(self, date, field):
-        # Figure out the weekday of the month's first day and the number
+        # Figure out the weekday of the month's first day and the number of days in that month
-        # of days in that month
         first_day_wday, last_day = monthrange(date.year, date.month)
 
         # Calculate which day of the month is the first of the target weekdays
@@ -160,23 +218,28 @@ class WeekdayPositionExpression(AllExpression):
         if self.option_num < 5:
             target_day = first_hit_day + self.option_num * 7
         else:
-            target_day = first_hit_day + ((last_day - first_hit_day) / 7) * 7
+            target_day = first_hit_day + ((last_day - first_hit_day) // 7) * 7
 
         if target_day <= last_day and target_day >= date.day:
             return target_day
 
+    def __eq__(self, other):
+        return (super(WeekdayPositionExpression, self).__eq__(other) and
+                self.option_num == other.option_num and self.weekday == other.weekday)
+
     def __str__(self):
         return '%s %s' % (self.options[self.option_num], WEEKDAYS[self.weekday])
 
     def __repr__(self):
-        return "%s('%s', '%s')" % (self.__class__.__name__, self.options[self.option_num], WEEKDAYS[self.weekday])
+        return "%s('%s', '%s')" % (self.__class__.__name__, self.options[self.option_num],
+                                   WEEKDAYS[self.weekday])
 
 
 class LastDayOfMonthExpression(AllExpression):
     value_re = re.compile(r'last', re.IGNORECASE)
 
     def __init__(self):
-        pass
+        super(LastDayOfMonthExpression, self).__init__(None)
 
     def get_next_value(self, date, field):
         return monthrange(date.year, date.month)[1]

lib/apscheduler/triggers/cron/fields.py

@@ -1,22 +1,26 @@
-"""
+"""Fields represent CronTrigger options which map to :class:`~datetime.datetime` fields."""
-Fields represent CronTrigger options which map to :class:`~datetime.datetime`
-fields.
-"""
 
 from calendar import monthrange
+import re
+
+import six
 
 from apscheduler.triggers.cron.expressions import (
-    AllExpression, RangeExpression, WeekdayPositionExpression, LastDayOfMonthExpression, WeekdayRangeExpression)
+    AllExpression, RangeExpression, WeekdayPositionExpression, LastDayOfMonthExpression,
+    WeekdayRangeExpression, MonthRangeExpression)
 
 
-__all__ = ('MIN_VALUES', 'MAX_VALUES', 'DEFAULT_VALUES', 'BaseField', 'WeekField', 'DayOfMonthField', 'DayOfWeekField')
+__all__ = ('MIN_VALUES', 'MAX_VALUES', 'DEFAULT_VALUES', 'BaseField', 'WeekField',
+           'DayOfMonthField', 'DayOfWeekField')
 
 
-MIN_VALUES = {'year': 1970, 'month': 1, 'day': 1, 'week': 1, 'day_of_week': 0, 'hour': 0, 'minute': 0, 'second': 0}
+MIN_VALUES = {'year': 1970, 'month': 1, 'day': 1, 'week': 1, 'day_of_week': 0, 'hour': 0,
-MAX_VALUES = {'year': 2 ** 63, 'month': 12, 'day:': 31, 'week': 53, 'day_of_week': 6, 'hour': 23, 'minute': 59,
+              'minute': 0, 'second': 0}
-              'second': 59}
+MAX_VALUES = {'year': 9999, 'month': 12, 'day': 31, 'week': 53, 'day_of_week': 6, 'hour': 23,
-DEFAULT_VALUES = {'year': '*', 'month': 1, 'day': 1, 'week': '*', 'day_of_week': '*', 'hour': 0, 'minute': 0,
+              'minute': 59, 'second': 59}
-                  'second': 0}
+DEFAULT_VALUES = {'year': '*', 'month': 1, 'day': 1, 'week': '*', 'day_of_week': '*', 'hour': 0,
+                  'minute': 0, 'second': 0}
+SEPARATOR = re.compile(' *, *')
 
 
 class BaseField(object):
@@ -50,23 +54,29 @@ class BaseField(object):
         self.expressions = []
 
         # Split a comma-separated expression list, if any
-        exprs = str(exprs).strip()
+        for expr in SEPARATOR.split(str(exprs).strip()):
-        if ',' in exprs:
+            self.compile_expression(expr)
-            for expr in exprs.split(','):
-                self.compile_expression(expr)
-        else:
-            self.compile_expression(exprs)
 
     def compile_expression(self, expr):
         for compiler in self.COMPILERS:
             match = compiler.value_re.match(expr)
             if match:
                 compiled_expr = compiler(**match.groupdict())
+
+                try:
+                    compiled_expr.validate_range(self.name)
+                except ValueError as e:
+                    exc = ValueError('Error validating expression {!r}: {}'.format(expr, e))
+                    six.raise_from(exc, None)
+
                 self.expressions.append(compiled_expr)
                 return
 
         raise ValueError('Unrecognized expression "%s" for field "%s"' % (expr, self.name))
 
+    def __eq__(self, other):
+        return isinstance(self, self.__class__) and self.expressions == other.expressions
+
     def __str__(self):
         expr_strings = (str(e) for e in self.expressions)
         return ','.join(expr_strings)
@@ -95,3 +105,7 @@ class DayOfWeekField(BaseField):
 
     def get_value(self, dateval):
         return dateval.weekday()
+
+
+class MonthField(BaseField):
+    COMPILERS = BaseField.COMPILERS + [MonthRangeExpression]

lib/apscheduler/triggers/date.py

@@ -14,15 +14,36 @@ class DateTrigger(BaseTrigger):
     :param datetime.tzinfo|str timezone: time zone for ``run_date`` if it doesn't have one already
     """
 
-    __slots__ = 'timezone', 'run_date'
+    __slots__ = 'run_date'
 
     def __init__(self, run_date=None, timezone=None):
         timezone = astimezone(timezone) or get_localzone()
-        self.run_date = convert_to_datetime(run_date or datetime.now(), timezone, 'run_date')
+        if run_date is not None:
+            self.run_date = convert_to_datetime(run_date, timezone, 'run_date')
+        else:
+            self.run_date = datetime.now(timezone)
 
     def get_next_fire_time(self, previous_fire_time, now):
         return self.run_date if previous_fire_time is None else None
 
+    def __getstate__(self):
+        return {
+            'version': 1,
+            'run_date': self.run_date
+        }
+
+    def __setstate__(self, state):
+        # This is for compatibility with APScheduler 3.0.x
+        if isinstance(state, tuple):
+            state = state[1]
+
+        if state.get('version', 1) > 1:
+            raise ValueError(
+                'Got serialized data for version %s of %s, but only version 1 can be handled' %
+                (state['version'], self.__class__.__name__))
+
+        self.run_date = state['run_date']
+
     def __str__(self):
         return 'date[%s]' % datetime_repr(self.run_date)
 

lib/apscheduler/triggers/interval.py

@@ -4,13 +4,15 @@ from math import ceil
 from tzlocal import get_localzone
 
 from apscheduler.triggers.base import BaseTrigger
-from apscheduler.util import convert_to_datetime, timedelta_seconds, datetime_repr, astimezone
+from apscheduler.util import (
+    convert_to_datetime, normalize, timedelta_seconds, datetime_repr,
+    astimezone)
 
 
 class IntervalTrigger(BaseTrigger):
     """
-    Triggers on specified intervals, starting on ``start_date`` if specified, ``datetime.now()`` + interval
+    Triggers on specified intervals, starting on ``start_date`` if specified, ``datetime.now()`` +
-    otherwise.
+    interval otherwise.
 
     :param int weeks: number of weeks to wait
     :param int days: number of days to wait
@@ -20,12 +22,15 @@ class IntervalTrigger(BaseTrigger):
     :param datetime|str start_date: starting point for the interval calculation
     :param datetime|str end_date: latest possible date/time to trigger on
     :param datetime.tzinfo|str timezone: time zone to use for the date/time calculations
+    :param int|None jitter: delay the job execution by ``jitter`` seconds at most
     """
 
-    __slots__ = 'timezone', 'start_date', 'end_date', 'interval'
+    __slots__ = 'timezone', 'start_date', 'end_date', 'interval', 'interval_length', 'jitter'
 
-    def __init__(self, weeks=0, days=0, hours=0, minutes=0, seconds=0, start_date=None, end_date=None, timezone=None):
+    def __init__(self, weeks=0, days=0, hours=0, minutes=0, seconds=0, start_date=None,
-        self.interval = timedelta(weeks=weeks, days=days, hours=hours, minutes=minutes, seconds=seconds)
+                 end_date=None, timezone=None, jitter=None):
+        self.interval = timedelta(weeks=weeks, days=days, hours=hours, minutes=minutes,
+                                  seconds=seconds)
         self.interval_length = timedelta_seconds(self.interval)
         if self.interval_length == 0:
             self.interval = timedelta(seconds=1)
@@ -33,9 +38,9 @@ class IntervalTrigger(BaseTrigger):
 
         if timezone:
             self.timezone = astimezone(timezone)
-        elif start_date and start_date.tzinfo:
+        elif isinstance(start_date, datetime) and start_date.tzinfo:
             self.timezone = start_date.tzinfo
-        elif end_date and end_date.tzinfo:
+        elif isinstance(end_date, datetime) and end_date.tzinfo:
             self.timezone = end_date.tzinfo
         else:
             self.timezone = get_localzone()
@@ -44,6 +49,8 @@ class IntervalTrigger(BaseTrigger):
         self.start_date = convert_to_datetime(start_date, self.timezone, 'start_date')
         self.end_date = convert_to_datetime(end_date, self.timezone, 'end_date')
 
+        self.jitter = jitter
+
     def get_next_fire_time(self, previous_fire_time, now):
         if previous_fire_time:
             next_fire_time = previous_fire_time + self.interval
@@ -54,12 +61,48 @@ class IntervalTrigger(BaseTrigger):
             next_interval_num = int(ceil(timediff_seconds / self.interval_length))
             next_fire_time = self.start_date + self.interval * next_interval_num
 
+        if self.jitter is not None:
+            next_fire_time = self._apply_jitter(next_fire_time, self.jitter, now)
+
         if not self.end_date or next_fire_time <= self.end_date:
-            return self.timezone.normalize(next_fire_time)
+            return normalize(next_fire_time)
+
+    def __getstate__(self):
+        return {
+            'version': 2,
+            'timezone': self.timezone,
+            'start_date': self.start_date,
+            'end_date': self.end_date,
+            'interval': self.interval,
+            'jitter': self.jitter,
+        }
+
+    def __setstate__(self, state):
+        # This is for compatibility with APScheduler 3.0.x
+        if isinstance(state, tuple):
+            state = state[1]
+
+        if state.get('version', 1) > 2:
+            raise ValueError(
+                'Got serialized data for version %s of %s, but only versions up to 2 can be '
+                'handled' % (state['version'], self.__class__.__name__))
+
+        self.timezone = state['timezone']
+        self.start_date = state['start_date']
+        self.end_date = state['end_date']
+        self.interval = state['interval']
+        self.interval_length = timedelta_seconds(self.interval)
+        self.jitter = state.get('jitter')
 
     def __str__(self):
         return 'interval[%s]' % str(self.interval)
 
     def __repr__(self):
-        return "<%s (interval=%r, start_date='%s')>" % (self.__class__.__name__, self.interval,
+        options = ['interval=%r' % self.interval, 'start_date=%r' % datetime_repr(self.start_date)]
-                                                        datetime_repr(self.start_date))
+        if self.end_date:
+            options.append("end_date=%r" % datetime_repr(self.end_date))
+        if self.jitter:
+            options.append('jitter=%s' % self.jitter)
+
+        return "<%s (%s, timezone='%s')>" % (
+            self.__class__.__name__, ', '.join(options), self.timezone)

lib/apscheduler/util.py

@@ -1,29 +1,36 @@
 """This module contains several handy functions primarily meant for internal use."""
 
+from __future__ import division
 
+from asyncio import iscoroutinefunction
 from datetime import date, datetime, time, timedelta, tzinfo
-from inspect import isfunction, ismethod, getargspec
 from calendar import timegm
+from functools import partial
+from inspect import isclass, ismethod
 import re
+import sys
 
-from pytz import timezone, utc
+from pytz import timezone, utc, FixedOffset
 import six
 
 try:
     from inspect import signature
 except ImportError:  # pragma: nocover
-    try:
+    from funcsigs import signature
-        from funcsigs import signature
+
-    except ImportError:
+try:
-        signature = None
+    from threading import TIMEOUT_MAX
+except ImportError:
+    TIMEOUT_MAX = 4294967  # Maximum value accepted by Event.wait() on Windows
 
 __all__ = ('asint', 'asbool', 'astimezone', 'convert_to_datetime', 'datetime_to_utc_timestamp',
-           'utc_timestamp_to_datetime', 'timedelta_seconds', 'datetime_ceil', 'get_callable_name', 'obj_to_ref',
+           'utc_timestamp_to_datetime', 'timedelta_seconds', 'datetime_ceil', 'get_callable_name',
-           'ref_to_obj', 'maybe_ref', 'repr_escape', 'check_callable_args')
+           'obj_to_ref', 'ref_to_obj', 'maybe_ref', 'repr_escape', 'check_callable_args',
+           'normalize', 'localize', 'TIMEOUT_MAX')
 
 
 class _Undefined(object):
-    def __bool__(self):
+    def __nonzero__(self):
         return False
 
     def __bool__(self):
@@ -32,17 +39,18 @@ class _Undefined(object):
     def __repr__(self):
         return '<undefined>'
 
+
 undefined = _Undefined()  #: a unique object that only signifies that no value is defined
 
 
 def asint(text):
     """
-    Safely converts a string to an integer, returning None if the string is None.
+    Safely converts a string to an integer, returning ``None`` if the string is ``None``.
 
     :type text: str
     :rtype: int
-    """
 
+    """
     if text is not None:
         return int(text)
 
@@ -52,8 +60,8 @@ def asbool(obj):
     Interprets an object as a boolean value.
 
     :rtype: bool
-    """
 
+    """
     if isinstance(obj, str):
         obj = obj.strip().lower()
         if obj in ('true', 'yes', 'on', 'y', 't', '1'):
@@ -69,15 +77,17 @@ def astimezone(obj):
     Interprets an object as a timezone.
 
     :rtype: tzinfo
-    """
 
+    """
     if isinstance(obj, six.string_types):
         return timezone(obj)
     if isinstance(obj, tzinfo):
-        if not hasattr(obj, 'localize') or not hasattr(obj, 'normalize'):
+        if obj.tzname(None) == 'local':
-            raise TypeError('Only timezones from the pytz library are supported')
+            raise ValueError(
-        if obj.zone == 'local':
+                'Unable to determine the name of the local timezone -- you must explicitly '
-            raise ValueError('Unable to determine the name of the local timezone -- use an explicit timezone instead')
+                'specify the name of the local timezone. Please refrain from using timezones like '
+                'EST to prevent problems with daylight saving time. Instead, use a locale based '
+                'timezone name (such as Europe/Helsinki).')
         return obj
     if obj is not None:
         raise TypeError('Expected tzinfo, got %s instead' % obj.__class__.__name__)
@@ -85,27 +95,30 @@ def astimezone(obj):
 
 _DATE_REGEX = re.compile(
     r'(?P<year>\d{4})-(?P<month>\d{1,2})-(?P<day>\d{1,2})'
-    r'(?: (?P<hour>\d{1,2}):(?P<minute>\d{1,2}):(?P<second>\d{1,2})'
+    r'(?:[ T](?P<hour>\d{1,2}):(?P<minute>\d{1,2}):(?P<second>\d{1,2})'
-    r'(?:\.(?P<microsecond>\d{1,6}))?)?')
+    r'(?:\.(?P<microsecond>\d{1,6}))?'
+    r'(?P<timezone>Z|[+-]\d\d:\d\d)?)?$')
 
 
 def convert_to_datetime(input, tz, arg_name):
     """
     Converts the given object to a timezone aware datetime object.
+
     If a timezone aware datetime object is passed, it is returned unmodified.
     If a native datetime object is passed, it is given the specified timezone.
     If the input is a string, it is parsed as a datetime with the given timezone.
 
-    Date strings are accepted in three different forms: date only (Y-m-d),
+    Date strings are accepted in three different forms: date only (Y-m-d), date with time
-    date with time (Y-m-d H:M:S) or with date+time with microseconds
+    (Y-m-d H:M:S) or with date+time with microseconds (Y-m-d H:M:S.micro). Additionally you can
-    (Y-m-d H:M:S.micro).
+    override the time zone by giving a specific offset in the format specified by ISO 8601:
+    Z (UTC), +HH:MM or -HH:MM.
 
     :param str|datetime input: the datetime or string to convert to a timezone aware datetime
     :param datetime.tzinfo tz: timezone to interpret ``input`` in
     :param str arg_name: the name of the argument (used in an error message)
     :rtype: datetime
-    """
 
+    """
     if input is None:
         return
     elif isinstance(input, datetime):
@@ -116,8 +129,17 @@ def convert_to_datetime(input, tz, arg_name):
         m = _DATE_REGEX.match(input)
         if not m:
             raise ValueError('Invalid date string')
-        values = [(k, int(v or 0)) for k, v in list(m.groupdict().items())]
+
-        values = dict(values)
+        values = m.groupdict()
+        tzname = values.pop('timezone')
+        if tzname == 'Z':
+            tz = utc
+        elif tzname:
+            hours, minutes = (int(x) for x in tzname[1:].split(':'))
+            sign = 1 if tzname[0] == '+' else -1
+            tz = FixedOffset(sign * (hours * 60 + minutes))
+
+        values = {k: int(v or 0) for k, v in values.items()}
         datetime_ = datetime(**values)
     else:
         raise TypeError('Unsupported type for %s: %s' % (arg_name, input.__class__.__name__))
@@ -125,14 +147,12 @@ def convert_to_datetime(input, tz, arg_name):
     if datetime_.tzinfo is not None:
         return datetime_
     if tz is None:
-        raise ValueError('The "tz" argument must be specified if %s has no timezone information' % arg_name)
+        raise ValueError(
+            'The "tz" argument must be specified if %s has no timezone information' % arg_name)
     if isinstance(tz, six.string_types):
         tz = timezone(tz)
 
-    try:
+    return localize(datetime_, tz)
-        return tz.localize(datetime_, is_dst=None)
-    except AttributeError:
-        raise TypeError('Only pytz timezones are supported (need the localize() and normalize() methods)')
 
 
 def datetime_to_utc_timestamp(timeval):
@@ -141,8 +161,8 @@ def datetime_to_utc_timestamp(timeval):
 
     :type timeval: datetime
     :rtype: float
-    """
 
+    """
     if timeval is not None:
         return timegm(timeval.utctimetuple()) + timeval.microsecond / 1000000
 
@@ -153,8 +173,8 @@ def utc_timestamp_to_datetime(timestamp):
 
     :type timestamp: float
     :rtype: datetime
-    """
 
+    """
     if timestamp is not None:
         return datetime.fromtimestamp(timestamp, utc)
 
@@ -165,8 +185,8 @@ def timedelta_seconds(delta):
 
     :type delta: timedelta
     :rtype: float
-    """
 
+    """
     return delta.days * 24 * 60 * 60 + delta.seconds + \
         delta.microseconds / 1000000.0
 
@@ -176,8 +196,8 @@ def datetime_ceil(dateval):
     Rounds the given datetime object upwards.
 
     :type dateval: datetime
-    """
 
+    """
     if dateval.microsecond > 0:
         return dateval + timedelta(seconds=1, microseconds=-dateval.microsecond)
     return dateval
@@ -192,8 +212,8 @@ def get_callable_name(func):
     Returns the best available display name for the given function/callable.
 
     :rtype: str
-    """
 
+    """
     # the easy case (on Python 3.3+)
     if hasattr(func, '__qualname__'):
         return func.__qualname__
@@ -201,7 +221,7 @@ def get_callable_name(func):
     # class methods, bound and unbound methods
     f_self = getattr(func, '__self__', None) or getattr(func, 'im_self', None)
     if f_self and hasattr(func, '__name__'):
-        f_class = f_self if isinstance(f_self, type) else f_self.__class__
+        f_class = f_self if isclass(f_self) else f_self.__class__
     else:
         f_class = getattr(func, 'im_class', None)
 
@@ -222,20 +242,35 @@ def get_callable_name(func):
 
 def obj_to_ref(obj):
     """
-    Returns the path to the given object.
+    Returns the path to the given callable.
 
     :rtype: str
+    :raises TypeError: if the given object is not callable
+    :raises ValueError: if the given object is a :class:`~functools.partial`, lambda or a nested
+        function
+
     """
+    if isinstance(obj, partial):
+        raise ValueError('Cannot create a reference to a partial()')
 
-    try:
+    name = get_callable_name(obj)
-        ref = '%s:%s' % (obj.__module__, get_callable_name(obj))
+    if '<lambda>' in name:
-        obj2 = ref_to_obj(ref)
+        raise ValueError('Cannot create a reference to a lambda')
-        if obj != obj2:
+    if '<locals>' in name:
-            raise ValueError
+        raise ValueError('Cannot create a reference to a nested function')
-    except Exception:
-        raise ValueError('Cannot determine the reference to %r' % obj)
 
-    return ref
+    if ismethod(obj):
+        if hasattr(obj, 'im_self') and obj.im_self:
+            # bound method
+            module = obj.im_self.__module__
+        elif hasattr(obj, 'im_class') and obj.im_class:
+            # unbound method
+            module = obj.im_class.__module__
+        else:
+            module = obj.__module__
+    else:
+        module = obj.__module__
+    return '%s:%s' % (module, name)
 
 
 def ref_to_obj(ref):
@@ -243,8 +278,8 @@ def ref_to_obj(ref):
     Returns the object pointed to by ``ref``.
 
     :type ref: str
-    """
 
+    """
     if not isinstance(ref, six.string_types):
         raise TypeError('References must be strings')
     if ':' not in ref:
@@ -252,12 +287,12 @@ def ref_to_obj(ref):
 
     modulename, rest = ref.split(':', 1)
     try:
-        obj = __import__(modulename)
+        obj = __import__(modulename, fromlist=[rest])
     except ImportError:
         raise LookupError('Error resolving reference %s: could not import module' % ref)
 
     try:
-        for name in modulename.split('.')[1:] + rest.split('.'):
+        for name in rest.split('.'):
             obj = getattr(obj, name)
         return obj
     except Exception:
@@ -268,8 +303,8 @@ def maybe_ref(ref):
     """
     Returns the object that the given reference points to, if it is indeed a reference.
     If it is not a reference, the object is returned as-is.
-    """
 
+    """
     if not isinstance(ref, str):
         return ref
     return ref_to_obj(ref)
@@ -281,7 +316,8 @@ if six.PY2:
             return string.encode('ascii', 'backslashreplace')
         return string
 else:
-    repr_escape = lambda string: string
+    def repr_escape(string):
+        return string
 
 
 def check_callable_args(func, args, kwargs):
@@ -290,70 +326,54 @@ def check_callable_args(func, args, kwargs):
 
     :type args: tuple
     :type kwargs: dict
-    """
 
+    """
     pos_kwargs_conflicts = []  # parameters that have a match in both args and kwargs
     positional_only_kwargs = []  # positional-only parameters that have a match in kwargs
     unsatisfied_args = []  # parameters in signature that don't have a match in args or kwargs
     unsatisfied_kwargs = []  # keyword-only arguments that don't have a match in kwargs
     unmatched_args = list(args)  # args that didn't match any of the parameters in the signature
-    unmatched_kwargs = list(kwargs)  # kwargs that didn't match any of the parameters in the signature
+    # kwargs that didn't match any of the parameters in the signature
-    has_varargs = has_var_kwargs = False  # indicates if the signature defines *args and **kwargs respectively
+    unmatched_kwargs = list(kwargs)
+    # indicates if the signature defines *args and **kwargs respectively
+    has_varargs = has_var_kwargs = False
 
-    if signature:
+    try:
-        try:
+        if sys.version_info >= (3, 5):
+            sig = signature(func, follow_wrapped=False)
+        else:
             sig = signature(func)
-        except ValueError:
+    except ValueError:
-            return  # signature() doesn't work against every kind of callable
+        # signature() doesn't work against every kind of callable
+        return
 
-        for param in six.itervalues(sig.parameters):
+    for param in six.itervalues(sig.parameters):
-            if param.kind == param.POSITIONAL_OR_KEYWORD:
+        if param.kind == param.POSITIONAL_OR_KEYWORD:
-                if param.name in unmatched_kwargs and unmatched_args:
+            if param.name in unmatched_kwargs and unmatched_args:
-                    pos_kwargs_conflicts.append(param.name)
+                pos_kwargs_conflicts.append(param.name)
-                elif unmatched_args:
-                    del unmatched_args[0]
-                elif param.name in unmatched_kwargs:
-                    unmatched_kwargs.remove(param.name)
-                elif param.default is param.empty:
-                    unsatisfied_args.append(param.name)
-            elif param.kind == param.POSITIONAL_ONLY:
-                if unmatched_args:
-                    del unmatched_args[0]
-                elif param.name in unmatched_kwargs:
-                    unmatched_kwargs.remove(param.name)
-                    positional_only_kwargs.append(param.name)
-                elif param.default is param.empty:
-                    unsatisfied_args.append(param.name)
-            elif param.kind == param.KEYWORD_ONLY:
-                if param.name in unmatched_kwargs:
-                    unmatched_kwargs.remove(param.name)
-                elif param.default is param.empty:
-                    unsatisfied_kwargs.append(param.name)
-            elif param.kind == param.VAR_POSITIONAL:
-                has_varargs = True
-            elif param.kind == param.VAR_KEYWORD:
-                has_var_kwargs = True
-    else:
-        if not isfunction(func) and not ismethod(func) and hasattr(func, '__call__'):
-            func = func.__call__
-
-        try:
-            argspec = getargspec(func)
-        except TypeError:
-            return  # getargspec() doesn't work certain callables
-
-        argspec_args = argspec.args if not ismethod(func) else argspec.args[1:]
-        has_varargs = bool(argspec.varargs)
-        has_var_kwargs = bool(argspec.keywords)
-        for arg, default in six.moves.zip_longest(argspec_args, argspec.defaults or (), fillvalue=undefined):
-            if arg in unmatched_kwargs and unmatched_args:
-                pos_kwargs_conflicts.append(arg)
             elif unmatched_args:
                 del unmatched_args[0]
-            elif arg in unmatched_kwargs:
+            elif param.name in unmatched_kwargs:
-                unmatched_kwargs.remove(arg)
+                unmatched_kwargs.remove(param.name)
-            elif default is undefined:
+            elif param.default is param.empty:
-                unsatisfied_args.append(arg)
+                unsatisfied_args.append(param.name)
+        elif param.kind == param.POSITIONAL_ONLY:
+            if unmatched_args:
+                del unmatched_args[0]
+            elif param.name in unmatched_kwargs:
+                unmatched_kwargs.remove(param.name)
+                positional_only_kwargs.append(param.name)
+            elif param.default is param.empty:
+                unsatisfied_args.append(param.name)
+        elif param.kind == param.KEYWORD_ONLY:
+            if param.name in unmatched_kwargs:
+                unmatched_kwargs.remove(param.name)
+            elif param.default is param.empty:
+                unsatisfied_kwargs.append(param.name)
+        elif param.kind == param.VAR_POSITIONAL:
+            has_varargs = True
+        elif param.kind == param.VAR_KEYWORD:
+            has_var_kwargs = True
 
     # Make sure there are no conflicts between args and kwargs
     if pos_kwargs_conflicts:
@@ -365,21 +385,46 @@ def check_callable_args(func, args, kwargs):
         raise ValueError('The following arguments cannot be given as keyword arguments: %s' %
                          ', '.join(positional_only_kwargs))
 
-    # Check that the number of positional arguments minus the number of matched kwargs matches the argspec
+    # Check that the number of positional arguments minus the number of matched kwargs matches the
+    # argspec
     if unsatisfied_args:
-        raise ValueError('The following arguments have not been supplied: %s' % ', '.join(unsatisfied_args))
+        raise ValueError('The following arguments have not been supplied: %s' %
+                         ', '.join(unsatisfied_args))
 
     # Check that all keyword-only arguments have been supplied
     if unsatisfied_kwargs:
-        raise ValueError('The following keyword-only arguments have not been supplied in kwargs: %s' %
+        raise ValueError(
-                         ', '.join(unsatisfied_kwargs))
+            'The following keyword-only arguments have not been supplied in kwargs: %s' %
+            ', '.join(unsatisfied_kwargs))
 
     # Check that the callable can accept the given number of positional arguments
     if not has_varargs and unmatched_args:
-        raise ValueError('The list of positional arguments is longer than the target callable can handle '
+        raise ValueError(
-                         '(allowed: %d, given in args: %d)' % (len(args) - len(unmatched_args), len(args)))
+            'The list of positional arguments is longer than the target callable can handle '
+            '(allowed: %d, given in args: %d)' % (len(args) - len(unmatched_args), len(args)))
 
     # Check that the callable can accept the given keyword arguments
     if not has_var_kwargs and unmatched_kwargs:
-        raise ValueError('The target callable does not accept the following keyword arguments: %s' %
+        raise ValueError(
-                         ', '.join(unmatched_kwargs))
+            'The target callable does not accept the following keyword arguments: %s' %
+            ', '.join(unmatched_kwargs))
+
+
+def iscoroutinefunction_partial(f):
+    while isinstance(f, partial):
+        f = f.func
+
+    # The asyncio version of iscoroutinefunction includes testing for @coroutine
+    # decorations vs. the inspect version which does not.
+    return iscoroutinefunction(f)
+
+
+def normalize(dt):
+    return datetime.fromtimestamp(dt.timestamp(), dt.tzinfo)
+
+
+def localize(dt, tzinfo):
+    if hasattr(tzinfo, 'localize'):
+        return tzinfo.localize(dt)
+
+    return normalize(dt.replace(tzinfo=tzinfo))

lib/biplist/__init__.py

@@ -1,803 +0,0 @@
-"""biplist -- a library for reading and writing binary property list files.
-
-Binary Property List (plist) files provide a faster and smaller serialization
-format for property lists on OS X. This is a library for generating binary
-plists which can be read by OS X, iOS, or other clients.
-
-The API models the plistlib API, and will call through to plistlib when
-XML serialization or deserialization is required.
-
-To generate plists with UID values, wrap the values with the Uid object. The
-value must be an int.
-
-To generate plists with NSData/CFData values, wrap the values with the
-Data object. The value must be a string.
-
-Date values can only be datetime.datetime objects.
-
-The exceptions InvalidPlistException and NotBinaryPlistException may be 
-thrown to indicate that the data cannot be serialized or deserialized as
-a binary plist.
-
-Plist generation example:
-    
-    from biplist import *
-    from datetime import datetime
-    plist = {'aKey':'aValue',
-             '0':1.322,
-             'now':datetime.now(),
-             'list':[1,2,3],
-             'tuple':('a','b','c')
-             }
-    try:
-        writePlist(plist, "example.plist")
-    except (InvalidPlistException, NotBinaryPlistException), e:
-        print "Something bad happened:", e
-
-Plist parsing example:
-
-    from biplist import *
-    try:
-        plist = readPlist("example.plist")
-        print plist
-    except (InvalidPlistException, NotBinaryPlistException), e:
-        print "Not a plist:", e
-"""
-
-import sys
-from collections import namedtuple
-import datetime
-import io
-import math
-import plistlib
-from struct import pack, unpack
-from struct import error as struct_error
-import sys
-import time
-
-try:
-    str
-    unicodeEmpty = r''
-except NameError:
-    str = str
-    unicodeEmpty = ''
-try:
-    int
-except NameError:
-    long = int
-try:
-    {}.iteritems
-    iteritems = lambda x: iter(x.items())
-except AttributeError:
-    iteritems = lambda x: list(x.items())
-
-__all__ = [
-    'Uid', 'Data', 'readPlist', 'writePlist', 'readPlistFromString',
-    'writePlistToString', 'InvalidPlistException', 'NotBinaryPlistException'
-]
-
-# Apple uses Jan 1, 2001 as a base for all plist date/times.
-apple_reference_date = datetime.datetime.utcfromtimestamp(978307200)
-
-class Uid(int):
-    """Wrapper around integers for representing UID values. This
-       is used in keyed archiving."""
-    def __repr__(self):
-        return "Uid(%d)" % self
-
-class Data(bytes):
-    """Wrapper around str types for representing Data values."""
-    pass
-
-class InvalidPlistException(Exception):
-    """Raised when the plist is incorrectly formatted."""
-    pass
-
-class NotBinaryPlistException(Exception):
-    """Raised when a binary plist was expected but not encountered."""
-    pass
-
-def readPlist(pathOrFile):
-    """Raises NotBinaryPlistException, InvalidPlistException"""
-    didOpen = False
-    result = None
-    if isinstance(pathOrFile, (bytes, str)):
-        pathOrFile = open(pathOrFile, 'rb')
-        didOpen = True
-    try:
-        reader = PlistReader(pathOrFile)
-        result = reader.parse()
-    except NotBinaryPlistException as e:
-        try:
-            pathOrFile.seek(0)
-            result = None
-            if hasattr(plistlib, 'loads'):
-                contents = None
-                if isinstance(pathOrFile, (bytes, str)):
-                    with open(pathOrFile, 'rb') as f:
-                        contents = f.read()
-                else:
-                    contents = pathOrFile.read()
-                result = plistlib.loads(contents)
-            else:
-                result = plistlib.readPlist(pathOrFile)
-            result = wrapDataObject(result, for_binary=True)
-        except Exception as e:
-            raise InvalidPlistException(e)
-    finally:
-        if didOpen:
-            pathOrFile.close()
-    return result
-
-def wrapDataObject(o, for_binary=False):
-    if isinstance(o, Data) and not for_binary:
-        v = sys.version_info
-        if not (v[0] >= 3 and v[1] >= 4):
-            o = plistlib.Data(o)
-    elif isinstance(o, (bytes, plistlib.Data)) and for_binary:
-        if hasattr(o, 'data'):
-            o = Data(o.data)
-    elif isinstance(o, tuple):
-        o = wrapDataObject(list(o), for_binary)
-        o = tuple(o)
-    elif isinstance(o, list):
-        for i in range(len(o)):
-            o[i] = wrapDataObject(o[i], for_binary)
-    elif isinstance(o, dict):
-        for k in o:
-            o[k] = wrapDataObject(o[k], for_binary)
-    return o
-
-def writePlist(rootObject, pathOrFile, binary=True):
-    if not binary:
-        rootObject = wrapDataObject(rootObject, binary)
-        if hasattr(plistlib, "dump"):
-            if isinstance(pathOrFile, (bytes, str)):
-                with open(pathOrFile, 'wb') as f:
-                    return plistlib.dump(rootObject, f)
-            else:
-                return plistlib.dump(rootObject, pathOrFile)
-        else:
-            return plistlib.writePlist(rootObject, pathOrFile)
-    else:
-        didOpen = False
-        if isinstance(pathOrFile, (bytes, str)):
-            pathOrFile = open(pathOrFile, 'wb')
-            didOpen = True
-        writer = PlistWriter(pathOrFile)
-        result = writer.writeRoot(rootObject)
-        if didOpen:
-            pathOrFile.close()
-        return result
-
-def readPlistFromString(data):
-    return readPlist(io.BytesIO(data))
-
-def writePlistToString(rootObject, binary=True):
-    if not binary:
-        rootObject = wrapDataObject(rootObject, binary)
-        if hasattr(plistlib, "dumps"):
-            return plistlib.dumps(rootObject)
-        elif hasattr(plistlib, "writePlistToBytes"):
-            return plistlib.writePlistToBytes(rootObject)
-        else:
-            return plistlib.writePlistToString(rootObject)
-    else:
-        ioObject = io.BytesIO()
-        writer = PlistWriter(ioObject)
-        writer.writeRoot(rootObject)
-        return ioObject.getvalue()
-
-def is_stream_binary_plist(stream):
-    stream.seek(0)
-    header = stream.read(7)
-    if header == b'bplist0':
-        return True
-    else:
-        return False
-
-PlistTrailer = namedtuple('PlistTrailer', 'offsetSize, objectRefSize, offsetCount, topLevelObjectNumber, offsetTableOffset')
-PlistByteCounts = namedtuple('PlistByteCounts', 'nullBytes, boolBytes, intBytes, realBytes, dateBytes, dataBytes, stringBytes, uidBytes, arrayBytes, setBytes, dictBytes')
-
-class PlistReader(object):
-    file = None
-    contents = ''
-    offsets = None
-    trailer = None
-    currentOffset = 0
-    
-    def __init__(self, fileOrStream):
-        """Raises NotBinaryPlistException."""
-        self.reset()
-        self.file = fileOrStream
-    
-    def parse(self):
-        return self.readRoot()
-    
-    def reset(self):
-        self.trailer = None
-        self.contents = ''
-        self.offsets = []
-        self.currentOffset = 0
-    
-    def readRoot(self):
-        result = None
-        self.reset()
-        # Get the header, make sure it's a valid file.
-        if not is_stream_binary_plist(self.file):
-            raise NotBinaryPlistException()
-        self.file.seek(0)
-        self.contents = self.file.read()
-        if len(self.contents) < 32:
-            raise InvalidPlistException("File is too short.")
-        trailerContents = self.contents[-32:]
-        try:
-            self.trailer = PlistTrailer._make(unpack("!xxxxxxBBQQQ", trailerContents))
-            offset_size = self.trailer.offsetSize * self.trailer.offsetCount
-            offset = self.trailer.offsetTableOffset
-            offset_contents = self.contents[offset:offset+offset_size]
-            offset_i = 0
-            while offset_i < self.trailer.offsetCount:
-                begin = self.trailer.offsetSize*offset_i
-                tmp_contents = offset_contents[begin:begin+self.trailer.offsetSize]
-                tmp_sized = self.getSizedInteger(tmp_contents, self.trailer.offsetSize)
-                self.offsets.append(tmp_sized)
-                offset_i += 1
-            self.setCurrentOffsetToObjectNumber(self.trailer.topLevelObjectNumber)
-            result = self.readObject()
-        except TypeError as e:
-            raise InvalidPlistException(e)
-        return result
-    
-    def setCurrentOffsetToObjectNumber(self, objectNumber):
-        self.currentOffset = self.offsets[objectNumber]
-    
-    def readObject(self):
-        result = None
-        tmp_byte = self.contents[self.currentOffset:self.currentOffset+1]
-        marker_byte = unpack("!B", tmp_byte)[0]
-        format = (marker_byte >> 4) & 0x0f
-        extra = marker_byte & 0x0f
-        self.currentOffset += 1
-        
-        def proc_extra(extra):
-            if extra == 0b1111:
-                #self.currentOffset += 1
-                extra = self.readObject()
-            return extra
-        
-        # bool, null, or fill byte
-        if format == 0b0000:
-            if extra == 0b0000:
-                result = None
-            elif extra == 0b1000:
-                result = False
-            elif extra == 0b1001:
-                result = True
-            elif extra == 0b1111:
-                pass # fill byte
-            else:
-                raise InvalidPlistException("Invalid object found at offset: %d" % (self.currentOffset - 1))
-        # int
-        elif format == 0b0001:
-            extra = proc_extra(extra)
-            result = self.readInteger(pow(2, extra))
-        # real
-        elif format == 0b0010:
-            extra = proc_extra(extra)
-            result = self.readReal(extra)
-        # date
-        elif format == 0b0011 and extra == 0b0011:
-            result = self.readDate()
-        # data
-        elif format == 0b0100:
-            extra = proc_extra(extra)
-            result = self.readData(extra)
-        # ascii string
-        elif format == 0b0101:
-            extra = proc_extra(extra)
-            result = self.readAsciiString(extra)
-        # Unicode string
-        elif format == 0b0110:
-            extra = proc_extra(extra)
-            result = self.readUnicode(extra)
-        # uid
-        elif format == 0b1000:
-            result = self.readUid(extra)
-        # array
-        elif format == 0b1010:
-            extra = proc_extra(extra)
-            result = self.readArray(extra)
-        # set
-        elif format == 0b1100:
-            extra = proc_extra(extra)
-            result = set(self.readArray(extra))
-        # dict
-        elif format == 0b1101:
-            extra = proc_extra(extra)
-            result = self.readDict(extra)
-        else:    
-            raise InvalidPlistException("Invalid object found: {format: %s, extra: %s}" % (bin(format), bin(extra)))
-        return result
-    
-    def readInteger(self, byteSize):
-        result = 0
-        original_offset = self.currentOffset
-        data = self.contents[self.currentOffset:self.currentOffset + byteSize]
-        result = self.getSizedInteger(data, byteSize, as_number=True)
-        self.currentOffset = original_offset + byteSize
-        return result
-    
-    def readReal(self, length):
-        result = 0.0
-        to_read = pow(2, length)
-        data = self.contents[self.currentOffset:self.currentOffset+to_read]
-        if length == 2: # 4 bytes
-            result = unpack('>f', data)[0]
-        elif length == 3: # 8 bytes
-            result = unpack('>d', data)[0]
-        else:
-            raise InvalidPlistException("Unknown real of length %d bytes" % to_read)
-        return result
-    
-    def readRefs(self, count):    
-        refs = []
-        i = 0
-        while i < count:
-            fragment = self.contents[self.currentOffset:self.currentOffset+self.trailer.objectRefSize]
-            ref = self.getSizedInteger(fragment, len(fragment))
-            refs.append(ref)
-            self.currentOffset += self.trailer.objectRefSize
-            i += 1
-        return refs
-    
-    def readArray(self, count):
-        result = []
-        values = self.readRefs(count)
-        i = 0
-        while i < len(values):
-            self.setCurrentOffsetToObjectNumber(values[i])
-            value = self.readObject()
-            result.append(value)
-            i += 1
-        return result
-    
-    def readDict(self, count):
-        result = {}
-        keys = self.readRefs(count)
-        values = self.readRefs(count)
-        i = 0
-        while i < len(keys):
-            self.setCurrentOffsetToObjectNumber(keys[i])
-            key = self.readObject()
-            self.setCurrentOffsetToObjectNumber(values[i])
-            value = self.readObject()
-            result[key] = value
-            i += 1
-        return result
-    
-    def readAsciiString(self, length):
-        result = unpack("!%ds" % length, self.contents[self.currentOffset:self.currentOffset+length])[0]
-        self.currentOffset += length
-        return result
-    
-    def readUnicode(self, length):
-        actual_length = length*2
-        data = self.contents[self.currentOffset:self.currentOffset+actual_length]
-        # unpack not needed?!! data = unpack(">%ds" % (actual_length), data)[0]
-        self.currentOffset += actual_length
-        return data.decode('utf_16_be')
-    
-    def readDate(self):
-        result = unpack(">d", self.contents[self.currentOffset:self.currentOffset+8])[0]
-        # Use timedelta to workaround time_t size limitation on 32-bit python.
-        result = datetime.timedelta(seconds=result) + apple_reference_date
-        self.currentOffset += 8
-        return result
-    
-    def readData(self, length):
-        result = self.contents[self.currentOffset:self.currentOffset+length]
-        self.currentOffset += length
-        return Data(result)
-    
-    def readUid(self, length):
-        return Uid(self.readInteger(length+1))
-    
-    def getSizedInteger(self, data, byteSize, as_number=False):
-        """Numbers of 8 bytes are signed integers when they refer to numbers, but unsigned otherwise."""
-        result = 0
-        # 1, 2, and 4 byte integers are unsigned
-        if byteSize == 1:
-            result = unpack('>B', data)[0]
-        elif byteSize == 2:
-            result = unpack('>H', data)[0]
-        elif byteSize == 4:
-            result = unpack('>L', data)[0]
-        elif byteSize == 8:
-            if as_number:
-                result = unpack('>q', data)[0]
-            else:
-                result = unpack('>Q', data)[0]
-        elif byteSize <= 16:
-            # Handle odd-sized or integers larger than 8 bytes
-            # Don't naively go over 16 bytes, in order to prevent infinite loops.
-            result = 0
-            if hasattr(int, 'from_bytes'):
-                result = int.from_bytes(data, 'big')
-            else:
-                for byte in data:
-                    result = (result << 8) | unpack('>B', byte)[0]
-        else:
-            raise InvalidPlistException("Encountered integer longer than 16 bytes.")
-        return result
-
-class HashableWrapper(object):
-    def __init__(self, value):
-        self.value = value
-    def __repr__(self):
-        return "<HashableWrapper: %s>" % [self.value]
-
-class BoolWrapper(object):
-    def __init__(self, value):
-        self.value = value
-    def __repr__(self):
-        return "<BoolWrapper: %s>" % self.value
-
-class FloatWrapper(object):
-    _instances = {}
-    def __new__(klass, value):
-        # Ensure FloatWrapper(x) for a given float x is always the same object
-        wrapper = klass._instances.get(value)
-        if wrapper is None:
-            wrapper = object.__new__(klass)
-            wrapper.value = value
-            klass._instances[value] = wrapper
-        return wrapper
-    def __repr__(self):
-        return "<FloatWrapper: %s>" % self.value
-
-class PlistWriter(object):
-    header = b'bplist00bybiplist1.0'
-    file = None
-    byteCounts = None
-    trailer = None
-    computedUniques = None
-    writtenReferences = None
-    referencePositions = None
-    wrappedTrue = None
-    wrappedFalse = None
-    
-    def __init__(self, file):
-        self.reset()
-        self.file = file
-        self.wrappedTrue = BoolWrapper(True)
-        self.wrappedFalse = BoolWrapper(False)
-
-    def reset(self):
-        self.byteCounts = PlistByteCounts(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0)
-        self.trailer = PlistTrailer(0, 0, 0, 0, 0)
-        
-        # A set of all the uniques which have been computed.
-        self.computedUniques = set()
-        # A list of all the uniques which have been written.
-        self.writtenReferences = {}
-        # A dict of the positions of the written uniques.
-        self.referencePositions = {}
-        
-    def positionOfObjectReference(self, obj):
-        """If the given object has been written already, return its
-           position in the offset table. Otherwise, return None."""
-        return self.writtenReferences.get(obj)
-        
-    def writeRoot(self, root):
-        """
-        Strategy is:
-        - write header
-        - wrap root object so everything is hashable
-        - compute size of objects which will be written
-          - need to do this in order to know how large the object refs
-            will be in the list/dict/set reference lists
-        - write objects
-          - keep objects in writtenReferences
-          - keep positions of object references in referencePositions
-          - write object references with the length computed previously
-        - computer object reference length
-        - write object reference positions
-        - write trailer
-        """
-        output = self.header
-        wrapped_root = self.wrapRoot(root)
-        should_reference_root = True#not isinstance(wrapped_root, HashableWrapper)
-        self.computeOffsets(wrapped_root, asReference=should_reference_root, isRoot=True)
-        self.trailer = self.trailer._replace(**{'objectRefSize':self.intSize(len(self.computedUniques))})
-        (_, output) = self.writeObjectReference(wrapped_root, output)
-        output = self.writeObject(wrapped_root, output, setReferencePosition=True)
-        
-        # output size at this point is an upper bound on how big the
-        # object reference offsets need to be.
-        self.trailer = self.trailer._replace(**{
-            'offsetSize':self.intSize(len(output)),
-            'offsetCount':len(self.computedUniques),
-            'offsetTableOffset':len(output),
-            'topLevelObjectNumber':0
-            })
-        
-        output = self.writeOffsetTable(output)
-        output += pack('!xxxxxxBBQQQ', *self.trailer)
-        self.file.write(output)
-
-    def wrapRoot(self, root):
-        if isinstance(root, bool):
-            if root is True:
-                return self.wrappedTrue
-            else:
-                return self.wrappedFalse
-        elif isinstance(root, float):
-            return FloatWrapper(root)
-        elif isinstance(root, set):
-            n = set()
-            for value in root:
-                n.add(self.wrapRoot(value))
-            return HashableWrapper(n)
-        elif isinstance(root, dict):
-            n = {}
-            for key, value in iteritems(root):
-                n[self.wrapRoot(key)] = self.wrapRoot(value)
-            return HashableWrapper(n)
-        elif isinstance(root, list):
-            n = []
-            for value in root:
-                n.append(self.wrapRoot(value))
-            return HashableWrapper(n)
-        elif isinstance(root, tuple):
-            n = tuple([self.wrapRoot(value) for value in root])
-            return HashableWrapper(n)
-        else:
-            return root
-
-    def incrementByteCount(self, field, incr=1):
-        self.byteCounts = self.byteCounts._replace(**{field:self.byteCounts.__getattribute__(field) + incr})
-
-    def computeOffsets(self, obj, asReference=False, isRoot=False):
-        def check_key(key):
-            if key is None:
-                raise InvalidPlistException('Dictionary keys cannot be null in plists.')
-            elif isinstance(key, Data):
-                raise InvalidPlistException('Data cannot be dictionary keys in plists.')
-            elif not isinstance(key, (bytes, str)):
-                raise InvalidPlistException('Keys must be strings.')
-        
-        def proc_size(size):
-            if size > 0b1110:
-                size += self.intSize(size)
-            return size
-        # If this should be a reference, then we keep a record of it in the
-        # uniques table.
-        if asReference:
-            if obj in self.computedUniques:
-                return
-            else:
-                self.computedUniques.add(obj)
-        
-        if obj is None:
-            self.incrementByteCount('nullBytes')
-        elif isinstance(obj, BoolWrapper):
-            self.incrementByteCount('boolBytes')
-        elif isinstance(obj, Uid):
-            size = self.intSize(obj)
-            self.incrementByteCount('uidBytes', incr=1+size)
-        elif isinstance(obj, int):
-            size = self.intSize(obj)
-            self.incrementByteCount('intBytes', incr=1+size)
-        elif isinstance(obj, FloatWrapper):
-            size = self.realSize(obj)
-            self.incrementByteCount('realBytes', incr=1+size)
-        elif isinstance(obj, datetime.datetime):    
-            self.incrementByteCount('dateBytes', incr=2)
-        elif isinstance(obj, Data):
-            size = proc_size(len(obj))
-            self.incrementByteCount('dataBytes', incr=1+size)
-        elif isinstance(obj, (str, bytes)):
-            size = proc_size(len(obj))
-            self.incrementByteCount('stringBytes', incr=1+size)
-        elif isinstance(obj, HashableWrapper):
-            obj = obj.value
-            if isinstance(obj, set):
-                size = proc_size(len(obj))
-                self.incrementByteCount('setBytes', incr=1+size)
-                for value in obj:
-                    self.computeOffsets(value, asReference=True)
-            elif isinstance(obj, (list, tuple)):
-                size = proc_size(len(obj))
-                self.incrementByteCount('arrayBytes', incr=1+size)
-                for value in obj:
-                    asRef = True
-                    self.computeOffsets(value, asReference=True)
-            elif isinstance(obj, dict):
-                size = proc_size(len(obj))
-                self.incrementByteCount('dictBytes', incr=1+size)
-                for key, value in iteritems(obj):
-                    check_key(key)
-                    self.computeOffsets(key, asReference=True)
-                    self.computeOffsets(value, asReference=True)
-        else:
-            raise InvalidPlistException("Unknown object type.")
-
-    def writeObjectReference(self, obj, output):
-        """Tries to write an object reference, adding it to the references
-           table. Does not write the actual object bytes or set the reference
-           position. Returns a tuple of whether the object was a new reference
-           (True if it was, False if it already was in the reference table)
-           and the new output.
-        """
-        position = self.positionOfObjectReference(obj)
-        if position is None:
-            self.writtenReferences[obj] = len(self.writtenReferences)
-            output += self.binaryInt(len(self.writtenReferences) - 1, byteSize=self.trailer.objectRefSize)
-            return (True, output)
-        else:
-            output += self.binaryInt(position, byteSize=self.trailer.objectRefSize)
-            return (False, output)
-
-    def writeObject(self, obj, output, setReferencePosition=False):
-        """Serializes the given object to the output. Returns output.
-           If setReferencePosition is True, will set the position the
-           object was written.
-        """
-        def proc_variable_length(format, length):
-            result = b''
-            if length > 0b1110:
-                result += pack('!B', (format << 4) | 0b1111)
-                result = self.writeObject(length, result)
-            else:
-                result += pack('!B', (format << 4) | length)
-            return result
-        
-        if isinstance(obj, str) and obj == unicodeEmpty:
-            # The Apple Plist decoder can't decode a zero length Unicode string.
-            obj = b''
-       
-        if setReferencePosition:
-            self.referencePositions[obj] = len(output)
-        
-        if obj is None:
-            output += pack('!B', 0b00000000)
-        elif isinstance(obj, BoolWrapper):
-            if obj.value is False:
-                output += pack('!B', 0b00001000)
-            else:
-                output += pack('!B', 0b00001001)
-        elif isinstance(obj, Uid):
-            size = self.intSize(obj)
-            output += pack('!B', (0b1000 << 4) | size - 1)
-            output += self.binaryInt(obj)
-        elif isinstance(obj, int):
-            byteSize = self.intSize(obj)
-            root = math.log(byteSize, 2)
-            output += pack('!B', (0b0001 << 4) | int(root))
-            output += self.binaryInt(obj, as_number=True)
-        elif isinstance(obj, FloatWrapper):
-            # just use doubles
-            output += pack('!B', (0b0010 << 4) | 3)
-            output += self.binaryReal(obj)
-        elif isinstance(obj, datetime.datetime):
-            timestamp = (obj - apple_reference_date).total_seconds()
-            output += pack('!B', 0b00110011)
-            output += pack('!d', float(timestamp))
-        elif isinstance(obj, Data):
-            output += proc_variable_length(0b0100, len(obj))
-            output += obj
-        elif isinstance(obj, str):
-            byteData = obj.encode('utf_16_be')
-            output += proc_variable_length(0b0110, len(byteData)//2)
-            output += byteData
-        elif isinstance(obj, bytes):
-            output += proc_variable_length(0b0101, len(obj))
-            output += obj
-        elif isinstance(obj, HashableWrapper):
-            obj = obj.value
-            if isinstance(obj, (set, list, tuple)):
-                if isinstance(obj, set):
-                    output += proc_variable_length(0b1100, len(obj))
-                else:
-                    output += proc_variable_length(0b1010, len(obj))
-            
-                objectsToWrite = []
-                for objRef in obj:
-                    (isNew, output) = self.writeObjectReference(objRef, output)
-                    if isNew:
-                        objectsToWrite.append(objRef)
-                for objRef in objectsToWrite:
-                    output = self.writeObject(objRef, output, setReferencePosition=True)
-            elif isinstance(obj, dict):
-                output += proc_variable_length(0b1101, len(obj))
-                keys = []
-                values = []
-                objectsToWrite = []
-                for key, value in iteritems(obj):
-                    keys.append(key)
-                    values.append(value)
-                for key in keys:
-                    (isNew, output) = self.writeObjectReference(key, output)
-                    if isNew:
-                        objectsToWrite.append(key)
-                for value in values:
-                    (isNew, output) = self.writeObjectReference(value, output)
-                    if isNew:
-                        objectsToWrite.append(value)
-                for objRef in objectsToWrite:
-                    output = self.writeObject(objRef, output, setReferencePosition=True)
-        return output
-    
-    def writeOffsetTable(self, output):
-        """Writes all of the object reference offsets."""
-        all_positions = []
-        writtenReferences = list(self.writtenReferences.items())
-        writtenReferences.sort(key=lambda x: x[1])
-        for obj,order in writtenReferences:
-            # Porting note: Elsewhere we deliberately replace empty unicdoe strings
-            # with empty binary strings, but the empty unicode string
-            # goes into writtenReferences.  This isn't an issue in Py2
-            # because u'' and b'' have the same hash; but it is in
-            # Py3, where they don't.
-            if bytes != str and obj == unicodeEmpty:
-                obj = b''
-            position = self.referencePositions.get(obj)
-            if position is None:
-                raise InvalidPlistException("Error while writing offsets table. Object not found. %s" % obj)
-            output += self.binaryInt(position, self.trailer.offsetSize)
-            all_positions.append(position)
-        return output
-    
-    def binaryReal(self, obj):
-        # just use doubles
-        result = pack('>d', obj.value)
-        return result
-    
-    def binaryInt(self, obj, byteSize=None, as_number=False):
-        result = b''
-        if byteSize is None:
-            byteSize = self.intSize(obj)
-        if byteSize == 1:
-            result += pack('>B', obj)
-        elif byteSize == 2:
-            result += pack('>H', obj)
-        elif byteSize == 4:
-            result += pack('>L', obj)
-        elif byteSize == 8:
-            if as_number:
-                result += pack('>q', obj)
-            else:
-                result += pack('>Q', obj)
-        elif byteSize <= 16:
-            try:
-                result = pack('>Q', 0) + pack('>Q', obj)
-            except struct_error as e:
-                raise InvalidPlistException("Unable to pack integer %d: %s" % (obj, e))
-        else:
-            raise InvalidPlistException("Core Foundation can't handle integers with size greater than 16 bytes.")
-        return result
-    
-    def intSize(self, obj):
-        """Returns the number of bytes necessary to store the given integer."""
-        # SIGNED
-        if obj < 0: # Signed integer, always 8 bytes
-            return 8
-        # UNSIGNED
-        elif obj <= 0xFF: # 1 byte
-            return 1
-        elif obj <= 0xFFFF: # 2 bytes
-            return 2
-        elif obj <= 0xFFFFFFFF: # 4 bytes
-            return 4
-        # SIGNED
-        # 0x7FFFFFFFFFFFFFFF is the max.
-        elif obj <= 0x7FFFFFFFFFFFFFFF: # 8 bytes signed
-            return 8
-        elif obj <= 0xffffffffffffffff: # 8 bytes unsigned
-            return 16
-        else:
-            raise InvalidPlistException("Core Foundation can't handle integers with size greater than 8 bytes.")
-    
-    def realSize(self, obj):
-        return 8

lib/cherrypy/_cpdispatch.py

@@ -206,12 +206,8 @@ except ImportError:
     def test_callable_spec(callable, args, kwargs):  # noqa: F811
         return None
 else:
-    getargspec = inspect.getargspec
+    def getargspec(callable):
-    # Python 3 requires using getfullargspec if
+        return inspect.getfullargspec(callable)[:4]
-    # keyword-only arguments are present
-    if hasattr(inspect, 'getfullargspec'):
-        def getargspec(callable):
-            return inspect.getfullargspec(callable)[:4]
 
 
 class LateParamPageHandler(PageHandler):

lib/cherrypy/_cperror.py

@@ -466,7 +466,7 @@ _HTTPErrorTemplate = '''<!DOCTYPE html PUBLIC
         <pre id="traceback">%(traceback)s</pre>
     <div id="powered_by">
       <span>
-        Powered by <a href="http://www.cherrypy.org">CherryPy %(version)s</a>
+        Powered by <a href="http://www.cherrypy.dev">CherryPy %(version)s</a>
       </span>
     </div>
     </body>
@@ -532,7 +532,8 @@ def get_error_page(status, **kwargs):
                     return result
             else:
                 # Load the template from this path.
-                template = io.open(error_page, newline='').read()
+                with io.open(error_page, newline='') as f:
+                    template = f.read()
         except Exception:
             e = _format_exception(*_exc_info())[-1]
             m = kwargs['message']

lib/cherrypy/_cpmodpy.py

@@ -339,11 +339,8 @@ LoadModule python_module modules/mod_python.so
                                      }
 
         mpconf = os.path.join(os.path.dirname(__file__), 'cpmodpy.conf')
-        f = open(mpconf, 'wb')
+        with open(mpconf, 'wb') as f:
-        try:
             f.write(conf_data)
-        finally:
-            f.close()
 
         response = read_process(self.apache_path, '-k start -f %s' % mpconf)
         self.ready = True

lib/cherrypy/_cprequest.py

@@ -169,7 +169,7 @@ def request_namespace(k, v):
 def response_namespace(k, v):
     """Attach response attributes declared in config."""
     # Provides config entries to set default response headers
-    # http://cherrypy.org/ticket/889
+    # http://cherrypy.dev/ticket/889
     if k[:8] == 'headers.':
         cherrypy.serving.response.headers[k.split('.', 1)[1]] = v
     else:
@@ -252,7 +252,7 @@ class Request(object):
     The query component of the Request-URI, a string of information to be
     interpreted by the resource. The query portion of a URI follows the
     path component, and is separated by a '?'. For example, the URI
-    'http://www.cherrypy.org/wiki?a=3&b=4' has the query component,
+    'http://www.cherrypy.dev/wiki?a=3&b=4' has the query component,
     'a=3&b=4'."""
 
     query_string_encoding = 'utf8'
@@ -742,6 +742,9 @@ class Request(object):
             if self.protocol >= (1, 1):
                 msg = "HTTP/1.1 requires a 'Host' request header."
                 raise cherrypy.HTTPError(400, msg)
+        else:
+            headers['Host'] = httputil.SanitizedHost(dict.get(headers, 'Host'))
+
         host = dict.get(headers, 'Host')
         if not host:
             host = self.local.name or self.local.ip

lib/cherrypy/lib/auth_digest.py

@@ -101,13 +101,12 @@ def get_ha1_file_htdigest(filename):
     """
     def get_ha1(realm, username):
         result = None
-        f = open(filename, 'r')
+        with open(filename, 'r') as f:
-        for line in f:
+            for line in f:
-            u, r, ha1 = line.rstrip().split(':')
+                u, r, ha1 = line.rstrip().split(':')
-            if u == username and r == realm:
+                if u == username and r == realm:
-                result = ha1
+                    result = ha1
-                break
+                    break
-        f.close()
         return result
 
     return get_ha1

lib/cherrypy/lib/covercp.py

@@ -334,9 +334,10 @@ class CoverStats(object):
         yield '</body></html>'
 
     def annotated_file(self, filename, statements, excluded, missing):
-        source = open(filename, 'r')
+        with open(filename, 'r') as source:
+            lines = source.readlines()
         buffer = []
-        for lineno, line in enumerate(source.readlines()):
+        for lineno, line in enumerate(lines):
             lineno += 1
             line = line.strip('\n\r')
             empty_the_buffer = True

lib/cherrypy/lib/httputil.py

@@ -516,3 +516,33 @@ class Host(object):
 
     def __repr__(self):
         return 'httputil.Host(%r, %r, %r)' % (self.ip, self.port, self.name)
+
+
+class SanitizedHost(str):
+    r"""
+    Wraps a raw host header received from the network in
+    a sanitized version that elides dangerous characters.
+
+    >>> SanitizedHost('foo\nbar')
+    'foobar'
+    >>> SanitizedHost('foo\nbar').raw
+    'foo\nbar'
+
+    A SanitizedInstance is only returned if sanitization was performed.
+
+    >>> isinstance(SanitizedHost('foobar'), SanitizedHost)
+    False
+    """
+    dangerous = re.compile(r'[\n\r]')
+
+    def __new__(cls, raw):
+        sanitized = cls._sanitize(raw)
+        if sanitized == raw:
+            return raw
+        instance = super().__new__(cls, sanitized)
+        instance.raw = raw
+        return instance
+
+    @classmethod
+    def _sanitize(cls, raw):
+        return cls.dangerous.sub('', raw)

lib/cherrypy/lib/reprconf.py

@@ -163,11 +163,8 @@ class Parser(configparser.ConfigParser):
             #     fp = open(filename)
             # except IOError:
             #     continue
-            fp = open(filename)
+            with open(filename) as fp:
-            try:
                 self._read(fp, filename)
-            finally:
-                fp.close()
 
     def as_dict(self, raw=False, vars=None):
         """Convert an INI file to a dictionary"""

lib/cherrypy/lib/sessions.py

@@ -516,11 +516,8 @@ class FileSession(Session):
         if path is None:
             path = self._get_file_path()
         try:
-            f = open(path, 'rb')
+            with open(path, 'rb') as f:
-            try:
                 return pickle.load(f)
-            finally:
-                f.close()
         except (IOError, EOFError):
             e = sys.exc_info()[1]
             if self.debug:
@@ -531,11 +528,8 @@ class FileSession(Session):
     def _save(self, expiration_time):
         assert self.locked, ('The session was saved without being locked.  '
                              "Check your tools' priority levels.")
-        f = open(self._get_file_path(), 'wb')
+        with open(self._get_file_path(), 'wb') as f:
-        try:
             pickle.dump((self._data, expiration_time), f, self.pickle_protocol)
-        finally:
-            f.close()
 
     def _delete(self):
         assert self.locked, ('The session deletion without being locked.  '

lib/cherrypy/process/plugins.py

@@ -436,7 +436,8 @@ class PIDFile(SimplePlugin):
         if self.finalized:
             self.bus.log('PID %r already written to %r.' % (pid, self.pidfile))
         else:
-            open(self.pidfile, 'wb').write(ntob('%s\n' % pid, 'utf8'))
+            with open(self.pidfile, 'wb') as f:
+                f.write(ntob('%s\n' % pid, 'utf8'))
             self.bus.log('PID %r written to %r.' % (pid, self.pidfile))
             self.finalized = True
     start.priority = 70

lib/importlib_resources/__init__.py

@@ -0,0 +1,36 @@
+"""Read resources contained within a package."""
+
+from ._common import (
+    as_file,
+    files,
+    Package,
+)
+
+from ._legacy import (
+    contents,
+    open_binary,
+    read_binary,
+    open_text,
+    read_text,
+    is_resource,
+    path,
+    Resource,
+)
+
+from importlib_resources.abc import ResourceReader
+
+
+__all__ = [
+    'Package',
+    'Resource',
+    'ResourceReader',
+    'as_file',
+    'contents',
+    'files',
+    'is_resource',
+    'open_binary',
+    'open_text',
+    'path',
+    'read_binary',
+    'read_text',
+]

lib/importlib_resources/_adapters.py

@@ -0,0 +1,170 @@
+from contextlib import suppress
+from io import TextIOWrapper
+
+from . import abc
+
+
+class SpecLoaderAdapter:
+    """
+    Adapt a package spec to adapt the underlying loader.
+    """
+
+    def __init__(self, spec, adapter=lambda spec: spec.loader):
+        self.spec = spec
+        self.loader = adapter(spec)
+
+    def __getattr__(self, name):
+        return getattr(self.spec, name)
+
+
+class TraversableResourcesLoader:
+    """
+    Adapt a loader to provide TraversableResources.
+    """
+
+    def __init__(self, spec):
+        self.spec = spec
+
+    def get_resource_reader(self, name):
+        return CompatibilityFiles(self.spec)._native()
+
+
+def _io_wrapper(file, mode='r', *args, **kwargs):
+    if mode == 'r':
+        return TextIOWrapper(file, *args, **kwargs)
+    elif mode == 'rb':
+        return file
+    raise ValueError(
+        "Invalid mode value '{}', only 'r' and 'rb' are supported".format(mode)
+    )
+
+
+class CompatibilityFiles:
+    """
+    Adapter for an existing or non-existent resource reader
+    to provide a compatibility .files().
+    """
+
+    class SpecPath(abc.Traversable):
+        """
+        Path tied to a module spec.
+        Can be read and exposes the resource reader children.
+        """
+
+        def __init__(self, spec, reader):
+            self._spec = spec
+            self._reader = reader
+
+        def iterdir(self):
+            if not self._reader:
+                return iter(())
+            return iter(
+                CompatibilityFiles.ChildPath(self._reader, path)
+                for path in self._reader.contents()
+            )
+
+        def is_file(self):
+            return False
+
+        is_dir = is_file
+
+        def joinpath(self, other):
+            if not self._reader:
+                return CompatibilityFiles.OrphanPath(other)
+            return CompatibilityFiles.ChildPath(self._reader, other)
+
+        @property
+        def name(self):
+            return self._spec.name
+
+        def open(self, mode='r', *args, **kwargs):
+            return _io_wrapper(self._reader.open_resource(None), mode, *args, **kwargs)
+
+    class ChildPath(abc.Traversable):
+        """
+        Path tied to a resource reader child.
+        Can be read but doesn't expose any meaningful children.
+        """
+
+        def __init__(self, reader, name):
+            self._reader = reader
+            self._name = name
+
+        def iterdir(self):
+            return iter(())
+
+        def is_file(self):
+            return self._reader.is_resource(self.name)
+
+        def is_dir(self):
+            return not self.is_file()
+
+        def joinpath(self, other):
+            return CompatibilityFiles.OrphanPath(self.name, other)
+
+        @property
+        def name(self):
+            return self._name
+
+        def open(self, mode='r', *args, **kwargs):
+            return _io_wrapper(
+                self._reader.open_resource(self.name), mode, *args, **kwargs
+            )
+
+    class OrphanPath(abc.Traversable):
+        """
+        Orphan path, not tied to a module spec or resource reader.
+        Can't be read and doesn't expose any meaningful children.
+        """
+
+        def __init__(self, *path_parts):
+            if len(path_parts) < 1:
+                raise ValueError('Need at least one path part to construct a path')
+            self._path = path_parts
+
+        def iterdir(self):
+            return iter(())
+
+        def is_file(self):
+            return False
+
+        is_dir = is_file
+
+        def joinpath(self, other):
+            return CompatibilityFiles.OrphanPath(*self._path, other)
+
+        @property
+        def name(self):
+            return self._path[-1]
+
+        def open(self, mode='r', *args, **kwargs):
+            raise FileNotFoundError("Can't open orphan path")
+
+    def __init__(self, spec):
+        self.spec = spec
+
+    @property
+    def _reader(self):
+        with suppress(AttributeError):
+            return self.spec.loader.get_resource_reader(self.spec.name)
+
+    def _native(self):
+        """
+        Return the native reader if it supports files().
+        """
+        reader = self._reader
+        return reader if hasattr(reader, 'files') else self
+
+    def __getattr__(self, attr):
+        return getattr(self._reader, attr)
+
+    def files(self):
+        return CompatibilityFiles.SpecPath(self.spec, self._reader)
+
+
+def wrap_spec(package):
+    """
+    Construct a package spec with traversable compatibility
+    on the spec/loader/reader.
+    """
+    return SpecLoaderAdapter(package.__spec__, TraversableResourcesLoader)

lib/importlib_resources/_common.py

@@ -0,0 +1,104 @@
+import os
+import pathlib
+import tempfile
+import functools
+import contextlib
+import types
+import importlib
+
+from typing import Union, Optional
+from .abc import ResourceReader, Traversable
+
+from ._compat import wrap_spec
+
+Package = Union[types.ModuleType, str]
+
+
+def files(package):
+    # type: (Package) -> Traversable
+    """
+    Get a Traversable resource from a package
+    """
+    return from_package(get_package(package))
+
+
+def get_resource_reader(package):
+    # type: (types.ModuleType) -> Optional[ResourceReader]
+    """
+    Return the package's loader if it's a ResourceReader.
+    """
+    # We can't use
+    # a issubclass() check here because apparently abc.'s __subclasscheck__()
+    # hook wants to create a weak reference to the object, but
+    # zipimport.zipimporter does not support weak references, resulting in a
+    # TypeError.  That seems terrible.
+    spec = package.__spec__
+    reader = getattr(spec.loader, 'get_resource_reader', None)  # type: ignore
+    if reader is None:
+        return None
+    return reader(spec.name)  # type: ignore
+
+
+def resolve(cand):
+    # type: (Package) -> types.ModuleType
+    return cand if isinstance(cand, types.ModuleType) else importlib.import_module(cand)
+
+
+def get_package(package):
+    # type: (Package) -> types.ModuleType
+    """Take a package name or module object and return the module.
+
+    Raise an exception if the resolved module is not a package.
+    """
+    resolved = resolve(package)
+    if wrap_spec(resolved).submodule_search_locations is None:
+        raise TypeError(f'{package!r} is not a package')
+    return resolved
+
+
+def from_package(package):
+    """
+    Return a Traversable object for the given package.
+
+    """
+    spec = wrap_spec(package)
+    reader = spec.loader.get_resource_reader(spec.name)
+    return reader.files()
+
+
+@contextlib.contextmanager
+def _tempfile(reader, suffix=''):
+    # Not using tempfile.NamedTemporaryFile as it leads to deeper 'try'
+    # blocks due to the need to close the temporary file to work on Windows
+    # properly.
+    fd, raw_path = tempfile.mkstemp(suffix=suffix)
+    try:
+        try:
+            os.write(fd, reader())
+        finally:
+            os.close(fd)
+        del reader
+        yield pathlib.Path(raw_path)
+    finally:
+        try:
+            os.remove(raw_path)
+        except FileNotFoundError:
+            pass
+
+
+@functools.singledispatch
+def as_file(path):
+    """
+    Given a Traversable object, return that object as a
+    path on the local file system in a context manager.
+    """
+    return _tempfile(path.read_bytes, suffix=path.name)
+
+
+@as_file.register(pathlib.Path)
+@contextlib.contextmanager
+def _(path):
+    """
+    Degenerate behavior for pathlib.Path objects.
+    """
+    yield path

lib/importlib_resources/_compat.py

@@ -0,0 +1,98 @@
+# flake8: noqa
+
+import abc
+import sys
+import pathlib
+from contextlib import suppress
+
+if sys.version_info >= (3, 10):
+    from zipfile import Path as ZipPath  # type: ignore
+else:
+    from zipp import Path as ZipPath  # type: ignore
+
+
+try:
+    from typing import runtime_checkable  # type: ignore
+except ImportError:
+
+    def runtime_checkable(cls):  # type: ignore
+        return cls
+
+
+try:
+    from typing import Protocol  # type: ignore
+except ImportError:
+    Protocol = abc.ABC  # type: ignore
+
+
+class TraversableResourcesLoader:
+    """
+    Adapt loaders to provide TraversableResources and other
+    compatibility.
+
+    Used primarily for Python 3.9 and earlier where the native
+    loaders do not yet implement TraversableResources.
+    """
+
+    def __init__(self, spec):
+        self.spec = spec
+
+    @property
+    def path(self):
+        return self.spec.origin
+
+    def get_resource_reader(self, name):
+        from . import readers, _adapters
+
+        def _zip_reader(spec):
+            with suppress(AttributeError):
+                return readers.ZipReader(spec.loader, spec.name)
+
+        def _namespace_reader(spec):
+            with suppress(AttributeError, ValueError):
+                return readers.NamespaceReader(spec.submodule_search_locations)
+
+        def _available_reader(spec):
+            with suppress(AttributeError):
+                return spec.loader.get_resource_reader(spec.name)
+
+        def _native_reader(spec):
+            reader = _available_reader(spec)
+            return reader if hasattr(reader, 'files') else None
+
+        def _file_reader(spec):
+            try:
+                path = pathlib.Path(self.path)
+            except TypeError:
+                return None
+            if path.exists():
+                return readers.FileReader(self)
+
+        return (
+            # native reader if it supplies 'files'
+            _native_reader(self.spec)
+            or
+            # local ZipReader if a zip module
+            _zip_reader(self.spec)
+            or
+            # local NamespaceReader if a namespace module
+            _namespace_reader(self.spec)
+            or
+            # local FileReader
+            _file_reader(self.spec)
+            # fallback - adapt the spec ResourceReader to TraversableReader
+            or _adapters.CompatibilityFiles(self.spec)
+        )
+
+
+def wrap_spec(package):
+    """
+    Construct a package spec with traversable compatibility
+    on the spec/loader/reader.
+
+    Supersedes _adapters.wrap_spec to use TraversableResourcesLoader
+    from above for older Python compatibility (<3.10).
+    """
+    from . import _adapters
+
+    return _adapters.SpecLoaderAdapter(package.__spec__, TraversableResourcesLoader)

lib/importlib_resources/_itertools.py

@@ -0,0 +1,35 @@
+from itertools import filterfalse
+
+from typing import (
+    Callable,
+    Iterable,
+    Iterator,
+    Optional,
+    Set,
+    TypeVar,
+    Union,
+)
+
+# Type and type variable definitions
+_T = TypeVar('_T')
+_U = TypeVar('_U')
+
+
+def unique_everseen(
+    iterable: Iterable[_T], key: Optional[Callable[[_T], _U]] = None
+) -> Iterator[_T]:
+    "List unique elements, preserving order. Remember all elements ever seen."
+    # unique_everseen('AAAABBBCCDAABBB') --> A B C D
+    # unique_everseen('ABBCcAD', str.lower) --> A B C D
+    seen: Set[Union[_T, _U]] = set()
+    seen_add = seen.add
+    if key is None:
+        for element in filterfalse(seen.__contains__, iterable):
+            seen_add(element)
+            yield element
+    else:
+        for element in iterable:
+            k = key(element)
+            if k not in seen:
+                seen_add(k)
+                yield element

lib/importlib_resources/_legacy.py

@@ -0,0 +1,121 @@
+import functools
+import os
+import pathlib
+import types
+import warnings
+
+from typing import Union, Iterable, ContextManager, BinaryIO, TextIO, Any
+
+from . import _common
+
+Package = Union[types.ModuleType, str]
+Resource = str
+
+
+def deprecated(func):
+    @functools.wraps(func)
+    def wrapper(*args, **kwargs):
+        warnings.warn(
+            f"{func.__name__} is deprecated. Use files() instead. "
+            "Refer to https://importlib-resources.readthedocs.io"
+            "/en/latest/using.html#migrating-from-legacy for migration advice.",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        return func(*args, **kwargs)
+
+    return wrapper
+
+
+def normalize_path(path):
+    # type: (Any) -> str
+    """Normalize a path by ensuring it is a string.
+
+    If the resulting string contains path separators, an exception is raised.
+    """
+    str_path = str(path)
+    parent, file_name = os.path.split(str_path)
+    if parent:
+        raise ValueError(f'{path!r} must be only a file name')
+    return file_name
+
+
+@deprecated
+def open_binary(package: Package, resource: Resource) -> BinaryIO:
+    """Return a file-like object opened for binary reading of the resource."""
+    return (_common.files(package) / normalize_path(resource)).open('rb')
+
+
+@deprecated
+def read_binary(package: Package, resource: Resource) -> bytes:
+    """Return the binary contents of the resource."""
+    return (_common.files(package) / normalize_path(resource)).read_bytes()
+
+
+@deprecated
+def open_text(
+    package: Package,
+    resource: Resource,
+    encoding: str = 'utf-8',
+    errors: str = 'strict',
+) -> TextIO:
+    """Return a file-like object opened for text reading of the resource."""
+    return (_common.files(package) / normalize_path(resource)).open(
+        'r', encoding=encoding, errors=errors
+    )
+
+
+@deprecated
+def read_text(
+    package: Package,
+    resource: Resource,
+    encoding: str = 'utf-8',
+    errors: str = 'strict',
+) -> str:
+    """Return the decoded string of the resource.
+
+    The decoding-related arguments have the same semantics as those of
+    bytes.decode().
+    """
+    with open_text(package, resource, encoding, errors) as fp:
+        return fp.read()
+
+
+@deprecated
+def contents(package: Package) -> Iterable[str]:
+    """Return an iterable of entries in `package`.
+
+    Note that not all entries are resources.  Specifically, directories are
+    not considered resources.  Use `is_resource()` on each entry returned here
+    to check if it is a resource or not.
+    """
+    return [path.name for path in _common.files(package).iterdir()]
+
+
+@deprecated
+def is_resource(package: Package, name: str) -> bool:
+    """True if `name` is a resource inside `package`.
+
+    Directories are *not* resources.
+    """
+    resource = normalize_path(name)
+    return any(
+        traversable.name == resource and traversable.is_file()
+        for traversable in _common.files(package).iterdir()
+    )
+
+
+@deprecated
+def path(
+    package: Package,
+    resource: Resource,
+) -> ContextManager[pathlib.Path]:
+    """A context manager providing a file path object to the resource.
+
+    If the resource does not already exist on its own on the file system,
+    a temporary file will be created. If the file was created, the file
+    will be deleted upon exiting the context manager (no exception is
+    raised if the file was deleted prior to the context manager
+    exiting).
+    """
+    return _common.as_file(_common.files(package) / normalize_path(resource))

lib/importlib_resources/abc.py

@@ -0,0 +1,137 @@
+import abc
+from typing import BinaryIO, Iterable, Text
+
+from ._compat import runtime_checkable, Protocol
+
+
+class ResourceReader(metaclass=abc.ABCMeta):
+    """Abstract base class for loaders to provide resource reading support."""
+
+    @abc.abstractmethod
+    def open_resource(self, resource: Text) -> BinaryIO:
+        """Return an opened, file-like object for binary reading.
+
+        The 'resource' argument is expected to represent only a file name.
+        If the resource cannot be found, FileNotFoundError is raised.
+        """
+        # This deliberately raises FileNotFoundError instead of
+        # NotImplementedError so that if this method is accidentally called,
+        # it'll still do the right thing.
+        raise FileNotFoundError
+
+    @abc.abstractmethod
+    def resource_path(self, resource: Text) -> Text:
+        """Return the file system path to the specified resource.
+
+        The 'resource' argument is expected to represent only a file name.
+        If the resource does not exist on the file system, raise
+        FileNotFoundError.
+        """
+        # This deliberately raises FileNotFoundError instead of
+        # NotImplementedError so that if this method is accidentally called,
+        # it'll still do the right thing.
+        raise FileNotFoundError
+
+    @abc.abstractmethod
+    def is_resource(self, path: Text) -> bool:
+        """Return True if the named 'path' is a resource.
+
+        Files are resources, directories are not.
+        """
+        raise FileNotFoundError
+
+    @abc.abstractmethod
+    def contents(self) -> Iterable[str]:
+        """Return an iterable of entries in `package`."""
+        raise FileNotFoundError
+
+
+@runtime_checkable
+class Traversable(Protocol):
+    """
+    An object with a subset of pathlib.Path methods suitable for
+    traversing directories and opening files.
+    """
+
+    @abc.abstractmethod
+    def iterdir(self):
+        """
+        Yield Traversable objects in self
+        """
+
+    def read_bytes(self):
+        """
+        Read contents of self as bytes
+        """
+        with self.open('rb') as strm:
+            return strm.read()
+
+    def read_text(self, encoding=None):
+        """
+        Read contents of self as text
+        """
+        with self.open(encoding=encoding) as strm:
+            return strm.read()
+
+    @abc.abstractmethod
+    def is_dir(self) -> bool:
+        """
+        Return True if self is a directory
+        """
+
+    @abc.abstractmethod
+    def is_file(self) -> bool:
+        """
+        Return True if self is a file
+        """
+
+    @abc.abstractmethod
+    def joinpath(self, child):
+        """
+        Return Traversable child in self
+        """
+
+    def __truediv__(self, child):
+        """
+        Return Traversable child in self
+        """
+        return self.joinpath(child)
+
+    @abc.abstractmethod
+    def open(self, mode='r', *args, **kwargs):
+        """
+        mode may be 'r' or 'rb' to open as text or binary. Return a handle
+        suitable for reading (same as pathlib.Path.open).
+
+        When opening as text, accepts encoding parameters such as those
+        accepted by io.TextIOWrapper.
+        """
+
+    @abc.abstractproperty
+    def name(self) -> str:
+        """
+        The base name of this object without any parent references.
+        """
+
+
+class TraversableResources(ResourceReader):
+    """
+    The required interface for providing traversable
+    resources.
+    """
+
+    @abc.abstractmethod
+    def files(self):
+        """Return a Traversable object for the loaded package."""
+
+    def open_resource(self, resource):
+        return self.files().joinpath(resource).open('rb')
+
+    def resource_path(self, resource):
+        raise FileNotFoundError(resource)
+
+    def is_resource(self, path):
+        return self.files().joinpath(path).is_file()
+
+    def contents(self):
+        return (item.name for item in self.files().iterdir())

lib/importlib_resources/readers.py

@@ -0,0 +1,122 @@
+import collections
+import pathlib
+import operator
+
+from . import abc
+
+from ._itertools import unique_everseen
+from ._compat import ZipPath
+
+
+def remove_duplicates(items):
+    return iter(collections.OrderedDict.fromkeys(items))
+
+
+class FileReader(abc.TraversableResources):
+    def __init__(self, loader):
+        self.path = pathlib.Path(loader.path).parent
+
+    def resource_path(self, resource):
+        """
+        Return the file system path to prevent
+        `resources.path()` from creating a temporary
+        copy.
+        """
+        return str(self.path.joinpath(resource))
+
+    def files(self):
+        return self.path
+
+
+class ZipReader(abc.TraversableResources):
+    def __init__(self, loader, module):
+        _, _, name = module.rpartition('.')
+        self.prefix = loader.prefix.replace('\\', '/') + name + '/'
+        self.archive = loader.archive
+
+    def open_resource(self, resource):
+        try:
+            return super().open_resource(resource)
+        except KeyError as exc:
+            raise FileNotFoundError(exc.args[0])
+
+    def is_resource(self, path):
+        # workaround for `zipfile.Path.is_file` returning true
+        # for non-existent paths.
+        target = self.files().joinpath(path)
+        return target.is_file() and target.exists()
+
+    def files(self):
+        return ZipPath(self.archive, self.prefix)
+
+
+class MultiplexedPath(abc.Traversable):
+    """
+    Given a series of Traversable objects, implement a merged
+    version of the interface across all objects. Useful for
+    namespace packages which may be multihomed at a single
+    name.
+    """
+
+    def __init__(self, *paths):
+        self._paths = list(map(pathlib.Path, remove_duplicates(paths)))
+        if not self._paths:
+            message = 'MultiplexedPath must contain at least one path'
+            raise FileNotFoundError(message)
+        if not all(path.is_dir() for path in self._paths):
+            raise NotADirectoryError('MultiplexedPath only supports directories')
+
+    def iterdir(self):
+        files = (file for path in self._paths for file in path.iterdir())
+        return unique_everseen(files, key=operator.attrgetter('name'))
+
+    def read_bytes(self):
+        raise FileNotFoundError(f'{self} is not a file')
+
+    def read_text(self, *args, **kwargs):
+        raise FileNotFoundError(f'{self} is not a file')
+
+    def is_dir(self):
+        return True
+
+    def is_file(self):
+        return False
+
+    def joinpath(self, child):
+        # first try to find child in current paths
+        for file in self.iterdir():
+            if file.name == child:
+                return file
+        # if it does not exist, construct it with the first path
+        return self._paths[0] / child
+
+    __truediv__ = joinpath
+
+    def open(self, *args, **kwargs):
+        raise FileNotFoundError(f'{self} is not a file')
+
+    @property
+    def name(self):
+        return self._paths[0].name
+
+    def __repr__(self):
+        paths = ', '.join(f"'{path}'" for path in self._paths)
+        return f'MultiplexedPath({paths})'
+
+
+class NamespaceReader(abc.TraversableResources):
+    def __init__(self, namespace_path):
+        if 'NamespacePath' not in str(namespace_path):
+            raise ValueError('Invalid path')
+        self.path = MultiplexedPath(*list(namespace_path))
+
+    def resource_path(self, resource):
+        """
+        Return the file system path to prevent
+        `resources.path()` from creating a temporary
+        copy.
+        """
+        return str(self.path.joinpath(resource))
+
+    def files(self):
+        return self.path

lib/importlib_resources/simple.py

@@ -0,0 +1,116 @@
+"""
+Interface adapters for low-level readers.
+"""
+
+import abc
+import io
+import itertools
+from typing import BinaryIO, List
+
+from .abc import Traversable, TraversableResources
+
+
+class SimpleReader(abc.ABC):
+    """
+    The minimum, low-level interface required from a resource
+    provider.
+    """
+
+    @abc.abstractproperty
+    def package(self):
+        # type: () -> str
+        """
+        The name of the package for which this reader loads resources.
+        """
+
+    @abc.abstractmethod
+    def children(self):
+        # type: () -> List['SimpleReader']
+        """
+        Obtain an iterable of SimpleReader for available
+        child containers (e.g. directories).
+        """
+
+    @abc.abstractmethod
+    def resources(self):
+        # type: () -> List[str]
+        """
+        Obtain available named resources for this virtual package.
+        """
+
+    @abc.abstractmethod
+    def open_binary(self, resource):
+        # type: (str) -> BinaryIO
+        """
+        Obtain a File-like for a named resource.
+        """
+
+    @property
+    def name(self):
+        return self.package.split('.')[-1]
+
+
+class ResourceHandle(Traversable):
+    """
+    Handle to a named resource in a ResourceReader.
+    """
+
+    def __init__(self, parent, name):
+        # type: (ResourceContainer, str) -> None
+        self.parent = parent
+        self.name = name  # type: ignore
+
+    def is_file(self):
+        return True
+
+    def is_dir(self):
+        return False
+
+    def open(self, mode='r', *args, **kwargs):
+        stream = self.parent.reader.open_binary(self.name)
+        if 'b' not in mode:
+            stream = io.TextIOWrapper(*args, **kwargs)
+        return stream
+
+    def joinpath(self, name):
+        raise RuntimeError("Cannot traverse into a resource")
+
+
+class ResourceContainer(Traversable):
+    """
+    Traversable container for a package's resources via its reader.
+    """
+
+    def __init__(self, reader):
+        # type: (SimpleReader) -> None
+        self.reader = reader
+
+    def is_dir(self):
+        return True
+
+    def is_file(self):
+        return False
+
+    def iterdir(self):
+        files = (ResourceHandle(self, name) for name in self.reader.resources)
+        dirs = map(ResourceContainer, self.reader.children())
+        return itertools.chain(files, dirs)
+
+    def open(self, *args, **kwargs):
+        raise IsADirectoryError()
+
+    def joinpath(self, name):
+        return next(
+            traversable for traversable in self.iterdir() if traversable.name == name
+        )
+
+
+class TraversableReader(TraversableResources, SimpleReader):
+    """
+    A TraversableResources based on SimpleReader. Resource providers
+    may derive from this class to provide the TraversableResources
+    interface by supplying the SimpleReader interface.
+    """
+
+    def files(self):
+        return ResourceContainer(self)

lib/mutagen/__init__.py

@@ -23,7 +23,7 @@ from mutagen._util import MutagenError
 from mutagen._file import FileType, StreamInfo, File
 from mutagen._tags import Tags, Metadata, PaddingInfo
 
-version = (1, 38, -1)
+version = (1, 45, 1)
 """Version tuple."""
 
 version_string = ".".join(map(str, version))

lib/mutagen/_compat.py

@@ -1,86 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright (C) 2013  Christoph Reiter
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-
-import sys
-
-
-PY2 = sys.version_info[0] == 2
-PY3 = not PY2
-
-if PY2:
-    from io import StringIO
-    BytesIO = StringIO
-    from io import StringIO as cBytesIO
-    
-
-    long_ = int
-    integer_types = (int, int)
-    string_types = (str, str)
-    text_type = str
-
-    xrange = xrange
-    cmp = cmp
-    chr_ = chr
-
-    def endswith(text, end):
-        return text.endswith(end)
-
-    iteritems = lambda d: iter(d.items())
-    itervalues = lambda d: iter(d.values())
-    iterkeys = lambda d: iter(d.keys())
-
-    iterbytes = lambda b: iter(b)
-
-    exec("def reraise(tp, value, tb):\n raise tp, value, tb")
-
-    def swap_to_string(cls):
-        if "__str__" in cls.__dict__:
-            cls.__unicode__ = cls.__str__
-
-        if "__bytes__" in cls.__dict__:
-            cls.__str__ = cls.__bytes__
-
-        return cls
-
-elif PY3:
-    from io import StringIO
-    StringIO = StringIO
-    from io import BytesIO
-    cBytesIO = BytesIO
-
-    long_ = int
-    integer_types = (int,)
-    string_types = (str,)
-    text_type = str
-
-    izip = zip
-    xrange = range
-    cmp = lambda a, b: (a > b) - (a < b)
-    chr_ = lambda x: bytes([x])
-
-    def endswith(text, end):
-        # usefull for paths which can be both, str and bytes
-        if isinstance(text, str):
-            if not isinstance(end, str):
-                end = end.decode("ascii")
-        else:
-            if not isinstance(end, bytes):
-                end = end.encode("ascii")
-        return text.endswith(end)
-
-    iteritems = lambda d: iter(list(d.items()))
-    itervalues = lambda d: iter(list(d.values()))
-    iterkeys = lambda d: iter(list(d.keys()))
-
-    iterbytes = lambda b: (bytes([v]) for v in b)
-
-    def reraise(tp, value, tb):
-        raise tp(value).with_traceback(tb)
-
-    def swap_to_string(cls):
-        return cls

lib/mutagen/_constants.py

@@ -8,197 +8,197 @@
 """Constants used by Mutagen."""
 
 GENRES = [
-    "Blues",
+    u"Blues",
-    "Classic Rock",
+    u"Classic Rock",
-    "Country",
+    u"Country",
-    "Dance",
+    u"Dance",
-    "Disco",
+    u"Disco",
-    "Funk",
+    u"Funk",
-    "Grunge",
+    u"Grunge",
-    "Hip-Hop",
+    u"Hip-Hop",
-    "Jazz",
+    u"Jazz",
-    "Metal",
+    u"Metal",
-    "New Age",
+    u"New Age",
-    "Oldies",
+    u"Oldies",
-    "Other",
+    u"Other",
-    "Pop",
+    u"Pop",
-    "R&B",
+    u"R&B",
-    "Rap",
+    u"Rap",
-    "Reggae",
+    u"Reggae",
-    "Rock",
+    u"Rock",
-    "Techno",
+    u"Techno",
-    "Industrial",
+    u"Industrial",
-    "Alternative",
+    u"Alternative",
-    "Ska",
+    u"Ska",
-    "Death Metal",
+    u"Death Metal",
-    "Pranks",
+    u"Pranks",
-    "Soundtrack",
+    u"Soundtrack",
-    "Euro-Techno",
+    u"Euro-Techno",
-    "Ambient",
+    u"Ambient",
-    "Trip-Hop",
+    u"Trip-Hop",
-    "Vocal",
+    u"Vocal",
-    "Jazz+Funk",
+    u"Jazz+Funk",
-    "Fusion",
+    u"Fusion",
-    "Trance",
+    u"Trance",
-    "Classical",
+    u"Classical",
-    "Instrumental",
+    u"Instrumental",
-    "Acid",
+    u"Acid",
-    "House",
+    u"House",
-    "Game",
+    u"Game",
-    "Sound Clip",
+    u"Sound Clip",
-    "Gospel",
+    u"Gospel",
-    "Noise",
+    u"Noise",
-    "Alt. Rock",
+    u"Alt. Rock",
-    "Bass",
+    u"Bass",
-    "Soul",
+    u"Soul",
-    "Punk",
+    u"Punk",
-    "Space",
+    u"Space",
-    "Meditative",
+    u"Meditative",
-    "Instrumental Pop",
+    u"Instrumental Pop",
-    "Instrumental Rock",
+    u"Instrumental Rock",
-    "Ethnic",
+    u"Ethnic",
-    "Gothic",
+    u"Gothic",
-    "Darkwave",
+    u"Darkwave",
-    "Techno-Industrial",
+    u"Techno-Industrial",
-    "Electronic",
+    u"Electronic",
-    "Pop-Folk",
+    u"Pop-Folk",
-    "Eurodance",
+    u"Eurodance",
-    "Dream",
+    u"Dream",
-    "Southern Rock",
+    u"Southern Rock",
-    "Comedy",
+    u"Comedy",
-    "Cult",
+    u"Cult",
-    "Gangsta Rap",
+    u"Gangsta Rap",
-    "Top 40",
+    u"Top 40",
-    "Christian Rap",
+    u"Christian Rap",
-    "Pop/Funk",
+    u"Pop/Funk",
-    "Jungle",
+    u"Jungle",
-    "Native American",
+    u"Native American",
-    "Cabaret",
+    u"Cabaret",
-    "New Wave",
+    u"New Wave",
-    "Psychedelic",
+    u"Psychedelic",
-    "Rave",
+    u"Rave",
-    "Showtunes",
+    u"Showtunes",
-    "Trailer",
+    u"Trailer",
-    "Lo-Fi",
+    u"Lo-Fi",
-    "Tribal",
+    u"Tribal",
-    "Acid Punk",
+    u"Acid Punk",
-    "Acid Jazz",
+    u"Acid Jazz",
-    "Polka",
+    u"Polka",
-    "Retro",
+    u"Retro",
-    "Musical",
+    u"Musical",
-    "Rock & Roll",
+    u"Rock & Roll",
-    "Hard Rock",
+    u"Hard Rock",
-    "Folk",
+    u"Folk",
-    "Folk-Rock",
+    u"Folk-Rock",
-    "National Folk",
+    u"National Folk",
-    "Swing",
+    u"Swing",
-    "Fast-Fusion",
+    u"Fast-Fusion",
-    "Bebop",
+    u"Bebop",
-    "Latin",
+    u"Latin",
-    "Revival",
+    u"Revival",
-    "Celtic",
+    u"Celtic",
-    "Bluegrass",
+    u"Bluegrass",
-    "Avantgarde",
+    u"Avantgarde",
-    "Gothic Rock",
+    u"Gothic Rock",
-    "Progressive Rock",
+    u"Progressive Rock",
-    "Psychedelic Rock",
+    u"Psychedelic Rock",
-    "Symphonic Rock",
+    u"Symphonic Rock",
-    "Slow Rock",
+    u"Slow Rock",
-    "Big Band",
+    u"Big Band",
-    "Chorus",
+    u"Chorus",
-    "Easy Listening",
+    u"Easy Listening",
-    "Acoustic",
+    u"Acoustic",
-    "Humour",
+    u"Humour",
-    "Speech",
+    u"Speech",
-    "Chanson",
+    u"Chanson",
-    "Opera",
+    u"Opera",
-    "Chamber Music",
+    u"Chamber Music",
-    "Sonata",
+    u"Sonata",
-    "Symphony",
+    u"Symphony",
-    "Booty Bass",
+    u"Booty Bass",
-    "Primus",
+    u"Primus",
-    "Porn Groove",
+    u"Porn Groove",
-    "Satire",
+    u"Satire",
-    "Slow Jam",
+    u"Slow Jam",
-    "Club",
+    u"Club",
-    "Tango",
+    u"Tango",
-    "Samba",
+    u"Samba",
-    "Folklore",
+    u"Folklore",
-    "Ballad",
+    u"Ballad",
-    "Power Ballad",
+    u"Power Ballad",
-    "Rhythmic Soul",
+    u"Rhythmic Soul",
-    "Freestyle",
+    u"Freestyle",
-    "Duet",
+    u"Duet",
-    "Punk Rock",
+    u"Punk Rock",
-    "Drum Solo",
+    u"Drum Solo",
-    "A Cappella",
+    u"A Cappella",
-    "Euro-House",
+    u"Euro-House",
-    "Dance Hall",
+    u"Dance Hall",
-    "Goa",
+    u"Goa",
-    "Drum & Bass",
+    u"Drum & Bass",
-    "Club-House",
+    u"Club-House",
-    "Hardcore",
+    u"Hardcore",
-    "Terror",
+    u"Terror",
-    "Indie",
+    u"Indie",
-    "BritPop",
+    u"BritPop",
-    "Afro-Punk",
+    u"Afro-Punk",
-    "Polsk Punk",
+    u"Polsk Punk",
-    "Beat",
+    u"Beat",
-    "Christian Gangsta Rap",
+    u"Christian Gangsta Rap",
-    "Heavy Metal",
+    u"Heavy Metal",
-    "Black Metal",
+    u"Black Metal",
-    "Crossover",
+    u"Crossover",
-    "Contemporary Christian",
+    u"Contemporary Christian",
-    "Christian Rock",
+    u"Christian Rock",
-    "Merengue",
+    u"Merengue",
-    "Salsa",
+    u"Salsa",
-    "Thrash Metal",
+    u"Thrash Metal",
-    "Anime",
+    u"Anime",
-    "JPop",
+    u"JPop",
-    "Synthpop",
+    u"Synthpop",
-    "Abstract",
+    u"Abstract",
-    "Art Rock",
+    u"Art Rock",
-    "Baroque",
+    u"Baroque",
-    "Bhangra",
+    u"Bhangra",
-    "Big Beat",
+    u"Big Beat",
-    "Breakbeat",
+    u"Breakbeat",
-    "Chillout",
+    u"Chillout",
-    "Downtempo",
+    u"Downtempo",
-    "Dub",
+    u"Dub",
-    "EBM",
+    u"EBM",
-    "Eclectic",
+    u"Eclectic",
-    "Electro",
+    u"Electro",
-    "Electroclash",
+    u"Electroclash",
-    "Emo",
+    u"Emo",
-    "Experimental",
+    u"Experimental",
-    "Garage",
+    u"Garage",
-    "Global",
+    u"Global",
-    "IDM",
+    u"IDM",
-    "Illbient",
+    u"Illbient",
-    "Industro-Goth",
+    u"Industro-Goth",
-    "Jam Band",
+    u"Jam Band",
-    "Krautrock",
+    u"Krautrock",
-    "Leftfield",
+    u"Leftfield",
-    "Lounge",
+    u"Lounge",
-    "Math Rock",
+    u"Math Rock",
-    "New Romantic",
+    u"New Romantic",
-    "Nu-Breakz",
+    u"Nu-Breakz",
-    "Post-Punk",
+    u"Post-Punk",
-    "Post-Rock",
+    u"Post-Rock",
-    "Psytrance",
+    u"Psytrance",
-    "Shoegaze",
+    u"Shoegaze",
-    "Space Rock",
+    u"Space Rock",
-    "Trop Rock",
+    u"Trop Rock",
-    "World Music",
+    u"World Music",
-    "Neoclassical",
+    u"Neoclassical",
-    "Audiobook",
+    u"Audiobook",
-    "Audio Theatre",
+    u"Audio Theatre",
-    "Neue Deutsche Welle",
+    u"Neue Deutsche Welle",
-    "Podcast",
+    u"Podcast",
-    "Indie Rock",
+    u"Indie Rock",
-    "G-Funk",
+    u"G-Funk",
-    "Dubstep",
+    u"Dubstep",
-    "Garage Rock",
+    u"Garage Rock",
-    "Psybient",
+    u"Psybient",
 ]
 """The ID3v1 genre list."""

lib/mutagen/_file.py

@@ -9,7 +9,6 @@
 import warnings
 
 from mutagen._util import DictMixin, loadfile
-from mutagen._compat import izip
 
 
 class FileType(DictMixin):
@@ -94,10 +93,10 @@ class FileType(DictMixin):
         if self.tags is None:
             return []
         else:
-            return list(self.tags.keys())
+            return self.tags.keys()
 
     @loadfile(writable=True)
-    def delete(self, filething):
+    def delete(self, filething=None):
         """delete(filething=None)
 
         Remove tags from a file.
@@ -113,14 +112,14 @@ class FileType(DictMixin):
         Does nothing if the file has no tags.
 
         Raises:
-            MutagenError: if deleting wasn't possible
+            mutagen.MutagenError: if deleting wasn't possible
         """
 
         if self.tags is not None:
             return self.tags.delete(filething)
 
     @loadfile(writable=True)
-    def save(self, filething, **kwargs):
+    def save(self, filething=None, **kwargs):
         """save(filething=None, **kwargs)
 
         Save metadata tags.
@@ -150,14 +149,15 @@ class FileType(DictMixin):
         """Adds new tags to the file.
 
         Raises:
-            MutagenError: if tags already exist or adding is not possible.
+            mutagen.MutagenError:
+                if tags already exist or adding is not possible.
         """
 
         raise NotImplementedError
 
     @property
     def mime(self):
-        """A list of mime types (`text`)"""
+        """A list of mime types (:class:`mutagen.text`)"""
 
         mimes = []
         for Kind in type(self).__mro__:
@@ -171,7 +171,7 @@ class FileType(DictMixin):
         """Returns a score for how likely the file can be parsed by this type.
 
         Args:
-            filename (path): a file path
+            filename (fspath): a file path
             fileobj (fileobj): a file object open in rb mode. Position is
                 undefined
             header (bytes): data of undefined length, starts with the start of
@@ -220,13 +220,13 @@ def File(filething, options=None, easy=False):
         filething (filething)
         options: Sequence of :class:`FileType` implementations,
             defaults to all included ones.
-        easy (bool):  If the easy wrappers should be returnd if available.
+        easy (bool):  If the easy wrappers should be returned if available.
             For example :class:`EasyMP3 <mp3.EasyMP3>` instead of
             :class:`MP3 <mp3.MP3>`.
 
     Returns:
         FileType: A FileType instance for the detected type or `None` in case
-            the type couln't be determined.
+            the type couldn't be determined.
 
     Raises:
         MutagenError: in case the detected type fails to load the file.
@@ -263,12 +263,16 @@ def File(filething, options=None, easy=False):
         from mutagen.optimfrog import OptimFROG
         from mutagen.aiff import AIFF
         from mutagen.aac import AAC
+        from mutagen.ac3 import AC3
         from mutagen.smf import SMF
+        from mutagen.tak import TAK
         from mutagen.dsf import DSF
+        from mutagen.dsdiff import DSDIFF
+        from mutagen.wave import WAVE
         options = [MP3, TrueAudio, OggTheora, OggSpeex, OggVorbis, OggFLAC,
                    FLAC, AIFF, APEv2File, MP4, ID3FileType, WavPack,
-                   Musepack, MonkeysAudio, OptimFROG, ASF, OggOpus, AAC,
+                   Musepack, MonkeysAudio, OptimFROG, ASF, OggOpus, AAC, AC3,
-                   SMF, DSF]
+                   SMF, TAK, DSF, DSDIFF, WAVE]
 
     if not options:
         return None

lib/mutagen/_iff.py

@@ -0,0 +1,387 @@
+# -*- coding: utf-8 -*-
+# Copyright (C) 2014  Evan Purkhiser
+#               2014  Ben Ockmore
+#               2017  Borewit
+#               2019-2020  Philipp Wolfer
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+
+"""Base classes for various IFF based formats (e.g. AIFF or RIFF)."""
+
+import sys
+
+from mutagen.id3 import ID3
+from mutagen.id3._util import ID3NoHeaderError, error as ID3Error
+from mutagen._util import (
+    MutagenError,
+    convert_error,
+    delete_bytes,
+    insert_bytes,
+    loadfile,
+    reraise,
+    resize_bytes,
+)
+
+
+class error(MutagenError):
+    pass
+
+
+class InvalidChunk(error):
+    pass
+
+
+class EmptyChunk(InvalidChunk):
+    pass
+
+
+def is_valid_chunk_id(id):
+    """ is_valid_chunk_id(FOURCC)
+
+    Arguments:
+        id (FOURCC)
+    Returns:
+        true if valid; otherwise false
+
+    Check if argument id is valid FOURCC type.
+    """
+
+    assert isinstance(id, str), \
+        'id is of type %s, must be str: %r' % (type(id), id)
+
+    return ((0 < len(id) <= 4) and (min(id) >= ' ') and
+            (max(id) <= '~'))
+
+
+#  Assert FOURCC formatted valid
+def assert_valid_chunk_id(id):
+    if not is_valid_chunk_id(id):
+        raise ValueError("IFF chunk ID must be four ASCII characters.")
+
+
+class IffChunk(object):
+    """Generic representation of a single IFF chunk.
+
+    IFF chunks always consist of an ID followed by the chunk size. The exact
+    format varies between different IFF based formats, e.g. AIFF uses
+    big-endian while RIFF uses little-endian.
+    """
+
+    # Chunk headers are usually 8 bytes long (4 for ID and 4 for the size)
+    HEADER_SIZE = 8
+
+    @classmethod
+    def parse_header(cls, header):
+        """Read ID and data_size from the given header.
+        Must be implemented in subclasses."""
+        raise error("Not implemented")
+
+    def write_new_header(self, id_, size):
+        """Write the chunk header with id_ and size to the file.
+        Must be implemented in subclasses. The data must be written
+        to the current position in self._fileobj."""
+        raise error("Not implemented")
+
+    def write_size(self):
+        """Write self.data_size to the file.
+        Must be implemented in subclasses. The data must be written
+        to the current position in self._fileobj."""
+        raise error("Not implemented")
+
+    @classmethod
+    def get_class(cls, id):
+        """Returns the class for a new chunk for a given ID.
+        Can be overridden in subclasses to implement specific chunk types."""
+        return cls
+
+    @classmethod
+    def parse(cls, fileobj, parent_chunk=None):
+        header = fileobj.read(cls.HEADER_SIZE)
+        if len(header) < cls.HEADER_SIZE:
+            raise EmptyChunk('Header size < %i' % cls.HEADER_SIZE)
+        id, data_size = cls.parse_header(header)
+        try:
+            id = id.decode('ascii').rstrip()
+        except UnicodeDecodeError as e:
+            raise InvalidChunk(e)
+
+        if not is_valid_chunk_id(id):
+            raise InvalidChunk('Invalid chunk ID %r' % id)
+
+        return cls.get_class(id)(fileobj, id, data_size, parent_chunk)
+
+    def __init__(self, fileobj, id, data_size, parent_chunk):
+        self._fileobj = fileobj
+        self.id = id
+        self.data_size = data_size
+        self.parent_chunk = parent_chunk
+        self.data_offset = fileobj.tell()
+        self.offset = self.data_offset - self.HEADER_SIZE
+        self._calculate_size()
+
+    def __repr__(self):
+        return ("<%s id=%s, offset=%i, size=%i, data_offset=%i, data_size=%i>"
+            % (type(self).__name__, self.id, self.offset, self.size,
+               self.data_offset, self.data_size))
+
+    def read(self):
+        """Read the chunks data"""
+
+        self._fileobj.seek(self.data_offset)
+        return self._fileobj.read(self.data_size)
+
+    def write(self, data):
+        """Write the chunk data"""
+
+        if len(data) > self.data_size:
+            raise ValueError
+
+        self._fileobj.seek(self.data_offset)
+        self._fileobj.write(data)
+        # Write the padding bytes
+        padding = self.padding()
+        if padding:
+            self._fileobj.seek(self.data_offset + self.data_size)
+            self._fileobj.write(b'\x00' * padding)
+
+    def delete(self):
+        """Removes the chunk from the file"""
+
+        delete_bytes(self._fileobj, self.size, self.offset)
+        if self.parent_chunk is not None:
+            self.parent_chunk._remove_subchunk(self)
+        self._fileobj.flush()
+
+    def _update_size(self, size_diff, changed_subchunk=None):
+        """Update the size of the chunk"""
+
+        old_size = self.size
+        self.data_size += size_diff
+        self._fileobj.seek(self.offset + 4)
+        self.write_size()
+        self._calculate_size()
+        if self.parent_chunk is not None:
+            self.parent_chunk._update_size(self.size - old_size, self)
+        if changed_subchunk:
+            self._update_sibling_offsets(
+                changed_subchunk, old_size - self.size)
+
+    def _calculate_size(self):
+        self.size = self.HEADER_SIZE + self.data_size + self.padding()
+        assert self.size % 2 == 0
+
+    def resize(self, new_data_size):
+        """Resize the file and update the chunk sizes"""
+
+        padding = new_data_size % 2
+        resize_bytes(self._fileobj, self.data_size + self.padding(),
+                     new_data_size + padding, self.data_offset)
+        size_diff = new_data_size - self.data_size
+        self._update_size(size_diff)
+        self._fileobj.flush()
+
+    def padding(self):
+        """Returns the number of padding bytes (0 or 1).
+        IFF chunks are required to be a even number in total length. If
+        data_size is odd a padding byte will be added at the end.
+        """
+        return self.data_size % 2
+
+
+class IffContainerChunkMixin():
+    """A IFF chunk containing other chunks.
+
+    A container chunk can have an additional name as the first 4 bytes of the
+    chunk data followed by an arbitrary number of subchunks. The root chunk of
+    the file is always a container chunk (e.g. the AIFF chunk or the FORM chunk
+    for RIFF) but there can be other types of container chunks (e.g. the LIST
+    chunks used in RIFF).
+    """
+
+    def parse_next_subchunk(self):
+        """"""
+        raise error("Not implemented")
+
+    def init_container(self, name_size=4):
+        # Lists can store an additional name identifier before the subchunks
+        self.__name_size = name_size
+        if self.data_size < name_size:
+            raise InvalidChunk(
+                'Container chunk data size < %i' % name_size)
+
+        # Read the container name
+        if name_size > 0:
+            try:
+                self.name = self._fileobj.read(name_size).decode('ascii')
+            except UnicodeDecodeError as e:
+                raise error(e)
+        else:
+            self.name = None
+
+        # Load all IFF subchunks
+        self.__subchunks = []
+
+    def subchunks(self):
+        """Returns a list of all subchunks.
+        The list is lazily loaded on first access.
+        """
+        if not self.__subchunks:
+            next_offset = self.data_offset + self.__name_size
+            while next_offset < self.offset + self.size:
+                self._fileobj.seek(next_offset)
+                try:
+                    chunk = self.parse_next_subchunk()
+                except EmptyChunk:
+                    break
+                except InvalidChunk:
+                    break
+                self.__subchunks.append(chunk)
+
+                # Calculate the location of the next chunk
+                next_offset = chunk.offset + chunk.size
+        return self.__subchunks
+
+    def insert_chunk(self, id_, data=None):
+        """Insert a new chunk at the end of the container chunk"""
+
+        if not is_valid_chunk_id(id_):
+            raise KeyError("Invalid IFF key.")
+
+        next_offset = self.offset + self.size
+        size = self.HEADER_SIZE
+        data_size = 0
+        if data:
+            data_size = len(data)
+            padding = data_size % 2
+            size += data_size + padding
+        insert_bytes(self._fileobj, size, next_offset)
+        self._fileobj.seek(next_offset)
+        self.write_new_header(id_.ljust(4).encode('ascii'), data_size)
+        self._fileobj.seek(next_offset)
+        chunk = self.parse_next_subchunk()
+        self._update_size(chunk.size)
+        if data:
+            chunk.write(data)
+        self.subchunks().append(chunk)
+        self._fileobj.flush()
+        return chunk
+
+    def __contains__(self, id_):
+        """Check if this chunk contains a specific subchunk."""
+        assert_valid_chunk_id(id_)
+        try:
+            self[id_]
+            return True
+        except KeyError:
+            return False
+
+    def __getitem__(self, id_):
+        """Get a subchunk by ID."""
+        assert_valid_chunk_id(id_)
+        found_chunk = None
+        for chunk in self.subchunks():
+            if chunk.id == id_:
+                found_chunk = chunk
+                break
+        else:
+            raise KeyError("No %r chunk found" % id_)
+        return found_chunk
+
+    def __delitem__(self, id_):
+        """Remove a chunk from the IFF file"""
+        assert_valid_chunk_id(id_)
+        self[id_].delete()
+
+    def _remove_subchunk(self, chunk):
+        assert chunk in self.__subchunks
+        self._update_size(-chunk.size, chunk)
+        self.__subchunks.remove(chunk)
+
+    def _update_sibling_offsets(self, changed_subchunk, size_diff):
+        """Update the offsets of subchunks after `changed_subchunk`.
+        """
+        index = self.__subchunks.index(changed_subchunk)
+        sibling_chunks = self.__subchunks[index + 1:len(self.__subchunks)]
+        for sibling in sibling_chunks:
+            sibling.offset -= size_diff
+            sibling.data_offset -= size_diff
+
+
+class IffFile:
+    """Representation of a IFF file"""
+
+    def __init__(self, chunk_cls, fileobj):
+        fileobj.seek(0)
+        self.root = chunk_cls.parse(fileobj)
+
+    def __contains__(self, id_):
+        """Check if the IFF file contains a specific chunk"""
+        return id_ in self.root
+
+    def __getitem__(self, id_):
+        """Get a chunk from the IFF file"""
+        return self.root[id_]
+
+    def __delitem__(self, id_):
+        """Remove a chunk from the IFF file"""
+        self.delete_chunk(id_)
+
+    def delete_chunk(self, id_):
+        """Remove a chunk from the IFF file"""
+        del self.root[id_]
+
+    def insert_chunk(self, id_, data=None):
+        """Insert a new chunk at the end of the IFF file"""
+        return self.root.insert_chunk(id_, data)
+
+
+class IffID3(ID3):
+    """A generic IFF file with ID3v2 tags"""
+
+    def _load_file(self, fileobj):
+        raise error("Not implemented")
+
+    def _pre_load_header(self, fileobj):
+        try:
+            fileobj.seek(self._load_file(fileobj)['ID3'].data_offset)
+        except (InvalidChunk, KeyError):
+            raise ID3NoHeaderError("No ID3 chunk")
+
+    @convert_error(IOError, error)
+    @loadfile(writable=True)
+    def save(self, filething=None, v2_version=4, v23_sep='/', padding=None):
+        """Save ID3v2 data to the IFF file"""
+
+        fileobj = filething.fileobj
+
+        iff_file = self._load_file(fileobj)
+
+        if 'ID3' not in iff_file:
+            iff_file.insert_chunk('ID3')
+
+        chunk = iff_file['ID3']
+
+        try:
+            data = self._prepare_data(
+                fileobj, chunk.data_offset, chunk.data_size, v2_version,
+                v23_sep, padding)
+        except ID3Error as e:
+            reraise(error, e, sys.exc_info()[2])
+
+        chunk.resize(len(data))
+        chunk.write(data)
+
+    @convert_error(IOError, error)
+    @loadfile(writable=True)
+    def delete(self, filething=None):
+        """Completely removes the ID3 chunk from the IFF file"""
+
+        try:
+            iff_file = self._load_file(filething.fileobj)
+            del iff_file['ID3']
+        except KeyError:
+            pass
+        self.clear()

lib/mutagen/_riff.py

@@ -0,0 +1,70 @@
+# -*- coding: utf-8 -*-
+# Copyright (C) 2017  Borewit
+# Copyright (C) 2019-2020  Philipp Wolfer
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+
+"""Resource Interchange File Format (RIFF)."""
+
+import struct
+from struct import pack
+
+from mutagen._iff import (
+    IffChunk,
+    IffContainerChunkMixin,
+    IffFile,
+    InvalidChunk,
+)
+
+
+class RiffChunk(IffChunk):
+    """Generic RIFF chunk"""
+
+    @classmethod
+    def parse_header(cls, header):
+        return struct.unpack('<4sI', header)
+
+    @classmethod
+    def get_class(cls, id):
+        if id in (u'LIST', u'RIFF'):
+            return RiffListChunk
+        else:
+            return cls
+
+    def write_new_header(self, id_, size):
+        self._fileobj.write(pack('<4sI', id_, size))
+
+    def write_size(self):
+        self._fileobj.write(pack('<I', self.data_size))
+
+
+class RiffListChunk(RiffChunk, IffContainerChunkMixin):
+    """A RIFF chunk containing other chunks.
+    This is either a 'LIST' or 'RIFF'
+    """
+
+    def parse_next_subchunk(self):
+        return RiffChunk.parse(self._fileobj, self)
+
+    def __init__(self, fileobj, id, data_size, parent_chunk):
+        if id not in (u'RIFF', u'LIST'):
+            raise InvalidChunk('Expected RIFF or LIST chunk, got %s' % id)
+
+        RiffChunk.__init__(self, fileobj, id, data_size, parent_chunk)
+        self.init_container()
+
+
+class RiffFile(IffFile):
+    """Representation of a RIFF file"""
+
+    def __init__(self, fileobj):
+        super().__init__(RiffChunk, fileobj)
+
+        if self.root.id != u'RIFF':
+            raise InvalidChunk("Root chunk must be a RIFF chunk, got %s"
+                               % self.root.id)
+
+        self.file_type = self.root.name

lib/mutagen/_senf/__init__.py

@@ -9,18 +9,20 @@
 # permit persons to whom the Software is furnished to do so, subject to
 # the following conditions:
 #
-# The above copyright notice and this permission notice shall be
+# The above copyright notice and this permission notice shall be included
-# included in all copies or substantial portions of the Software.
+# in all copies or substantial portions of the Software.
-
+#
-import os
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-if os.name != "nt":
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
-    # make imports work
+# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
-    _winapi = object()
+# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
 from ._fsnative import fsnative, path2fsn, fsn2text, fsn2bytes, \
-    bytes2fsn, uri2fsn, fsn2uri, text2fsn
+    bytes2fsn, uri2fsn, fsn2uri, text2fsn, fsn2norm
-from ._print import print_, input_
+from ._print import print_, input_, supports_ansi_escape_codes
 from ._stdlib import sep, pathsep, curdir, pardir, altsep, extsep, devnull, \
     defpath, getcwd, expanduser, expandvars
 from ._argv import argv
@@ -30,10 +32,11 @@ from ._temp import mkstemp, gettempdir, gettempprefix, mkdtemp
 
 fsnative, print_, getcwd, getenv, unsetenv, putenv, environ, expandvars, \
     path2fsn, fsn2text, fsn2bytes, bytes2fsn, uri2fsn, fsn2uri, mkstemp, \
-    gettempdir, gettempprefix, mkdtemp, input_, expanduser, text2fsn
+    gettempdir, gettempprefix, mkdtemp, input_, expanduser, text2fsn, \
+    supports_ansi_escape_codes, fsn2norm
 
 
-version = (1, 2, 2)
+version = (1, 4, 2)
 """Tuple[`int`, `int`, `int`]: The version tuple (major, minor, micro)"""
 
 

lib/mutagen/_senf/__init__.pyi

@@ -0,0 +1,104 @@
+import sys
+import os
+
+from typing import Text, Union, Any, Optional, Tuple, List, Dict
+
+if sys.version_info[0] == 2:
+    _pathlike = Union[Text, bytes]
+else:
+    _pathlike = Union[Text, bytes, 'os.PathLike[Any]']
+_uri = Union[Text, str]
+
+if sys.version_info[0] == 2:
+    if sys.platform == "win32":
+        _base = Text
+    else:
+        _base = bytes
+else:
+    _base = Text
+
+class fsnative(_base):
+    def __init__(self, object: Text=u"") -> None:
+        ...
+
+_fsnative = Union[fsnative, _base]
+
+if sys.platform == "win32":
+    _bytes_default_encoding = str
+else:
+    _bytes_default_encoding = Optional[str]
+
+def path2fsn(path: _pathlike) -> _fsnative:
+    ...
+
+def fsn2text(path: _fsnative, strict: bool=False) -> Text:
+    ...
+
+def text2fsn(text: Text) -> _fsnative:
+    ...
+
+def fsn2bytes(path: _fsnative, encoding: _bytes_default_encoding="utf-8") -> bytes:
+    ...
+
+def bytes2fsn(data: bytes, encoding: _bytes_default_encoding="utf-8") -> _fsnative:
+    ...
+
+def uri2fsn(uri: _uri) -> _fsnative:
+    ...
+
+def fsn2uri(path: _fsnative) -> Text:
+    ...
+
+def fsn2norm(path: _fsnative) -> _fsnative:
+    ...
+
+sep: _fsnative
+pathsep: _fsnative
+curdir: _fsnative
+pardir: _fsnative
+altsep: _fsnative
+extsep: _fsnative
+devnull: _fsnative
+defpath: _fsnative
+
+def getcwd() -> _fsnative:
+    ...
+
+def getenv(key: _pathlike, value: Optional[_fsnative]=None) -> Optional[_fsnative]:
+    ...
+
+def putenv(key: _pathlike, value: _pathlike):
+    ...
+
+def unsetenv(key: _pathlike) -> None:
+    ...
+
+def supports_ansi_escape_codes(fd: int) -> bool:
+    ...
+
+def expandvars(path: _pathlike) -> _fsnative:
+    ...
+
+def expanduser(path: _pathlike) -> _fsnative:
+    ...
+
+environ: Dict[_fsnative,_fsnative]
+argv: List[_fsnative]
+
+def gettempdir() -> _fsnative:
+    pass
+
+def mkstemp(suffix: Optional[_pathlike]=None, prefix: Optional[_pathlike]=None, dir: Optional[_pathlike]=None, text: bool=False) -> Tuple[int, _fsnative]:
+    ...
+
+def mkdtemp(suffix: Optional[_pathlike]=None, prefix: Optional[_pathlike]=None, dir: Optional[_pathlike]=None) -> _fsnative:
+    ...
+
+version_string: str
+
+version: Tuple[int, int, int]
+
+print_ = print
+
+def input_(prompt: Any=None) -> _fsnative:
+    ...

lib/mutagen/_senf/_argv.py

@@ -9,12 +9,23 @@
 # permit persons to whom the Software is furnished to do so, subject to
 # the following conditions:
 #
-# The above copyright notice and this permission notice shall be
+# The above copyright notice and this permission notice shall be included
-# included in all copies or substantial portions of the Software.
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
 import sys
 import ctypes
-import collections
+try:
+    from collections import abc
+except ImportError:
+    import collections as abc  # type: ignore
 from functools import total_ordering
 
 from ._compat import PY2, string_types
@@ -49,7 +60,7 @@ def _get_win_argv():
 
 
 @total_ordering
-class Argv(collections.MutableSequence):
+class Argv(abc.MutableSequence):
     """List[`fsnative`]: Like `sys.argv` but contains unicode
     keys and values under Windows + Python 2.