Merge branch 'develop'

* fix: Update parsing logic for updated TPB proxy html structure

---------

Co-authored-by: rembo10 <rembo10@users.noreply.github.com>

CHANGELOG.md

@@ -1,5 +1,39 @@
 # Changelog
 
+## v0.6.3
+ Released 26 May 2024
+
+Highlights:
+* Hotfix for searcher not returning results
+
+The full list of commits can be found [here](https://github.com/rembo10/headphones/compare/v0.6.2...v0.6.3).
+
+## v0.6.2
+ Released 26 May 2024
+
+Highlights:
+* Added soulseek support
+* Added bandcamp support
+* Changes and dependency updates to work with Python >= 3.12
+
+The full list of commits can be found [here](https://github.com/rembo10/headphones/compare/v0.6.1...v0.6.2).
+
+## v0.6.1
+R eleased 26 November 2023
+
+Highlights:
+* Dependency updates to work with > Python 3.11
+
+The full list of commits can be found [here](https://github.com/rembo10/headphones/compare/v0.6.0...v0.6.1).
+
+## v0.6.0
+Released 13 November 2022
+
+Highlights:
+* Updated to python 3
+
+The full list of commits can be found [here](https://github.com/rembo10/headphones/compare/v0.5.20...v0.6.0).
+
 ## v0.5.20
 Released 15 October 2021
 

Headphones.py

@@ -17,8 +17,8 @@
 import os
 import sys
 
-if sys.version_info <= (3, 5):
-    sys.stdout.write("Headphones requires Python >= 3.5\n")
+if sys.version_info <= (3, 6):
+    sys.stdout.write("Headphones requires Python >= 3.7\n")
     sys.exit(1)
 
 # Ensure lib added to path, before any other imports

data/interfaces/default/config.html

@@ -310,6 +310,16 @@
                                     <input type="text" name="usenet_retention" value="${config['usenet_retention']}" size="5">
                                 </div>
                             </fieldset>
+                            <fieldset title="Method for downloading Bandcamp.com files.">
+                                <legend>Bandcamp</legend>
+                                <div class="row">
+                                    <label title="Path to folder where Headphones can store raw downloads from Bandcamp.com.">
+                                        Bandcamp Directory
+                                    </label>
+                                    <input type="text" name="bandcamp_dir" value="${config['bandcamp_dir']}" size="50">
+                                    <small>Full path where raw MP3s will be stored, e.g. /Users/name/Downloads/bandcamp</small>
+                                </div>
+                            </fieldset>
                         </td>
                         <td>
                             <fieldset title="Method for downloading torrent files.">
@@ -317,7 +327,7 @@
                                 <input type="radio" name="torrent_downloader" id="torrent_downloader_blackhole" value="0" ${config['torrent_downloader_blackhole']}> Black Hole
                                 <input type="radio" name="torrent_downloader" id="torrent_downloader_transmission" value="1" ${config['torrent_downloader_transmission']}> Transmission
                                 <input type="radio" name="torrent_downloader" id="torrent_downloader_utorrent" value="2" ${config['torrent_downloader_utorrent']}> uTorrent (Beta)
-                                <input type="radio" name="torrent_downloader" id="torrent_downloader_deluge" value="3" ${config['torrent_downloader_deluge']}> Deluge (Beta)
+                                <input type="radio" name="torrent_downloader" id="torrent_downloader_deluge" value="3" ${config['torrent_downloader_deluge']}> Deluge
                                 <input type="radio" name="torrent_downloader" id="torrent_downloader_qbittorrent" value="4" ${config['torrent_downloader_qbittorrent']}> QBitTorrent
                             </fieldset>
                             <fieldset id="torrent_blackhole_options">
@@ -438,6 +448,11 @@
                                     <input type="text" name="deluge_label" value="${config['deluge_label']}" size="30">
                                     <small>Labels shouldn't contain spaces (requires Label plugin)</small>
                                 </div>
+                                <div class="row">
+                                    <label>Download Directory</label>
+                                    <input type="text" name="deluge_download_directory" value="${config['deluge_download_directory']}" size="30">
+                                    <small>Directory where Deluge should download to</small>
+                                </div>
                                 <div class="row">
                                     <label>Move When Completed</label>
                                     <input type="text" name="deluge_done_directory" value="${config['deluge_done_directory']}" size="30">
@@ -467,7 +482,33 @@
                                     <label>Prefer</label>
                                     <input type="radio" name="prefer_torrents" id="prefer_torrents_0" value="0" ${config['prefer_torrents_0']}>NZBs
                                     <input type="radio" name="prefer_torrents" id="prefer_torrents_1" value="1" ${config['prefer_torrents_1']}>Torrents
-                                    <input type="radio" name="prefer_torrents" id="prefer_torrents_2" value="2" ${config['prefer_torrents_2']}>No Preference
+                                    <input type="radio" name="prefer_torrents" id="prefer_torrents_2" value="2" ${config['prefer_torrents_2']}>Soulseek
+                                    <input type="radio" name="prefer_torrents" id="prefer_torrents_3" value="3" ${config['prefer_torrents_3']}>No Preference
+                                </div>
+                            </fieldset>
+                        </td>
+                        <td>
+                            <fieldset>
+                                <legend>Soulseek</legend>
+                                <div class="row">
+                                    <label>Soulseek API URL</label>
+                                    <input type="text" name="soulseek_api_url" value="${config['soulseek_api_url']}" size="50">
+                                </div>
+                                <div class="row">
+                                    <label>Soulseek API KEY</label>
+                                    <input type="text" name="soulseek_api_key" value="${config['soulseek_api_key']}" size="20">
+                                </div>
+                                <div class="row">
+                                    <label title="Path to folder where Headphones can find the downloads.">
+                                        Soulseek Download Dir:
+                                    </label>
+                                    <input type="text" name="soulseek_download_dir" value="${config['soulseek_download_dir']}" size="50">
+                                </div>
+                                <div class="row">
+                                    <label title="Path to folder where Headphones can find the downloads.">
+                                        Soulseek Incomplete Download Dir:
+                                    </label>
+                                    <input type="text" name="soulseek_incomplete_download_dir" value="${config['soulseek_incomplete_download_dir']}" size="50">
                                 </div>
                             </fieldset>
                         </td>
@@ -579,6 +620,19 @@
                                     </div>
                                 </div>
                             </fieldset>
+                            <fieldset>
+                                <legend>Other</legend>
+                                <fieldset>
+                                    <div class="row checkbox left">
+                                        <input id="use_bandcamp" type="checkbox" class="bigcheck" name="use_bandcamp" value="1" ${config['use_bandcamp']} /><label for="use_bandcamp"><span class="option">Bandcamp</span></label>
+                                    </div>
+                                </fieldset>
+                                <fieldset>
+                                    <div class="row checkbox left">
+                                        <input id="use_soulseek" type="checkbox" class="bigcheck" name="use_soulseek" value="1" ${config['use_soulseek']} /><label for="use_soulseek"><span class="option">Soulseek</span></label>
+                                    </div>
+                                </fieldset>
+                            </fieldset>
                         </td>
                         <td>
                             <fieldset>
@@ -1370,17 +1424,20 @@
                             <div class="row">
                                 <label>File Format</label>
                                 <input type="text" name="file_format" value="${config['file_format']}" size="43">
-                                <small>Use: $Disc/$disc (disc #), $Track/$track (track #), $Title/$title, $Artist/$artist, $Album/$album and $Year/$year. Put optional variables in curly braces, use single-quote marks to escape curly braces literally ('{', '}').</small>
+                                <small>Use: In addition to the above, there is also $Title/$title (track title), $Track (track #), $Disc (disc #), $DiscTotal.</small>
                             </div>
-                            <div class="checkbox row clearfix">
+                            <div class="checkbox row left clearfix nopad">
                                 <input type="checkbox" name="file_underscores" id="file_underscores" value="1" ${config['file_underscores']}/><label>Use underscores instead of spaces</label>
                             </div>
+                            <div class="checkbox row left clearfix nopad">
+                                <input type="checkbox" name="rename_single_disc_ignore" id="rename_single_disc_ignore" value="1" ${config['rename_single_disc_ignore']}/><label>Don't include disc# for single disc albums</label>
+                            </div>
                         </fieldset>
 
                         <fieldset>
                             <legend>Re-Encoding Options</legend>
                             <small class="heading"><i class="fa fa-info-circle"></i> Note: this option requires the lame, ffmpeg or xld encoder</small>
-                            <div class="checkbox row clearfix">
+                            <div class="checkbox row left clearfix nopad">
                                 <input type="checkbox" name="music_encoder" id="music_encoder" value="1" ${config['music_encoder']}/><label>Re-encode downloads during postprocessing</label>
                             </div>
                             <div id="encoderoptions" class="row clearfix checkbox">
@@ -1651,6 +1708,16 @@
                                 </div>
                             </fieldset>
 
+                            <fieldset>
+                                <legend>Last.fm</legend>
+                                <div id="lastfmoptions">
+                                    <div class="row">
+                                        <label>API Key</label>
+                                        <input type="text" name="lastfm_apikey" value="${config['lastfm_apikey']}" size="40" />
+                                    </div>
+                                </div>
+                            </fieldset>
+
                             <fieldset>
                                 <legend>Songkick</legend>
                                 <div class="row checkbox">

data/interfaces/default/history.html

@@ -56,6 +56,8 @@
                 fileid = 'torrent'
             if item['URL'].find('codeshy') != -1:
                 fileid = 'nzb'
+            if item['URL'].find('bandcamp') != -1:
+                fileid = 'bandcamp'
 
             folder = 'Folder: ' + item['FolderName']
 

headphones/bandcamp.py

@@ -0,0 +1,166 @@
+#  This file is part of Headphones.
+#
+#  Headphones is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  Headphones is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with Headphones.  If not, see <http://www.gnu.org/licenses/>
+
+import headphones
+import json
+import os
+import re
+
+from headphones import logger, helpers, metadata, request
+from headphones.common import USER_AGENT
+from headphones.types import Result
+
+from mediafile import MediaFile, UnreadableFileError
+from bs4 import BeautifulSoup
+from bs4 import FeatureNotFound
+
+
+def search(album, albumlength=None, page=1, resultlist=None):
+    dic = {'...': '', ' & ': ' ', ' = ': ' ', '?': '', '$': 's', ' + ': ' ',
+           '"': '', ',': '', '*': '', '.': '', ':': ''}
+    if resultlist is None:
+        resultlist = []
+
+    cleanalbum = helpers.latinToAscii(
+        helpers.replace_all(album['AlbumTitle'], dic)
+        ).strip()
+    cleanartist = helpers.latinToAscii(
+        helpers.replace_all(album['ArtistName'], dic)
+        ).strip()
+
+    headers = {'User-Agent': USER_AGENT}
+    params = {
+        "page": page,
+        "q": cleanalbum,
+    }
+    logger.info("Looking up https://bandcamp.com/search with {}".format(
+        params))
+    content = request.request_content(
+        url='https://bandcamp.com/search',
+        params=params,
+        headers=headers
+        ).decode('utf8')
+    try:
+        soup = BeautifulSoup(content, "html5lib")
+    except FeatureNotFound:
+        soup = BeautifulSoup(content, "html.parser")
+
+    for item in soup.find_all("li", class_="searchresult"):
+        type = item.find('div', class_='itemtype').text.strip().lower()
+        if type == "album":
+            data = parse_album(item)
+
+            cleanartist_found = helpers.latinToAscii(data['artist'])
+            cleanalbum_found = helpers.latinToAscii(data['album'])
+
+            logger.debug(u"{} - {}".format(data['album'], cleanalbum_found))
+
+            logger.debug("Comparing {} to {}".format(
+                cleanalbum, cleanalbum_found))
+            if (cleanartist.lower() == cleanartist_found.lower() and
+                    cleanalbum.lower() == cleanalbum_found.lower()):
+                resultlist.append(Result(
+                    data['title'], data['size'], data['url'],
+                    'bandcamp', 'bandcamp', True))
+        else:
+            continue
+
+    if(soup.find('a', class_='next')):
+        page += 1
+        logger.debug("Calling next page ({})".format(page))
+        search(album, albumlength=albumlength,
+               page=page, resultlist=resultlist)
+
+    return resultlist
+
+
+def download(album, bestqual):
+    html = request.request_content(url=bestqual.url).decode('utf-8')
+    trackinfo = []
+    try:
+        trackinfo = json.loads(
+            re.search(r"trackinfo&quot;:(\[.*?\]),", html)
+            .group(1)
+            .replace('&quot;', '"'))
+    except ValueError as e:
+        logger.warn("Couldn't load json: {}".format(e))
+
+    directory = os.path.join(
+        headphones.CONFIG.BANDCAMP_DIR,
+        u'{} - {}'.format(
+            album['ArtistName'].replace('/', '_'),
+            album['AlbumTitle'].replace('/', '_')))
+    directory = helpers.latinToAscii(directory)
+
+    if not os.path.exists(directory):
+        try:
+            os.makedirs(directory)
+        except Exception as e:
+            logger.warn("Could not create directory ({})".format(e))
+
+    index = 1
+    for track in trackinfo:
+        filename = helpers.replace_illegal_chars(
+                    u'{:02d} - {}.mp3'.format(index, track['title']))
+        fullname = os.path.join(directory.encode('utf-8'),
+                                filename.encode('utf-8'))
+        logger.debug("Downloading to {}".format(fullname))
+
+        if 'file' in track and track['file'] != None and 'mp3-128' in track['file']:
+            content = request.request_content(track['file']['mp3-128'])
+            open(fullname, 'wb').write(content)
+            try:
+                f = MediaFile(fullname)
+                date, year = metadata._date_year(album)
+                f.update({
+                    'artist': album['ArtistName'].encode('utf-8'),
+                    'album': album['AlbumTitle'].encode('utf-8'),
+                    'title': track['title'].encode('utf-8'),
+                    'track': track['track_num'],
+                    'tracktotal': len(trackinfo),
+                    'year': year,
+                })
+                f.save()
+            except UnreadableFileError as ex:
+                logger.warn("MediaFile couldn't parse: %s (%s)",
+                            fullname,
+                            str(ex))
+
+        index += 1
+
+    return directory
+
+
+def parse_album(item):
+    album = item.find('div', class_='heading').text.strip()
+    artist = item.find('div', class_='subhead').text.strip().replace("by ", "")
+    released = item.find('div', class_='released').text.strip().replace(
+        "released ", "")
+    year = re.search(r"(\d{4})", released).group(1)
+
+    url = item.find('div', class_='heading').find('a')['href'].split("?")[0]
+
+    length = item.find('div', class_='length').text.strip()
+    tracks, minutes = length.split(",")
+    tracks = tracks.replace(" tracks", "").replace(" track", "").strip()
+    minutes = minutes.replace(" minutes", "").strip()
+    # bandcamp offers mp3 128b with should be 960KB/minute
+    size = int(minutes) * 983040
+
+    data = {"title": u'{} - {} [{}]'.format(artist, album, year),
+            "artist": artist, "album": album,
+            "url": url, "size": size}
+
+    return data

headphones/common.py

@@ -102,36 +102,6 @@ class Quality:
 
         return (anyQualities, bestQualities)
 
-    @staticmethod
-    def nameQuality(name):
-
-        def checkName(list, func):
-            return func([re.search(x, name, re.I) for x in list])
-
-        name = os.path.basename(name)
-
-        # if we have our exact text then assume we put it there
-        for x in Quality.qualityStrings:
-            if x == Quality.UNKNOWN:
-                continue
-
-            regex = '\W' + Quality.qualityStrings[x].replace(' ', '\W') + '\W'
-            regex_match = re.search(regex, name, re.I)
-            if regex_match:
-                return x
-
-        # TODO: fix quality checking here
-        if checkName(["mp3", "192"], any) and not checkName(["flac"], all):
-            return Quality.B192
-        elif checkName(["mp3", "256"], any) and not checkName(["flac"], all):
-            return Quality.B256
-        elif checkName(["mp3", "vbr"], any) and not checkName(["flac"], all):
-            return Quality.VBR
-        elif checkName(["mp3", "320"], any) and not checkName(["flac"], all):
-            return Quality.B320
-        else:
-            return Quality.UNKNOWN
-
     @staticmethod
     def assumeQuality(name):
         if name.lower().endswith(".mp3"):
@@ -158,13 +128,6 @@ class Quality:
 
         return (Quality.NONE, status)
 
-    @staticmethod
-    def statusFromName(name, assume=True):
-        quality = Quality.nameQuality(name)
-        if assume and quality == Quality.UNKNOWN:
-            quality = Quality.assumeQuality(name)
-        return Quality.compositeStatus(DOWNLOADED, quality)
-
     DOWNLOADED = None
     SNATCHED = None
     SNATCHED_PROPER = None

headphones/config.py

@@ -80,6 +80,7 @@ _CONFIG_DEFINITIONS = {
     'DELUGE_PASSWORD': (str, 'Deluge', ''),
     'DELUGE_LABEL': (str, 'Deluge', ''),
     'DELUGE_DONE_DIRECTORY': (str, 'Deluge', ''),
+    'DELUGE_DOWNLOAD_DIRECTORY': (str, 'Deluge', ''),
     'DELUGE_PAUSED': (int, 'Deluge', 0),
     'DESTINATION_DIR': (str, 'General', ''),
     'DETECT_BITRATE': (int, 'General', 0),
@@ -155,9 +156,10 @@ _CONFIG_DEFINITIONS = {
     'KEEP_TORRENT_FILES': (int, 'General', 0),
     'KEEP_TORRENT_FILES_DIR': (path, 'General', ''),
     'LASTFM_USERNAME': (str, 'General', ''),
+    'LASTFM_APIKEY': (str, 'General', ''),
     'LAUNCH_BROWSER': (int, 'General', 1),
     'LIBRARYSCAN': (int, 'General', 1),
-    'LIBRARYSCAN_INTERVAL': (int, 'General', 300),
+    'LIBRARYSCAN_INTERVAL': (int, 'General', 24),
     'LMS_ENABLED': (int, 'LMS', 0),
     'LMS_HOST': (str, 'LMS', ''),
     'LOG_DIR': (path, 'General', ''),
@@ -240,6 +242,7 @@ _CONFIG_DEFINITIONS = {
     'QBITTORRENT_PASSWORD': (str, 'QBitTorrent', ''),
     'QBITTORRENT_USERNAME': (str, 'QBitTorrent', ''),
     'RENAME_FILES': (int, 'General', 0),
+    'RENAME_SINGLE_DISC_IGNORE': (int, 'General', 0),
     'RENAME_UNPROCESSED': (bool_int, 'General', 1),
     'RENAME_FROZEN': (bool_int, 'General', 1),
     'REPLACE_EXISTING_FOLDERS': (int, 'General', 0),
@@ -267,6 +270,11 @@ _CONFIG_DEFINITIONS = {
     'SONGKICK_ENABLED': (int, 'Songkick', 1),
     'SONGKICK_FILTER_ENABLED': (int, 'Songkick', 0),
     'SONGKICK_LOCATION': (str, 'Songkick', ''),
+    'SOULSEEK_API_URL': (str, 'Soulseek', ''),
+    'SOULSEEK_API_KEY': (str, 'Soulseek', ''),
+    'SOULSEEK_DOWNLOAD_DIR': (str, 'Soulseek', ''),
+    'SOULSEEK_INCOMPLETE_DOWNLOAD_DIR': (str, 'Soulseek', ''),
+    'SOULSEEK': (int, 'Soulseek', 0),
     'SUBSONIC_ENABLED': (int, 'Subsonic', 0),
     'SUBSONIC_HOST': (str, 'Subsonic', ''),
     'SUBSONIC_PASSWORD': (str, 'Subsonic', ''),
@@ -315,7 +323,9 @@ _CONFIG_DEFINITIONS = {
     'XBMC_PASSWORD': (str, 'XBMC', ''),
     'XBMC_UPDATE': (int, 'XBMC', 0),
     'XBMC_USERNAME': (str, 'XBMC', ''),
-    'XLDPROFILE': (str, 'General', '')
+    'XLDPROFILE': (str, 'General', ''),
+    'BANDCAMP': (int, 'General', 0),
+    'BANDCAMP_DIR': (path, 'General', '')
 }
 
 
@@ -327,7 +337,7 @@ class Config(object):
     def __init__(self, config_file):
         """ Initialize the config with values from a file """
         self._config_file = config_file
-        self._config = ConfigParser()
+        self._config = ConfigParser(interpolation=None)
         self._config.read(self._config_file)
         for key in list(_CONFIG_DEFINITIONS.keys()):
             self.check_setting(key)
@@ -363,12 +373,12 @@ class Config(object):
 
         try:
             my_val = definition_type(self._config[section][ini_key])
-            # ConfigParser interprets empty strings in the config
+            # ConfigParser interprets quotes in the config
             # literally, so we need to sanitize it. It's not really
             # a config upgrade, since a user can at any time put 
-            # some_key = ''
-            if my_val == '""' or my_val == "''":
-                my_val = ''
+            # some_key = 'some_val'
+            if type(my_val) in [str, path]:
+                my_val = my_val.strip('"').strip("'")
         except Exception:
             my_val = default
             self._config[section][ini_key] = str(my_val)
@@ -376,7 +386,7 @@ class Config(object):
 
     def write(self):
         """ Make a copy of the stored config and write it to the configured file """
-        new_config = ConfigParser()
+        new_config = ConfigParser(interpolation=None)
 
         # first copy over everything from the old config, even if it is not
         # correctly defined to keep from losing data

headphones/db.py

@@ -117,7 +117,7 @@ class DBConnection:
                     break
 
             except sqlite3.OperationalError as e:
-                if "unable to open database file" in e.message or "database is locked" in e.message:
+                if "unable to open database file" in str(e) or "database is locked" in str(e):
                     dberror = e
                     if args is None:
                         logger.debug('Database error: %s. Query: %s', e, query)

headphones/deluge.py

@@ -58,12 +58,12 @@ def _scrubber(text):
     if scrub_logs:
         try:
             # URL parameter values
-            text = re.sub('=[0-9a-zA-Z]*', '=REMOVED', text)
+            text = re.sub(r'=[0-9a-zA-Z]*', r'=REMOVED', text)
             # Local host with port
             # text = re.sub('\:\/\/.*\:', '://REMOVED:', text) # just host
-            text = re.sub('\:\/\/.*\:[0-9]*', '://REMOVED:', text)
+            text = re.sub(r'\:\/\/.*\:[0-9]*', r'://REMOVED:', text)
             # Session cookie
-            text = re.sub("_session_id'\: '.*'", "_session_id': 'REMOVED'", text)
+            text = re.sub(r"_session_id'\: '.*'", r"_session_id': 'REMOVED'", text)
             # Local Windows user path
             if text.lower().startswith('c:\\users\\'):
                 k = text.split('\\')
@@ -128,9 +128,9 @@ def addTorrent(link, data=None, name=None):
                 # Extract torrent name from .torrent
                 try:
                     logger.debug('Deluge: Getting torrent name length')
-                    name_length = int(re.findall('name([0-9]*)\:.*?\:', str(torrentfile))[0])
+                    name_length = int(re.findall(r'name([0-9]*)\:.*?\:', str(torrentfile))[0])
                     logger.debug('Deluge: Getting torrent name')
-                    name = re.findall('name[0-9]*\:(.*?)\:', str(torrentfile))[0][:name_length]
+                    name = re.findall(r'name[0-9]*\:(.*?)\:', str(torrentfile))[0][:name_length]
                 except Exception as e:
                     logger.debug('Deluge: Could not get torrent name, getting file name')
                     # get last part of link/path (name only)
@@ -160,9 +160,9 @@ def addTorrent(link, data=None, name=None):
                 # Extract torrent name from .torrent
                 try:
                     logger.debug('Deluge: Getting torrent name length')
-                    name_length = int(re.findall('name([0-9]*)\:.*?\:', str(torrentfile))[0])
+                    name_length = int(re.findall(r'name([0-9]*)\:.*?\:', str(torrentfile))[0])
                     logger.debug('Deluge: Getting torrent name')
-                    name = re.findall('name[0-9]*\:(.*?)\:', str(torrentfile))[0][:name_length]
+                    name = re.findall(r'name[0-9]*\:(.*?)\:', str(torrentfile))[0][:name_length]
                 except Exception as e:
                     logger.debug('Deluge: Could not get torrent name, getting file name')
                     # get last part of link/path (name only)
@@ -466,19 +466,56 @@ def _add_torrent_url(result):
 
 def _add_torrent_file(result):
     logger.debug('Deluge: Adding file')
+
+    options = {}
+
+    if headphones.CONFIG.DELUGE_DOWNLOAD_DIRECTORY:
+        options['download_location'] = headphones.CONFIG.DELUGE_DOWNLOAD_DIRECTORY
+
+    if headphones.CONFIG.DELUGE_DONE_DIRECTORY or headphones.CONFIG.DOWNLOAD_TORRENT_DIR:
+        options['move_completed'] = 1
+        if headphones.CONFIG.DELUGE_DONE_DIRECTORY:
+            options['move_completed_path'] = headphones.CONFIG.DELUGE_DONE_DIRECTORY
+        else:
+            options['move_completed_path'] = headphones.CONFIG.DOWNLOAD_TORRENT_DIR
+
+    if headphones.CONFIG.DELUGE_PAUSED:
+        options['add_paused'] = headphones.CONFIG.DELUGE_PAUSED
+
     if not any(delugeweb_auth):
         _get_auth()
     try:
         # content is torrent file contents that needs to be encoded to base64
         post_data = json.dumps({"method": "core.add_torrent_file",
                                 "params": [result['name'] + '.torrent',
-                                b64encode(result['content']).decode(), {}],
+                                b64encode(result['content'].encode('utf8')),
+                                options],
                                 "id": 2})
         response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth,
             verify=deluge_verify_cert, headers=headers)
         result['hash'] = json.loads(response.text)['result']
         logger.debug('Deluge: Response was %s' % str(json.loads(response.text)))
         return json.loads(response.text)['result']
+    except UnicodeDecodeError:
+        try:
+            # content is torrent file contents that needs to be encoded to base64
+            # this time let's try leaving the encoding as is
+            logger.debug('Deluge: There was a decoding issue, let\'s try again')
+            post_data = json.dumps({"method": "core.add_torrent_file",
+                                    "params": [result['name'].decode('utf8') + '.torrent',
+                                    b64encode(result['content']),
+                                    options],
+                                    "id": 22})
+            response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth,
+                verify=deluge_verify_cert, headers=headers)
+            result['hash'] = json.loads(response.text)['result']
+            logger.debug('Deluge: Response was %s' % str(json.loads(response.text)))
+            return json.loads(response.text)['result']
+        except Exception as e:
+            logger.error('Deluge: Adding torrent file failed after decode: %s' % str(e))
+            formatted_lines = traceback.format_exc().splitlines()
+            logger.error('; '.join(formatted_lines))
+            return False
     except Exception as e:
         logger.error('Deluge: Adding torrent file failed: %s' % str(e))
         formatted_lines = traceback.format_exc().splitlines()
@@ -566,61 +603,3 @@ def setSeedRatio(result):
         return None
 
 
-def setTorrentPath(result):
-    logger.debug('Deluge: Setting download path')
-    if not any(delugeweb_auth):
-        _get_auth()
-
-    try:
-        if headphones.CONFIG.DELUGE_DONE_DIRECTORY or headphones.CONFIG.DOWNLOAD_TORRENT_DIR:
-            post_data = json.dumps({"method": "core.set_torrent_move_completed",
-                                    "params": [result['hash'], True],
-                                    "id": 7})
-            response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth,
-                verify=deluge_verify_cert, headers=headers)
-
-            if headphones.CONFIG.DELUGE_DONE_DIRECTORY:
-                move_to = headphones.CONFIG.DELUGE_DONE_DIRECTORY
-            else:
-                move_to = headphones.CONFIG.DOWNLOAD_TORRENT_DIR
-
-            if not os.path.exists(move_to):
-                logger.debug('Deluge: %s directory doesn\'t exist, let\'s create it' % move_to)
-                os.makedirs(move_to)
-            post_data = json.dumps({"method": "core.set_torrent_move_completed_path",
-                                    "params": [result['hash'], move_to],
-                                    "id": 8})
-            response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth,
-                verify=deluge_verify_cert, headers=headers)
-
-            return not json.loads(response.text)['error']
-
-        return True
-    except Exception as e:
-        logger.error('Deluge: Setting torrent move-to directory failed: %s' % str(e))
-        formatted_lines = traceback.format_exc().splitlines()
-        logger.error('; '.join(formatted_lines))
-        return None
-
-
-def setTorrentPause(result):
-    logger.debug('Deluge: Pausing torrent')
-    if not any(delugeweb_auth):
-        _get_auth()
-
-    try:
-        if headphones.CONFIG.DELUGE_PAUSED:
-            post_data = json.dumps({"method": "core.pause_torrent",
-                                    "params": [[result['hash']]],
-                                    "id": 9})
-            response = requests.post(delugeweb_url, data=post_data.encode('utf-8'), cookies=delugeweb_auth,
-                verify=deluge_verify_cert, headers=headers)
-
-            return not json.loads(response.text)['error']
-
-        return True
-    except Exception as e:
-        logger.error('Deluge: Setting torrent paused failed: %s' % str(e))
-        formatted_lines = traceback.format_exc().splitlines()
-        logger.error('; '.join(formatted_lines))
-        return None

headphones/getXldProfile.py

@@ -1,6 +1,6 @@
 import os.path
 
-import biplist
+import plistlib
 from headphones import logger
 
 
@@ -14,8 +14,9 @@ def getXldProfile(xldProfile):
 
     # Get xld preferences plist
     try:
-        preferences = biplist.readPlist(expanded)
-    except (biplist.InvalidPlistException, biplist.NotBinaryPlistException) as e:
+        with open(expanded, 'rb') as _f:
+            preferences = plistlib.load(_f)
+    except Exception as e:
         logger.error("Error reading xld preferences plist: %s", e)
         return (xldProfileNotFound, None, None)
 

headphones/helpers.py

@@ -14,25 +14,25 @@
 #  You should have received a copy of the GNU General Public License
 #  along with Headphones.  If not, see <http://www.gnu.org/licenses/>.
 
-from operator import itemgetter
-import unicodedata
-import datetime
+import os
+import re
 import shutil
-import time
 import sys
 import tempfile
-import glob
+import time
+import unicodedata
+from contextlib import contextmanager
+from datetime import datetime, date
+from fnmatch import fnmatch
+from functools import cmp_to_key
+from glob import glob
+from operator import itemgetter
 
 from beets import logging as beetslogging
-import six
-from contextlib import contextmanager
-
-import fnmatch
-import functools
-import re
-import os
 from mediafile import MediaFile, FileTypeError, UnreadableFileError
+from six import text_type
 from unidecode import unidecode
+
 import headphones
 
 
@@ -52,7 +52,14 @@ def cmp(x, y):
 
     https://portingguide.readthedocs.io/en/latest/comparisons.html#the-cmp-function
     """
-    return (x > y) - (x < y)
+    if x is None and y is None:
+        return 0
+    elif x is None:
+        return -1
+    elif y is None:
+        return 1
+    else:
+        return (x > y) - (x < y)
 
 def multikeysort(items, columns):
     comparers = [
@@ -67,7 +74,7 @@ def multikeysort(items, columns):
         else:
             return 0
 
-    return sorted(items, key=functools.cmp_to_key(comparer))
+    return sorted(items, key=cmp_to_key(comparer))
 
 
 def checked(variable):
@@ -149,28 +156,25 @@ def convert_seconds(s):
 
 
 def today():
-    today = datetime.date.today()
-    yyyymmdd = datetime.date.isoformat(today)
-    return yyyymmdd
+    return date.isoformat(date.today())
 
 
 def now():
-    now = datetime.datetime.now()
+    now = datetime.now()
     return now.strftime("%Y-%m-%d %H:%M:%S")
 
 
-def get_age(date):
-    try:
-        split_date = date.split('-')
-    except:
+def is_valid_date(d):
+    if not d:
         return False
+    else:
+        return bool(re.match(r'\d{4}-\d{2}-\d{2}', d))
 
-    try:
-        days_old = int(split_date[0]) * 365 + int(split_date[1]) * 30 + int(split_date[2])
-    except (IndexError, ValueError):
-        days_old = False
 
-    return days_old
+def age(d):
+    '''Requires a valid date'''
+    delta = date.today() - date.fromisoformat(d)
+    return delta.days
 
 
 def bytes_to_mb(bytes):
@@ -180,7 +184,7 @@ def bytes_to_mb(bytes):
 
 
 def mb_to_bytes(mb_str):
-    result = re.search('^(\d+(?:\.\d+)?)\s?(?:mb)?', mb_str, flags=re.I)
+    result = re.search(r"^(\d+(?:\.\d+)?)\s?(?:mb)?", mb_str, flags=re.I)
     if result:
         return int(float(result.group(1)) * 1048576)
 
@@ -249,9 +253,9 @@ def replace_all(text, dic):
 
 def replace_illegal_chars(string, type="file"):
     if type == "file":
-        string = re.sub('[\?"*:|<>/]', '_', string)
+        string = re.sub(r"[\?\"*:|<>/]", "_", string)
     if type == "folder":
-        string = re.sub('[:\?<>"|*]', '_', string)
+        string = re.sub(r"[:\?<>\"|*]", "_", string)
     return string
 
 
@@ -382,7 +386,7 @@ def clean_musicbrainz_name(s, return_as_string=True):
 
 
 def cleanTitle(title):
-    title = re.sub('[\.\-\/\_]', ' ', title).lower()
+    title = re.sub(r"[\.\-\/\_]", " ", title).lower()
 
     # Strip out extra whitespace
     title = ' '.join(title.split())
@@ -502,7 +506,7 @@ def path_match_patterns(path, patterns):
     """
 
     for pattern in patterns:
-        if fnmatch.fnmatch(path, pattern):
+        if fnmatch(path, pattern):
             return True
 
     # No match
@@ -708,7 +712,7 @@ def preserve_torrent_directory(albumpath, forced=False, single=False):
             workdir = os.path.join(tempdir, prefix)
             workdir = re.sub(r'\[', '[[]', workdir)
             workdir = re.sub(r'(?<!\[)\]', '[]]', workdir)
-            if len(glob.glob(workdir + '*/')) >= 3:
+            if len(glob(workdir + '*/')) >= 3:
                 logger.error(
                     "Looks like a temp directory has previously been created "
                     "for this albumpath, not continuing "
@@ -1027,7 +1031,7 @@ class BeetsLogCapture(beetslogging.Handler):
         self.messages = []
 
     def emit(self, record):
-        self.messages.append(six.text_type(record.msg))
+        self.messages.append(text_type(record.msg))
 
 
 @contextmanager
@@ -1039,3 +1043,17 @@ def capture_beets_log(logger='beets'):
         yield capture.messages
     finally:
         log.removeHandler(capture)
+
+def have_pct_have_total(db_artist):
+    have_tracks = db_artist['HaveTracks'] or 0
+    total_tracks = db_artist['TotalTracks'] or 0 
+    have_pct = have_tracks / total_tracks if total_tracks else 0
+    return (have_pct, total_tracks)
+
+
+def has_token(title, token):
+    return bool(
+            re.search(rf'(?:\W|^)+{token}(?:\W|$)+', 
+            title,
+            re.IGNORECASE | re.UNICODE)
+        )

headphones/helpers_test.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 from .unittestcompat import TestCase
-from headphones.helpers import clean_name
+from headphones.helpers import clean_name, is_valid_date, age, has_token
 
 
 class HelpersTest(TestCase):
@@ -46,3 +46,28 @@ class HelpersTest(TestCase):
         self.assertEqual(
             test, expected, "check clean_name() with narrow non-ascii input"
         )
+
+    def test_is_valid_date(date):
+        test_cases = [
+            ('2021-11-12', True, "check is_valid_date returns True for valid date"),
+            (None, False, "check is_valid_date returns False for None"),
+            ('2021-11', False, "check is_valid_date returns False for incomplete"),
+            ('2021', False, "check is_valid_date returns False for incomplete")
+        ]
+        for input, expected, desc in test_cases:
+            self.assertEqual(is_valid_date(input), expected, desc)
+
+    def test_has_token(self):
+        """helpers: has_token()"""
+        self.assertEqual(
+            has_token("a cat ran", "cat"), 
+            True, 
+            "return True if token is in string"
+        )
+        self.assertEqual(
+            has_token("acatran", "cat"),
+            False,
+            "return False if token is part of another word"
+        )
+
+

headphones/importer.py

@@ -102,12 +102,7 @@ def artistlist_to_mbids(artistlist, forced=False):
             myDB.action('DELETE from newartists WHERE ArtistName=?', [artist])
 
     # Update the similar artist tag cloud:
-    logger.info('Updating artist information from Last.fm')
-
-    try:
-        lastfm.getSimilar()
-    except Exception as e:
-        logger.warn('Failed to update artist information from Last.fm: %s' % e)
+    lastfm.getSimilar()
 
 
 def addArtistIDListToDB(artistidlist):
@@ -245,7 +240,7 @@ def addArtisttoDB(artistid, extrasonly=False, forcefull=False, type="artist"):
         rgid = rg['id']
         skip_log = 0
         # Make a user configurable variable to skip update of albums with release dates older than this date (in days)
-        pause_delta = headphones.CONFIG.MB_IGNORE_AGE
+        ignore_age = headphones.CONFIG.MB_IGNORE_AGE
 
         rg_exists = myDB.action("SELECT * from albums WHERE AlbumID=?", [rg['id']]).fetchone()
 
@@ -274,18 +269,18 @@ def addArtisttoDB(artistid, extrasonly=False, forcefull=False, type="artist"):
                     if len(check_release_date) == 10:
                         release_date = check_release_date
                     elif len(check_release_date) == 7:
-                        release_date = check_release_date + "-31"
+                        release_date = check_release_date + "-27"
                     elif len(check_release_date) == 4:
-                        release_date = check_release_date + "-12-31"
+                        release_date = check_release_date + "-12-27"
                     else:
                         release_date = today
-                    if helpers.get_age(today) - helpers.get_age(release_date) < pause_delta:
+                    if helpers.age(release_date) < ignore_age:
                         logger.info("[%s] Now updating: %s (Release Date <%s Days)",
-                                    artist['artist_name'], rg['title'], pause_delta)
+                                    artist['artist_name'], rg['title'], ignore_age)
                         new_releases = mb.get_new_releases(rgid, includeExtras, True)
                     else:
                         logger.info("[%s] Skipping: %s (Release Date >%s Days)",
-                                    artist['artist_name'], rg['title'], pause_delta)
+                                    artist['artist_name'], rg['title'], ignore_age)
                         skip_log = 1
                         new_releases = 0
 
@@ -450,14 +445,9 @@ def addArtisttoDB(artistid, extrasonly=False, forcefull=False, type="artist"):
 
                 if headphones.CONFIG.AUTOWANT_ALL:
                     newValueDict['Status'] = "Wanted"
-                elif album['ReleaseDate'] > today and headphones.CONFIG.AUTOWANT_UPCOMING:
-                    newValueDict['Status'] = "Wanted"
-                # Sometimes "new" albums are added to musicbrainz after their release date, so let's try to catch these
-                # The first test just makes sure we have year-month-day
-                elif helpers.get_age(album['ReleaseDate']) and helpers.get_age(
-                        today) - helpers.get_age(
-                        album['ReleaseDate']) < 21 and headphones.CONFIG.AUTOWANT_UPCOMING:
-                    newValueDict['Status'] = "Wanted"
+                elif headphones.CONFIG.AUTOWANT_UPCOMING:
+                    if helpers.is_valid_date(album['ReleaseDate']) and helpers.age(album['ReleaseDate']) < 21:
+                        newValueDict['Status'] = "Wanted"
                 else:
                     newValueDict['Status'] = "Skipped"
 

headphones/lastfm.py

@@ -23,7 +23,7 @@ from headphones import db, logger, request
 TIMEOUT = 60.0  # seconds
 REQUEST_LIMIT = 1.0 / 5  # seconds
 ENTRY_POINT = "https://ws.audioscrobbler.com/2.0/"
-API_KEY = "395e6ec6bb557382fc41fde867bce66f"
+APP_API_KEY = "395e6ec6bb557382fc41fde867bce66f"
 
 # Required for API request limit
 lastfm_lock = headphones.lock.TimedLock(REQUEST_LIMIT)
@@ -31,7 +31,7 @@ lastfm_lock = headphones.lock.TimedLock(REQUEST_LIMIT)
 
 def request_lastfm(method, **kwargs):
     """
-    Call a Last.FM API method. Automatically sets the method and API key. Method
+    Call a Last.fm API method. Automatically sets the method and API key. Method
     will return the result if no error occured.
 
     By default, this method will request the JSON format, since it is more
@@ -40,35 +40,42 @@ def request_lastfm(method, **kwargs):
 
     # Prepare request
     kwargs["method"] = method
-    kwargs.setdefault("api_key", API_KEY)
+    kwargs.setdefault("api_key", headphones.CONFIG.LASTFM_APIKEY or APP_API_KEY)
     kwargs.setdefault("format", "json")
 
     # Send request
-    logger.debug("Calling Last.FM method: %s", method)
-    logger.debug("Last.FM call parameters: %s", kwargs)
+    logger.debug("Calling Last.fm method: %s", method)
+    logger.debug("Last.fm call parameters: %s", kwargs)
 
     data = request.request_json(ENTRY_POINT, timeout=TIMEOUT, params=kwargs, lock=lastfm_lock)
 
     # Parse response and check for errors.
     if not data:
-        logger.error("Error calling Last.FM method: %s", method)
+        logger.error("Error calling Last.fm method: %s", method)
         return
 
     if "error" in data:
-        logger.debug("Last.FM returned an error: %s", data["message"])
+        logger.debug("Last.fm returned an error: %s", data["message"])
         return
 
     return data
 
 
 def getSimilar():
-    myDB = db.DBConnection()
-    results = myDB.select("SELECT ArtistID from artists ORDER BY HaveTracks DESC")
+    if not headphones.CONFIG.LASTFM_APIKEY:
+        logger.info(
+            'To update the Similar Artists cloud tag, create a Last.fm application api key '
+            'and add it under the Advanced config tab'
+        )
+        return
 
-    logger.info("Fetching similar artists from Last.FM for tag cloud")
+    myDB = db.DBConnection()
+    results = myDB.select("SELECT ArtistID from artists ORDER BY HaveTracks DESC LIMIT 10")
+
+    logger.info("Fetching similar artists from Last.fm for tag cloud")
     artistlist = []
 
-    for result in results[:12]:
+    for result in results:
         data = request_lastfm("artist.getsimilar", mbid=result["ArtistId"])
 
         if data and "similarartists" in data:
@@ -85,7 +92,7 @@ def getSimilar():
                     artistlist.append((artist_name, artist_mbid))
 
     # Add new artists to tag cloud
-    logger.debug("Fetched %d artists from Last.FM", len(artistlist))
+    logger.debug("Fetched %d artists from Last.fm", len(artistlist))
     count = defaultdict(int)
 
     for artist, mbid in artistlist:
@@ -103,7 +110,7 @@ def getSimilar():
 
         myDB.action("INSERT INTO lastfmcloud VALUES( ?, ?, ?)", [artist_name, artist_mbid, count])
 
-    logger.debug("Inserted %d artists into Last.FM tag cloud", len(top_list))
+    logger.debug("Inserted %d artists into Last.fm tag cloud", len(top_list))
 
 
 def getArtists():
@@ -111,16 +118,16 @@ def getArtists():
     results = myDB.select("SELECT ArtistID from artists")
 
     if not headphones.CONFIG.LASTFM_USERNAME:
-        logger.warn("Last.FM username not set, not importing artists.")
+        logger.warn("Last.fm username not set, not importing artists.")
         return
 
-    logger.info("Fetching artists from Last.FM for username: %s", headphones.CONFIG.LASTFM_USERNAME)
+    logger.info("Fetching artists from Last.fm for username: %s", headphones.CONFIG.LASTFM_USERNAME)
     data = request_lastfm("library.getartists", limit=1000, user=headphones.CONFIG.LASTFM_USERNAME)
 
     if data and "artists" in data:
         artistlist = []
         artists = data["artists"]["artist"]
-        logger.debug("Fetched %d artists from Last.FM", len(artists))
+        logger.debug("Fetched %d artists from Last.fm", len(artists))
 
         for artist in artists:
             artist_mbid = artist["mbid"]
@@ -133,20 +140,20 @@ def getArtists():
         for artistid in artistlist:
             importer.addArtisttoDB(artistid)
 
-        logger.info("Imported %d new artists from Last.FM", len(artistlist))
+        logger.info("Imported %d new artists from Last.fm", len(artistlist))
 
 
 def getTagTopArtists(tag, limit=50):
     myDB = db.DBConnection()
     results = myDB.select("SELECT ArtistID from artists")
 
-    logger.info("Fetching top artists from Last.FM for tag: %s", tag)
+    logger.info("Fetching top artists from Last.fm for tag: %s", tag)
     data = request_lastfm("tag.gettopartists", limit=limit, tag=tag)
 
     if data and "topartists" in data:
         artistlist = []
         artists = data["topartists"]["artist"]
-        logger.debug("Fetched %d artists from Last.FM", len(artists))
+        logger.debug("Fetched %d artists from Last.fm", len(artists))
 
         for artist in artists:
             try:
@@ -162,4 +169,4 @@ def getTagTopArtists(tag, limit=50):
         for artistid in artistlist:
             importer.addArtisttoDB(artistid)
 
-        logger.debug("Added %d new artists from Last.FM", len(artistlist))
+        logger.debug("Added %d new artists from Last.fm", len(artistlist))

headphones/librarysync.py

@@ -77,9 +77,9 @@ def libraryScan(dir=None, append=False, ArtistID=None, ArtistName=None,
                 if track['ArtistName']:
                     # Make sure deleted files get accounted for when updating artist track counts
                     new_artists.append(track['ArtistName'])
-                myDB.action('DELETE FROM have WHERE Location=?', [Track['Location']])
+                myDB.action('DELETE FROM have WHERE Location=?', [track['Location']])
                 logger.info(
-                    f"{Track['Location']} removed from Headphones, as it "
+                    f"{track['Location']} removed from Headphones, as it "
                     f"is no longer on disk"
                 )
 

headphones/mb.py

@@ -14,20 +14,14 @@
 #  along with Headphones.  If not, see <http://www.gnu.org/licenses/>.
 
 
-from headphones import logger, db, helpers
+from collections import OrderedDict
+
+import musicbrainzngs
 
 import headphones
-import musicbrainzngs
 import headphones.lock
+from headphones import logger, db, helpers
 
-try:
-    # pylint:disable=E0611
-    # ignore this error because we are catching the ImportError
-    from collections import OrderedDict
-    # pylint:enable=E0611
-except ImportError:
-    # Python 2.6.x fallback, from libs
-    from ordereddict import OrderedDict
 
 mb_lock = headphones.lock.TimedLock(0)
 
@@ -97,7 +91,7 @@ def findArtist(name, limit=1):
         try:
             artistResults = musicbrainzngs.search_artists(limit=limit, **criteria)['artist-list']
         except ValueError as e:
-            if "at least one query term is required" in e.message:
+            if "at least one query term is required" in str(e):
                 logger.error(
                     "Tried to search without a term, or an empty one. Provided artist (probably emtpy): %s",
                     name)

headphones/metadata.py

@@ -79,6 +79,7 @@ class Vars:
     Metadata $variable names (only ones set explicitly by headphones).
     """
     DISC = '$Disc'
+    DISC_TOTAL = '$DiscTotal'
     TRACK = '$Track'
     TITLE = '$Title'
     ARTIST = '$Artist'
@@ -171,7 +172,7 @@ def _lower(s):
     return None
 
 
-def file_metadata(path, release):
+def file_metadata(path, release, single_disc_ignore=False):
     # type: (str,sqlite3.Row)->Tuple[Mapping[str,str],bool]
     """
     Prepare metadata dictionary for path substitution, based on file name,
@@ -194,7 +195,13 @@ def file_metadata(path, release):
     _row_to_dict(release, res)
 
     date, year = _date_year(release)
-    if not f.disc:
+
+    if not f.disctotal or (f.disctotal == 1 and single_disc_ignore):
+        disc_total = ''
+    else:
+        disc_total = '%d' % f.disctotal
+
+    if not f.disc or (f.disctotal == 1 and single_disc_ignore):
         disc_number = ''
     else:
         disc_number = '%d' % f.disc
@@ -226,6 +233,7 @@ def file_metadata(path, release):
     album_title = release['AlbumTitle']
     override_values = {
         Vars.DISC: disc_number,
+        Vars.DISC_TOTAL: disc_total,
         Vars.TRACK: track_number,
         Vars.TITLE: title,
         Vars.ARTIST: artist_name,

headphones/music_encoder.py

@@ -30,7 +30,6 @@ from . import getXldProfile
 
 
 def encode(albumPath):
-    print(albumPath)
     use_xld = headphones.CONFIG.ENCODER == 'xld'
 
     # Return if xld details not found

headphones/nzbget.py

@@ -70,7 +70,8 @@ def sendNZB(nzb):
     nzbcontent64 = None
     if nzb.resultType == "nzbdata":
         data = nzb.extraInfo[0]
-        nzbcontent64 = standard_b64encode(data)
+        # NZBGet needs a string, not bytes
+        nzbcontent64 = standard_b64encode(data).decode("utf-8")
 
     logger.info("Sending NZB to NZBget")
     logger.debug("URL: " + url)

headphones/postprocessor.py

@@ -27,7 +27,7 @@ from beets import config as beetsconfig
 from beets import logging as beetslogging
 from mediafile import MediaFile, FileTypeError, UnreadableFileError
 from beetsplug import lyrics as beetslyrics
-from headphones import notifiers, utorrent, transmission, deluge, qbittorrent
+from headphones import notifiers, utorrent, transmission, deluge, qbittorrent, soulseek
 from headphones import db, albumart, librarysync
 from headphones import logger, helpers, mb, music_encoder
 from headphones import metadata
@@ -36,18 +36,44 @@ postprocessor_lock = threading.Lock()
 
 
 def checkFolder():
-    logger.debug("Checking download folder for completed downloads (only snatched ones).")
+    logger.info("Checking download folder for completed downloads (only snatched ones).")
 
     with postprocessor_lock:
         myDB = db.DBConnection()
         snatched = myDB.select('SELECT * from snatched WHERE Status="Snatched"')
 
+        # If soulseek is used, this part will get the status from the soulseek api and return completed and errored albums
+        completed_albums, errored_albums = set(), set()
+        if any(album['Kind'] == 'soulseek' for album in snatched):
+            completed_albums, errored_albums = soulseek.download_completed()
+
         for album in snatched:
             if album['FolderName']:
                 folder_name = album['FolderName']
                 single = False
-                if album['Kind'] == 'nzb':
-                    download_dir = headphones.CONFIG.DOWNLOAD_DIR
+                if album['Kind'] == 'soulseek':
+                    if folder_name in errored_albums:
+                        # If the album had any tracks with errors in it, the whole download is considered faulty. Status will be reset to wanted.
+                        logger.info(f"Album with folder '{folder_name}' had errors during download. Setting status to 'Wanted'.")
+                        myDB.action('UPDATE albums SET Status="Wanted" WHERE AlbumID=? AND Status="Snatched"', (album['AlbumID'],))
+                        
+                        # Folder will be removed from configured complete and Incomplete directory
+                        complete_path = os.path.join(headphones.CONFIG.SOULSEEK_DOWNLOAD_DIR, folder_name)
+                        incomplete_path = os.path.join(headphones.CONFIG.SOULSEEK_INCOMPLETE_DOWNLOAD_DIR, folder_name)
+                        for path in [complete_path, incomplete_path]:
+                            try:
+                                shutil.rmtree(path)
+                            except Exception as e:
+                                pass
+                        continue
+                    elif folder_name in completed_albums:
+                        download_dir = headphones.CONFIG.SOULSEEK_DOWNLOAD_DIR
+                    else:
+                        continue
+                elif album['Kind'] == 'nzb':
+                    download_dir = headphones.CONFIG.DOWNLOAD_DIR   
+                elif album['Kind'] == 'bandcamp':
+                    download_dir = headphones.CONFIG.BANDCAMP_DIR 
                 else:
                     if headphones.CONFIG.DELUGE_DONE_DIRECTORY and headphones.CONFIG.TORRENT_DOWNLOADER == 3:
                         download_dir = headphones.CONFIG.DELUGE_DONE_DIRECTORY
@@ -65,7 +91,6 @@ def checkFolder():
                             folder_name = torrent_folder_name
 
                 if folder_name:
-                    print(folder_name)
                     album_path = os.path.join(download_dir, folder_name)
                     logger.debug("Checking if %s exists" % album_path)
 
@@ -80,7 +105,6 @@ def checkFolder():
 
 
 def verify(albumid, albumpath, Kind=None, forced=False, keep_original_folder=False, single=False):
-    print(albumpath)
     myDB = db.DBConnection()
     release = myDB.action('SELECT * from albums WHERE AlbumID=?', [albumid]).fetchone()
     tracks = myDB.select('SELECT * from tracks WHERE AlbumID=?', [albumid])
@@ -291,7 +315,7 @@ def verify(albumid, albumpath, Kind=None, forced=False, keep_original_folder=Fal
     logger.debug('Metadata check failed. Verifying filenames...')
     for downloaded_track in downloaded_track_list:
         track_name = os.path.splitext(downloaded_track)[0]
-        split_track_name = re.sub('[\.\-\_]', ' ', track_name).lower()
+        split_track_name = re.sub(r'[\.\-\_]', r' ', track_name).lower()
         for track in tracks:
 
             if not track['TrackTitle']:
@@ -1087,7 +1111,11 @@ def renameFiles(albumpath, downloaded_track_list, release):
     # Until tagging works better I'm going to rely on the already provided metadata
 
     for downloaded_track in downloaded_track_list:
-        md, from_metadata = metadata.file_metadata(downloaded_track, release)
+        md, from_metadata = metadata.file_metadata(
+            downloaded_track,
+            release,
+            headphones.CONFIG.RENAME_SINGLE_DISC_IGNORE
+        )
         if md is None:
             # unable to parse media file, skip file
             continue
@@ -1168,8 +1196,14 @@ def forcePostProcess(dir=None, expand_subfolders=True, album_dir=None, keep_orig
             download_dirs.append(dir)
         if headphones.CONFIG.DOWNLOAD_DIR and not dir:
             download_dirs.append(headphones.CONFIG.DOWNLOAD_DIR)
+        if headphones.CONFIG.SOULSEEK_DOWNLOAD_DIR and not dir:
+            download_dirs.append(headphones.CONFIG.SOULSEEK_DOWNLOAD_DIR)
         if headphones.CONFIG.DOWNLOAD_TORRENT_DIR and not dir:
-            download_dirs.append(headphones.CONFIG.DOWNLOAD_TORRENT_DIR)
+            download_dirs.append(
+                headphones.CONFIG.DOWNLOAD_TORRENT_DIR.encode(headphones.SYS_ENCODING, 'replace'))
+        if headphones.CONFIG.BANDCAMP and not dir:
+            download_dirs.append(
+                headphones.CONFIG.BANDCAMP_DIR.encode(headphones.SYS_ENCODING, 'replace'))
 
         # If DOWNLOAD_DIR and DOWNLOAD_TORRENT_DIR are the same, remove the duplicate to prevent us from trying to process the same folder twice.
         download_dirs = list(set(download_dirs))

headphones/rutracker.py

@@ -11,6 +11,7 @@ from bs4 import BeautifulSoup
 
 import headphones
 from headphones import logger
+from headphones.types import Result
 
 
 class Rutracker(object):
@@ -41,19 +42,22 @@ class Rutracker(object):
             'login_password': headphones.CONFIG.RUTRACKER_PASSWORD,
             'login': b'\xc2\xf5\xee\xe4'  # '%C2%F5%EE%E4'
         }
+        headers = {
+            'User-Agent' : 'Headphones'   
+        }
 
         logger.info("Attempting to log in to rutracker...")
 
         try:
-            r = self.session.post(loginpage, data=post_params, timeout=self.timeout, allow_redirects=False)
+            r = self.session.post(loginpage, data=post_params, timeout=self.timeout, allow_redirects=False, headers=headers)
             # try again
             if not self.has_bb_session_cookie(r):
                 time.sleep(10)
                 if headphones.CONFIG.RUTRACKER_COOKIE:
                     logger.info("Attempting to log in using predefined cookie...")
-                    r = self.session.post(loginpage, data=post_params, timeout=self.timeout, allow_redirects=False, cookies={'bb_session': headphones.CONFIG.RUTRACKER_COOKIE})
+                    r = self.session.post(loginpage, data=post_params, timeout=self.timeout, allow_redirects=False, headers=headers, cookies={'bb_session': headphones.CONFIG.RUTRACKER_COOKIE})
                 else:
-                    r = self.session.post(loginpage, data=post_params, timeout=self.timeout, allow_redirects=False)
+                    r = self.session.post(loginpage, data=post_params, timeout=self.timeout, allow_redirects=False, headers=headers)
             if self.has_bb_session_cookie(r):
                 self.loggedin = True
                 logger.info("Successfully logged in to rutracker")
@@ -112,7 +116,10 @@ class Rutracker(object):
         Parse the search results and return valid torrent list
         """
         try:
-            headers = {'Referer': self.search_referer}
+            headers = {
+                'Referer': self.search_referer,
+                'User-Agent' : 'Headphones'   
+            }
             r = self.session.get(url=searchurl, headers=headers, timeout=self.timeout)
             soup = BeautifulSoup(r.content, 'html.parser')
 
@@ -160,7 +167,7 @@ class Rutracker(object):
                     torrent_id = dict([part.split('=') for part in urlparse(url)[4].split('&')])[
                         't']
                     topicurl = 'https://rutracker.org/forum/viewtopic.php?t=' + torrent_id
-                    rulist.append((title, size, topicurl, 'rutracker.org', 'torrent', True))
+                    rulist.append(Result(title, size, url, 'rutracker.org', 'torrent', True))
                 else:
                     logger.info("%s is larger than the maxsize or has too little seeders for this category, "
                                 "skipping. (Size: %i bytes, Seeders: %i)" % (title, size, int(seeds)))
@@ -182,7 +189,10 @@ class Rutracker(object):
         downloadurl = 'https://rutracker.org/forum/dl.php?t=' + torrent_id
         cookie = {'bb_dl': torrent_id}
         try:
-            headers = {'Referer': url}
+            headers = {
+                'Referer': url,
+                'User-Agent' : 'Headphones'   
+            }
             r = self.session.post(url=downloadurl, cookies=cookie, headers=headers,
                                   timeout=self.timeout)
             return r.content

headphones/soulseek.py

@@ -0,0 +1,185 @@
+from collections import defaultdict, namedtuple
+import os
+import time
+import slskd_api
+import headphones
+from headphones import logger
+from datetime import datetime, timedelta
+
+Result = namedtuple('Result', ['title', 'size', 'user', 'provider', 'type', 'matches', 'bandwidth', 'hasFreeUploadSlot', 'queueLength', 'files', 'kind', 'url', 'folder'])
+
+def initialize_soulseek_client():
+    host = headphones.CONFIG.SOULSEEK_API_URL
+    api_key = headphones.CONFIG.SOULSEEK_API_KEY
+    return slskd_api.SlskdClient(host=host, api_key=api_key)
+
+    # Search logic, calling search and processing fucntions
+def search(artist, album, year, num_tracks, losslessOnly):
+    client = initialize_soulseek_client()
+    
+    # Stage 1: Search with artist, album, year, and num_tracks
+    results = execute_search(client, artist, album, year, losslessOnly)
+    processed_results = process_results(results, losslessOnly, num_tracks)
+    if processed_results:
+        return processed_results
+    
+    # Stage 2: If Stage 1 fails, search with artist, album, and num_tracks (excluding year)
+    logger.info("Soulseek search stage 1 did not meet criteria. Retrying without year...")
+    results = execute_search(client, artist, album, None, losslessOnly)
+    processed_results = process_results(results, losslessOnly, num_tracks)
+    if processed_results:
+        return processed_results
+    
+    # Stage 3: Final attempt, search only with artist and album
+    logger.info("Soulseek search stage 2 did not meet criteria. Final attempt with only artist and album.")
+    results = execute_search(client, artist, album, None, losslessOnly)
+    processed_results = process_results(results, losslessOnly, num_tracks, ignore_track_count=True)
+    
+    return processed_results
+
+def execute_search(client, artist, album, year, losslessOnly):
+    search_text = f"{artist} {album}"
+    if year:
+        search_text += f" {year}"
+    if losslessOnly:
+        search_text += ".flac"
+
+    # Actual search
+    search_response = client.searches.search_text(searchText=search_text, filterResponses=True)
+    search_id = search_response.get('id')
+    
+    # Wait for search completion and return response
+    while not client.searches.state(id=search_id).get('isComplete'):
+        time.sleep(2)
+    
+    return client.searches.search_responses(id=search_id)
+
+# Processing the search result passed
+def process_results(results, losslessOnly, num_tracks, ignore_track_count=False):
+    valid_extensions = {'.flac'} if losslessOnly else {'.mp3', '.flac'}
+    albums = defaultdict(lambda: {'files': [], 'user': None, 'hasFreeUploadSlot': None, 'queueLength': None, 'uploadSpeed': None})
+
+    # Extract info from the api response and combine files at album level
+    for result in results:
+        user = result.get('username')
+        hasFreeUploadSlot = result.get('hasFreeUploadSlot')
+        queueLength = result.get('queueLength')
+        uploadSpeed = result.get('uploadSpeed')
+
+        # Only handle .mp3 and .flac
+        for file in result.get('files', []):
+            filename = file.get('filename')
+            file_extension = os.path.splitext(filename)[1].lower()
+            if file_extension in valid_extensions:
+                album_directory = os.path.dirname(filename)
+                albums[album_directory]['files'].append(file)
+
+                # Update metadata only once per album_directory
+                if albums[album_directory]['user'] is None:
+                    albums[album_directory].update({
+                        'user': user,
+                        'hasFreeUploadSlot': hasFreeUploadSlot,
+                        'queueLength': queueLength,
+                        'uploadSpeed': uploadSpeed,
+                    })
+
+    # Filter albums based on num_tracks, add bunch of useful info to the compiled album
+    final_results = []
+    for directory, album_data in albums.items():
+        if ignore_track_count or len(album_data['files']) == num_tracks:
+            album_title = os.path.basename(directory)
+            total_size = sum(file.get('size', 0) for file in album_data['files'])
+            final_results.append(Result(
+                title=album_title,
+                size=int(total_size),
+                user=album_data['user'],
+                provider="soulseek",
+                type="soulseek",
+                matches=True,
+                bandwidth=album_data['uploadSpeed'],
+                hasFreeUploadSlot=album_data['hasFreeUploadSlot'],
+                queueLength=album_data['queueLength'],
+                files=album_data['files'],
+                kind='soulseek',
+                url='http://thisisnot.needed', # URL is needed in other parts of the program.
+                folder=os.path.basename(directory)
+            ))
+
+    return final_results
+
+
+def download(user, filelist):
+    client = initialize_soulseek_client()
+    client.transfers.enqueue(username=user, files=filelist)
+
+
+def download_completed():
+    client = initialize_soulseek_client()
+    all_downloads = client.transfers.get_all_downloads(includeRemoved=False)
+    album_completion_tracker = {}  # Tracks completion state of each album's songs
+    album_errored_tracker = {}  # Tracks albums with errored downloads
+
+    # Anything older than 24 hours will be canceled
+    cutoff_time = datetime.now() - timedelta(hours=24)
+
+    # Identify errored and completed albums
+    for download in all_downloads:
+        directories = download.get('directories', [])
+        for directory in directories:
+            album_part = directory.get('directory', '').split('\\')[-1]
+            files = directory.get('files', [])
+            for file_data in files:
+                state = file_data.get('state', '')
+                requested_at_str = file_data.get('requestedAt', '1900-01-01 00:00:00')
+                requested_at = parse_datetime(requested_at_str)
+
+                # Initialize or update album entry in trackers
+                if album_part not in album_completion_tracker:
+                    album_completion_tracker[album_part] = {'total': 0, 'completed': 0, 'errored': 0}
+                if album_part not in album_errored_tracker:
+                    album_errored_tracker[album_part] = False
+
+                album_completion_tracker[album_part]['total'] += 1
+
+                if 'Completed, Succeeded' in state:
+                    album_completion_tracker[album_part]['completed'] += 1
+                elif 'Completed, Errored' in state or requested_at < cutoff_time:
+                    album_completion_tracker[album_part]['errored'] += 1
+                    album_errored_tracker[album_part] = True  # Mark album as having errored downloads
+
+    # Identify errored albums
+    errored_albums = {album for album, errored in album_errored_tracker.items() if errored}
+
+    # Cancel downloads for errored albums
+    for download in all_downloads:
+        directories = download.get('directories', [])
+        for directory in directories:
+            album_part = directory.get('directory', '').split('\\')[-1]
+            files = directory.get('files', [])
+            for file_data in files:
+                if album_part in errored_albums:
+                    # Extract 'id' and 'username' for each file to cancel the download
+                    file_id = file_data.get('id', '')
+                    username = file_data.get('username', '')
+                    success = client.transfers.cancel_download(username, file_id)
+                    if not success:
+                        print(f"Failed to cancel download for file ID: {file_id}")
+
+    # Clear completed/canceled/errored stuff from client downloads
+    try:
+        client.transfers.remove_completed_downloads()
+    except Exception as e:
+        print(f"Failed to remove completed downloads: {e}")
+
+    # Identify completed albums
+    completed_albums = {album for album, counts in album_completion_tracker.items() if counts['total'] == counts['completed']}
+
+    # Return both completed and errored albums
+    return completed_albums, errored_albums
+
+
+def parse_datetime(datetime_string):
+    # Parse the datetime api response
+    if '.' in datetime_string:
+        datetime_string = datetime_string[:datetime_string.index('.')+7]
+    return datetime.strptime(datetime_string, '%Y-%m-%dT%H:%M:%S.%f')

headphones/transmission.py

@@ -15,7 +15,7 @@
 
 import time
 import json
-import base64
+from base64 import b64encode
 import urllib.parse
 import os
 
@@ -36,10 +36,10 @@ def addTorrent(link, data=None):
 
     if link.endswith('.torrent') and not link.startswith(('http', 'magnet')) or data:
         if data:
-            metainfo = str(base64.b64encode(data))
+            metainfo = b64encode(data).decode("utf-8")
         else:
             with open(link, 'rb') as f:
-                metainfo = str(base64.b64encode(f.read()))
+                metainfo = b64encode(f.read()).decode("utf-8")
         arguments = {'metainfo': metainfo, 'download-dir': headphones.CONFIG.DOWNLOAD_TORRENT_DIR}
     else:
         arguments = {'filename': link, 'download-dir': headphones.CONFIG.DOWNLOAD_TORRENT_DIR}
@@ -205,5 +205,4 @@ def torrentAction(method, arguments):
             continue
 
         resp_json = response.json()
-        print(resp_json)
         return resp_json

headphones/types.py

@@ -0,0 +1,10 @@
+from dataclasses import dataclass
+
+@dataclass(frozen=True)
+class Result:
+    title: str
+    size: int
+    url: str
+    provider: str
+    kind: str
+    matches: bool

headphones/webserve.py

@@ -15,34 +15,46 @@
 
 # NZBGet support added by CurlyMo <curlymoo1@gmail.com> as a part of XBian - XBMC on the Raspberry Pi
 
-from operator import itemgetter
-import threading
-import secrets
-import random
-import urllib.request, urllib.parse, urllib.error
 import json
-import time
-import sys
-from html import escape as html_escape
-import urllib.request, urllib.error, urllib.parse
-
 import os
+import random
 import re
-from headphones import logger, searcher, db, importer, mb, lastfm, librarysync, helpers, notifiers, crier
-from headphones.helpers import checked, radio, today, clean_name
-from mako.lookup import TemplateLookup
-from mako import exceptions
-import headphones
-import cherrypy
+import secrets
+import sys
+import threading
+import time
+from collections import OrderedDict
+from dataclasses import asdict
+from html import escape as html_escape
+from operator import itemgetter
+from urllib import parse
 
-try:
-    # pylint:disable=E0611
-    # ignore this error because we are catching the ImportError
-    from collections import OrderedDict
-    # pylint:enable=E0611
-except ImportError:
-    # Python 2.6.x fallback, from libs
-    from ordereddict import OrderedDict
+import cherrypy
+from mako import exceptions
+from mako.lookup import TemplateLookup
+
+import headphones
+from headphones import (
+    crier,
+    db,
+    importer,
+    lastfm,
+    librarysync,
+    logger,
+    mb,
+    notifiers,
+    searcher,
+)
+from headphones.helpers import (
+    checked,
+    clean_name,
+    have_pct_have_total,
+    pattern_substitute,
+    radio,
+    replace_illegal_chars,
+    today,
+)
+from headphones.types import Result
 
 
 def serve_template(templatename, **kwargs):
@@ -326,9 +338,9 @@ class WebInterface(object):
                   '$first': firstchar.lower(),
                   }
 
-        folder = helpers.pattern_substitute(folder_format.strip(), values, normalize=True)
+        folder = pattern_substitute(folder_format.strip(), values, normalize=True)
 
-        folder = helpers.replace_illegal_chars(folder, type="folder")
+        folder = replace_illegal_chars(folder, type="folder")
         folder = folder.replace('./', '_/').replace('/.', '/_')
 
         if folder.endswith('.'):
@@ -440,40 +452,27 @@ class WebInterface(object):
     @cherrypy.expose
     @cherrypy.tools.json_out()
     def choose_specific_download(self, AlbumID):
-        results = searcher.searchforalbum(AlbumID, choose_specific_download=True)
-
-        data = []
-
-        for result in results:
-            result_dict = {
-                'title': result[0],
-                'size': result[1],
-                'url': result[2],
-                'provider': result[3],
-                'kind': result[4],
-                'matches': result[5]
-            }
-            data.append(result_dict)
-        return data
+        results = searcher.searchforalbum(AlbumID, choose_specific_download=True) or []
+        return list(map(asdict, results))
 
     @cherrypy.expose
     @cherrypy.tools.json_out()
     def download_specific_release(self, AlbumID, title, size, url, provider, kind, **kwargs):
         # Handle situations where the torrent url contains arguments that are parsed
         if kwargs:
-            url = urllib.parse.quote(url, safe=":?/=&") + '&' + urllib.parse.urlencode(kwargs)
+            url = parse.quote(url, safe=":?/=&") + '&' + parse.urlencode(kwargs)
         try:
-            result = [(title, int(size), url, provider, kind)]
+            result = [Result(title, int(size), url, provider, kind, True)]
         except ValueError:
-            result = [(title, float(size), url, provider, kind)]
+            result = [Result(title, float(size), url, provider, kind, True)]
 
         logger.info("Making sure we can download the chosen result")
-        (data, bestqual) = searcher.preprocess(result)
+        data, result = searcher.preprocess(result)
 
-        if data and bestqual:
+        if data and result:
             myDB = db.DBConnection()
             album = myDB.action('SELECT * from albums WHERE AlbumID=?', [AlbumID]).fetchone()
-            searcher.send_to_downloader(data, bestqual, album)
+            searcher.send_to_downloader(data, result, album)
             return {'result': 'success'}
         else:
             return {'result': 'failure'}
@@ -586,7 +585,7 @@ class WebInterface(object):
         for albums in have_albums:
             # Have to skip over manually matched tracks
             if albums['ArtistName'] and albums['AlbumTitle'] and albums['TrackTitle']:
-                original_clean = helpers.clean_name(
+                original_clean = clean_name(
                     albums['ArtistName'] + " " + albums['AlbumTitle'] + " " + albums['TrackTitle'])
                 # else:
                 #     original_clean = None
@@ -633,8 +632,8 @@ class WebInterface(object):
                 (artist, album))
 
         elif action == "matchArtist":
-            existing_artist_clean = helpers.clean_name(existing_artist).lower()
-            new_artist_clean = helpers.clean_name(new_artist).lower()
+            existing_artist_clean = clean_name(existing_artist).lower()
+            new_artist_clean = clean_name(new_artist).lower()
             if new_artist_clean != existing_artist_clean:
                 have_tracks = myDB.action(
                     'SELECT Matched, CleanName, Location, BitRate, Format FROM have WHERE ArtistName=?',
@@ -678,10 +677,10 @@ class WebInterface(object):
                     "Artist %s already named appropriately; nothing to modify" % existing_artist)
 
         elif action == "matchAlbum":
-            existing_artist_clean = helpers.clean_name(existing_artist).lower()
-            new_artist_clean = helpers.clean_name(new_artist).lower()
-            existing_album_clean = helpers.clean_name(existing_album).lower()
-            new_album_clean = helpers.clean_name(new_album).lower()
+            existing_artist_clean = clean_name(existing_artist).lower()
+            new_artist_clean = clean_name(new_artist).lower()
+            existing_album_clean = clean_name(existing_album).lower()
+            new_album_clean = clean_name(new_album).lower()
             existing_clean_string = existing_artist_clean + " " + existing_album_clean
             new_clean_string = new_artist_clean + " " + new_album_clean
             if existing_clean_string != new_clean_string:
@@ -737,7 +736,7 @@ class WebInterface(object):
             'SELECT ArtistName, AlbumTitle, TrackTitle, CleanName, Matched from have')
         for albums in manualalbums:
             if albums['ArtistName'] and albums['AlbumTitle'] and albums['TrackTitle']:
-                original_clean = helpers.clean_name(
+                original_clean = clean_name(
                     albums['ArtistName'] + " " + albums['AlbumTitle'] + " " + albums['TrackTitle'])
                 if albums['Matched'] == "Ignored" or albums['Matched'] == "Manual" or albums[
                         'CleanName'] != original_clean:
@@ -778,7 +777,7 @@ class WebInterface(object):
                 [artist])
             update_count = 0
             for tracks in update_clean:
-                original_clean = helpers.clean_name(
+                original_clean = clean_name(
                     tracks['ArtistName'] + " " + tracks['AlbumTitle'] + " " + tracks[
                         'TrackTitle']).lower()
                 album = tracks['AlbumTitle']
@@ -810,7 +809,7 @@ class WebInterface(object):
                 (artist, album))
             update_count = 0
             for tracks in update_clean:
-                original_clean = helpers.clean_name(
+                original_clean = clean_name(
                     tracks['ArtistName'] + " " + tracks['AlbumTitle'] + " " + tracks[
                         'TrackTitle']).lower()
                 track_title = tracks['TrackTitle']
@@ -1018,9 +1017,7 @@ class WebInterface(object):
             totalcount = myDB.select('SELECT COUNT(*) from artists')[0][0]
 
         if sortbyhavepercent:
-            filtered.sort(key=lambda x: (
-                float(x['HaveTracks']) / x['TotalTracks'] if x['TotalTracks'] > 0 else 0.0,
-                x['HaveTracks'] if x['HaveTracks'] else 0.0), reverse=sSortDir_0 == "asc")
+            filtered.sort(key=have_pct_have_total, reverse=sSortDir_0 == "asc")
 
         # can't figure out how to change the datatables default sorting order when its using an ajax datasource so ill
         # just reverse it here and the first click on the "Latest Album" header will sort by descending release date
@@ -1186,6 +1183,7 @@ class WebInterface(object):
             "deluge_password": headphones.CONFIG.DELUGE_PASSWORD,
             "deluge_label": headphones.CONFIG.DELUGE_LABEL,
             "deluge_done_directory": headphones.CONFIG.DELUGE_DONE_DIRECTORY,
+            "deluge_download_directory": headphones.CONFIG.DELUGE_DOWNLOAD_DIRECTORY,
             "deluge_paused": checked(headphones.CONFIG.DELUGE_PAUSED),
             "utorrent_host": headphones.CONFIG.UTORRENT_HOST,
             "utorrent_username": headphones.CONFIG.UTORRENT_USERNAME,
@@ -1200,6 +1198,8 @@ class WebInterface(object):
             "torrent_downloader_deluge": radio(headphones.CONFIG.TORRENT_DOWNLOADER, 3),
             "torrent_downloader_qbittorrent": radio(headphones.CONFIG.TORRENT_DOWNLOADER, 4),
             "download_dir": headphones.CONFIG.DOWNLOAD_DIR,
+            "soulseek_download_dir": headphones.CONFIG.SOULSEEK_DOWNLOAD_DIR,
+            "soulseek_incomplete_download_dir": headphones.CONFIG.SOULSEEK_INCOMPLETE_DOWNLOAD_DIR,
             "use_blackhole": checked(headphones.CONFIG.BLACKHOLE),
             "blackhole_dir": headphones.CONFIG.BLACKHOLE_DIR,
             "usenet_retention": headphones.CONFIG.USENET_RETENTION,
@@ -1271,6 +1271,7 @@ class WebInterface(object):
             "cue_split_shntool_path": headphones.CONFIG.CUE_SPLIT_SHNTOOL_PATH,
             "move_files": checked(headphones.CONFIG.MOVE_FILES),
             "rename_files": checked(headphones.CONFIG.RENAME_FILES),
+            "rename_single_disc_ignore": checked(headphones.CONFIG.RENAME_SINGLE_DISC_IGNORE),
             "correct_metadata": checked(headphones.CONFIG.CORRECT_METADATA),
             "cleanup_files": checked(headphones.CONFIG.CLEANUP_FILES),
             "keep_nfo": checked(headphones.CONFIG.KEEP_NFO),
@@ -1298,6 +1299,7 @@ class WebInterface(object):
             "prefer_torrents_0": radio(headphones.CONFIG.PREFER_TORRENTS, 0),
             "prefer_torrents_1": radio(headphones.CONFIG.PREFER_TORRENTS, 1),
             "prefer_torrents_2": radio(headphones.CONFIG.PREFER_TORRENTS, 2),
+            "prefer_torrents_3": radio(headphones.CONFIG.PREFER_TORRENTS, 3),
             "magnet_links_0": radio(headphones.CONFIG.MAGNET_LINKS, 0),
             "magnet_links_1": radio(headphones.CONFIG.MAGNET_LINKS, 1),
             "magnet_links_2": radio(headphones.CONFIG.MAGNET_LINKS, 2),
@@ -1387,6 +1389,7 @@ class WebInterface(object):
             "custompass": headphones.CONFIG.CUSTOMPASS,
             "hpuser": headphones.CONFIG.HPUSER,
             "hppass": headphones.CONFIG.HPPASS,
+            "lastfm_apikey": headphones.CONFIG.LASTFM_APIKEY,
             "songkick_enabled": checked(headphones.CONFIG.SONGKICK_ENABLED),
             "songkick_apikey": headphones.CONFIG.SONGKICK_APIKEY,
             "songkick_location": headphones.CONFIG.SONGKICK_LOCATION,
@@ -1414,7 +1417,12 @@ class WebInterface(object):
             "join_enabled": checked(headphones.CONFIG.JOIN_ENABLED),
             "join_onsnatch": checked(headphones.CONFIG.JOIN_ONSNATCH),
             "join_apikey": headphones.CONFIG.JOIN_APIKEY,
-            "join_deviceid": headphones.CONFIG.JOIN_DEVICEID
+            "join_deviceid": headphones.CONFIG.JOIN_DEVICEID,
+            "use_bandcamp": checked(headphones.CONFIG.BANDCAMP),
+            "bandcamp_dir": headphones.CONFIG.BANDCAMP_DIR,
+            'soulseek_api_url': headphones.CONFIG.SOULSEEK_API_URL,
+            'soulseek_api_key': headphones.CONFIG.SOULSEEK_API_KEY,
+            'use_soulseek': checked(headphones.CONFIG.SOULSEEK)
         }
 
         for k, v in config.items():
@@ -1463,8 +1471,8 @@ class WebInterface(object):
             "use_waffles", "use_rutracker",
             "use_orpheus", "use_redacted", "redacted_use_fltoken", "preferred_bitrate_allow_lossless",
             "detect_bitrate", "ignore_clean_releases", "freeze_db", "cue_split", "move_files",
-            "rename_files", "correct_metadata", "cleanup_files", "keep_nfo", "add_album_art",
-            "embed_album_art", "embed_lyrics",
+            "rename_files", "rename_single_disc_ignore", "correct_metadata", "cleanup_files",
+            "keep_nfo", "add_album_art", "embed_album_art", "embed_lyrics",
             "replace_existing_folders", "keep_original_folder", "file_underscores",
             "include_extras", "official_releases_only",
             "wait_until_release_date", "autowant_upcoming", "autowant_all",
@@ -1483,7 +1491,7 @@ class WebInterface(object):
             "songkick_enabled", "songkick_filter_enabled",
             "mpc_enabled", "email_enabled", "email_ssl", "email_tls", "email_onsnatch",
             "customauth", "idtag", "deluge_paused",
-            "join_enabled", "join_onsnatch"
+            "join_enabled", "join_onsnatch", "use_bandcamp"
         ]
         for checked_config in checked_configs:
             if checked_config not in kwargs:

lib/apscheduler/__init__.py

@@ -1,5 +1,10 @@
-version_info = (3, 0, 1)
-version = '3.0.1'
-release = '3.0.1'
+from pkg_resources import get_distribution, DistributionNotFound
 
-__version__ = release  # PEP 396
+try:
+    release = get_distribution('APScheduler').version.split('-')[0]
+except DistributionNotFound:
+    release = '3.5.0'
+
+version_info = tuple(int(x) if x.isdigit() else x for x in release.split('.'))
+version = __version__ = '.'.join(str(x) for x in version_info[:3])
+del get_distribution, DistributionNotFound

lib/apscheduler/events.py

@@ -1,25 +1,33 @@
-__all__ = ('EVENT_SCHEDULER_START', 'EVENT_SCHEDULER_SHUTDOWN', 'EVENT_EXECUTOR_ADDED', 'EVENT_EXECUTOR_REMOVED',
-           'EVENT_JOBSTORE_ADDED', 'EVENT_JOBSTORE_REMOVED', 'EVENT_ALL_JOBS_REMOVED', 'EVENT_JOB_ADDED',
-           'EVENT_JOB_REMOVED', 'EVENT_JOB_MODIFIED', 'EVENT_JOB_EXECUTED', 'EVENT_JOB_ERROR', 'EVENT_JOB_MISSED',
-           'SchedulerEvent', 'JobEvent', 'JobExecutionEvent')
+__all__ = ('EVENT_SCHEDULER_STARTED', 'EVENT_SCHEDULER_SHUTDOWN', 'EVENT_SCHEDULER_PAUSED',
+           'EVENT_SCHEDULER_RESUMED', 'EVENT_EXECUTOR_ADDED', 'EVENT_EXECUTOR_REMOVED',
+           'EVENT_JOBSTORE_ADDED', 'EVENT_JOBSTORE_REMOVED', 'EVENT_ALL_JOBS_REMOVED',
+           'EVENT_JOB_ADDED', 'EVENT_JOB_REMOVED', 'EVENT_JOB_MODIFIED', 'EVENT_JOB_EXECUTED',
+           'EVENT_JOB_ERROR', 'EVENT_JOB_MISSED', 'EVENT_JOB_SUBMITTED', 'EVENT_JOB_MAX_INSTANCES',
+           'SchedulerEvent', 'JobEvent', 'JobExecutionEvent', 'JobSubmissionEvent')
 
 
-EVENT_SCHEDULER_START = 1
-EVENT_SCHEDULER_SHUTDOWN = 2
-EVENT_EXECUTOR_ADDED = 4
-EVENT_EXECUTOR_REMOVED = 8
-EVENT_JOBSTORE_ADDED = 16
-EVENT_JOBSTORE_REMOVED = 32
-EVENT_ALL_JOBS_REMOVED = 64
-EVENT_JOB_ADDED = 128
-EVENT_JOB_REMOVED = 256
-EVENT_JOB_MODIFIED = 512
-EVENT_JOB_EXECUTED = 1024
-EVENT_JOB_ERROR = 2048
-EVENT_JOB_MISSED = 4096
-EVENT_ALL = (EVENT_SCHEDULER_START | EVENT_SCHEDULER_SHUTDOWN | EVENT_JOBSTORE_ADDED | EVENT_JOBSTORE_REMOVED |
+EVENT_SCHEDULER_STARTED = EVENT_SCHEDULER_START = 2 ** 0
+EVENT_SCHEDULER_SHUTDOWN = 2 ** 1
+EVENT_SCHEDULER_PAUSED = 2 ** 2
+EVENT_SCHEDULER_RESUMED = 2 ** 3
+EVENT_EXECUTOR_ADDED = 2 ** 4
+EVENT_EXECUTOR_REMOVED = 2 ** 5
+EVENT_JOBSTORE_ADDED = 2 ** 6
+EVENT_JOBSTORE_REMOVED = 2 ** 7
+EVENT_ALL_JOBS_REMOVED = 2 ** 8
+EVENT_JOB_ADDED = 2 ** 9
+EVENT_JOB_REMOVED = 2 ** 10
+EVENT_JOB_MODIFIED = 2 ** 11
+EVENT_JOB_EXECUTED = 2 ** 12
+EVENT_JOB_ERROR = 2 ** 13
+EVENT_JOB_MISSED = 2 ** 14
+EVENT_JOB_SUBMITTED = 2 ** 15
+EVENT_JOB_MAX_INSTANCES = 2 ** 16
+EVENT_ALL = (EVENT_SCHEDULER_STARTED | EVENT_SCHEDULER_SHUTDOWN | EVENT_SCHEDULER_PAUSED |
+             EVENT_SCHEDULER_RESUMED | EVENT_EXECUTOR_ADDED | EVENT_EXECUTOR_REMOVED |
+             EVENT_JOBSTORE_ADDED | EVENT_JOBSTORE_REMOVED | EVENT_ALL_JOBS_REMOVED |
              EVENT_JOB_ADDED | EVENT_JOB_REMOVED | EVENT_JOB_MODIFIED | EVENT_JOB_EXECUTED |
-             EVENT_JOB_ERROR | EVENT_JOB_MISSED)
+             EVENT_JOB_ERROR | EVENT_JOB_MISSED | EVENT_JOB_SUBMITTED | EVENT_JOB_MAX_INSTANCES)
 
 
 class SchedulerEvent(object):
@@ -55,9 +63,21 @@ class JobEvent(SchedulerEvent):
         self.jobstore = jobstore
 
 
+class JobSubmissionEvent(JobEvent):
+    """
+    An event that concerns the submission of a job to its executor.
+
+    :ivar scheduled_run_times: a list of datetimes when the job was intended to run
+    """
+
+    def __init__(self, code, job_id, jobstore, scheduled_run_times):
+        super(JobSubmissionEvent, self).__init__(code, job_id, jobstore)
+        self.scheduled_run_times = scheduled_run_times
+
+
 class JobExecutionEvent(JobEvent):
     """
-    An event that concerns the execution of individual jobs.
+    An event that concerns the running of a job within its executor.
 
     :ivar scheduled_run_time: the time when the job was scheduled to be run
     :ivar retval: the return value of the successfully executed job
@@ -65,7 +85,8 @@ class JobExecutionEvent(JobEvent):
     :ivar traceback: a formatted traceback for the exception
     """
 
-    def __init__(self, code, job_id, jobstore, scheduled_run_time, retval=None, exception=None, traceback=None):
+    def __init__(self, code, job_id, jobstore, scheduled_run_time, retval=None, exception=None,
+                 traceback=None):
         super(JobExecutionEvent, self).__init__(code, job_id, jobstore)
         self.scheduled_run_time = scheduled_run_time
         self.retval = retval

lib/apscheduler/executors/asyncio.py

@@ -1,28 +1,52 @@
+from __future__ import absolute_import
 
 import sys
 
 from apscheduler.executors.base import BaseExecutor, run_job
+from apscheduler.executors.base_py3 import run_coroutine_job
+from apscheduler.util import iscoroutinefunction_partial
 
 
 class AsyncIOExecutor(BaseExecutor):
     """
     Runs jobs in the default executor of the event loop.
 
+    If the job function is a native coroutine function, it is scheduled to be run directly in the
+    event loop as soon as possible. All other functions are run in the event loop's default
+    executor which is usually a thread pool.
+
     Plugin alias: ``asyncio``
     """
 
     def start(self, scheduler, alias):
         super(AsyncIOExecutor, self).start(scheduler, alias)
         self._eventloop = scheduler._eventloop
+        self._pending_futures = set()
+
+    def shutdown(self, wait=True):
+        # There is no way to honor wait=True without converting this method into a coroutine method
+        for f in self._pending_futures:
+            if not f.done():
+                f.cancel()
+
+        self._pending_futures.clear()
 
     def _do_submit_job(self, job, run_times):
         def callback(f):
+            self._pending_futures.discard(f)
             try:
                 events = f.result()
-            except:
+            except BaseException:
                 self._run_job_error(job.id, *sys.exc_info()[1:])
             else:
                 self._run_job_success(job.id, events)
 
-        f = self._eventloop.run_in_executor(None, run_job, job, job._jobstore_alias, run_times, self._logger.name)
+        if iscoroutinefunction_partial(job.func):
+            coro = run_coroutine_job(job, job._jobstore_alias, run_times, self._logger.name)
+            f = self._eventloop.create_task(coro)
+        else:
+            f = self._eventloop.run_in_executor(None, run_job, job, job._jobstore_alias, run_times,
+                                                self._logger.name)
+
         f.add_done_callback(callback)
+        self._pending_futures.add(f)

lib/apscheduler/executors/base.py

@@ -8,13 +8,15 @@ import sys
 from pytz import utc
 import six
 
-from apscheduler.events import JobExecutionEvent, EVENT_JOB_MISSED, EVENT_JOB_ERROR, EVENT_JOB_EXECUTED
+from apscheduler.events import (
+    JobExecutionEvent, EVENT_JOB_MISSED, EVENT_JOB_ERROR, EVENT_JOB_EXECUTED)
 
 
 class MaxInstancesReachedError(Exception):
     def __init__(self, job):
         super(MaxInstancesReachedError, self).__init__(
-            'Job "%s" has already reached its maximum number of instances (%d)' % (job.id, job.max_instances))
+            'Job "%s" has already reached its maximum number of instances (%d)' %
+            (job.id, job.max_instances))
 
 
 class BaseExecutor(six.with_metaclass(ABCMeta, object)):
@@ -30,13 +32,14 @@ class BaseExecutor(six.with_metaclass(ABCMeta, object)):
 
     def start(self, scheduler, alias):
         """
-        Called by the scheduler when the scheduler is being started or when the executor is being added to an already
-        running scheduler.
+        Called by the scheduler when the scheduler is being started or when the executor is being
+        added to an already running scheduler.
 
-        :param apscheduler.schedulers.base.BaseScheduler scheduler: the scheduler that is starting this executor
+        :param apscheduler.schedulers.base.BaseScheduler scheduler: the scheduler that is starting
+            this executor
         :param str|unicode alias: alias of this executor as it was assigned to the scheduler
-        """
 
+        """
         self._scheduler = scheduler
         self._lock = scheduler._create_lock()
         self._logger = logging.getLogger('apscheduler.executors.%s' % alias)
@@ -45,7 +48,8 @@ class BaseExecutor(six.with_metaclass(ABCMeta, object)):
         """
         Shuts down this executor.
 
-        :param bool wait: ``True`` to wait until all submitted jobs have been executed
+        :param bool wait: ``True`` to wait until all submitted jobs
+            have been executed
         """
 
     def submit_job(self, job, run_times):
@@ -53,10 +57,12 @@ class BaseExecutor(six.with_metaclass(ABCMeta, object)):
         Submits job for execution.
 
         :param Job job: job to execute
-        :param list[datetime] run_times: list of datetimes specifying when the job should have been run
-        :raises MaxInstancesReachedError: if the maximum number of allowed instances for this job has been reached
-        """
+        :param list[datetime] run_times: list of datetimes specifying
+            when the job should have been run
+        :raises MaxInstancesReachedError: if the maximum number of
+            allowed instances for this job has been reached
 
+        """
         assert self._lock is not None, 'This executor has not been started yet'
         with self._lock:
             if self._instances[job.id] >= job.max_instances:
@@ -70,50 +76,71 @@ class BaseExecutor(six.with_metaclass(ABCMeta, object)):
         """Performs the actual task of scheduling `run_job` to be called."""
 
     def _run_job_success(self, job_id, events):
-        """Called by the executor with the list of generated events when `run_job` has been successfully called."""
+        """
+        Called by the executor with the list of generated events when :func:`run_job` has been
+        successfully called.
 
+        """
         with self._lock:
             self._instances[job_id] -= 1
+            if self._instances[job_id] == 0:
+                del self._instances[job_id]
 
         for event in events:
             self._scheduler._dispatch_event(event)
 
     def _run_job_error(self, job_id, exc, traceback=None):
-        """Called by the executor with the exception if there is an error calling `run_job`."""
-
+        """Called by the executor with the exception if there is an error  calling `run_job`."""
         with self._lock:
             self._instances[job_id] -= 1
+            if self._instances[job_id] == 0:
+                del self._instances[job_id]
 
         exc_info = (exc.__class__, exc, traceback)
         self._logger.error('Error running job %s', job_id, exc_info=exc_info)
 
 
 def run_job(job, jobstore_alias, run_times, logger_name):
-    """Called by executors to run the job. Returns a list of scheduler events to be dispatched by the scheduler."""
+    """
+    Called by executors to run the job. Returns a list of scheduler events to be dispatched by the
+    scheduler.
 
+    """
     events = []
     logger = logging.getLogger(logger_name)
     for run_time in run_times:
-        # See if the job missed its run time window, and handle possible misfires accordingly
+        # See if the job missed its run time window, and handle
+        # possible misfires accordingly
         if job.misfire_grace_time is not None:
             difference = datetime.now(utc) - run_time
             grace_time = timedelta(seconds=job.misfire_grace_time)
             if difference > grace_time:
-                events.append(JobExecutionEvent(EVENT_JOB_MISSED, job.id, jobstore_alias, run_time))
+                events.append(JobExecutionEvent(EVENT_JOB_MISSED, job.id, jobstore_alias,
+                                                run_time))
                 logger.warning('Run time of job "%s" was missed by %s', job, difference)
                 continue
 
         logger.info('Running job "%s" (scheduled at %s)', job, run_time)
         try:
             retval = job.func(*job.args, **job.kwargs)
-        except:
+        except BaseException:
             exc, tb = sys.exc_info()[1:]
             formatted_tb = ''.join(format_tb(tb))
-            events.append(JobExecutionEvent(EVENT_JOB_ERROR, job.id, jobstore_alias, run_time, exception=exc,
-                                            traceback=formatted_tb))
+            events.append(JobExecutionEvent(EVENT_JOB_ERROR, job.id, jobstore_alias, run_time,
+                                            exception=exc, traceback=formatted_tb))
             logger.exception('Job "%s" raised an exception', job)
+
+            # This is to prevent cyclic references that would lead to memory leaks
+            if six.PY2:
+                sys.exc_clear()
+                del tb
+            else:
+                import traceback
+                traceback.clear_frames(tb)
+                del tb
         else:
-            events.append(JobExecutionEvent(EVENT_JOB_EXECUTED, job.id, jobstore_alias, run_time, retval=retval))
+            events.append(JobExecutionEvent(EVENT_JOB_EXECUTED, job.id, jobstore_alias, run_time,
+                                            retval=retval))
             logger.info('Job "%s" executed successfully', job)
 
     return events

lib/apscheduler/executors/base_py3.py

@@ -0,0 +1,43 @@
+import logging
+import sys
+import traceback
+from datetime import datetime, timedelta
+from traceback import format_tb
+
+from pytz import utc
+
+from apscheduler.events import (
+    JobExecutionEvent, EVENT_JOB_MISSED, EVENT_JOB_ERROR, EVENT_JOB_EXECUTED)
+
+
+async def run_coroutine_job(job, jobstore_alias, run_times, logger_name):
+    """Coroutine version of run_job()."""
+    events = []
+    logger = logging.getLogger(logger_name)
+    for run_time in run_times:
+        # See if the job missed its run time window, and handle possible misfires accordingly
+        if job.misfire_grace_time is not None:
+            difference = datetime.now(utc) - run_time
+            grace_time = timedelta(seconds=job.misfire_grace_time)
+            if difference > grace_time:
+                events.append(JobExecutionEvent(EVENT_JOB_MISSED, job.id, jobstore_alias,
+                                                run_time))
+                logger.warning('Run time of job "%s" was missed by %s', job, difference)
+                continue
+
+        logger.info('Running job "%s" (scheduled at %s)', job, run_time)
+        try:
+            retval = await job.func(*job.args, **job.kwargs)
+        except BaseException:
+            exc, tb = sys.exc_info()[1:]
+            formatted_tb = ''.join(format_tb(tb))
+            events.append(JobExecutionEvent(EVENT_JOB_ERROR, job.id, jobstore_alias, run_time,
+                                            exception=exc, traceback=formatted_tb))
+            logger.exception('Job "%s" raised an exception', job)
+            traceback.clear_frames(tb)
+        else:
+            events.append(JobExecutionEvent(EVENT_JOB_EXECUTED, job.id, jobstore_alias, run_time,
+                                            retval=retval))
+            logger.info('Job "%s" executed successfully', job)
+
+    return events

lib/apscheduler/executors/debug.py

@@ -5,7 +5,8 @@ from apscheduler.executors.base import BaseExecutor, run_job
 
 class DebugExecutor(BaseExecutor):
     """
-    A special executor that executes the target callable directly instead of deferring it to a thread or process.
+    A special executor that executes the target callable directly instead of deferring it to a
+    thread or process.
 
     Plugin alias: ``debug``
     """
@@ -13,7 +14,7 @@ class DebugExecutor(BaseExecutor):
     def _do_submit_job(self, job, run_times):
         try:
             events = run_job(job, job._jobstore_alias, run_times, self._logger.name)
-        except:
+        except BaseException:
             self._run_job_error(job.id, *sys.exc_info()[1:])
         else:
             self._run_job_success(job.id, events)

lib/apscheduler/executors/gevent.py

@@ -1,4 +1,4 @@
-
+from __future__ import absolute_import
 import sys
 
 from apscheduler.executors.base import BaseExecutor, run_job
@@ -21,9 +21,10 @@ class GeventExecutor(BaseExecutor):
         def callback(greenlet):
             try:
                 events = greenlet.get()
-            except:
+            except BaseException:
                 self._run_job_error(job.id, *sys.exc_info()[1:])
             else:
                 self._run_job_success(job.id, events)
 
-        gevent.spawn(run_job, job, job._jobstore_alias, run_times, self._logger.name).link(callback)
+        gevent.spawn(run_job, job, job._jobstore_alias, run_times, self._logger.name).\
+            link(callback)

lib/apscheduler/executors/pool.py

@@ -3,6 +3,11 @@ import concurrent.futures
 
 from apscheduler.executors.base import BaseExecutor, run_job
 
+try:
+    from concurrent.futures.process import BrokenProcessPool
+except ImportError:
+    BrokenProcessPool = None
+
 
 class BasePoolExecutor(BaseExecutor):
     @abstractmethod
@@ -19,7 +24,13 @@ class BasePoolExecutor(BaseExecutor):
             else:
                 self._run_job_success(job.id, f.result())
 
-        f = self._pool.submit(run_job, job, job._jobstore_alias, run_times, self._logger.name)
+        try:
+            f = self._pool.submit(run_job, job, job._jobstore_alias, run_times, self._logger.name)
+        except BrokenProcessPool:
+            self._logger.warning('Process pool is broken; replacing pool with a fresh instance')
+            self._pool = self._pool.__class__(self._pool._max_workers)
+            f = self._pool.submit(run_job, job, job._jobstore_alias, run_times, self._logger.name)
+
         f.add_done_callback(callback)
 
     def shutdown(self, wait=True):
@@ -33,10 +44,13 @@ class ThreadPoolExecutor(BasePoolExecutor):
     Plugin alias: ``threadpool``
 
     :param max_workers: the maximum number of spawned threads.
+    :param pool_kwargs: dict of keyword arguments to pass to the underlying
+        ThreadPoolExecutor constructor
     """
 
-    def __init__(self, max_workers=10):
-        pool = concurrent.futures.ThreadPoolExecutor(int(max_workers))
+    def __init__(self, max_workers=10, pool_kwargs=None):
+        pool_kwargs = pool_kwargs or {}
+        pool = concurrent.futures.ThreadPoolExecutor(int(max_workers), **pool_kwargs)
         super(ThreadPoolExecutor, self).__init__(pool)
 
 
@@ -47,8 +61,11 @@ class ProcessPoolExecutor(BasePoolExecutor):
     Plugin alias: ``processpool``
 
     :param max_workers: the maximum number of spawned processes.
+    :param pool_kwargs: dict of keyword arguments to pass to the underlying
+        ProcessPoolExecutor constructor
     """
 
-    def __init__(self, max_workers=10):
-        pool = concurrent.futures.ProcessPoolExecutor(int(max_workers))
+    def __init__(self, max_workers=10, pool_kwargs=None):
+        pool_kwargs = pool_kwargs or {}
+        pool = concurrent.futures.ProcessPoolExecutor(int(max_workers), **pool_kwargs)
         super(ProcessPoolExecutor, self).__init__(pool)

lib/apscheduler/executors/tornado.py

@@ -0,0 +1,54 @@
+from __future__ import absolute_import
+
+import sys
+from concurrent.futures import ThreadPoolExecutor
+
+from tornado.gen import convert_yielded
+
+from apscheduler.executors.base import BaseExecutor, run_job
+
+try:
+    from apscheduler.executors.base_py3 import run_coroutine_job
+    from apscheduler.util import iscoroutinefunction_partial
+except ImportError:
+    def iscoroutinefunction_partial(func):
+        return False
+
+
+class TornadoExecutor(BaseExecutor):
+    """
+    Runs jobs either in a thread pool or directly on the I/O loop.
+
+    If the job function is a native coroutine function, it is scheduled to be run directly in the
+    I/O loop as soon as possible. All other functions are run in a thread pool.
+
+    Plugin alias: ``tornado``
+
+    :param int max_workers: maximum number of worker threads in the thread pool
+    """
+
+    def __init__(self, max_workers=10):
+        super(TornadoExecutor, self).__init__()
+        self.executor = ThreadPoolExecutor(max_workers)
+
+    def start(self, scheduler, alias):
+        super(TornadoExecutor, self).start(scheduler, alias)
+        self._ioloop = scheduler._ioloop
+
+    def _do_submit_job(self, job, run_times):
+        def callback(f):
+            try:
+                events = f.result()
+            except BaseException:
+                self._run_job_error(job.id, *sys.exc_info()[1:])
+            else:
+                self._run_job_success(job.id, events)
+
+        if iscoroutinefunction_partial(job.func):
+            f = run_coroutine_job(job, job._jobstore_alias, run_times, self._logger.name)
+        else:
+            f = self.executor.submit(run_job, job, job._jobstore_alias, run_times,
+                                     self._logger.name)
+
+        f = convert_yielded(f)
+        f.add_done_callback(callback)

lib/apscheduler/executors/twisted.py

@@ -1,4 +1,4 @@
-
+from __future__ import absolute_import
 
 from apscheduler.executors.base import BaseExecutor, run_job
 
@@ -21,5 +21,5 @@ class TwistedExecutor(BaseExecutor):
             else:
                 self._run_job_error(job.id, result.value, result.tb)
 
-        self._reactor.getThreadPool().callInThreadWithCallback(callback, run_job, job, job._jobstore_alias, run_times,
-                                                               self._logger.name)
+        self._reactor.getThreadPool().callInThreadWithCallback(
+            callback, run_job, job, job._jobstore_alias, run_times, self._logger.name)

lib/apscheduler/job.py

@@ -1,11 +1,17 @@
-from collections.abc import Iterable, Mapping
+from inspect import ismethod, isclass
 from uuid import uuid4
 
 import six
 
 from apscheduler.triggers.base import BaseTrigger
-from apscheduler.util import ref_to_obj, obj_to_ref, datetime_repr, repr_escape, get_callable_name, check_callable_args, \
-    convert_to_datetime
+from apscheduler.util import (
+    ref_to_obj, obj_to_ref, datetime_repr, repr_escape, get_callable_name, check_callable_args,
+    convert_to_datetime)
+
+try:
+    from collections.abc import Iterable, Mapping
+except ImportError:
+    from collections import Iterable, Mapping
 
 
 class Job(object):
@@ -21,13 +27,20 @@ class Job(object):
     :var bool coalesce: whether to only run the job once when several run times are due
     :var trigger: the trigger object that controls the schedule of this job
     :var str executor: the name of the executor that will run this job
-    :var int misfire_grace_time: the time (in seconds) how much this job's execution is allowed to be late
-    :var int max_instances: the maximum number of concurrently executing instances allowed for this job
+    :var int misfire_grace_time: the time (in seconds) how much this job's execution is allowed to
+        be late (``None`` means "allow the job to run no matter how late it is")
+    :var int max_instances: the maximum number of concurrently executing instances allowed for this
+        job
     :var datetime.datetime next_run_time: the next scheduled run time of this job
+
+    .. note::
+        The ``misfire_grace_time`` has some non-obvious effects on job execution. See the
+        :ref:`missed-job-executions` section in the documentation for an in-depth explanation.
     """
 
-    __slots__ = ('_scheduler', '_jobstore_alias', 'id', 'trigger', 'executor', 'func', 'func_ref', 'args', 'kwargs',
-                 'name', 'misfire_grace_time', 'coalesce', 'max_instances', 'next_run_time')
+    __slots__ = ('_scheduler', '_jobstore_alias', 'id', 'trigger', 'executor', 'func', 'func_ref',
+                 'args', 'kwargs', 'name', 'misfire_grace_time', 'coalesce', 'max_instances',
+                 'next_run_time', '__weakref__')
 
     def __init__(self, scheduler, id=None, **kwargs):
         super(Job, self).__init__()
@@ -38,53 +51,69 @@ class Job(object):
     def modify(self, **changes):
         """
         Makes the given changes to this job and saves it in the associated job store.
+
         Accepted keyword arguments are the same as the variables on this class.
 
         .. seealso:: :meth:`~apscheduler.schedulers.base.BaseScheduler.modify_job`
-        """
 
+        :return Job: this job instance
+
+        """
         self._scheduler.modify_job(self.id, self._jobstore_alias, **changes)
+        return self
 
     def reschedule(self, trigger, **trigger_args):
         """
         Shortcut for switching the trigger on this job.
 
         .. seealso:: :meth:`~apscheduler.schedulers.base.BaseScheduler.reschedule_job`
-        """
 
+        :return Job: this job instance
+
+        """
         self._scheduler.reschedule_job(self.id, self._jobstore_alias, trigger, **trigger_args)
+        return self
 
     def pause(self):
         """
         Temporarily suspend the execution of this job.
 
         .. seealso:: :meth:`~apscheduler.schedulers.base.BaseScheduler.pause_job`
-        """
 
+        :return Job: this job instance
+
+        """
         self._scheduler.pause_job(self.id, self._jobstore_alias)
+        return self
 
     def resume(self):
         """
         Resume the schedule of this job if previously paused.
 
         .. seealso:: :meth:`~apscheduler.schedulers.base.BaseScheduler.resume_job`
-        """
 
+        :return Job: this job instance
+
+        """
         self._scheduler.resume_job(self.id, self._jobstore_alias)
+        return self
 
     def remove(self):
         """
         Unschedules this job and removes it from its associated job store.
 
         .. seealso:: :meth:`~apscheduler.schedulers.base.BaseScheduler.remove_job`
-        """
 
+        """
         self._scheduler.remove_job(self.id, self._jobstore_alias)
 
     @property
     def pending(self):
-        """Returns ``True`` if the referenced job is still waiting to be added to its designated job store."""
+        """
+        Returns ``True`` if the referenced job is still waiting to be added to its designated job
+        store.
 
+        """
         return self._jobstore_alias is None
 
     #
@@ -97,8 +126,8 @@ class Job(object):
 
         :type now: datetime.datetime
         :rtype: list[datetime.datetime]
-        """
 
+        """
         run_times = []
         next_run_time = self.next_run_time
         while next_run_time and next_run_time <= now:
@@ -108,8 +137,11 @@ class Job(object):
         return run_times
 
     def _modify(self, **changes):
-        """Validates the changes to the Job and makes the modifications if and only if all of them validate."""
+        """
+        Validates the changes to the Job and makes the modifications if and only if all of them
+        validate.
 
+        """
         approved = {}
 
         if 'id' in changes:
@@ -125,7 +157,7 @@ class Job(object):
             args = changes.pop('args') if 'args' in changes else self.args
             kwargs = changes.pop('kwargs') if 'kwargs' in changes else self.kwargs
 
-            if isinstance(func, str):
+            if isinstance(func, six.string_types):
                 func_ref = func
                 func = ref_to_obj(func)
             elif callable(func):
@@ -177,7 +209,8 @@ class Job(object):
         if 'trigger' in changes:
             trigger = changes.pop('trigger')
             if not isinstance(trigger, BaseTrigger):
-                raise TypeError('Expected a trigger instance, got %s instead' % trigger.__class__.__name__)
+                raise TypeError('Expected a trigger instance, got %s instead' %
+                                trigger.__class__.__name__)
 
             approved['trigger'] = trigger
 
@@ -189,10 +222,12 @@ class Job(object):
 
         if 'next_run_time' in changes:
             value = changes.pop('next_run_time')
-            approved['next_run_time'] = convert_to_datetime(value, self._scheduler.timezone, 'next_run_time')
+            approved['next_run_time'] = convert_to_datetime(value, self._scheduler.timezone,
+                                                            'next_run_time')
 
         if changes:
-            raise AttributeError('The following are not modifiable attributes of Job: %s' % ', '.join(changes))
+            raise AttributeError('The following are not modifiable attributes of Job: %s' %
+                                 ', '.join(changes))
 
         for key, value in six.iteritems(approved):
             setattr(self, key, value)
@@ -200,9 +235,18 @@ class Job(object):
     def __getstate__(self):
         # Don't allow this Job to be serialized if the function reference could not be determined
         if not self.func_ref:
-            raise ValueError('This Job cannot be serialized since the reference to its callable (%r) could not be '
-                             'determined. Consider giving a textual reference (module:function name) instead.' %
-                             (self.func,))
+            raise ValueError(
+                'This Job cannot be serialized since the reference to its callable (%r) could not '
+                'be determined. Consider giving a textual reference (module:function name) '
+                'instead.' % (self.func,))
+
+        # Instance methods cannot survive serialization as-is, so store the "self" argument
+        # explicitly
+        func = self.func
+        if ismethod(func) and not isclass(func.__self__) and obj_to_ref(func) == self.func_ref:
+            args = (func.__self__,) + tuple(self.args)
+        else:
+            args = self.args
 
         return {
             'version': 1,
@@ -210,7 +254,7 @@ class Job(object):
             'func': self.func_ref,
             'trigger': self.trigger,
             'executor': self.executor,
-            'args': self.args,
+            'args': args,
             'kwargs': self.kwargs,
             'name': self.name,
             'misfire_grace_time': self.misfire_grace_time,
@@ -221,7 +265,8 @@ class Job(object):
 
     def __setstate__(self, state):
         if state.get('version', 1) > 1:
-            raise ValueError('Job has version %s, but only version 1 can be handled' % state['version'])
+            raise ValueError('Job has version %s, but only version 1 can be handled' %
+                             state['version'])
 
         self.id = state['id']
         self.func_ref = state['func']
@@ -245,8 +290,13 @@ class Job(object):
         return '<Job (id=%s name=%s)>' % (repr_escape(self.id), repr_escape(self.name))
 
     def __str__(self):
-        return '%s (trigger: %s, next run at: %s)' % (repr_escape(self.name), repr_escape(str(self.trigger)),
-                                                      datetime_repr(self.next_run_time))
+        return repr_escape(self.__unicode__())
 
     def __unicode__(self):
-        return six.u('%s (trigger: %s, next run at: %s)') % (self.name, self.trigger, datetime_repr(self.next_run_time))
+        if hasattr(self, 'next_run_time'):
+            status = ('next run at: ' + datetime_repr(self.next_run_time) if
+                      self.next_run_time else 'paused')
+        else:
+            status = 'pending'
+
+        return u'%s (trigger: %s, %s)' % (self.name, self.trigger, status)

lib/apscheduler/jobstores/base.py

@@ -8,23 +8,27 @@ class JobLookupError(KeyError):
     """Raised when the job store cannot find a job for update or removal."""
 
     def __init__(self, job_id):
-        super(JobLookupError, self).__init__(six.u('No job by the id of %s was found') % job_id)
+        super(JobLookupError, self).__init__(u'No job by the id of %s was found' % job_id)
 
 
 class ConflictingIdError(KeyError):
     """Raised when the uniqueness of job IDs is being violated."""
 
     def __init__(self, job_id):
-        super(ConflictingIdError, self).__init__(six.u('Job identifier (%s) conflicts with an existing job') % job_id)
+        super(ConflictingIdError, self).__init__(
+            u'Job identifier (%s) conflicts with an existing job' % job_id)
 
 
 class TransientJobError(ValueError):
-    """Raised when an attempt to add transient (with no func_ref) job to a persistent job store is detected."""
+    """
+    Raised when an attempt to add transient (with no func_ref) job to a persistent job store is
+    detected.
+    """
 
     def __init__(self, job_id):
         super(TransientJobError, self).__init__(
-            six.u('Job (%s) cannot be added to this job store because a reference to the callable could not be '
-                  'determined.') % job_id)
+            u'Job (%s) cannot be added to this job store because a reference to the callable '
+            u'could not be determined.' % job_id)
 
 
 class BaseJobStore(six.with_metaclass(ABCMeta)):
@@ -36,10 +40,11 @@ class BaseJobStore(six.with_metaclass(ABCMeta)):
 
     def start(self, scheduler, alias):
         """
-        Called by the scheduler when the scheduler is being started or when the job store is being added to an already
-        running scheduler.
+        Called by the scheduler when the scheduler is being started or when the job store is being
+        added to an already running scheduler.
 
-        :param apscheduler.schedulers.base.BaseScheduler scheduler: the scheduler that is starting this job store
+        :param apscheduler.schedulers.base.BaseScheduler scheduler: the scheduler that is starting
+            this job store
         :param str|unicode alias: alias of this job store as it was assigned to the scheduler
         """
 
@@ -50,13 +55,22 @@ class BaseJobStore(six.with_metaclass(ABCMeta)):
     def shutdown(self):
         """Frees any resources still bound to this job store."""
 
+    def _fix_paused_jobs_sorting(self, jobs):
+        for i, job in enumerate(jobs):
+            if job.next_run_time is not None:
+                if i > 0:
+                    paused_jobs = jobs[:i]
+                    del jobs[:i]
+                    jobs.extend(paused_jobs)
+                break
+
     @abstractmethod
     def lookup_job(self, job_id):
         """
         Returns a specific job, or ``None`` if it isn't found..
 
-        The job store is responsible for setting the ``scheduler`` and ``jobstore`` attributes of the returned job to
-        point to the scheduler and itself, respectively.
+        The job store is responsible for setting the ``scheduler`` and ``jobstore`` attributes of
+        the returned job to point to the scheduler and itself, respectively.
 
         :param str|unicode job_id: identifier of the job
         :rtype: Job
@@ -75,7 +89,8 @@ class BaseJobStore(six.with_metaclass(ABCMeta)):
     @abstractmethod
     def get_next_run_time(self):
         """
-        Returns the earliest run time of all the jobs stored in this job store, or ``None`` if there are no active jobs.
+        Returns the earliest run time of all the jobs stored in this job store, or ``None`` if
+        there are no active jobs.
 
         :rtype: datetime.datetime
         """
@@ -83,11 +98,12 @@ class BaseJobStore(six.with_metaclass(ABCMeta)):
     @abstractmethod
     def get_all_jobs(self):
         """
-        Returns a list of all jobs in this job store. The returned jobs should be sorted by next run time (ascending).
-        Paused jobs (next_run_time is None) should be sorted last.
+        Returns a list of all jobs in this job store.
+        The returned jobs should be sorted by next run time (ascending).
+        Paused jobs (next_run_time == None) should be sorted last.
 
-        The job store is responsible for setting the ``scheduler`` and ``jobstore`` attributes of the returned jobs to
-        point to the scheduler and itself, respectively.
+        The job store is responsible for setting the ``scheduler`` and ``jobstore`` attributes of
+        the returned jobs to point to the scheduler and itself, respectively.
 
         :rtype: list[Job]
         """

lib/apscheduler/jobstores/memory.py

@@ -1,4 +1,4 @@
-
+from __future__ import absolute_import
 
 from apscheduler.jobstores.base import BaseJobStore, JobLookupError, ConflictingIdError
 from apscheduler.util import datetime_to_utc_timestamp
@@ -13,7 +13,8 @@ class MemoryJobStore(BaseJobStore):
 
     def __init__(self):
         super(MemoryJobStore, self).__init__()
-        self._jobs = []  # list of (job, timestamp), sorted by next_run_time and job id (ascending)
+        # list of (job, timestamp), sorted by next_run_time and job id (ascending)
+        self._jobs = []
         self._jobs_index = {}  # id -> (job, timestamp) lookup table
 
     def lookup_job(self, job_id):
@@ -80,13 +81,13 @@ class MemoryJobStore(BaseJobStore):
 
     def _get_job_index(self, timestamp, job_id):
         """
-        Returns the index of the given job, or if it's not found, the index where the job should be inserted based on
-        the given timestamp.
+        Returns the index of the given job, or if it's not found, the index where the job should be
+        inserted based on the given timestamp.
 
         :type timestamp: int
         :type job_id: str
-        """
 
+        """
         lo, hi = 0, len(self._jobs)
         timestamp = float('inf') if timestamp is None else timestamp
         while lo < hi:

lib/apscheduler/jobstores/mongodb.py

@@ -1,11 +1,12 @@
-
+from __future__ import absolute_import
+import warnings
 
 from apscheduler.jobstores.base import BaseJobStore, JobLookupError, ConflictingIdError
 from apscheduler.util import maybe_ref, datetime_to_utc_timestamp, utc_timestamp_to_datetime
 from apscheduler.job import Job
 
 try:
-    import pickle as pickle
+    import cPickle as pickle
 except ImportError:  # pragma: nocover
     import pickle
 
@@ -19,16 +20,18 @@ except ImportError:  # pragma: nocover
 
 class MongoDBJobStore(BaseJobStore):
     """
-    Stores jobs in a MongoDB database. Any leftover keyword arguments are directly passed to pymongo's `MongoClient
+    Stores jobs in a MongoDB database. Any leftover keyword arguments are directly passed to
+    pymongo's `MongoClient
     <http://api.mongodb.org/python/current/api/pymongo/mongo_client.html#pymongo.mongo_client.MongoClient>`_.
 
     Plugin alias: ``mongodb``
 
     :param str database: database to store jobs in
     :param str collection: collection to store jobs in
-    :param client: a :class:`~pymongo.mongo_client.MongoClient` instance to use instead of providing connection
-                   arguments
-    :param int pickle_protocol: pickle protocol level to use (for serialization), defaults to the highest available
+    :param client: a :class:`~pymongo.mongo_client.MongoClient` instance to use instead of
+        providing connection arguments
+    :param int pickle_protocol: pickle protocol level to use (for serialization), defaults to the
+        highest available
     """
 
     def __init__(self, database='apscheduler', collection='jobs', client=None,
@@ -42,13 +45,22 @@ class MongoDBJobStore(BaseJobStore):
             raise ValueError('The "collection" parameter must not be empty')
 
         if client:
-            self.connection = maybe_ref(client)
+            self.client = maybe_ref(client)
         else:
             connect_args.setdefault('w', 1)
-            self.connection = MongoClient(**connect_args)
+            self.client = MongoClient(**connect_args)
 
-        self.collection = self.connection[database][collection]
-        self.collection.ensure_index('next_run_time', sparse=True)
+        self.collection = self.client[database][collection]
+
+    def start(self, scheduler, alias):
+        super(MongoDBJobStore, self).start(scheduler, alias)
+        self.collection.create_index('next_run_time', sparse=True)
+
+    @property
+    def connection(self):
+        warnings.warn('The "connection" member is deprecated -- use "client" instead',
+                      DeprecationWarning)
+        return self.client
 
     def lookup_job(self, job_id):
         document = self.collection.find_one(job_id, ['job_state'])
@@ -59,16 +71,19 @@ class MongoDBJobStore(BaseJobStore):
         return self._get_jobs({'next_run_time': {'$lte': timestamp}})
 
     def get_next_run_time(self):
-        document = self.collection.find_one({'next_run_time': {'$ne': None}}, fields=['next_run_time'],
+        document = self.collection.find_one({'next_run_time': {'$ne': None}},
+                                            projection=['next_run_time'],
                                             sort=[('next_run_time', ASCENDING)])
         return utc_timestamp_to_datetime(document['next_run_time']) if document else None
 
     def get_all_jobs(self):
-        return self._get_jobs({})
+        jobs = self._get_jobs({})
+        self._fix_paused_jobs_sorting(jobs)
+        return jobs
 
     def add_job(self, job):
         try:
-            self.collection.insert({
+            self.collection.insert_one({
                 '_id': job.id,
                 'next_run_time': datetime_to_utc_timestamp(job.next_run_time),
                 'job_state': Binary(pickle.dumps(job.__getstate__(), self.pickle_protocol))
@@ -81,20 +96,20 @@ class MongoDBJobStore(BaseJobStore):
             'next_run_time': datetime_to_utc_timestamp(job.next_run_time),
             'job_state': Binary(pickle.dumps(job.__getstate__(), self.pickle_protocol))
         }
-        result = self.collection.update({'_id': job.id}, {'$set': changes})
-        if result and result['n'] == 0:
-            raise JobLookupError(id)
+        result = self.collection.update_one({'_id': job.id}, {'$set': changes})
+        if result and result.matched_count == 0:
+            raise JobLookupError(job.id)
 
     def remove_job(self, job_id):
-        result = self.collection.remove(job_id)
-        if result and result['n'] == 0:
+        result = self.collection.delete_one({'_id': job_id})
+        if result and result.deleted_count == 0:
             raise JobLookupError(job_id)
 
     def remove_all_jobs(self):
-        self.collection.remove()
+        self.collection.delete_many({})
 
     def shutdown(self):
-        self.connection.disconnect()
+        self.client.close()
 
     def _reconstitute_job(self, job_state):
         job_state = pickle.loads(job_state)
@@ -107,18 +122,20 @@ class MongoDBJobStore(BaseJobStore):
     def _get_jobs(self, conditions):
         jobs = []
         failed_job_ids = []
-        for document in self.collection.find(conditions, ['_id', 'job_state'], sort=[('next_run_time', ASCENDING)]):
+        for document in self.collection.find(conditions, ['_id', 'job_state'],
+                                             sort=[('next_run_time', ASCENDING)]):
             try:
                 jobs.append(self._reconstitute_job(document['job_state']))
-            except:
-                self._logger.exception('Unable to restore job "%s" -- removing it', document['_id'])
+            except BaseException:
+                self._logger.exception('Unable to restore job "%s" -- removing it',
+                                       document['_id'])
                 failed_job_ids.append(document['_id'])
 
         # Remove all the jobs we failed to restore
         if failed_job_ids:
-            self.collection.remove({'_id': {'$in': failed_job_ids}})
+            self.collection.delete_many({'_id': {'$in': failed_job_ids}})
 
         return jobs
 
     def __repr__(self):
-        return '<%s (client=%s)>' % (self.__class__.__name__, self.connection)
+        return '<%s (client=%s)>' % (self.__class__.__name__, self.client)

lib/apscheduler/jobstores/redis.py

@@ -1,5 +1,7 @@
+from __future__ import absolute_import
+from datetime import datetime
 
-
+from pytz import utc
 import six
 
 from apscheduler.jobstores.base import BaseJobStore, JobLookupError, ConflictingIdError
@@ -7,26 +9,28 @@ from apscheduler.util import datetime_to_utc_timestamp, utc_timestamp_to_datetim
 from apscheduler.job import Job
 
 try:
-    import pickle as pickle
+    import cPickle as pickle
 except ImportError:  # pragma: nocover
     import pickle
 
 try:
-    from redis import StrictRedis
+    from redis import Redis
 except ImportError:  # pragma: nocover
     raise ImportError('RedisJobStore requires redis installed')
 
 
 class RedisJobStore(BaseJobStore):
     """
-    Stores jobs in a Redis database. Any leftover keyword arguments are directly passed to redis's StrictRedis.
+    Stores jobs in a Redis database. Any leftover keyword arguments are directly passed to redis's
+    :class:`~redis.StrictRedis`.
 
     Plugin alias: ``redis``
 
     :param int db: the database number to store jobs in
     :param str jobs_key: key to store jobs in
     :param str run_times_key: key to store the jobs' run times in
-    :param int pickle_protocol: pickle protocol level to use (for serialization), defaults to the highest available
+    :param int pickle_protocol: pickle protocol level to use (for serialization), defaults to the
+        highest available
     """
 
     def __init__(self, db=0, jobs_key='apscheduler.jobs', run_times_key='apscheduler.run_times',
@@ -43,7 +47,7 @@ class RedisJobStore(BaseJobStore):
         self.pickle_protocol = pickle_protocol
         self.jobs_key = jobs_key
         self.run_times_key = run_times_key
-        self.redis = StrictRedis(db=int(db), **connect_args)
+        self.redis = Redis(db=int(db), **connect_args)
 
     def lookup_job(self, job_id):
         job_state = self.redis.hget(self.jobs_key, job_id)
@@ -65,7 +69,8 @@ class RedisJobStore(BaseJobStore):
     def get_all_jobs(self):
         job_states = self.redis.hgetall(self.jobs_key)
         jobs = self._reconstitute_jobs(six.iteritems(job_states))
-        return sorted(jobs, key=lambda job: job.next_run_time)
+        paused_sort_key = datetime(9999, 12, 31, tzinfo=utc)
+        return sorted(jobs, key=lambda job: job.next_run_time or paused_sort_key)
 
     def add_job(self, job):
         if self.redis.hexists(self.jobs_key, job.id):
@@ -73,8 +78,12 @@ class RedisJobStore(BaseJobStore):
 
         with self.redis.pipeline() as pipe:
             pipe.multi()
-            pipe.hset(self.jobs_key, job.id, pickle.dumps(job.__getstate__(), self.pickle_protocol))
-            pipe.zadd(self.run_times_key, datetime_to_utc_timestamp(job.next_run_time), job.id)
+            pipe.hset(self.jobs_key, job.id, pickle.dumps(job.__getstate__(),
+                                                          self.pickle_protocol))
+            if job.next_run_time:
+                pipe.zadd(self.run_times_key,
+                          {job.id: datetime_to_utc_timestamp(job.next_run_time)})
+
             pipe.execute()
 
     def update_job(self, job):
@@ -82,11 +91,14 @@ class RedisJobStore(BaseJobStore):
             raise JobLookupError(job.id)
 
         with self.redis.pipeline() as pipe:
-            pipe.hset(self.jobs_key, job.id, pickle.dumps(job.__getstate__(), self.pickle_protocol))
+            pipe.hset(self.jobs_key, job.id, pickle.dumps(job.__getstate__(),
+                                                          self.pickle_protocol))
             if job.next_run_time:
-                pipe.zadd(self.run_times_key, datetime_to_utc_timestamp(job.next_run_time), job.id)
+                pipe.zadd(self.run_times_key,
+                          {job.id: datetime_to_utc_timestamp(job.next_run_time)})
             else:
                 pipe.zrem(self.run_times_key, job.id)
+
             pipe.execute()
 
     def remove_job(self, job_id):
@@ -121,7 +133,7 @@ class RedisJobStore(BaseJobStore):
         for job_id, job_state in job_states:
             try:
                 jobs.append(self._reconstitute_job(job_state))
-            except:
+            except BaseException:
                 self._logger.exception('Unable to restore job "%s" -- removing it', job_id)
                 failed_job_ids.append(job_id)
 

lib/apscheduler/jobstores/rethinkdb.py

@@ -0,0 +1,155 @@
+from __future__ import absolute_import
+
+from apscheduler.jobstores.base import BaseJobStore, JobLookupError, ConflictingIdError
+from apscheduler.util import maybe_ref, datetime_to_utc_timestamp, utc_timestamp_to_datetime
+from apscheduler.job import Job
+
+try:
+    import cPickle as pickle
+except ImportError:  # pragma: nocover
+    import pickle
+
+try:
+    from rethinkdb import RethinkDB
+except ImportError:  # pragma: nocover
+    raise ImportError('RethinkDBJobStore requires rethinkdb installed')
+
+
+class RethinkDBJobStore(BaseJobStore):
+    """
+    Stores jobs in a RethinkDB database. Any leftover keyword arguments are directly passed to
+    rethinkdb's `RethinkdbClient <http://www.rethinkdb.com/api/#connect>`_.
+
+    Plugin alias: ``rethinkdb``
+
+    :param str database: database to store jobs in
+    :param str collection: collection to store jobs in
+    :param client: a :class:`rethinkdb.net.Connection` instance to use instead of providing
+        connection arguments
+    :param int pickle_protocol: pickle protocol level to use (for serialization), defaults to the
+        highest available
+    """
+
+    def __init__(self, database='apscheduler', table='jobs', client=None,
+                 pickle_protocol=pickle.HIGHEST_PROTOCOL, **connect_args):
+        super(RethinkDBJobStore, self).__init__()
+
+        if not database:
+            raise ValueError('The "database" parameter must not be empty')
+        if not table:
+            raise ValueError('The "table" parameter must not be empty')
+
+        self.database = database
+        self.table_name = table
+        self.table = None
+        self.client = client
+        self.pickle_protocol = pickle_protocol
+        self.connect_args = connect_args
+        self.r = RethinkDB()
+        self.conn = None
+
+    def start(self, scheduler, alias):
+        super(RethinkDBJobStore, self).start(scheduler, alias)
+
+        if self.client:
+            self.conn = maybe_ref(self.client)
+        else:
+            self.conn = self.r.connect(db=self.database, **self.connect_args)
+
+        if self.database not in self.r.db_list().run(self.conn):
+            self.r.db_create(self.database).run(self.conn)
+
+        if self.table_name not in self.r.table_list().run(self.conn):
+            self.r.table_create(self.table_name).run(self.conn)
+
+        if 'next_run_time' not in self.r.table(self.table_name).index_list().run(self.conn):
+            self.r.table(self.table_name).index_create('next_run_time').run(self.conn)
+
+        self.table = self.r.db(self.database).table(self.table_name)
+
+    def lookup_job(self, job_id):
+        results = list(self.table.get_all(job_id).pluck('job_state').run(self.conn))
+        return self._reconstitute_job(results[0]['job_state']) if results else None
+
+    def get_due_jobs(self, now):
+        return self._get_jobs(self.r.row['next_run_time'] <= datetime_to_utc_timestamp(now))
+
+    def get_next_run_time(self):
+        results = list(
+            self.table
+            .filter(self.r.row['next_run_time'] != None)  # noqa
+            .order_by(self.r.asc('next_run_time'))
+            .map(lambda x: x['next_run_time'])
+            .limit(1)
+            .run(self.conn)
+        )
+        return utc_timestamp_to_datetime(results[0]) if results else None
+
+    def get_all_jobs(self):
+        jobs = self._get_jobs()
+        self._fix_paused_jobs_sorting(jobs)
+        return jobs
+
+    def add_job(self, job):
+        job_dict = {
+            'id': job.id,
+            'next_run_time': datetime_to_utc_timestamp(job.next_run_time),
+            'job_state': self.r.binary(pickle.dumps(job.__getstate__(), self.pickle_protocol))
+        }
+        results = self.table.insert(job_dict).run(self.conn)
+        if results['errors'] > 0:
+            raise ConflictingIdError(job.id)
+
+    def update_job(self, job):
+        changes = {
+            'next_run_time': datetime_to_utc_timestamp(job.next_run_time),
+            'job_state': self.r.binary(pickle.dumps(job.__getstate__(), self.pickle_protocol))
+        }
+        results = self.table.get_all(job.id).update(changes).run(self.conn)
+        skipped = False in map(lambda x: results[x] == 0, results.keys())
+        if results['skipped'] > 0 or results['errors'] > 0 or not skipped:
+            raise JobLookupError(job.id)
+
+    def remove_job(self, job_id):
+        results = self.table.get_all(job_id).delete().run(self.conn)
+        if results['deleted'] + results['skipped'] != 1:
+            raise JobLookupError(job_id)
+
+    def remove_all_jobs(self):
+        self.table.delete().run(self.conn)
+
+    def shutdown(self):
+        self.conn.close()
+
+    def _reconstitute_job(self, job_state):
+        job_state = pickle.loads(job_state)
+        job = Job.__new__(Job)
+        job.__setstate__(job_state)
+        job._scheduler = self._scheduler
+        job._jobstore_alias = self._alias
+        return job
+
+    def _get_jobs(self, predicate=None):
+        jobs = []
+        failed_job_ids = []
+        query = (self.table.filter(self.r.row['next_run_time'] != None).filter(predicate)  # noqa
+                 if predicate else self.table)
+        query = query.order_by('next_run_time', 'id').pluck('id', 'job_state')
+
+        for document in query.run(self.conn):
+            try:
+                jobs.append(self._reconstitute_job(document['job_state']))
+            except Exception:
+                self._logger.exception('Unable to restore job "%s" -- removing it', document['id'])
+                failed_job_ids.append(document['id'])
+
+        # Remove all the jobs we failed to restore
+        if failed_job_ids:
+            self.r.expr(failed_job_ids).for_each(
+                lambda job_id: self.table.get_all(job_id).delete()).run(self.conn)
+
+        return jobs
+
+    def __repr__(self):
+        connection = self.conn
+        return '<%s (connection=%s)>' % (self.__class__.__name__, connection)

lib/apscheduler/jobstores/sqlalchemy.py

@@ -1,38 +1,47 @@
-
+from __future__ import absolute_import
 
 from apscheduler.jobstores.base import BaseJobStore, JobLookupError, ConflictingIdError
 from apscheduler.util import maybe_ref, datetime_to_utc_timestamp, utc_timestamp_to_datetime
 from apscheduler.job import Job
 
 try:
-    import pickle as pickle
+    import cPickle as pickle
 except ImportError:  # pragma: nocover
     import pickle
 
 try:
-    from sqlalchemy import create_engine, Table, Column, MetaData, Unicode, Float, LargeBinary, select
+    from sqlalchemy import (
+        create_engine, Table, Column, MetaData, Unicode, Float, LargeBinary, select, and_)
     from sqlalchemy.exc import IntegrityError
+    from sqlalchemy.sql.expression import null
 except ImportError:  # pragma: nocover
     raise ImportError('SQLAlchemyJobStore requires SQLAlchemy installed')
 
 
 class SQLAlchemyJobStore(BaseJobStore):
     """
-    Stores jobs in a database table using SQLAlchemy. The table will be created if it doesn't exist in the database.
+    Stores jobs in a database table using SQLAlchemy.
+    The table will be created if it doesn't exist in the database.
 
     Plugin alias: ``sqlalchemy``
 
-    :param str url: connection string (see `SQLAlchemy documentation
-                    <http://docs.sqlalchemy.org/en/latest/core/engines.html?highlight=create_engine#database-urls>`_
-                    on this)
-    :param engine: an SQLAlchemy Engine to use instead of creating a new one based on ``url``
+    :param str url: connection string (see
+        :ref:`SQLAlchemy documentation <sqlalchemy:database_urls>` on this)
+    :param engine: an SQLAlchemy :class:`~sqlalchemy.engine.Engine` to use instead of creating a
+        new one based on ``url``
     :param str tablename: name of the table to store jobs in
-    :param metadata: a :class:`~sqlalchemy.MetaData` instance to use instead of creating a new one
-    :param int pickle_protocol: pickle protocol level to use (for serialization), defaults to the highest available
+    :param metadata: a :class:`~sqlalchemy.schema.MetaData` instance to use instead of creating a
+        new one
+    :param int pickle_protocol: pickle protocol level to use (for serialization), defaults to the
+        highest available
+    :param str tableschema: name of the (existing) schema in the target database where the table
+        should be
+    :param dict engine_options: keyword arguments to :func:`~sqlalchemy.create_engine`
+        (ignored if ``engine`` is given)
     """
 
     def __init__(self, url=None, engine=None, tablename='apscheduler_jobs', metadata=None,
-                 pickle_protocol=pickle.HIGHEST_PROTOCOL):
+                 pickle_protocol=pickle.HIGHEST_PROTOCOL, tableschema=None, engine_options=None):
         super(SQLAlchemyJobStore, self).__init__()
         self.pickle_protocol = pickle_protocol
         metadata = maybe_ref(metadata) or MetaData()
@@ -40,37 +49,46 @@ class SQLAlchemyJobStore(BaseJobStore):
         if engine:
             self.engine = maybe_ref(engine)
         elif url:
-            self.engine = create_engine(url)
+            self.engine = create_engine(url, **(engine_options or {}))
         else:
             raise ValueError('Need either "engine" or "url" defined')
 
-        # 191 = max key length in MySQL for InnoDB/utf8mb4 tables, 25 = precision that translates to an 8-byte float
+        # 191 = max key length in MySQL for InnoDB/utf8mb4 tables,
+        # 25 = precision that translates to an 8-byte float
         self.jobs_t = Table(
             tablename, metadata,
-            Column('id', Unicode(191, _warn_on_bytestring=False), primary_key=True),
+            Column('id', Unicode(191), primary_key=True),
             Column('next_run_time', Float(25), index=True),
-            Column('job_state', LargeBinary, nullable=False)
+            Column('job_state', LargeBinary, nullable=False),
+            schema=tableschema
         )
 
+    def start(self, scheduler, alias):
+        super(SQLAlchemyJobStore, self).start(scheduler, alias)
         self.jobs_t.create(self.engine, True)
 
     def lookup_job(self, job_id):
-        selectable = select([self.jobs_t.c.job_state]).where(self.jobs_t.c.id == job_id)
-        job_state = self.engine.execute(selectable).scalar()
-        return self._reconstitute_job(job_state) if job_state else None
+        selectable = select(self.jobs_t.c.job_state).where(self.jobs_t.c.id == job_id)
+        with self.engine.begin() as connection:
+            job_state = connection.execute(selectable).scalar()
+            return self._reconstitute_job(job_state) if job_state else None
 
     def get_due_jobs(self, now):
         timestamp = datetime_to_utc_timestamp(now)
         return self._get_jobs(self.jobs_t.c.next_run_time <= timestamp)
 
     def get_next_run_time(self):
-        selectable = select([self.jobs_t.c.next_run_time]).where(self.jobs_t.c.next_run_time != None).\
+        selectable = select(self.jobs_t.c.next_run_time).\
+            where(self.jobs_t.c.next_run_time != null()).\
             order_by(self.jobs_t.c.next_run_time).limit(1)
-        next_run_time = self.engine.execute(selectable).scalar()
-        return utc_timestamp_to_datetime(next_run_time)
+        with self.engine.begin() as connection:
+            next_run_time = connection.execute(selectable).scalar()
+            return utc_timestamp_to_datetime(next_run_time)
 
     def get_all_jobs(self):
-        return self._get_jobs()
+        jobs = self._get_jobs()
+        self._fix_paused_jobs_sorting(jobs)
+        return jobs
 
     def add_job(self, job):
         insert = self.jobs_t.insert().values(**{
@@ -78,29 +96,33 @@ class SQLAlchemyJobStore(BaseJobStore):
             'next_run_time': datetime_to_utc_timestamp(job.next_run_time),
             'job_state': pickle.dumps(job.__getstate__(), self.pickle_protocol)
         })
-        try:
-            self.engine.execute(insert)
-        except IntegrityError:
-            raise ConflictingIdError(job.id)
+        with self.engine.begin() as connection:
+            try:
+                connection.execute(insert)
+            except IntegrityError:
+                raise ConflictingIdError(job.id)
 
     def update_job(self, job):
         update = self.jobs_t.update().values(**{
             'next_run_time': datetime_to_utc_timestamp(job.next_run_time),
             'job_state': pickle.dumps(job.__getstate__(), self.pickle_protocol)
         }).where(self.jobs_t.c.id == job.id)
-        result = self.engine.execute(update)
-        if result.rowcount == 0:
-            raise JobLookupError(id)
+        with self.engine.begin() as connection:
+            result = connection.execute(update)
+            if result.rowcount == 0:
+                raise JobLookupError(job.id)
 
     def remove_job(self, job_id):
         delete = self.jobs_t.delete().where(self.jobs_t.c.id == job_id)
-        result = self.engine.execute(delete)
-        if result.rowcount == 0:
-            raise JobLookupError(job_id)
+        with self.engine.begin() as connection:
+            result = connection.execute(delete)
+            if result.rowcount == 0:
+                raise JobLookupError(job_id)
 
     def remove_all_jobs(self):
         delete = self.jobs_t.delete()
-        self.engine.execute(delete)
+        with self.engine.begin() as connection:
+            connection.execute(delete)
 
     def shutdown(self):
         self.engine.dispose()
@@ -116,20 +138,22 @@ class SQLAlchemyJobStore(BaseJobStore):
 
     def _get_jobs(self, *conditions):
         jobs = []
-        selectable = select([self.jobs_t.c.id, self.jobs_t.c.job_state]).order_by(self.jobs_t.c.next_run_time)
-        selectable = selectable.where(*conditions) if conditions else selectable
+        selectable = select(self.jobs_t.c.id, self.jobs_t.c.job_state).\
+            order_by(self.jobs_t.c.next_run_time)
+        selectable = selectable.where(and_(*conditions)) if conditions else selectable
         failed_job_ids = set()
-        for row in self.engine.execute(selectable):
-            try:
-                jobs.append(self._reconstitute_job(row.job_state))
-            except:
-                self._logger.exception('Unable to restore job "%s" -- removing it', row.id)
-                failed_job_ids.add(row.id)
+        with self.engine.begin() as connection:
+            for row in connection.execute(selectable):
+                try:
+                    jobs.append(self._reconstitute_job(row.job_state))
+                except BaseException:
+                    self._logger.exception('Unable to restore job "%s" -- removing it', row.id)
+                    failed_job_ids.add(row.id)
 
-        # Remove all the jobs we failed to restore
-        if failed_job_ids:
-            delete = self.jobs_t.delete().where(self.jobs_t.c.id.in_(failed_job_ids))
-            self.engine.execute(delete)
+            # Remove all the jobs we failed to restore
+            if failed_job_ids:
+                delete = self.jobs_t.delete().where(self.jobs_t.c.id.in_(failed_job_ids))
+                connection.execute(delete)
 
         return jobs
 

lib/apscheduler/jobstores/zookeeper.py

@@ -0,0 +1,178 @@
+from __future__ import absolute_import
+
+from datetime import datetime
+
+from pytz import utc
+from kazoo.exceptions import NoNodeError, NodeExistsError
+
+from apscheduler.jobstores.base import BaseJobStore, JobLookupError, ConflictingIdError
+from apscheduler.util import maybe_ref, datetime_to_utc_timestamp, utc_timestamp_to_datetime
+from apscheduler.job import Job
+
+try:
+    import cPickle as pickle
+except ImportError:  # pragma: nocover
+    import pickle
+
+try:
+    from kazoo.client import KazooClient
+except ImportError:  # pragma: nocover
+    raise ImportError('ZooKeeperJobStore requires Kazoo installed')
+
+
+class ZooKeeperJobStore(BaseJobStore):
+    """
+    Stores jobs in a ZooKeeper tree. Any leftover keyword arguments are directly passed to
+    kazoo's `KazooClient
+    <http://kazoo.readthedocs.io/en/latest/api/client.html>`_.
+
+    Plugin alias: ``zookeeper``
+
+    :param str path: path to store jobs in
+    :param client: a :class:`~kazoo.client.KazooClient` instance to use instead of
+        providing connection arguments
+    :param int pickle_protocol: pickle protocol level to use (for serialization), defaults to the
+        highest available
+    """
+
+    def __init__(self, path='/apscheduler', client=None, close_connection_on_exit=False,
+                 pickle_protocol=pickle.HIGHEST_PROTOCOL, **connect_args):
+        super(ZooKeeperJobStore, self).__init__()
+        self.pickle_protocol = pickle_protocol
+        self.close_connection_on_exit = close_connection_on_exit
+
+        if not path:
+            raise ValueError('The "path" parameter must not be empty')
+
+        self.path = path
+
+        if client:
+            self.client = maybe_ref(client)
+        else:
+            self.client = KazooClient(**connect_args)
+        self._ensured_path = False
+
+    def _ensure_paths(self):
+        if not self._ensured_path:
+            self.client.ensure_path(self.path)
+        self._ensured_path = True
+
+    def start(self, scheduler, alias):
+        super(ZooKeeperJobStore, self).start(scheduler, alias)
+        if not self.client.connected:
+            self.client.start()
+
+    def lookup_job(self, job_id):
+        self._ensure_paths()
+        node_path = self.path + "/" + str(job_id)
+        try:
+            content, _ = self.client.get(node_path)
+            doc = pickle.loads(content)
+            job = self._reconstitute_job(doc['job_state'])
+            return job
+        except BaseException:
+            return None
+
+    def get_due_jobs(self, now):
+        timestamp = datetime_to_utc_timestamp(now)
+        jobs = [job_def['job'] for job_def in self._get_jobs()
+                if job_def['next_run_time'] is not None and job_def['next_run_time'] <= timestamp]
+        return jobs
+
+    def get_next_run_time(self):
+        next_runs = [job_def['next_run_time'] for job_def in self._get_jobs()
+                     if job_def['next_run_time'] is not None]
+        return utc_timestamp_to_datetime(min(next_runs)) if len(next_runs) > 0 else None
+
+    def get_all_jobs(self):
+        jobs = [job_def['job'] for job_def in self._get_jobs()]
+        self._fix_paused_jobs_sorting(jobs)
+        return jobs
+
+    def add_job(self, job):
+        self._ensure_paths()
+        node_path = self.path + "/" + str(job.id)
+        value = {
+            'next_run_time': datetime_to_utc_timestamp(job.next_run_time),
+            'job_state': job.__getstate__()
+        }
+        data = pickle.dumps(value, self.pickle_protocol)
+        try:
+            self.client.create(node_path, value=data)
+        except NodeExistsError:
+            raise ConflictingIdError(job.id)
+
+    def update_job(self, job):
+        self._ensure_paths()
+        node_path = self.path + "/" + str(job.id)
+        changes = {
+            'next_run_time': datetime_to_utc_timestamp(job.next_run_time),
+            'job_state': job.__getstate__()
+        }
+        data = pickle.dumps(changes, self.pickle_protocol)
+        try:
+            self.client.set(node_path, value=data)
+        except NoNodeError:
+            raise JobLookupError(job.id)
+
+    def remove_job(self, job_id):
+        self._ensure_paths()
+        node_path = self.path + "/" + str(job_id)
+        try:
+            self.client.delete(node_path)
+        except NoNodeError:
+            raise JobLookupError(job_id)
+
+    def remove_all_jobs(self):
+        try:
+            self.client.delete(self.path, recursive=True)
+        except NoNodeError:
+            pass
+        self._ensured_path = False
+
+    def shutdown(self):
+        if self.close_connection_on_exit:
+            self.client.stop()
+            self.client.close()
+
+    def _reconstitute_job(self, job_state):
+        job_state = job_state
+        job = Job.__new__(Job)
+        job.__setstate__(job_state)
+        job._scheduler = self._scheduler
+        job._jobstore_alias = self._alias
+        return job
+
+    def _get_jobs(self):
+        self._ensure_paths()
+        jobs = []
+        failed_job_ids = []
+        all_ids = self.client.get_children(self.path)
+        for node_name in all_ids:
+            try:
+                node_path = self.path + "/" + node_name
+                content, _ = self.client.get(node_path)
+                doc = pickle.loads(content)
+                job_def = {
+                    'job_id': node_name,
+                    'next_run_time': doc['next_run_time'] if doc['next_run_time'] else None,
+                    'job_state': doc['job_state'],
+                    'job': self._reconstitute_job(doc['job_state']),
+                    'creation_time': _.ctime
+                }
+                jobs.append(job_def)
+            except BaseException:
+                self._logger.exception('Unable to restore job "%s" -- removing it' % node_name)
+                failed_job_ids.append(node_name)
+
+        # Remove all the jobs we failed to restore
+        if failed_job_ids:
+            for failed_id in failed_job_ids:
+                self.remove_job(failed_id)
+        paused_sort_key = datetime(9999, 12, 31, tzinfo=utc)
+        return sorted(jobs, key=lambda job_def: (job_def['job'].next_run_time or paused_sort_key,
+                                                 job_def['creation_time']))
+
+    def __repr__(self):
+        self._logger.exception('<%s (client=%s)>' % (self.__class__.__name__, self.client))
+        return '<%s (client=%s)>' % (self.__class__.__name__, self.client)

lib/apscheduler/schedulers/asyncio.py

@@ -1,22 +1,16 @@
-
-from functools import wraps
+from __future__ import absolute_import
+import asyncio
+from functools import wraps, partial
 
 from apscheduler.schedulers.base import BaseScheduler
 from apscheduler.util import maybe_ref
 
-try:
-    import asyncio
-except ImportError:  # pragma: nocover
-    try:
-        import trollius as asyncio
-    except ImportError:
-        raise ImportError('AsyncIOScheduler requires either Python 3.4 or the asyncio package installed')
-
 
 def run_in_event_loop(func):
     @wraps(func)
     def wrapper(self, *args, **kwargs):
-        self._eventloop.call_soon_threadsafe(func, self, *args, **kwargs)
+        wrapped = partial(func, self, *args, **kwargs)
+        self._eventloop.call_soon_threadsafe(wrapped)
     return wrapper
 
 
@@ -24,6 +18,8 @@ class AsyncIOScheduler(BaseScheduler):
     """
     A scheduler that runs on an asyncio (:pep:`3156`) event loop.
 
+    The default executor can run jobs based on native coroutines (``async def``).
+
     Extra options:
 
     ============== =============================================================
@@ -34,9 +30,11 @@ class AsyncIOScheduler(BaseScheduler):
     _eventloop = None
     _timeout = None
 
-    def start(self):
-        super(AsyncIOScheduler, self).start()
-        self.wakeup()
+    def start(self, paused=False):
+        if not self._eventloop:
+            self._eventloop = asyncio.get_event_loop()
+
+        super(AsyncIOScheduler, self).start(paused)
 
     @run_in_event_loop
     def shutdown(self, wait=True):
@@ -44,7 +42,7 @@ class AsyncIOScheduler(BaseScheduler):
         self._stop_timer()
 
     def _configure(self, config):
-        self._eventloop = maybe_ref(config.pop('event_loop', None)) or asyncio.get_event_loop()
+        self._eventloop = maybe_ref(config.pop('event_loop', None))
         super(AsyncIOScheduler, self)._configure(config)
 
     def _start_timer(self, wait_seconds):

lib/apscheduler/schedulers/background.py

@@ -1,3 +1,4 @@
+from __future__ import absolute_import
 
 from threading import Thread, Event
 
@@ -13,11 +14,12 @@ class BackgroundScheduler(BlockingScheduler):
 
     Extra options:
 
-    ========== ============================================================================================
-    ``daemon`` Set the ``daemon`` option in the background thread (defaults to ``True``,
-               see `the documentation <https://docs.python.org/3.4/library/threading.html#thread-objects>`_
+    ========== =============================================================================
+    ``daemon`` Set the ``daemon`` option in the background thread (defaults to ``True``, see
+               `the documentation
+               <https://docs.python.org/3.4/library/threading.html#thread-objects>`_
                for further details)
-    ========== ============================================================================================
+    ========== =============================================================================
     """
 
     _thread = None
@@ -26,14 +28,16 @@ class BackgroundScheduler(BlockingScheduler):
         self._daemon = asbool(config.pop('daemon', True))
         super(BackgroundScheduler, self)._configure(config)
 
-    def start(self):
-        BaseScheduler.start(self)
-        self._event = Event()
+    def start(self, *args, **kwargs):
+        if self._event is None or self._event.is_set():
+            self._event = Event()
+
+        BaseScheduler.start(self, *args, **kwargs)
         self._thread = Thread(target=self._main_loop, name='APScheduler')
         self._thread.daemon = self._daemon
         self._thread.start()
 
-    def shutdown(self, wait=True):
-        super(BackgroundScheduler, self).shutdown(wait)
+    def shutdown(self, *args, **kwargs):
+        super(BackgroundScheduler, self).shutdown(*args, **kwargs)
         self._thread.join()
         del self._thread

lib/apscheduler/schedulers/blocking.py

@@ -1,21 +1,23 @@
+from __future__ import absolute_import
 
 from threading import Event
 
-from apscheduler.schedulers.base import BaseScheduler
+from apscheduler.schedulers.base import BaseScheduler, STATE_STOPPED
+from apscheduler.util import TIMEOUT_MAX
 
 
 class BlockingScheduler(BaseScheduler):
     """
-    A scheduler that runs in the foreground (:meth:`~apscheduler.schedulers.base.BaseScheduler.start` will block).
+    A scheduler that runs in the foreground
+    (:meth:`~apscheduler.schedulers.base.BaseScheduler.start` will block).
     """
-
-    MAX_WAIT_TIME = 4294967  # Maximum value accepted by Event.wait() on Windows
-
     _event = None
 
-    def start(self):
-        super(BlockingScheduler, self).start()
-        self._event = Event()
+    def start(self, *args, **kwargs):
+        if self._event is None or self._event.is_set():
+            self._event = Event()
+
+        super(BlockingScheduler, self).start(*args, **kwargs)
         self._main_loop()
 
     def shutdown(self, wait=True):
@@ -23,10 +25,11 @@ class BlockingScheduler(BaseScheduler):
         self._event.set()
 
     def _main_loop(self):
-        while self.running:
-            wait_seconds = self._process_jobs()
-            self._event.wait(wait_seconds if wait_seconds is not None else self.MAX_WAIT_TIME)
+        wait_seconds = TIMEOUT_MAX
+        while self.state != STATE_STOPPED:
+            self._event.wait(wait_seconds)
             self._event.clear()
+            wait_seconds = self._process_jobs()
 
     def wakeup(self):
         self._event.set()

lib/apscheduler/schedulers/gevent.py

@@ -1,4 +1,4 @@
-
+from __future__ import absolute_import
 
 from apscheduler.schedulers.blocking import BlockingScheduler
 from apscheduler.schedulers.base import BaseScheduler
@@ -16,14 +16,14 @@ class GeventScheduler(BlockingScheduler):
 
     _greenlet = None
 
-    def start(self):
-        BaseScheduler.start(self)
+    def start(self, *args, **kwargs):
         self._event = Event()
+        BaseScheduler.start(self, *args, **kwargs)
         self._greenlet = gevent.spawn(self._main_loop)
         return self._greenlet
 
-    def shutdown(self, wait=True):
-        super(GeventScheduler, self).shutdown(wait)
+    def shutdown(self, *args, **kwargs):
+        super(GeventScheduler, self).shutdown(*args, **kwargs)
         self._greenlet.join()
         del self._greenlet
 

lib/apscheduler/schedulers/qt.py

@@ -1,17 +1,24 @@
-
+from __future__ import absolute_import
 
 from apscheduler.schedulers.base import BaseScheduler
 
 try:
     from PyQt5.QtCore import QObject, QTimer
-except ImportError:  # pragma: nocover
+except (ImportError, RuntimeError):  # pragma: nocover
     try:
         from PyQt4.QtCore import QObject, QTimer
     except ImportError:
         try:
-            from PySide.QtCore import QObject, QTimer  # flake8: noqa
+            from PySide6.QtCore import QObject, QTimer  # noqa
         except ImportError:
-            raise ImportError('QtScheduler requires either PyQt5, PyQt4 or PySide installed')
+            try:
+                from PySide2.QtCore import QObject, QTimer  # noqa
+            except ImportError:
+                try:
+                    from PySide.QtCore import QObject, QTimer  # noqa
+                except ImportError:
+                    raise ImportError('QtScheduler requires either PyQt5, PyQt4, PySide6, PySide2 '
+                                      'or PySide installed')
 
 
 class QtScheduler(BaseScheduler):
@@ -19,18 +26,15 @@ class QtScheduler(BaseScheduler):
 
     _timer = None
 
-    def start(self):
-        super(QtScheduler, self).start()
-        self.wakeup()
-
-    def shutdown(self, wait=True):
-        super(QtScheduler, self).shutdown(wait)
+    def shutdown(self, *args, **kwargs):
+        super(QtScheduler, self).shutdown(*args, **kwargs)
         self._stop_timer()
 
     def _start_timer(self, wait_seconds):
         self._stop_timer()
         if wait_seconds is not None:
-            self._timer = QTimer.singleShot(wait_seconds * 1000, self._process_jobs)
+            wait_time = min(int(wait_seconds * 1000), 2147483647)
+            self._timer = QTimer.singleShot(wait_time, self._process_jobs)
 
     def _stop_timer(self):
         if self._timer:

lib/apscheduler/schedulers/tornado.py

@@ -1,3 +1,4 @@
+from __future__ import absolute_import
 
 from datetime import timedelta
 from functools import wraps
@@ -22,6 +23,8 @@ class TornadoScheduler(BaseScheduler):
     """
     A scheduler that runs on a Tornado IOLoop.
 
+    The default executor can run jobs based on native coroutines (``async def``).
+
     =========== ===============================================================
     ``io_loop`` Tornado IOLoop instance to use (defaults to the global IO loop)
     =========== ===============================================================
@@ -30,10 +33,6 @@ class TornadoScheduler(BaseScheduler):
     _ioloop = None
     _timeout = None
 
-    def start(self):
-        super(TornadoScheduler, self).start()
-        self.wakeup()
-
     @run_in_ioloop
     def shutdown(self, wait=True):
         super(TornadoScheduler, self).shutdown(wait)
@@ -53,6 +52,10 @@ class TornadoScheduler(BaseScheduler):
             self._ioloop.remove_timeout(self._timeout)
             del self._timeout
 
+    def _create_default_executor(self):
+        from apscheduler.executors.tornado import TornadoExecutor
+        return TornadoExecutor()
+
     @run_in_ioloop
     def wakeup(self):
         self._stop_timer()

lib/apscheduler/schedulers/twisted.py

@@ -1,3 +1,4 @@
+from __future__ import absolute_import
 
 from functools import wraps
 
@@ -35,10 +36,6 @@ class TwistedScheduler(BaseScheduler):
         self._reactor = maybe_ref(config.pop('reactor', default_reactor))
         super(TwistedScheduler, self)._configure(config)
 
-    def start(self):
-        super(TwistedScheduler, self).start()
-        self.wakeup()
-
     @run_in_reactor
     def shutdown(self, wait=True):
         super(TwistedScheduler, self).shutdown(wait)

lib/apscheduler/triggers/base.py

@@ -1,4 +1,6 @@
 from abc import ABCMeta, abstractmethod
+from datetime import timedelta
+import random
 
 import six
 
@@ -6,11 +8,30 @@ import six
 class BaseTrigger(six.with_metaclass(ABCMeta)):
     """Abstract base class that defines the interface that every trigger must implement."""
 
+    __slots__ = ()
+
     @abstractmethod
     def get_next_fire_time(self, previous_fire_time, now):
         """
-        Returns the next datetime to fire on, If no such datetime can be calculated, returns ``None``.
+        Returns the next datetime to fire on, If no such datetime can be calculated, returns
+        ``None``.
 
         :param datetime.datetime previous_fire_time: the previous time the trigger was fired
         :param datetime.datetime now: current datetime
         """
+
+    def _apply_jitter(self, next_fire_time, jitter, now):
+        """
+        Randomize ``next_fire_time`` by adding a random value (the jitter).
+
+        :param datetime.datetime|None next_fire_time: next fire time without jitter applied. If
+            ``None``, returns ``None``.
+        :param int|None jitter: maximum number of seconds to add to ``next_fire_time``
+            (if ``None`` or ``0``, returns ``next_fire_time``)
+        :param datetime.datetime now: current datetime
+        :return datetime.datetime|None: next fire time with a jitter.
+        """
+        if next_fire_time is None or not jitter:
+            return next_fire_time
+
+        return next_fire_time + timedelta(seconds=random.uniform(0, jitter))

lib/apscheduler/triggers/combining.py

@@ -0,0 +1,95 @@
+from apscheduler.triggers.base import BaseTrigger
+from apscheduler.util import obj_to_ref, ref_to_obj
+
+
+class BaseCombiningTrigger(BaseTrigger):
+    __slots__ = ('triggers', 'jitter')
+
+    def __init__(self, triggers, jitter=None):
+        self.triggers = triggers
+        self.jitter = jitter
+
+    def __getstate__(self):
+        return {
+            'version': 1,
+            'triggers': [(obj_to_ref(trigger.__class__), trigger.__getstate__())
+                         for trigger in self.triggers],
+            'jitter': self.jitter
+        }
+
+    def __setstate__(self, state):
+        if state.get('version', 1) > 1:
+            raise ValueError(
+                'Got serialized data for version %s of %s, but only versions up to 1 can be '
+                'handled' % (state['version'], self.__class__.__name__))
+
+        self.jitter = state['jitter']
+        self.triggers = []
+        for clsref, state in state['triggers']:
+            cls = ref_to_obj(clsref)
+            trigger = cls.__new__(cls)
+            trigger.__setstate__(state)
+            self.triggers.append(trigger)
+
+    def __repr__(self):
+        return '<{}({}{})>'.format(self.__class__.__name__, self.triggers,
+                                   ', jitter={}'.format(self.jitter) if self.jitter else '')
+
+
+class AndTrigger(BaseCombiningTrigger):
+    """
+    Always returns the earliest next fire time that all the given triggers can agree on.
+    The trigger is considered to be finished when any of the given triggers has finished its
+    schedule.
+
+    Trigger alias: ``and``
+
+    :param list triggers: triggers to combine
+    :param int|None jitter: delay the job execution by ``jitter`` seconds at most
+    """
+
+    __slots__ = ()
+
+    def get_next_fire_time(self, previous_fire_time, now):
+        while True:
+            fire_times = [trigger.get_next_fire_time(previous_fire_time, now)
+                          for trigger in self.triggers]
+            if None in fire_times:
+                return None
+            elif min(fire_times) == max(fire_times):
+                return self._apply_jitter(fire_times[0], self.jitter, now)
+            else:
+                now = max(fire_times)
+
+    def __str__(self):
+        return 'and[{}]'.format(', '.join(str(trigger) for trigger in self.triggers))
+
+
+class OrTrigger(BaseCombiningTrigger):
+    """
+    Always returns the earliest next fire time produced by any of the given triggers.
+    The trigger is considered finished when all the given triggers have finished their schedules.
+
+    Trigger alias: ``or``
+
+    :param list triggers: triggers to combine
+    :param int|None jitter: delay the job execution by ``jitter`` seconds at most
+
+    .. note:: Triggers that depends on the previous fire time, such as the interval trigger, may
+        seem to behave strangely since they are always passed the previous fire time produced by
+        any of the given triggers.
+    """
+
+    __slots__ = ()
+
+    def get_next_fire_time(self, previous_fire_time, now):
+        fire_times = [trigger.get_next_fire_time(previous_fire_time, now)
+                      for trigger in self.triggers]
+        fire_times = [fire_time for fire_time in fire_times if fire_time is not None]
+        if fire_times:
+            return self._apply_jitter(min(fire_times), self.jitter, now)
+        else:
+            return None
+
+    def __str__(self):
+        return 'or[{}]'.format(', '.join(str(trigger) for trigger in self.triggers))

lib/apscheduler/triggers/cron/__init__.py

@@ -4,17 +4,20 @@ from tzlocal import get_localzone
 import six
 
 from apscheduler.triggers.base import BaseTrigger
-from apscheduler.triggers.cron.fields import BaseField, WeekField, DayOfMonthField, DayOfWeekField, DEFAULT_VALUES
-from apscheduler.util import datetime_ceil, convert_to_datetime, datetime_repr, astimezone
+from apscheduler.triggers.cron.fields import (
+    BaseField, MonthField, WeekField, DayOfMonthField, DayOfWeekField, DEFAULT_VALUES)
+from apscheduler.util import (
+    datetime_ceil, convert_to_datetime, datetime_repr, astimezone, localize, normalize)
 
 
 class CronTrigger(BaseTrigger):
     """
-    Triggers when current time matches all specified time constraints, similarly to how the UNIX cron scheduler works.
+    Triggers when current time matches all specified time constraints,
+    similarly to how the UNIX cron scheduler works.
 
     :param int|str year: 4-digit year
     :param int|str month: month (1-12)
-    :param int|str day: day of the (1-31)
+    :param int|str day: day of month (1-31)
     :param int|str week: ISO week (1-53)
     :param int|str day_of_week: number or name of weekday (0-6 or mon,tue,wed,thu,fri,sat,sun)
     :param int|str hour: hour (0-23)
@@ -22,8 +25,9 @@ class CronTrigger(BaseTrigger):
     :param int|str second: second (0-59)
     :param datetime|str start_date: earliest possible date/time to trigger on (inclusive)
     :param datetime|str end_date: latest possible date/time to trigger on (inclusive)
-    :param datetime.tzinfo|str timezone: time zone to use for the date/time calculations
-                                         (defaults to scheduler timezone)
+    :param datetime.tzinfo|str timezone: time zone to use for the date/time calculations (defaults
+        to scheduler timezone)
+    :param int|None jitter: delay the job execution by ``jitter`` seconds at most
 
     .. note:: The first weekday is always **monday**.
     """
@@ -31,7 +35,7 @@ class CronTrigger(BaseTrigger):
     FIELD_NAMES = ('year', 'month', 'day', 'week', 'day_of_week', 'hour', 'minute', 'second')
     FIELDS_MAP = {
         'year': BaseField,
-        'month': BaseField,
+        'month': MonthField,
         'week': WeekField,
         'day': DayOfMonthField,
         'day_of_week': DayOfWeekField,
@@ -40,15 +44,16 @@ class CronTrigger(BaseTrigger):
         'second': BaseField
     }
 
-    __slots__ = 'timezone', 'start_date', 'end_date', 'fields'
+    __slots__ = 'timezone', 'start_date', 'end_date', 'fields', 'jitter'
 
-    def __init__(self, year=None, month=None, day=None, week=None, day_of_week=None, hour=None, minute=None,
-                 second=None, start_date=None, end_date=None, timezone=None):
+    def __init__(self, year=None, month=None, day=None, week=None, day_of_week=None, hour=None,
+                 minute=None, second=None, start_date=None, end_date=None, timezone=None,
+                 jitter=None):
         if timezone:
             self.timezone = astimezone(timezone)
-        elif start_date and start_date.tzinfo:
+        elif isinstance(start_date, datetime) and start_date.tzinfo:
             self.timezone = start_date.tzinfo
-        elif end_date and end_date.tzinfo:
+        elif isinstance(end_date, datetime) and end_date.tzinfo:
             self.timezone = end_date.tzinfo
         else:
             self.timezone = get_localzone()
@@ -56,6 +61,8 @@ class CronTrigger(BaseTrigger):
         self.start_date = convert_to_datetime(start_date, self.timezone, 'start_date')
         self.end_date = convert_to_datetime(end_date, self.timezone, 'end_date')
 
+        self.jitter = jitter
+
         values = dict((key, value) for (key, value) in six.iteritems(locals())
                       if key in self.FIELD_NAMES and value is not None)
         self.fields = []
@@ -76,13 +83,35 @@ class CronTrigger(BaseTrigger):
             field = field_class(field_name, exprs, is_default)
             self.fields.append(field)
 
+    @classmethod
+    def from_crontab(cls, expr, timezone=None):
+        """
+        Create a :class:`~CronTrigger` from a standard crontab expression.
+
+        See https://en.wikipedia.org/wiki/Cron for more information on the format accepted here.
+
+        :param expr: minute, hour, day of month, month, day of week
+        :param datetime.tzinfo|str timezone: time zone to use for the date/time calculations (
+            defaults to scheduler timezone)
+        :return: a :class:`~CronTrigger` instance
+
+        """
+        values = expr.split()
+        if len(values) != 5:
+            raise ValueError('Wrong number of fields; got {}, expected 5'.format(len(values)))
+
+        return cls(minute=values[0], hour=values[1], day=values[2], month=values[3],
+                   day_of_week=values[4], timezone=timezone)
+
     def _increment_field_value(self, dateval, fieldnum):
         """
-        Increments the designated field and resets all less significant fields to their minimum values.
+        Increments the designated field and resets all less significant fields to their minimum
+        values.
 
         :type dateval: datetime
         :type fieldnum: int
-        :return: a tuple containing the new date, and the number of the field that was actually incremented
+        :return: a tuple containing the new date, and the number of the field that was actually
+            incremented
         :rtype: tuple
         """
 
@@ -115,7 +144,7 @@ class CronTrigger(BaseTrigger):
                     i += 1
 
         difference = datetime(**values) - dateval.replace(tzinfo=None)
-        return self.timezone.normalize(dateval + difference), fieldnum
+        return normalize(dateval + difference), fieldnum
 
     def _set_field_value(self, dateval, fieldnum, new_value):
         values = {}
@@ -128,12 +157,13 @@ class CronTrigger(BaseTrigger):
                 else:
                     values[field.name] = new_value
 
-        difference = datetime(**values) - dateval.replace(tzinfo=None)
-        return self.timezone.normalize(dateval + difference)
+        return localize(datetime(**values), self.timezone)
 
     def get_next_fire_time(self, previous_fire_time, now):
         if previous_fire_time:
-            start_date = max(now, previous_fire_time + timedelta(microseconds=1))
+            start_date = min(now, previous_fire_time + timedelta(microseconds=1))
+            if start_date == previous_fire_time:
+                start_date += timedelta(microseconds=1)
         else:
             start_date = max(now, self.start_date) if self.start_date else now
 
@@ -163,7 +193,34 @@ class CronTrigger(BaseTrigger):
                 return None
 
         if fieldnum >= 0:
-            return next_date
+            next_date = self._apply_jitter(next_date, self.jitter, now)
+            return min(next_date, self.end_date) if self.end_date else next_date
+
+    def __getstate__(self):
+        return {
+            'version': 2,
+            'timezone': self.timezone,
+            'start_date': self.start_date,
+            'end_date': self.end_date,
+            'fields': self.fields,
+            'jitter': self.jitter,
+        }
+
+    def __setstate__(self, state):
+        # This is for compatibility with APScheduler 3.0.x
+        if isinstance(state, tuple):
+            state = state[1]
+
+        if state.get('version', 1) > 2:
+            raise ValueError(
+                'Got serialized data for version %s of %s, but only versions up to 2 can be '
+                'handled' % (state['version'], self.__class__.__name__))
+
+        self.timezone = state['timezone']
+        self.start_date = state['start_date']
+        self.end_date = state['end_date']
+        self.fields = state['fields']
+        self.jitter = state.get('jitter')
 
     def __str__(self):
         options = ["%s='%s'" % (f.name, f) for f in self.fields if not f.is_default]
@@ -172,5 +229,11 @@ class CronTrigger(BaseTrigger):
     def __repr__(self):
         options = ["%s='%s'" % (f.name, f) for f in self.fields if not f.is_default]
         if self.start_date:
-            options.append("start_date='%s'" % datetime_repr(self.start_date))
-        return '<%s (%s)>' % (self.__class__.__name__, ', '.join(options))
+            options.append("start_date=%r" % datetime_repr(self.start_date))
+        if self.end_date:
+            options.append("end_date=%r" % datetime_repr(self.end_date))
+        if self.jitter:
+            options.append('jitter=%s' % self.jitter)
+
+        return "<%s (%s, timezone='%s')>" % (
+            self.__class__.__name__, ', '.join(options), self.timezone)

lib/apscheduler/triggers/cron/expressions.py

@@ -1,17 +1,16 @@
-"""
-This module contains the expressions applicable for CronTrigger's fields.
-"""
+"""This module contains the expressions applicable for CronTrigger's fields."""
 
 from calendar import monthrange
 import re
 
 from apscheduler.util import asint
 
-__all__ = ('AllExpression', 'RangeExpression', 'WeekdayRangeExpression', 'WeekdayPositionExpression',
-           'LastDayOfMonthExpression')
+__all__ = ('AllExpression', 'RangeExpression', 'WeekdayRangeExpression',
+           'WeekdayPositionExpression', 'LastDayOfMonthExpression')
 
 
 WEEKDAYS = ['mon', 'tue', 'wed', 'thu', 'fri', 'sat', 'sun']
+MONTHS = ['jan', 'feb', 'mar', 'apr', 'may', 'jun', 'jul', 'aug', 'sep', 'oct', 'nov', 'dec']
 
 
 class AllExpression(object):
@@ -22,6 +21,14 @@ class AllExpression(object):
         if self.step == 0:
             raise ValueError('Increment must be higher than 0')
 
+    def validate_range(self, field_name):
+        from apscheduler.triggers.cron.fields import MIN_VALUES, MAX_VALUES
+
+        value_range = MAX_VALUES[field_name] - MIN_VALUES[field_name]
+        if self.step and self.step > value_range:
+            raise ValueError('the step value ({}) is higher than the total range of the '
+                             'expression ({})'.format(self.step, value_range))
+
     def get_next_value(self, date, field):
         start = field.get_value(date)
         minval = field.get_min(date)
@@ -37,6 +44,9 @@ class AllExpression(object):
         if next <= maxval:
             return next
 
+    def __eq__(self, other):
+        return isinstance(other, self.__class__) and self.step == other.step
+
     def __str__(self):
         if self.step:
             return '*/%d' % self.step
@@ -51,7 +61,7 @@ class RangeExpression(AllExpression):
         r'(?P<first>\d+)(?:-(?P<last>\d+))?(?:/(?P<step>\d+))?$')
 
     def __init__(self, first, last=None, step=None):
-        AllExpression.__init__(self, step)
+        super(RangeExpression, self).__init__(step)
         first = asint(first)
         last = asint(last)
         if last is None and step is None:
@@ -61,25 +71,41 @@ class RangeExpression(AllExpression):
         self.first = first
         self.last = last
 
+    def validate_range(self, field_name):
+        from apscheduler.triggers.cron.fields import MIN_VALUES, MAX_VALUES
+
+        super(RangeExpression, self).validate_range(field_name)
+        if self.first < MIN_VALUES[field_name]:
+            raise ValueError('the first value ({}) is lower than the minimum value ({})'
+                             .format(self.first, MIN_VALUES[field_name]))
+        if self.last is not None and self.last > MAX_VALUES[field_name]:
+            raise ValueError('the last value ({}) is higher than the maximum value ({})'
+                             .format(self.last, MAX_VALUES[field_name]))
+        value_range = (self.last or MAX_VALUES[field_name]) - self.first
+        if self.step and self.step > value_range:
+            raise ValueError('the step value ({}) is higher than the total range of the '
+                             'expression ({})'.format(self.step, value_range))
+
     def get_next_value(self, date, field):
-        start = field.get_value(date)
+        startval = field.get_value(date)
         minval = field.get_min(date)
         maxval = field.get_max(date)
 
         # Apply range limits
         minval = max(minval, self.first)
-        if self.last is not None:
-            maxval = min(maxval, self.last)
-        start = max(start, minval)
+        maxval = min(maxval, self.last) if self.last is not None else maxval
+        nextval = max(minval, startval)
 
-        if not self.step:
-            next = start
-        else:
-            distance_to_next = (self.step - (start - minval)) % self.step
-            next = start + distance_to_next
+        # Apply the step if defined
+        if self.step:
+            distance_to_next = (self.step - (nextval - minval)) % self.step
+            nextval += distance_to_next
 
-        if next <= maxval:
-            return next
+        return nextval if nextval <= maxval else None
+
+    def __eq__(self, other):
+        return (isinstance(other, self.__class__) and self.first == other.first and
+                self.last == other.last)
 
     def __str__(self):
         if self.last != self.first and self.last is not None:
@@ -100,6 +126,37 @@ class RangeExpression(AllExpression):
         return "%s(%s)" % (self.__class__.__name__, ', '.join(args))
 
 
+class MonthRangeExpression(RangeExpression):
+    value_re = re.compile(r'(?P<first>[a-z]+)(?:-(?P<last>[a-z]+))?', re.IGNORECASE)
+
+    def __init__(self, first, last=None):
+        try:
+            first_num = MONTHS.index(first.lower()) + 1
+        except ValueError:
+            raise ValueError('Invalid month name "%s"' % first)
+
+        if last:
+            try:
+                last_num = MONTHS.index(last.lower()) + 1
+            except ValueError:
+                raise ValueError('Invalid month name "%s"' % last)
+        else:
+            last_num = None
+
+        super(MonthRangeExpression, self).__init__(first_num, last_num)
+
+    def __str__(self):
+        if self.last != self.first and self.last is not None:
+            return '%s-%s' % (MONTHS[self.first - 1], MONTHS[self.last - 1])
+        return MONTHS[self.first - 1]
+
+    def __repr__(self):
+        args = ["'%s'" % MONTHS[self.first]]
+        if self.last != self.first and self.last is not None:
+            args.append("'%s'" % MONTHS[self.last - 1])
+        return "%s(%s)" % (self.__class__.__name__, ', '.join(args))
+
+
 class WeekdayRangeExpression(RangeExpression):
     value_re = re.compile(r'(?P<first>[a-z]+)(?:-(?P<last>[a-z]+))?', re.IGNORECASE)
 
@@ -117,7 +174,7 @@ class WeekdayRangeExpression(RangeExpression):
         else:
             last_num = None
 
-        RangeExpression.__init__(self, first_num, last_num)
+        super(WeekdayRangeExpression, self).__init__(first_num, last_num)
 
     def __str__(self):
         if self.last != self.first and self.last is not None:
@@ -133,9 +190,11 @@ class WeekdayRangeExpression(RangeExpression):
 
 class WeekdayPositionExpression(AllExpression):
     options = ['1st', '2nd', '3rd', '4th', '5th', 'last']
-    value_re = re.compile(r'(?P<option_name>%s) +(?P<weekday_name>(?:\d+|\w+))' % '|'.join(options), re.IGNORECASE)
+    value_re = re.compile(r'(?P<option_name>%s) +(?P<weekday_name>(?:\d+|\w+))' %
+                          '|'.join(options), re.IGNORECASE)
 
     def __init__(self, option_name, weekday_name):
+        super(WeekdayPositionExpression, self).__init__(None)
         try:
             self.option_num = self.options.index(option_name.lower())
         except ValueError:
@@ -147,8 +206,7 @@ class WeekdayPositionExpression(AllExpression):
             raise ValueError('Invalid weekday name "%s"' % weekday_name)
 
     def get_next_value(self, date, field):
-        # Figure out the weekday of the month's first day and the number
-        # of days in that month
+        # Figure out the weekday of the month's first day and the number of days in that month
         first_day_wday, last_day = monthrange(date.year, date.month)
 
         # Calculate which day of the month is the first of the target weekdays
@@ -160,23 +218,28 @@ class WeekdayPositionExpression(AllExpression):
         if self.option_num < 5:
             target_day = first_hit_day + self.option_num * 7
         else:
-            target_day = first_hit_day + ((last_day - first_hit_day) / 7) * 7
+            target_day = first_hit_day + ((last_day - first_hit_day) // 7) * 7
 
         if target_day <= last_day and target_day >= date.day:
             return target_day
 
+    def __eq__(self, other):
+        return (super(WeekdayPositionExpression, self).__eq__(other) and
+                self.option_num == other.option_num and self.weekday == other.weekday)
+
     def __str__(self):
         return '%s %s' % (self.options[self.option_num], WEEKDAYS[self.weekday])
 
     def __repr__(self):
-        return "%s('%s', '%s')" % (self.__class__.__name__, self.options[self.option_num], WEEKDAYS[self.weekday])
+        return "%s('%s', '%s')" % (self.__class__.__name__, self.options[self.option_num],
+                                   WEEKDAYS[self.weekday])
 
 
 class LastDayOfMonthExpression(AllExpression):
     value_re = re.compile(r'last', re.IGNORECASE)
 
     def __init__(self):
-        pass
+        super(LastDayOfMonthExpression, self).__init__(None)
 
     def get_next_value(self, date, field):
         return monthrange(date.year, date.month)[1]

lib/apscheduler/triggers/cron/fields.py

@@ -1,22 +1,26 @@
-"""
-Fields represent CronTrigger options which map to :class:`~datetime.datetime`
-fields.
-"""
+"""Fields represent CronTrigger options which map to :class:`~datetime.datetime` fields."""
 
 from calendar import monthrange
+import re
+
+import six
 
 from apscheduler.triggers.cron.expressions import (
-    AllExpression, RangeExpression, WeekdayPositionExpression, LastDayOfMonthExpression, WeekdayRangeExpression)
+    AllExpression, RangeExpression, WeekdayPositionExpression, LastDayOfMonthExpression,
+    WeekdayRangeExpression, MonthRangeExpression)
 
 
-__all__ = ('MIN_VALUES', 'MAX_VALUES', 'DEFAULT_VALUES', 'BaseField', 'WeekField', 'DayOfMonthField', 'DayOfWeekField')
+__all__ = ('MIN_VALUES', 'MAX_VALUES', 'DEFAULT_VALUES', 'BaseField', 'WeekField',
+           'DayOfMonthField', 'DayOfWeekField')
 
 
-MIN_VALUES = {'year': 1970, 'month': 1, 'day': 1, 'week': 1, 'day_of_week': 0, 'hour': 0, 'minute': 0, 'second': 0}
-MAX_VALUES = {'year': 2 ** 63, 'month': 12, 'day:': 31, 'week': 53, 'day_of_week': 6, 'hour': 23, 'minute': 59,
-              'second': 59}
-DEFAULT_VALUES = {'year': '*', 'month': 1, 'day': 1, 'week': '*', 'day_of_week': '*', 'hour': 0, 'minute': 0,
-                  'second': 0}
+MIN_VALUES = {'year': 1970, 'month': 1, 'day': 1, 'week': 1, 'day_of_week': 0, 'hour': 0,
+              'minute': 0, 'second': 0}
+MAX_VALUES = {'year': 9999, 'month': 12, 'day': 31, 'week': 53, 'day_of_week': 6, 'hour': 23,
+              'minute': 59, 'second': 59}
+DEFAULT_VALUES = {'year': '*', 'month': 1, 'day': 1, 'week': '*', 'day_of_week': '*', 'hour': 0,
+                  'minute': 0, 'second': 0}
+SEPARATOR = re.compile(' *, *')
 
 
 class BaseField(object):
@@ -50,23 +54,29 @@ class BaseField(object):
         self.expressions = []
 
         # Split a comma-separated expression list, if any
-        exprs = str(exprs).strip()
-        if ',' in exprs:
-            for expr in exprs.split(','):
-                self.compile_expression(expr)
-        else:
-            self.compile_expression(exprs)
+        for expr in SEPARATOR.split(str(exprs).strip()):
+            self.compile_expression(expr)
 
     def compile_expression(self, expr):
         for compiler in self.COMPILERS:
             match = compiler.value_re.match(expr)
             if match:
                 compiled_expr = compiler(**match.groupdict())
+
+                try:
+                    compiled_expr.validate_range(self.name)
+                except ValueError as e:
+                    exc = ValueError('Error validating expression {!r}: {}'.format(expr, e))
+                    six.raise_from(exc, None)
+
                 self.expressions.append(compiled_expr)
                 return
 
         raise ValueError('Unrecognized expression "%s" for field "%s"' % (expr, self.name))
 
+    def __eq__(self, other):
+        return isinstance(self, self.__class__) and self.expressions == other.expressions
+
     def __str__(self):
         expr_strings = (str(e) for e in self.expressions)
         return ','.join(expr_strings)
@@ -95,3 +105,7 @@ class DayOfWeekField(BaseField):
 
     def get_value(self, dateval):
         return dateval.weekday()
+
+
+class MonthField(BaseField):
+    COMPILERS = BaseField.COMPILERS + [MonthRangeExpression]

lib/apscheduler/triggers/date.py

@@ -14,15 +14,36 @@ class DateTrigger(BaseTrigger):
     :param datetime.tzinfo|str timezone: time zone for ``run_date`` if it doesn't have one already
     """
 
-    __slots__ = 'timezone', 'run_date'
+    __slots__ = 'run_date'
 
     def __init__(self, run_date=None, timezone=None):
         timezone = astimezone(timezone) or get_localzone()
-        self.run_date = convert_to_datetime(run_date or datetime.now(), timezone, 'run_date')
+        if run_date is not None:
+            self.run_date = convert_to_datetime(run_date, timezone, 'run_date')
+        else:
+            self.run_date = datetime.now(timezone)
 
     def get_next_fire_time(self, previous_fire_time, now):
         return self.run_date if previous_fire_time is None else None
 
+    def __getstate__(self):
+        return {
+            'version': 1,
+            'run_date': self.run_date
+        }
+
+    def __setstate__(self, state):
+        # This is for compatibility with APScheduler 3.0.x
+        if isinstance(state, tuple):
+            state = state[1]
+
+        if state.get('version', 1) > 1:
+            raise ValueError(
+                'Got serialized data for version %s of %s, but only version 1 can be handled' %
+                (state['version'], self.__class__.__name__))
+
+        self.run_date = state['run_date']
+
     def __str__(self):
         return 'date[%s]' % datetime_repr(self.run_date)
 

lib/apscheduler/triggers/interval.py

@@ -4,13 +4,15 @@ from math import ceil
 from tzlocal import get_localzone
 
 from apscheduler.triggers.base import BaseTrigger
-from apscheduler.util import convert_to_datetime, timedelta_seconds, datetime_repr, astimezone
+from apscheduler.util import (
+    convert_to_datetime, normalize, timedelta_seconds, datetime_repr,
+    astimezone)
 
 
 class IntervalTrigger(BaseTrigger):
     """
-    Triggers on specified intervals, starting on ``start_date`` if specified, ``datetime.now()`` + interval
-    otherwise.
+    Triggers on specified intervals, starting on ``start_date`` if specified, ``datetime.now()`` +
+    interval otherwise.
 
     :param int weeks: number of weeks to wait
     :param int days: number of days to wait
@@ -20,12 +22,15 @@ class IntervalTrigger(BaseTrigger):
     :param datetime|str start_date: starting point for the interval calculation
     :param datetime|str end_date: latest possible date/time to trigger on
     :param datetime.tzinfo|str timezone: time zone to use for the date/time calculations
+    :param int|None jitter: delay the job execution by ``jitter`` seconds at most
     """
 
-    __slots__ = 'timezone', 'start_date', 'end_date', 'interval'
+    __slots__ = 'timezone', 'start_date', 'end_date', 'interval', 'interval_length', 'jitter'
 
-    def __init__(self, weeks=0, days=0, hours=0, minutes=0, seconds=0, start_date=None, end_date=None, timezone=None):
-        self.interval = timedelta(weeks=weeks, days=days, hours=hours, minutes=minutes, seconds=seconds)
+    def __init__(self, weeks=0, days=0, hours=0, minutes=0, seconds=0, start_date=None,
+                 end_date=None, timezone=None, jitter=None):
+        self.interval = timedelta(weeks=weeks, days=days, hours=hours, minutes=minutes,
+                                  seconds=seconds)
         self.interval_length = timedelta_seconds(self.interval)
         if self.interval_length == 0:
             self.interval = timedelta(seconds=1)
@@ -33,9 +38,9 @@ class IntervalTrigger(BaseTrigger):
 
         if timezone:
             self.timezone = astimezone(timezone)
-        elif start_date and start_date.tzinfo:
+        elif isinstance(start_date, datetime) and start_date.tzinfo:
             self.timezone = start_date.tzinfo
-        elif end_date and end_date.tzinfo:
+        elif isinstance(end_date, datetime) and end_date.tzinfo:
             self.timezone = end_date.tzinfo
         else:
             self.timezone = get_localzone()
@@ -44,6 +49,8 @@ class IntervalTrigger(BaseTrigger):
         self.start_date = convert_to_datetime(start_date, self.timezone, 'start_date')
         self.end_date = convert_to_datetime(end_date, self.timezone, 'end_date')
 
+        self.jitter = jitter
+
     def get_next_fire_time(self, previous_fire_time, now):
         if previous_fire_time:
             next_fire_time = previous_fire_time + self.interval
@@ -54,12 +61,48 @@ class IntervalTrigger(BaseTrigger):
             next_interval_num = int(ceil(timediff_seconds / self.interval_length))
             next_fire_time = self.start_date + self.interval * next_interval_num
 
+        if self.jitter is not None:
+            next_fire_time = self._apply_jitter(next_fire_time, self.jitter, now)
+
         if not self.end_date or next_fire_time <= self.end_date:
-            return self.timezone.normalize(next_fire_time)
+            return normalize(next_fire_time)
+
+    def __getstate__(self):
+        return {
+            'version': 2,
+            'timezone': self.timezone,
+            'start_date': self.start_date,
+            'end_date': self.end_date,
+            'interval': self.interval,
+            'jitter': self.jitter,
+        }
+
+    def __setstate__(self, state):
+        # This is for compatibility with APScheduler 3.0.x
+        if isinstance(state, tuple):
+            state = state[1]
+
+        if state.get('version', 1) > 2:
+            raise ValueError(
+                'Got serialized data for version %s of %s, but only versions up to 2 can be '
+                'handled' % (state['version'], self.__class__.__name__))
+
+        self.timezone = state['timezone']
+        self.start_date = state['start_date']
+        self.end_date = state['end_date']
+        self.interval = state['interval']
+        self.interval_length = timedelta_seconds(self.interval)
+        self.jitter = state.get('jitter')
 
     def __str__(self):
         return 'interval[%s]' % str(self.interval)
 
     def __repr__(self):
-        return "<%s (interval=%r, start_date='%s')>" % (self.__class__.__name__, self.interval,
-                                                        datetime_repr(self.start_date))
+        options = ['interval=%r' % self.interval, 'start_date=%r' % datetime_repr(self.start_date)]
+        if self.end_date:
+            options.append("end_date=%r" % datetime_repr(self.end_date))
+        if self.jitter:
+            options.append('jitter=%s' % self.jitter)
+
+        return "<%s (%s, timezone='%s')>" % (
+            self.__class__.__name__, ', '.join(options), self.timezone)

lib/apscheduler/util.py

@@ -1,29 +1,36 @@
 """This module contains several handy functions primarily meant for internal use."""
 
+from __future__ import division
 
+from asyncio import iscoroutinefunction
 from datetime import date, datetime, time, timedelta, tzinfo
-from inspect import isfunction, ismethod, getargspec
 from calendar import timegm
+from functools import partial
+from inspect import isclass, ismethod
 import re
+import sys
 
-from pytz import timezone, utc
+from pytz import timezone, utc, FixedOffset
 import six
 
 try:
     from inspect import signature
 except ImportError:  # pragma: nocover
-    try:
-        from funcsigs import signature
-    except ImportError:
-        signature = None
+    from funcsigs import signature
+
+try:
+    from threading import TIMEOUT_MAX
+except ImportError:
+    TIMEOUT_MAX = 4294967  # Maximum value accepted by Event.wait() on Windows
 
 __all__ = ('asint', 'asbool', 'astimezone', 'convert_to_datetime', 'datetime_to_utc_timestamp',
-           'utc_timestamp_to_datetime', 'timedelta_seconds', 'datetime_ceil', 'get_callable_name', 'obj_to_ref',
-           'ref_to_obj', 'maybe_ref', 'repr_escape', 'check_callable_args')
+           'utc_timestamp_to_datetime', 'timedelta_seconds', 'datetime_ceil', 'get_callable_name',
+           'obj_to_ref', 'ref_to_obj', 'maybe_ref', 'repr_escape', 'check_callable_args',
+           'normalize', 'localize', 'TIMEOUT_MAX')
 
 
 class _Undefined(object):
-    def __bool__(self):
+    def __nonzero__(self):
         return False
 
     def __bool__(self):
@@ -32,17 +39,18 @@ class _Undefined(object):
     def __repr__(self):
         return '<undefined>'
 
+
 undefined = _Undefined()  #: a unique object that only signifies that no value is defined
 
 
 def asint(text):
     """
-    Safely converts a string to an integer, returning None if the string is None.
+    Safely converts a string to an integer, returning ``None`` if the string is ``None``.
 
     :type text: str
     :rtype: int
-    """
 
+    """
     if text is not None:
         return int(text)
 
@@ -52,8 +60,8 @@ def asbool(obj):
     Interprets an object as a boolean value.
 
     :rtype: bool
-    """
 
+    """
     if isinstance(obj, str):
         obj = obj.strip().lower()
         if obj in ('true', 'yes', 'on', 'y', 't', '1'):
@@ -69,15 +77,17 @@ def astimezone(obj):
     Interprets an object as a timezone.
 
     :rtype: tzinfo
-    """
 
+    """
     if isinstance(obj, six.string_types):
         return timezone(obj)
     if isinstance(obj, tzinfo):
-        if not hasattr(obj, 'localize') or not hasattr(obj, 'normalize'):
-            raise TypeError('Only timezones from the pytz library are supported')
-        if obj.zone == 'local':
-            raise ValueError('Unable to determine the name of the local timezone -- use an explicit timezone instead')
+        if obj.tzname(None) == 'local':
+            raise ValueError(
+                'Unable to determine the name of the local timezone -- you must explicitly '
+                'specify the name of the local timezone. Please refrain from using timezones like '
+                'EST to prevent problems with daylight saving time. Instead, use a locale based '
+                'timezone name (such as Europe/Helsinki).')
         return obj
     if obj is not None:
         raise TypeError('Expected tzinfo, got %s instead' % obj.__class__.__name__)
@@ -85,27 +95,30 @@ def astimezone(obj):
 
 _DATE_REGEX = re.compile(
     r'(?P<year>\d{4})-(?P<month>\d{1,2})-(?P<day>\d{1,2})'
-    r'(?: (?P<hour>\d{1,2}):(?P<minute>\d{1,2}):(?P<second>\d{1,2})'
-    r'(?:\.(?P<microsecond>\d{1,6}))?)?')
+    r'(?:[ T](?P<hour>\d{1,2}):(?P<minute>\d{1,2}):(?P<second>\d{1,2})'
+    r'(?:\.(?P<microsecond>\d{1,6}))?'
+    r'(?P<timezone>Z|[+-]\d\d:\d\d)?)?$')
 
 
 def convert_to_datetime(input, tz, arg_name):
     """
     Converts the given object to a timezone aware datetime object.
+
     If a timezone aware datetime object is passed, it is returned unmodified.
     If a native datetime object is passed, it is given the specified timezone.
     If the input is a string, it is parsed as a datetime with the given timezone.
 
-    Date strings are accepted in three different forms: date only (Y-m-d),
-    date with time (Y-m-d H:M:S) or with date+time with microseconds
-    (Y-m-d H:M:S.micro).
+    Date strings are accepted in three different forms: date only (Y-m-d), date with time
+    (Y-m-d H:M:S) or with date+time with microseconds (Y-m-d H:M:S.micro). Additionally you can
+    override the time zone by giving a specific offset in the format specified by ISO 8601:
+    Z (UTC), +HH:MM or -HH:MM.
 
     :param str|datetime input: the datetime or string to convert to a timezone aware datetime
     :param datetime.tzinfo tz: timezone to interpret ``input`` in
     :param str arg_name: the name of the argument (used in an error message)
     :rtype: datetime
-    """
 
+    """
     if input is None:
         return
     elif isinstance(input, datetime):
@@ -116,8 +129,17 @@ def convert_to_datetime(input, tz, arg_name):
         m = _DATE_REGEX.match(input)
         if not m:
             raise ValueError('Invalid date string')
-        values = [(k, int(v or 0)) for k, v in list(m.groupdict().items())]
-        values = dict(values)
+
+        values = m.groupdict()
+        tzname = values.pop('timezone')
+        if tzname == 'Z':
+            tz = utc
+        elif tzname:
+            hours, minutes = (int(x) for x in tzname[1:].split(':'))
+            sign = 1 if tzname[0] == '+' else -1
+            tz = FixedOffset(sign * (hours * 60 + minutes))
+
+        values = {k: int(v or 0) for k, v in values.items()}
         datetime_ = datetime(**values)
     else:
         raise TypeError('Unsupported type for %s: %s' % (arg_name, input.__class__.__name__))
@@ -125,14 +147,12 @@ def convert_to_datetime(input, tz, arg_name):
     if datetime_.tzinfo is not None:
         return datetime_
     if tz is None:
-        raise ValueError('The "tz" argument must be specified if %s has no timezone information' % arg_name)
+        raise ValueError(
+            'The "tz" argument must be specified if %s has no timezone information' % arg_name)
     if isinstance(tz, six.string_types):
         tz = timezone(tz)
 
-    try:
-        return tz.localize(datetime_, is_dst=None)
-    except AttributeError:
-        raise TypeError('Only pytz timezones are supported (need the localize() and normalize() methods)')
+    return localize(datetime_, tz)
 
 
 def datetime_to_utc_timestamp(timeval):
@@ -141,8 +161,8 @@ def datetime_to_utc_timestamp(timeval):
 
     :type timeval: datetime
     :rtype: float
-    """
 
+    """
     if timeval is not None:
         return timegm(timeval.utctimetuple()) + timeval.microsecond / 1000000
 
@@ -153,8 +173,8 @@ def utc_timestamp_to_datetime(timestamp):
 
     :type timestamp: float
     :rtype: datetime
-    """
 
+    """
     if timestamp is not None:
         return datetime.fromtimestamp(timestamp, utc)
 
@@ -165,8 +185,8 @@ def timedelta_seconds(delta):
 
     :type delta: timedelta
     :rtype: float
-    """
 
+    """
     return delta.days * 24 * 60 * 60 + delta.seconds + \
         delta.microseconds / 1000000.0
 
@@ -176,8 +196,8 @@ def datetime_ceil(dateval):
     Rounds the given datetime object upwards.
 
     :type dateval: datetime
-    """
 
+    """
     if dateval.microsecond > 0:
         return dateval + timedelta(seconds=1, microseconds=-dateval.microsecond)
     return dateval
@@ -192,8 +212,8 @@ def get_callable_name(func):
     Returns the best available display name for the given function/callable.
 
     :rtype: str
-    """
 
+    """
     # the easy case (on Python 3.3+)
     if hasattr(func, '__qualname__'):
         return func.__qualname__
@@ -201,7 +221,7 @@ def get_callable_name(func):
     # class methods, bound and unbound methods
     f_self = getattr(func, '__self__', None) or getattr(func, 'im_self', None)
     if f_self and hasattr(func, '__name__'):
-        f_class = f_self if isinstance(f_self, type) else f_self.__class__
+        f_class = f_self if isclass(f_self) else f_self.__class__
     else:
         f_class = getattr(func, 'im_class', None)
 
@@ -222,20 +242,35 @@ def get_callable_name(func):
 
 def obj_to_ref(obj):
     """
-    Returns the path to the given object.
+    Returns the path to the given callable.
 
     :rtype: str
+    :raises TypeError: if the given object is not callable
+    :raises ValueError: if the given object is a :class:`~functools.partial`, lambda or a nested
+        function
+
     """
+    if isinstance(obj, partial):
+        raise ValueError('Cannot create a reference to a partial()')
 
-    try:
-        ref = '%s:%s' % (obj.__module__, get_callable_name(obj))
-        obj2 = ref_to_obj(ref)
-        if obj != obj2:
-            raise ValueError
-    except Exception:
-        raise ValueError('Cannot determine the reference to %r' % obj)
+    name = get_callable_name(obj)
+    if '<lambda>' in name:
+        raise ValueError('Cannot create a reference to a lambda')
+    if '<locals>' in name:
+        raise ValueError('Cannot create a reference to a nested function')
 
-    return ref
+    if ismethod(obj):
+        if hasattr(obj, 'im_self') and obj.im_self:
+            # bound method
+            module = obj.im_self.__module__
+        elif hasattr(obj, 'im_class') and obj.im_class:
+            # unbound method
+            module = obj.im_class.__module__
+        else:
+            module = obj.__module__
+    else:
+        module = obj.__module__
+    return '%s:%s' % (module, name)
 
 
 def ref_to_obj(ref):
@@ -243,8 +278,8 @@ def ref_to_obj(ref):
     Returns the object pointed to by ``ref``.
 
     :type ref: str
-    """
 
+    """
     if not isinstance(ref, six.string_types):
         raise TypeError('References must be strings')
     if ':' not in ref:
@@ -252,12 +287,12 @@ def ref_to_obj(ref):
 
     modulename, rest = ref.split(':', 1)
     try:
-        obj = __import__(modulename)
+        obj = __import__(modulename, fromlist=[rest])
     except ImportError:
         raise LookupError('Error resolving reference %s: could not import module' % ref)
 
     try:
-        for name in modulename.split('.')[1:] + rest.split('.'):
+        for name in rest.split('.'):
             obj = getattr(obj, name)
         return obj
     except Exception:
@@ -268,8 +303,8 @@ def maybe_ref(ref):
     """
     Returns the object that the given reference points to, if it is indeed a reference.
     If it is not a reference, the object is returned as-is.
-    """
 
+    """
     if not isinstance(ref, str):
         return ref
     return ref_to_obj(ref)
@@ -281,7 +316,8 @@ if six.PY2:
             return string.encode('ascii', 'backslashreplace')
         return string
 else:
-    repr_escape = lambda string: string
+    def repr_escape(string):
+        return string
 
 
 def check_callable_args(func, args, kwargs):
@@ -290,70 +326,54 @@ def check_callable_args(func, args, kwargs):
 
     :type args: tuple
     :type kwargs: dict
-    """
 
+    """
     pos_kwargs_conflicts = []  # parameters that have a match in both args and kwargs
     positional_only_kwargs = []  # positional-only parameters that have a match in kwargs
     unsatisfied_args = []  # parameters in signature that don't have a match in args or kwargs
     unsatisfied_kwargs = []  # keyword-only arguments that don't have a match in kwargs
     unmatched_args = list(args)  # args that didn't match any of the parameters in the signature
-    unmatched_kwargs = list(kwargs)  # kwargs that didn't match any of the parameters in the signature
-    has_varargs = has_var_kwargs = False  # indicates if the signature defines *args and **kwargs respectively
+    # kwargs that didn't match any of the parameters in the signature
+    unmatched_kwargs = list(kwargs)
+    # indicates if the signature defines *args and **kwargs respectively
+    has_varargs = has_var_kwargs = False
 
-    if signature:
-        try:
+    try:
+        if sys.version_info >= (3, 5):
+            sig = signature(func, follow_wrapped=False)
+        else:
             sig = signature(func)
-        except ValueError:
-            return  # signature() doesn't work against every kind of callable
+    except ValueError:
+        # signature() doesn't work against every kind of callable
+        return
 
-        for param in six.itervalues(sig.parameters):
-            if param.kind == param.POSITIONAL_OR_KEYWORD:
-                if param.name in unmatched_kwargs and unmatched_args:
-                    pos_kwargs_conflicts.append(param.name)
-                elif unmatched_args:
-                    del unmatched_args[0]
-                elif param.name in unmatched_kwargs:
-                    unmatched_kwargs.remove(param.name)
-                elif param.default is param.empty:
-                    unsatisfied_args.append(param.name)
-            elif param.kind == param.POSITIONAL_ONLY:
-                if unmatched_args:
-                    del unmatched_args[0]
-                elif param.name in unmatched_kwargs:
-                    unmatched_kwargs.remove(param.name)
-                    positional_only_kwargs.append(param.name)
-                elif param.default is param.empty:
-                    unsatisfied_args.append(param.name)
-            elif param.kind == param.KEYWORD_ONLY:
-                if param.name in unmatched_kwargs:
-                    unmatched_kwargs.remove(param.name)
-                elif param.default is param.empty:
-                    unsatisfied_kwargs.append(param.name)
-            elif param.kind == param.VAR_POSITIONAL:
-                has_varargs = True
-            elif param.kind == param.VAR_KEYWORD:
-                has_var_kwargs = True
-    else:
-        if not isfunction(func) and not ismethod(func) and hasattr(func, '__call__'):
-            func = func.__call__
-
-        try:
-            argspec = getargspec(func)
-        except TypeError:
-            return  # getargspec() doesn't work certain callables
-
-        argspec_args = argspec.args if not ismethod(func) else argspec.args[1:]
-        has_varargs = bool(argspec.varargs)
-        has_var_kwargs = bool(argspec.keywords)
-        for arg, default in six.moves.zip_longest(argspec_args, argspec.defaults or (), fillvalue=undefined):
-            if arg in unmatched_kwargs and unmatched_args:
-                pos_kwargs_conflicts.append(arg)
+    for param in six.itervalues(sig.parameters):
+        if param.kind == param.POSITIONAL_OR_KEYWORD:
+            if param.name in unmatched_kwargs and unmatched_args:
+                pos_kwargs_conflicts.append(param.name)
             elif unmatched_args:
                 del unmatched_args[0]
-            elif arg in unmatched_kwargs:
-                unmatched_kwargs.remove(arg)
-            elif default is undefined:
-                unsatisfied_args.append(arg)
+            elif param.name in unmatched_kwargs:
+                unmatched_kwargs.remove(param.name)
+            elif param.default is param.empty:
+                unsatisfied_args.append(param.name)
+        elif param.kind == param.POSITIONAL_ONLY:
+            if unmatched_args:
+                del unmatched_args[0]
+            elif param.name in unmatched_kwargs:
+                unmatched_kwargs.remove(param.name)
+                positional_only_kwargs.append(param.name)
+            elif param.default is param.empty:
+                unsatisfied_args.append(param.name)
+        elif param.kind == param.KEYWORD_ONLY:
+            if param.name in unmatched_kwargs:
+                unmatched_kwargs.remove(param.name)
+            elif param.default is param.empty:
+                unsatisfied_kwargs.append(param.name)
+        elif param.kind == param.VAR_POSITIONAL:
+            has_varargs = True
+        elif param.kind == param.VAR_KEYWORD:
+            has_var_kwargs = True
 
     # Make sure there are no conflicts between args and kwargs
     if pos_kwargs_conflicts:
@@ -365,21 +385,46 @@ def check_callable_args(func, args, kwargs):
         raise ValueError('The following arguments cannot be given as keyword arguments: %s' %
                          ', '.join(positional_only_kwargs))
 
-    # Check that the number of positional arguments minus the number of matched kwargs matches the argspec
+    # Check that the number of positional arguments minus the number of matched kwargs matches the
+    # argspec
     if unsatisfied_args:
-        raise ValueError('The following arguments have not been supplied: %s' % ', '.join(unsatisfied_args))
+        raise ValueError('The following arguments have not been supplied: %s' %
+                         ', '.join(unsatisfied_args))
 
     # Check that all keyword-only arguments have been supplied
     if unsatisfied_kwargs:
-        raise ValueError('The following keyword-only arguments have not been supplied in kwargs: %s' %
-                         ', '.join(unsatisfied_kwargs))
+        raise ValueError(
+            'The following keyword-only arguments have not been supplied in kwargs: %s' %
+            ', '.join(unsatisfied_kwargs))
 
     # Check that the callable can accept the given number of positional arguments
     if not has_varargs and unmatched_args:
-        raise ValueError('The list of positional arguments is longer than the target callable can handle '
-                         '(allowed: %d, given in args: %d)' % (len(args) - len(unmatched_args), len(args)))
+        raise ValueError(
+            'The list of positional arguments is longer than the target callable can handle '
+            '(allowed: %d, given in args: %d)' % (len(args) - len(unmatched_args), len(args)))
 
     # Check that the callable can accept the given keyword arguments
     if not has_var_kwargs and unmatched_kwargs:
-        raise ValueError('The target callable does not accept the following keyword arguments: %s' %
-                         ', '.join(unmatched_kwargs))
+        raise ValueError(
+            'The target callable does not accept the following keyword arguments: %s' %
+            ', '.join(unmatched_kwargs))
+
+
+def iscoroutinefunction_partial(f):
+    while isinstance(f, partial):
+        f = f.func
+
+    # The asyncio version of iscoroutinefunction includes testing for @coroutine
+    # decorations vs. the inspect version which does not.
+    return iscoroutinefunction(f)
+
+
+def normalize(dt):
+    return datetime.fromtimestamp(dt.timestamp(), dt.tzinfo)
+
+
+def localize(dt, tzinfo):
+    if hasattr(tzinfo, 'localize'):
+        return tzinfo.localize(dt)
+
+    return normalize(dt.replace(tzinfo=tzinfo))

lib/biplist/__init__.py

@@ -1,803 +0,0 @@
-"""biplist -- a library for reading and writing binary property list files.
-
-Binary Property List (plist) files provide a faster and smaller serialization
-format for property lists on OS X. This is a library for generating binary
-plists which can be read by OS X, iOS, or other clients.
-
-The API models the plistlib API, and will call through to plistlib when
-XML serialization or deserialization is required.
-
-To generate plists with UID values, wrap the values with the Uid object. The
-value must be an int.
-
-To generate plists with NSData/CFData values, wrap the values with the
-Data object. The value must be a string.
-
-Date values can only be datetime.datetime objects.
-
-The exceptions InvalidPlistException and NotBinaryPlistException may be 
-thrown to indicate that the data cannot be serialized or deserialized as
-a binary plist.
-
-Plist generation example:
-    
-    from biplist import *
-    from datetime import datetime
-    plist = {'aKey':'aValue',
-             '0':1.322,
-             'now':datetime.now(),
-             'list':[1,2,3],
-             'tuple':('a','b','c')
-             }
-    try:
-        writePlist(plist, "example.plist")
-    except (InvalidPlistException, NotBinaryPlistException), e:
-        print "Something bad happened:", e
-
-Plist parsing example:
-
-    from biplist import *
-    try:
-        plist = readPlist("example.plist")
-        print plist
-    except (InvalidPlistException, NotBinaryPlistException), e:
-        print "Not a plist:", e
-"""
-
-import sys
-from collections import namedtuple
-import datetime
-import io
-import math
-import plistlib
-from struct import pack, unpack
-from struct import error as struct_error
-import sys
-import time
-
-try:
-    str
-    unicodeEmpty = r''
-except NameError:
-    str = str
-    unicodeEmpty = ''
-try:
-    int
-except NameError:
-    long = int
-try:
-    {}.iteritems
-    iteritems = lambda x: iter(x.items())
-except AttributeError:
-    iteritems = lambda x: list(x.items())
-
-__all__ = [
-    'Uid', 'Data', 'readPlist', 'writePlist', 'readPlistFromString',
-    'writePlistToString', 'InvalidPlistException', 'NotBinaryPlistException'
-]
-
-# Apple uses Jan 1, 2001 as a base for all plist date/times.
-apple_reference_date = datetime.datetime.utcfromtimestamp(978307200)
-
-class Uid(int):
-    """Wrapper around integers for representing UID values. This
-       is used in keyed archiving."""
-    def __repr__(self):
-        return "Uid(%d)" % self
-
-class Data(bytes):
-    """Wrapper around str types for representing Data values."""
-    pass
-
-class InvalidPlistException(Exception):
-    """Raised when the plist is incorrectly formatted."""
-    pass
-
-class NotBinaryPlistException(Exception):
-    """Raised when a binary plist was expected but not encountered."""
-    pass
-
-def readPlist(pathOrFile):
-    """Raises NotBinaryPlistException, InvalidPlistException"""
-    didOpen = False
-    result = None
-    if isinstance(pathOrFile, (bytes, str)):
-        pathOrFile = open(pathOrFile, 'rb')
-        didOpen = True
-    try:
-        reader = PlistReader(pathOrFile)
-        result = reader.parse()
-    except NotBinaryPlistException as e:
-        try:
-            pathOrFile.seek(0)
-            result = None
-            if hasattr(plistlib, 'loads'):
-                contents = None
-                if isinstance(pathOrFile, (bytes, str)):
-                    with open(pathOrFile, 'rb') as f:
-                        contents = f.read()
-                else:
-                    contents = pathOrFile.read()
-                result = plistlib.loads(contents)
-            else:
-                result = plistlib.readPlist(pathOrFile)
-            result = wrapDataObject(result, for_binary=True)
-        except Exception as e:
-            raise InvalidPlistException(e)
-    finally:
-        if didOpen:
-            pathOrFile.close()
-    return result
-
-def wrapDataObject(o, for_binary=False):
-    if isinstance(o, Data) and not for_binary:
-        v = sys.version_info
-        if not (v[0] >= 3 and v[1] >= 4):
-            o = plistlib.Data(o)
-    elif isinstance(o, (bytes, plistlib.Data)) and for_binary:
-        if hasattr(o, 'data'):
-            o = Data(o.data)
-    elif isinstance(o, tuple):
-        o = wrapDataObject(list(o), for_binary)
-        o = tuple(o)
-    elif isinstance(o, list):
-        for i in range(len(o)):
-            o[i] = wrapDataObject(o[i], for_binary)
-    elif isinstance(o, dict):
-        for k in o:
-            o[k] = wrapDataObject(o[k], for_binary)
-    return o
-
-def writePlist(rootObject, pathOrFile, binary=True):
-    if not binary:
-        rootObject = wrapDataObject(rootObject, binary)
-        if hasattr(plistlib, "dump"):
-            if isinstance(pathOrFile, (bytes, str)):
-                with open(pathOrFile, 'wb') as f:
-                    return plistlib.dump(rootObject, f)
-            else:
-                return plistlib.dump(rootObject, pathOrFile)
-        else:
-            return plistlib.writePlist(rootObject, pathOrFile)
-    else:
-        didOpen = False
-        if isinstance(pathOrFile, (bytes, str)):
-            pathOrFile = open(pathOrFile, 'wb')
-            didOpen = True
-        writer = PlistWriter(pathOrFile)
-        result = writer.writeRoot(rootObject)
-        if didOpen:
-            pathOrFile.close()
-        return result
-
-def readPlistFromString(data):
-    return readPlist(io.BytesIO(data))
-
-def writePlistToString(rootObject, binary=True):
-    if not binary:
-        rootObject = wrapDataObject(rootObject, binary)
-        if hasattr(plistlib, "dumps"):
-            return plistlib.dumps(rootObject)
-        elif hasattr(plistlib, "writePlistToBytes"):
-            return plistlib.writePlistToBytes(rootObject)
-        else:
-            return plistlib.writePlistToString(rootObject)
-    else:
-        ioObject = io.BytesIO()
-        writer = PlistWriter(ioObject)
-        writer.writeRoot(rootObject)
-        return ioObject.getvalue()
-
-def is_stream_binary_plist(stream):
-    stream.seek(0)
-    header = stream.read(7)
-    if header == b'bplist0':
-        return True
-    else:
-        return False
-
-PlistTrailer = namedtuple('PlistTrailer', 'offsetSize, objectRefSize, offsetCount, topLevelObjectNumber, offsetTableOffset')
-PlistByteCounts = namedtuple('PlistByteCounts', 'nullBytes, boolBytes, intBytes, realBytes, dateBytes, dataBytes, stringBytes, uidBytes, arrayBytes, setBytes, dictBytes')
-
-class PlistReader(object):
-    file = None
-    contents = ''
-    offsets = None
-    trailer = None
-    currentOffset = 0
-    
-    def __init__(self, fileOrStream):
-        """Raises NotBinaryPlistException."""
-        self.reset()
-        self.file = fileOrStream
-    
-    def parse(self):
-        return self.readRoot()
-    
-    def reset(self):
-        self.trailer = None
-        self.contents = ''
-        self.offsets = []
-        self.currentOffset = 0
-    
-    def readRoot(self):
-        result = None
-        self.reset()
-        # Get the header, make sure it's a valid file.
-        if not is_stream_binary_plist(self.file):
-            raise NotBinaryPlistException()
-        self.file.seek(0)
-        self.contents = self.file.read()
-        if len(self.contents) < 32:
-            raise InvalidPlistException("File is too short.")
-        trailerContents = self.contents[-32:]
-        try:
-            self.trailer = PlistTrailer._make(unpack("!xxxxxxBBQQQ", trailerContents))
-            offset_size = self.trailer.offsetSize * self.trailer.offsetCount
-            offset = self.trailer.offsetTableOffset
-            offset_contents = self.contents[offset:offset+offset_size]
-            offset_i = 0
-            while offset_i < self.trailer.offsetCount:
-                begin = self.trailer.offsetSize*offset_i
-                tmp_contents = offset_contents[begin:begin+self.trailer.offsetSize]
-                tmp_sized = self.getSizedInteger(tmp_contents, self.trailer.offsetSize)
-                self.offsets.append(tmp_sized)
-                offset_i += 1
-            self.setCurrentOffsetToObjectNumber(self.trailer.topLevelObjectNumber)
-            result = self.readObject()
-        except TypeError as e:
-            raise InvalidPlistException(e)
-        return result
-    
-    def setCurrentOffsetToObjectNumber(self, objectNumber):
-        self.currentOffset = self.offsets[objectNumber]
-    
-    def readObject(self):
-        result = None
-        tmp_byte = self.contents[self.currentOffset:self.currentOffset+1]
-        marker_byte = unpack("!B", tmp_byte)[0]
-        format = (marker_byte >> 4) & 0x0f
-        extra = marker_byte & 0x0f
-        self.currentOffset += 1
-        
-        def proc_extra(extra):
-            if extra == 0b1111:
-                #self.currentOffset += 1
-                extra = self.readObject()
-            return extra
-        
-        # bool, null, or fill byte
-        if format == 0b0000:
-            if extra == 0b0000:
-                result = None
-            elif extra == 0b1000:
-                result = False
-            elif extra == 0b1001:
-                result = True
-            elif extra == 0b1111:
-                pass # fill byte
-            else:
-                raise InvalidPlistException("Invalid object found at offset: %d" % (self.currentOffset - 1))
-        # int
-        elif format == 0b0001:
-            extra = proc_extra(extra)
-            result = self.readInteger(pow(2, extra))
-        # real
-        elif format == 0b0010:
-            extra = proc_extra(extra)
-            result = self.readReal(extra)
-        # date
-        elif format == 0b0011 and extra == 0b0011:
-            result = self.readDate()
-        # data
-        elif format == 0b0100:
-            extra = proc_extra(extra)
-            result = self.readData(extra)
-        # ascii string
-        elif format == 0b0101:
-            extra = proc_extra(extra)
-            result = self.readAsciiString(extra)
-        # Unicode string
-        elif format == 0b0110:
-            extra = proc_extra(extra)
-            result = self.readUnicode(extra)
-        # uid
-        elif format == 0b1000:
-            result = self.readUid(extra)
-        # array
-        elif format == 0b1010:
-            extra = proc_extra(extra)
-            result = self.readArray(extra)
-        # set
-        elif format == 0b1100:
-            extra = proc_extra(extra)
-            result = set(self.readArray(extra))
-        # dict
-        elif format == 0b1101:
-            extra = proc_extra(extra)
-            result = self.readDict(extra)
-        else:    
-            raise InvalidPlistException("Invalid object found: {format: %s, extra: %s}" % (bin(format), bin(extra)))
-        return result
-    
-    def readInteger(self, byteSize):
-        result = 0
-        original_offset = self.currentOffset
-        data = self.contents[self.currentOffset:self.currentOffset + byteSize]
-        result = self.getSizedInteger(data, byteSize, as_number=True)
-        self.currentOffset = original_offset + byteSize
-        return result
-    
-    def readReal(self, length):
-        result = 0.0
-        to_read = pow(2, length)
-        data = self.contents[self.currentOffset:self.currentOffset+to_read]
-        if length == 2: # 4 bytes
-            result = unpack('>f', data)[0]
-        elif length == 3: # 8 bytes
-            result = unpack('>d', data)[0]
-        else:
-            raise InvalidPlistException("Unknown real of length %d bytes" % to_read)
-        return result
-    
-    def readRefs(self, count):    
-        refs = []
-        i = 0
-        while i < count:
-            fragment = self.contents[self.currentOffset:self.currentOffset+self.trailer.objectRefSize]
-            ref = self.getSizedInteger(fragment, len(fragment))
-            refs.append(ref)
-            self.currentOffset += self.trailer.objectRefSize
-            i += 1
-        return refs
-    
-    def readArray(self, count):
-        result = []
-        values = self.readRefs(count)
-        i = 0
-        while i < len(values):
-            self.setCurrentOffsetToObjectNumber(values[i])
-            value = self.readObject()
-            result.append(value)
-            i += 1
-        return result
-    
-    def readDict(self, count):
-        result = {}
-        keys = self.readRefs(count)
-        values = self.readRefs(count)
-        i = 0
-        while i < len(keys):
-            self.setCurrentOffsetToObjectNumber(keys[i])
-            key = self.readObject()
-            self.setCurrentOffsetToObjectNumber(values[i])
-            value = self.readObject()
-            result[key] = value
-            i += 1
-        return result
-    
-    def readAsciiString(self, length):
-        result = unpack("!%ds" % length, self.contents[self.currentOffset:self.currentOffset+length])[0]
-        self.currentOffset += length
-        return result
-    
-    def readUnicode(self, length):
-        actual_length = length*2
-        data = self.contents[self.currentOffset:self.currentOffset+actual_length]
-        # unpack not needed?!! data = unpack(">%ds" % (actual_length), data)[0]
-        self.currentOffset += actual_length
-        return data.decode('utf_16_be')
-    
-    def readDate(self):
-        result = unpack(">d", self.contents[self.currentOffset:self.currentOffset+8])[0]
-        # Use timedelta to workaround time_t size limitation on 32-bit python.
-        result = datetime.timedelta(seconds=result) + apple_reference_date
-        self.currentOffset += 8
-        return result
-    
-    def readData(self, length):
-        result = self.contents[self.currentOffset:self.currentOffset+length]
-        self.currentOffset += length
-        return Data(result)
-    
-    def readUid(self, length):
-        return Uid(self.readInteger(length+1))
-    
-    def getSizedInteger(self, data, byteSize, as_number=False):
-        """Numbers of 8 bytes are signed integers when they refer to numbers, but unsigned otherwise."""
-        result = 0
-        # 1, 2, and 4 byte integers are unsigned
-        if byteSize == 1:
-            result = unpack('>B', data)[0]
-        elif byteSize == 2:
-            result = unpack('>H', data)[0]
-        elif byteSize == 4:
-            result = unpack('>L', data)[0]
-        elif byteSize == 8:
-            if as_number:
-                result = unpack('>q', data)[0]
-            else:
-                result = unpack('>Q', data)[0]
-        elif byteSize <= 16:
-            # Handle odd-sized or integers larger than 8 bytes
-            # Don't naively go over 16 bytes, in order to prevent infinite loops.
-            result = 0
-            if hasattr(int, 'from_bytes'):
-                result = int.from_bytes(data, 'big')
-            else:
-                for byte in data:
-                    result = (result << 8) | unpack('>B', byte)[0]
-        else:
-            raise InvalidPlistException("Encountered integer longer than 16 bytes.")
-        return result
-
-class HashableWrapper(object):
-    def __init__(self, value):
-        self.value = value
-    def __repr__(self):
-        return "<HashableWrapper: %s>" % [self.value]
-
-class BoolWrapper(object):
-    def __init__(self, value):
-        self.value = value
-    def __repr__(self):
-        return "<BoolWrapper: %s>" % self.value
-
-class FloatWrapper(object):
-    _instances = {}
-    def __new__(klass, value):
-        # Ensure FloatWrapper(x) for a given float x is always the same object
-        wrapper = klass._instances.get(value)
-        if wrapper is None:
-            wrapper = object.__new__(klass)
-            wrapper.value = value
-            klass._instances[value] = wrapper
-        return wrapper
-    def __repr__(self):
-        return "<FloatWrapper: %s>" % self.value
-
-class PlistWriter(object):
-    header = b'bplist00bybiplist1.0'
-    file = None
-    byteCounts = None
-    trailer = None
-    computedUniques = None
-    writtenReferences = None
-    referencePositions = None
-    wrappedTrue = None
-    wrappedFalse = None
-    
-    def __init__(self, file):
-        self.reset()
-        self.file = file
-        self.wrappedTrue = BoolWrapper(True)
-        self.wrappedFalse = BoolWrapper(False)
-
-    def reset(self):
-        self.byteCounts = PlistByteCounts(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0)
-        self.trailer = PlistTrailer(0, 0, 0, 0, 0)
-        
-        # A set of all the uniques which have been computed.
-        self.computedUniques = set()
-        # A list of all the uniques which have been written.
-        self.writtenReferences = {}
-        # A dict of the positions of the written uniques.
-        self.referencePositions = {}
-        
-    def positionOfObjectReference(self, obj):
-        """If the given object has been written already, return its
-           position in the offset table. Otherwise, return None."""
-        return self.writtenReferences.get(obj)
-        
-    def writeRoot(self, root):
-        """
-        Strategy is:
-        - write header
-        - wrap root object so everything is hashable
-        - compute size of objects which will be written
-          - need to do this in order to know how large the object refs
-            will be in the list/dict/set reference lists
-        - write objects
-          - keep objects in writtenReferences
-          - keep positions of object references in referencePositions
-          - write object references with the length computed previously
-        - computer object reference length
-        - write object reference positions
-        - write trailer
-        """
-        output = self.header
-        wrapped_root = self.wrapRoot(root)
-        should_reference_root = True#not isinstance(wrapped_root, HashableWrapper)
-        self.computeOffsets(wrapped_root, asReference=should_reference_root, isRoot=True)
-        self.trailer = self.trailer._replace(**{'objectRefSize':self.intSize(len(self.computedUniques))})
-        (_, output) = self.writeObjectReference(wrapped_root, output)
-        output = self.writeObject(wrapped_root, output, setReferencePosition=True)
-        
-        # output size at this point is an upper bound on how big the
-        # object reference offsets need to be.
-        self.trailer = self.trailer._replace(**{
-            'offsetSize':self.intSize(len(output)),
-            'offsetCount':len(self.computedUniques),
-            'offsetTableOffset':len(output),
-            'topLevelObjectNumber':0
-            })
-        
-        output = self.writeOffsetTable(output)
-        output += pack('!xxxxxxBBQQQ', *self.trailer)
-        self.file.write(output)
-
-    def wrapRoot(self, root):
-        if isinstance(root, bool):
-            if root is True:
-                return self.wrappedTrue
-            else:
-                return self.wrappedFalse
-        elif isinstance(root, float):
-            return FloatWrapper(root)
-        elif isinstance(root, set):
-            n = set()
-            for value in root:
-                n.add(self.wrapRoot(value))
-            return HashableWrapper(n)
-        elif isinstance(root, dict):
-            n = {}
-            for key, value in iteritems(root):
-                n[self.wrapRoot(key)] = self.wrapRoot(value)
-            return HashableWrapper(n)
-        elif isinstance(root, list):
-            n = []
-            for value in root:
-                n.append(self.wrapRoot(value))
-            return HashableWrapper(n)
-        elif isinstance(root, tuple):
-            n = tuple([self.wrapRoot(value) for value in root])
-            return HashableWrapper(n)
-        else:
-            return root
-
-    def incrementByteCount(self, field, incr=1):
-        self.byteCounts = self.byteCounts._replace(**{field:self.byteCounts.__getattribute__(field) + incr})
-
-    def computeOffsets(self, obj, asReference=False, isRoot=False):
-        def check_key(key):
-            if key is None:
-                raise InvalidPlistException('Dictionary keys cannot be null in plists.')
-            elif isinstance(key, Data):
-                raise InvalidPlistException('Data cannot be dictionary keys in plists.')
-            elif not isinstance(key, (bytes, str)):
-                raise InvalidPlistException('Keys must be strings.')
-        
-        def proc_size(size):
-            if size > 0b1110:
-                size += self.intSize(size)
-            return size
-        # If this should be a reference, then we keep a record of it in the
-        # uniques table.
-        if asReference:
-            if obj in self.computedUniques:
-                return
-            else:
-                self.computedUniques.add(obj)
-        
-        if obj is None:
-            self.incrementByteCount('nullBytes')
-        elif isinstance(obj, BoolWrapper):
-            self.incrementByteCount('boolBytes')
-        elif isinstance(obj, Uid):
-            size = self.intSize(obj)
-            self.incrementByteCount('uidBytes', incr=1+size)
-        elif isinstance(obj, int):
-            size = self.intSize(obj)
-            self.incrementByteCount('intBytes', incr=1+size)
-        elif isinstance(obj, FloatWrapper):
-            size = self.realSize(obj)
-            self.incrementByteCount('realBytes', incr=1+size)
-        elif isinstance(obj, datetime.datetime):    
-            self.incrementByteCount('dateBytes', incr=2)
-        elif isinstance(obj, Data):
-            size = proc_size(len(obj))
-            self.incrementByteCount('dataBytes', incr=1+size)
-        elif isinstance(obj, (str, bytes)):
-            size = proc_size(len(obj))
-            self.incrementByteCount('stringBytes', incr=1+size)
-        elif isinstance(obj, HashableWrapper):
-            obj = obj.value
-            if isinstance(obj, set):
-                size = proc_size(len(obj))
-                self.incrementByteCount('setBytes', incr=1+size)
-                for value in obj:
-                    self.computeOffsets(value, asReference=True)
-            elif isinstance(obj, (list, tuple)):
-                size = proc_size(len(obj))
-                self.incrementByteCount('arrayBytes', incr=1+size)
-                for value in obj:
-                    asRef = True
-                    self.computeOffsets(value, asReference=True)
-            elif isinstance(obj, dict):
-                size = proc_size(len(obj))
-                self.incrementByteCount('dictBytes', incr=1+size)
-                for key, value in iteritems(obj):
-                    check_key(key)
-                    self.computeOffsets(key, asReference=True)
-                    self.computeOffsets(value, asReference=True)
-        else:
-            raise InvalidPlistException("Unknown object type.")
-
-    def writeObjectReference(self, obj, output):
-        """Tries to write an object reference, adding it to the references
-           table. Does not write the actual object bytes or set the reference
-           position. Returns a tuple of whether the object was a new reference
-           (True if it was, False if it already was in the reference table)
-           and the new output.
-        """
-        position = self.positionOfObjectReference(obj)
-        if position is None:
-            self.writtenReferences[obj] = len(self.writtenReferences)
-            output += self.binaryInt(len(self.writtenReferences) - 1, byteSize=self.trailer.objectRefSize)
-            return (True, output)
-        else:
-            output += self.binaryInt(position, byteSize=self.trailer.objectRefSize)
-            return (False, output)
-
-    def writeObject(self, obj, output, setReferencePosition=False):
-        """Serializes the given object to the output. Returns output.
-           If setReferencePosition is True, will set the position the
-           object was written.
-        """
-        def proc_variable_length(format, length):
-            result = b''
-            if length > 0b1110:
-                result += pack('!B', (format << 4) | 0b1111)
-                result = self.writeObject(length, result)
-            else:
-                result += pack('!B', (format << 4) | length)
-            return result
-        
-        if isinstance(obj, str) and obj == unicodeEmpty:
-            # The Apple Plist decoder can't decode a zero length Unicode string.
-            obj = b''
-       
-        if setReferencePosition:
-            self.referencePositions[obj] = len(output)
-        
-        if obj is None:
-            output += pack('!B', 0b00000000)
-        elif isinstance(obj, BoolWrapper):
-            if obj.value is False:
-                output += pack('!B', 0b00001000)
-            else:
-                output += pack('!B', 0b00001001)
-        elif isinstance(obj, Uid):
-            size = self.intSize(obj)
-            output += pack('!B', (0b1000 << 4) | size - 1)
-            output += self.binaryInt(obj)
-        elif isinstance(obj, int):
-            byteSize = self.intSize(obj)
-            root = math.log(byteSize, 2)
-            output += pack('!B', (0b0001 << 4) | int(root))
-            output += self.binaryInt(obj, as_number=True)
-        elif isinstance(obj, FloatWrapper):
-            # just use doubles
-            output += pack('!B', (0b0010 << 4) | 3)
-            output += self.binaryReal(obj)
-        elif isinstance(obj, datetime.datetime):
-            timestamp = (obj - apple_reference_date).total_seconds()
-            output += pack('!B', 0b00110011)
-            output += pack('!d', float(timestamp))
-        elif isinstance(obj, Data):
-            output += proc_variable_length(0b0100, len(obj))
-            output += obj
-        elif isinstance(obj, str):
-            byteData = obj.encode('utf_16_be')
-            output += proc_variable_length(0b0110, len(byteData)//2)
-            output += byteData
-        elif isinstance(obj, bytes):
-            output += proc_variable_length(0b0101, len(obj))
-            output += obj
-        elif isinstance(obj, HashableWrapper):
-            obj = obj.value
-            if isinstance(obj, (set, list, tuple)):
-                if isinstance(obj, set):
-                    output += proc_variable_length(0b1100, len(obj))
-                else:
-                    output += proc_variable_length(0b1010, len(obj))
-            
-                objectsToWrite = []
-                for objRef in obj:
-                    (isNew, output) = self.writeObjectReference(objRef, output)
-                    if isNew:
-                        objectsToWrite.append(objRef)
-                for objRef in objectsToWrite:
-                    output = self.writeObject(objRef, output, setReferencePosition=True)
-            elif isinstance(obj, dict):
-                output += proc_variable_length(0b1101, len(obj))
-                keys = []
-                values = []
-                objectsToWrite = []
-                for key, value in iteritems(obj):
-                    keys.append(key)
-                    values.append(value)
-                for key in keys:
-                    (isNew, output) = self.writeObjectReference(key, output)
-                    if isNew:
-                        objectsToWrite.append(key)
-                for value in values:
-                    (isNew, output) = self.writeObjectReference(value, output)
-                    if isNew:
-                        objectsToWrite.append(value)
-                for objRef in objectsToWrite:
-                    output = self.writeObject(objRef, output, setReferencePosition=True)
-        return output
-    
-    def writeOffsetTable(self, output):
-        """Writes all of the object reference offsets."""
-        all_positions = []
-        writtenReferences = list(self.writtenReferences.items())
-        writtenReferences.sort(key=lambda x: x[1])
-        for obj,order in writtenReferences:
-            # Porting note: Elsewhere we deliberately replace empty unicdoe strings
-            # with empty binary strings, but the empty unicode string
-            # goes into writtenReferences.  This isn't an issue in Py2
-            # because u'' and b'' have the same hash; but it is in
-            # Py3, where they don't.
-            if bytes != str and obj == unicodeEmpty:
-                obj = b''
-            position = self.referencePositions.get(obj)
-            if position is None:
-                raise InvalidPlistException("Error while writing offsets table. Object not found. %s" % obj)
-            output += self.binaryInt(position, self.trailer.offsetSize)
-            all_positions.append(position)
-        return output
-    
-    def binaryReal(self, obj):
-        # just use doubles
-        result = pack('>d', obj.value)
-        return result
-    
-    def binaryInt(self, obj, byteSize=None, as_number=False):
-        result = b''
-        if byteSize is None:
-            byteSize = self.intSize(obj)
-        if byteSize == 1:
-            result += pack('>B', obj)
-        elif byteSize == 2:
-            result += pack('>H', obj)
-        elif byteSize == 4:
-            result += pack('>L', obj)
-        elif byteSize == 8:
-            if as_number:
-                result += pack('>q', obj)
-            else:
-                result += pack('>Q', obj)
-        elif byteSize <= 16:
-            try:
-                result = pack('>Q', 0) + pack('>Q', obj)
-            except struct_error as e:
-                raise InvalidPlistException("Unable to pack integer %d: %s" % (obj, e))
-        else:
-            raise InvalidPlistException("Core Foundation can't handle integers with size greater than 16 bytes.")
-        return result
-    
-    def intSize(self, obj):
-        """Returns the number of bytes necessary to store the given integer."""
-        # SIGNED
-        if obj < 0: # Signed integer, always 8 bytes
-            return 8
-        # UNSIGNED
-        elif obj <= 0xFF: # 1 byte
-            return 1
-        elif obj <= 0xFFFF: # 2 bytes
-            return 2
-        elif obj <= 0xFFFFFFFF: # 4 bytes
-            return 4
-        # SIGNED
-        # 0x7FFFFFFFFFFFFFFF is the max.
-        elif obj <= 0x7FFFFFFFFFFFFFFF: # 8 bytes signed
-            return 8
-        elif obj <= 0xffffffffffffffff: # 8 bytes unsigned
-            return 16
-        else:
-            raise InvalidPlistException("Core Foundation can't handle integers with size greater than 8 bytes.")
-    
-    def realSize(self, obj):
-        return 8

lib/cherrypy/__init__.py

@@ -57,9 +57,11 @@ These API's are described in the `CherryPy specification
 """
 
 try:
-    import pkg_resources
+    import importlib.metadata as importlib_metadata
 except ImportError:
-    pass
+    # fall back for python <= 3.7
+    # This try/except can be removed with py <= 3.7 support
+    import importlib_metadata
 
 from threading import local as _local
 
@@ -109,7 +111,7 @@ tree = _cptree.Tree()
 
 
 try:
-    __version__ = pkg_resources.require('cherrypy')[0].version
+    __version__ = importlib_metadata.version('cherrypy')
 except Exception:
     __version__ = 'unknown'
 
@@ -181,24 +183,28 @@ def quickstart(root=None, script_name='', config=None):
 class _Serving(_local):
     """An interface for registering request and response objects.
 
-    Rather than have a separate "thread local" object for the request and
-    the response, this class works as a single threadlocal container for
-    both objects (and any others which developers wish to define). In this
-    way, we can easily dump those objects when we stop/start a new HTTP
-    conversation, yet still refer to them as module-level globals in a
-    thread-safe way.
+    Rather than have a separate "thread local" object for the request
+    and the response, this class works as a single threadlocal container
+    for both objects (and any others which developers wish to define).
+    In this way, we can easily dump those objects when we stop/start a
+    new HTTP conversation, yet still refer to them as module-level
+    globals in a thread-safe way.
     """
 
     request = _cprequest.Request(_httputil.Host('127.0.0.1', 80),
                                  _httputil.Host('127.0.0.1', 1111))
+    """The request object for the current thread.
+
+    In the main thread, and any threads which are not receiving HTTP
+    requests, this is None.
     """
-    The request object for the current thread. In the main thread,
-    and any threads which are not receiving HTTP requests, this is None."""
 
     response = _cprequest.Response()
+    """The response object for the current thread.
+
+    In the main thread, and any threads which are not receiving HTTP
+    requests, this is None.
     """
-    The response object for the current thread. In the main thread,
-    and any threads which are not receiving HTTP requests, this is None."""
 
     def load(self, request, response):
         self.request = request
@@ -316,8 +322,8 @@ class _GlobalLogManager(_cplogging.LogManager):
     def __call__(self, *args, **kwargs):
         """Log the given message to the app.log or global log.
 
-        Log the given message to the app.log or global
-        log as appropriate.
+        Log the given message to the app.log or global log as
+        appropriate.
         """
         # Do NOT use try/except here. See
         # https://github.com/cherrypy/cherrypy/issues/945
@@ -330,8 +336,8 @@ class _GlobalLogManager(_cplogging.LogManager):
     def access(self):
         """Log an access message to the app.log or global log.
 
-        Log the given message to the app.log or global
-        log as appropriate.
+        Log the given message to the app.log or global log as
+        appropriate.
         """
         try:
             return request.app.log.access()

lib/cherrypy/_cpchecker.py

@@ -313,7 +313,10 @@ class Checker(object):
 
     # -------------------- Specific config warnings -------------------- #
     def check_localhost(self):
-        """Warn if any socket_host is 'localhost'. See #711."""
+        """Warn if any socket_host is 'localhost'.
+
+        See #711.
+        """
         for k, v in cherrypy.config.items():
             if k == 'server.socket_host' and v == 'localhost':
                 warnings.warn("The use of 'localhost' as a socket host can "

lib/cherrypy/_cpconfig.py

@@ -1,5 +1,4 @@
-"""
-Configuration system for CherryPy.
+"""Configuration system for CherryPy.
 
 Configuration in CherryPy is implemented via dictionaries. Keys are strings
 which name the mapped value, which may be of any type.
@@ -132,8 +131,8 @@ def _if_filename_register_autoreload(ob):
 def merge(base, other):
     """Merge one app config (from a dict, file, or filename) into another.
 
-    If the given config is a filename, it will be appended to
-    the list of files to monitor for "autoreload" changes.
+    If the given config is a filename, it will be appended to the list
+    of files to monitor for "autoreload" changes.
     """
     _if_filename_register_autoreload(other)
 

lib/cherrypy/_cpdispatch.py

@@ -1,9 +1,10 @@
 """CherryPy dispatchers.
 
 A 'dispatcher' is the object which looks up the 'page handler' callable
-and collects config for the current request based on the path_info, other
-request attributes, and the application architecture. The core calls the
-dispatcher as early as possible, passing it a 'path_info' argument.
+and collects config for the current request based on the path_info,
+other request attributes, and the application architecture. The core
+calls the dispatcher as early as possible, passing it a 'path_info'
+argument.
 
 The default dispatcher discovers the page handler by matching path_info
 to a hierarchical arrangement of objects, starting at request.app.root.
@@ -21,7 +22,6 @@ import cherrypy
 
 
 class PageHandler(object):
-
     """Callable which sets response.body."""
 
     def __init__(self, callable, *args, **kwargs):
@@ -64,8 +64,7 @@ class PageHandler(object):
 
 
 def test_callable_spec(callable, callable_args, callable_kwargs):
-    """
-    Inspect callable and test to see if the given args are suitable for it.
+    """Inspect callable and test to see if the given args are suitable for it.
 
     When an error occurs during the handler's invoking stage there are 2
     erroneous cases:
@@ -206,12 +205,8 @@ except ImportError:
     def test_callable_spec(callable, args, kwargs):  # noqa: F811
         return None
 else:
-    getargspec = inspect.getargspec
-    # Python 3 requires using getfullargspec if
-    # keyword-only arguments are present
-    if hasattr(inspect, 'getfullargspec'):
-        def getargspec(callable):
-            return inspect.getfullargspec(callable)[:4]
+    def getargspec(callable):
+        return inspect.getfullargspec(callable)[:4]
 
 
 class LateParamPageHandler(PageHandler):
@@ -256,16 +251,16 @@ else:
 
 
 class Dispatcher(object):
-
     """CherryPy Dispatcher which walks a tree of objects to find a handler.
 
-    The tree is rooted at cherrypy.request.app.root, and each hierarchical
-    component in the path_info argument is matched to a corresponding nested
-    attribute of the root object. Matching handlers must have an 'exposed'
-    attribute which evaluates to True. The special method name "index"
-    matches a URI which ends in a slash ("/"). The special method name
-    "default" may match a portion of the path_info (but only when no longer
-    substring of the path_info matches some other object).
+    The tree is rooted at cherrypy.request.app.root, and each
+    hierarchical component in the path_info argument is matched to a
+    corresponding nested attribute of the root object. Matching handlers
+    must have an 'exposed' attribute which evaluates to True. The
+    special method name "index" matches a URI which ends in a slash
+    ("/"). The special method name "default" may match a portion of the
+    path_info (but only when no longer substring of the path_info
+    matches some other object).
 
     This is the default, built-in dispatcher for CherryPy.
     """
@@ -310,9 +305,9 @@ class Dispatcher(object):
 
         The second object returned will be a list of names which are
         'virtual path' components: parts of the URL which are dynamic,
-        and were not used when looking up the handler.
-        These virtual path components are passed to the handler as
-        positional arguments.
+        and were not used when looking up the handler. These virtual
+        path components are passed to the handler as positional
+        arguments.
         """
         request = cherrypy.serving.request
         app = request.app
@@ -452,13 +447,11 @@ class Dispatcher(object):
 
 
 class MethodDispatcher(Dispatcher):
-
     """Additional dispatch based on cherrypy.request.method.upper().
 
-    Methods named GET, POST, etc will be called on an exposed class.
-    The method names must be all caps; the appropriate Allow header
-    will be output showing all capitalized method names as allowable
-    HTTP verbs.
+    Methods named GET, POST, etc will be called on an exposed class. The
+    method names must be all caps; the appropriate Allow header will be
+    output showing all capitalized method names as allowable HTTP verbs.
 
     Note that the containing class must be exposed, not the methods.
     """
@@ -496,16 +489,14 @@ class MethodDispatcher(Dispatcher):
 
 
 class RoutesDispatcher(object):
-
     """A Routes based dispatcher for CherryPy."""
 
     def __init__(self, full_result=False, **mapper_options):
-        """
-        Routes dispatcher
+        """Routes dispatcher.
 
-        Set full_result to True if you wish the controller
-        and the action to be passed on to the page handler
-        parameters. By default they won't be.
+        Set full_result to True if you wish the controller and the
+        action to be passed on to the page handler parameters. By
+        default they won't be.
         """
         import routes
         self.full_result = full_result
@@ -621,8 +612,7 @@ def XMLRPCDispatcher(next_dispatcher=Dispatcher()):
 
 def VirtualHost(next_dispatcher=Dispatcher(), use_x_forwarded_host=True,
                 **domains):
-    """
-    Select a different handler based on the Host header.
+    """Select a different handler based on the Host header.
 
     This can be useful when running multiple sites within one CP server.
     It allows several domains to point to different parts of a single

lib/cherrypy/_cperror.py

@@ -136,19 +136,17 @@ from cherrypy.lib import httputil as _httputil
 
 
 class CherryPyException(Exception):
-
     """A base class for CherryPy exceptions."""
     pass
 
 
 class InternalRedirect(CherryPyException):
-
     """Exception raised to switch to the handler for a different URL.
 
-    This exception will redirect processing to another path within the site
-    (without informing the client). Provide the new path as an argument when
-    raising the exception. Provide any params in the querystring for the new
-    URL.
+    This exception will redirect processing to another path within the
+    site (without informing the client). Provide the new path as an
+    argument when raising the exception. Provide any params in the
+    querystring for the new URL.
     """
 
     def __init__(self, path, query_string=''):
@@ -173,7 +171,6 @@ class InternalRedirect(CherryPyException):
 
 
 class HTTPRedirect(CherryPyException):
-
     """Exception raised when the request should be redirected.
 
     This exception will force a HTTP redirect to the URL or URL's you give it.
@@ -202,7 +199,7 @@ class HTTPRedirect(CherryPyException):
     """The list of URL's to emit."""
 
     encoding = 'utf-8'
-    """The encoding when passed urls are not native strings"""
+    """The encoding when passed urls are not native strings."""
 
     def __init__(self, urls, status=None, encoding=None):
         self.urls = abs_urls = [
@@ -230,8 +227,7 @@ class HTTPRedirect(CherryPyException):
 
     @classproperty
     def default_status(cls):
-        """
-        The default redirect status for the request.
+        """The default redirect status for the request.
 
         RFC 2616 indicates a 301 response code fits our goal; however,
         browser support for 301 is quite messy. Use 302/303 instead. See
@@ -249,8 +245,9 @@ class HTTPRedirect(CherryPyException):
         """Modify cherrypy.response status, headers, and body to represent
         self.
 
-        CherryPy uses this internally, but you can also use it to create an
-        HTTPRedirect object and set its output without *raising* the exception.
+        CherryPy uses this internally, but you can also use it to create
+        an HTTPRedirect object and set its output without *raising* the
+        exception.
         """
         response = cherrypy.serving.response
         response.status = status = self.status
@@ -339,7 +336,6 @@ def clean_headers(status):
 
 
 class HTTPError(CherryPyException):
-
     """Exception used to return an HTTP error code (4xx-5xx) to the client.
 
     This exception can be used to automatically send a response using a
@@ -358,7 +354,9 @@ class HTTPError(CherryPyException):
     """
 
     status = None
-    """The HTTP status code. May be of type int or str (with a Reason-Phrase).
+    """The HTTP status code.
+
+    May be of type int or str (with a Reason-Phrase).
     """
 
     code = None
@@ -386,8 +384,9 @@ class HTTPError(CherryPyException):
         """Modify cherrypy.response status, headers, and body to represent
         self.
 
-        CherryPy uses this internally, but you can also use it to create an
-        HTTPError object and set its output without *raising* the exception.
+        CherryPy uses this internally, but you can also use it to create
+        an HTTPError object and set its output without *raising* the
+        exception.
         """
         response = cherrypy.serving.response
 
@@ -426,11 +425,10 @@ class HTTPError(CherryPyException):
 
 
 class NotFound(HTTPError):
-
     """Exception raised when a URL could not be mapped to any handler (404).
 
-    This is equivalent to raising
-    :class:`HTTPError("404 Not Found") <cherrypy._cperror.HTTPError>`.
+    This is equivalent to raising :class:`HTTPError("404 Not Found")
+    <cherrypy._cperror.HTTPError>`.
     """
 
     def __init__(self, path=None):
@@ -466,7 +464,7 @@ _HTTPErrorTemplate = '''<!DOCTYPE html PUBLIC
         <pre id="traceback">%(traceback)s</pre>
     <div id="powered_by">
       <span>
-        Powered by <a href="http://www.cherrypy.org">CherryPy %(version)s</a>
+        Powered by <a href="http://www.cherrypy.dev">CherryPy %(version)s</a>
       </span>
     </div>
     </body>
@@ -477,8 +475,8 @@ _HTTPErrorTemplate = '''<!DOCTYPE html PUBLIC
 def get_error_page(status, **kwargs):
     """Return an HTML page, containing a pretty error response.
 
-    status should be an int or a str.
-    kwargs will be interpolated into the page template.
+    status should be an int or a str. kwargs will be interpolated into
+    the page template.
     """
     try:
         code, reason, message = _httputil.valid_status(status)
@@ -532,7 +530,8 @@ def get_error_page(status, **kwargs):
                     return result
             else:
                 # Load the template from this path.
-                template = io.open(error_page, newline='').read()
+                with io.open(error_page, newline='') as f:
+                    template = f.read()
         except Exception:
             e = _format_exception(*_exc_info())[-1]
             m = kwargs['message']
@@ -594,8 +593,8 @@ def bare_error(extrabody=None):
     """Produce status, headers, body for a critical error.
 
     Returns a triple without calling any other questionable functions,
-    so it should be as error-free as possible. Call it from an HTTP server
-    if you get errors outside of the request.
+    so it should be as error-free as possible. Call it from an HTTP
+    server if you get errors outside of the request.
 
     If extrabody is None, a friendly but rather unhelpful error message
     is set in the body. If extrabody is a string, it will be appended

lib/cherrypy/_cplogging.py

@@ -123,7 +123,6 @@ logfmt = logging.Formatter('%(message)s')
 
 
 class NullHandler(logging.Handler):
-
     """A no-op logging handler to silence the logging.lastResort handler."""
 
     def handle(self, record):
@@ -137,15 +136,16 @@ class NullHandler(logging.Handler):
 
 
 class LogManager(object):
-
     """An object to assist both simple and advanced logging.
 
     ``cherrypy.log`` is an instance of this class.
     """
 
     appid = None
-    """The id() of the Application object which owns this log manager. If this
-    is a global log manager, appid is None."""
+    """The id() of the Application object which owns this log manager.
+
+    If this is a global log manager, appid is None.
+    """
 
     error_log = None
     """The actual :class:`logging.Logger` instance for error messages."""
@@ -317,8 +317,8 @@ class LogManager(object):
     def screen(self):
         """Turn stderr/stdout logging on or off.
 
-        If you set this to True, it'll add the appropriate StreamHandler for
-        you. If you set it to False, it will remove the handler.
+        If you set this to True, it'll add the appropriate StreamHandler
+        for you. If you set it to False, it will remove the handler.
         """
         h = self._get_builtin_handler
         has_h = h(self.error_log, 'screen') or h(self.access_log, 'screen')
@@ -414,7 +414,6 @@ class LogManager(object):
 
 
 class WSGIErrorHandler(logging.Handler):
-
     "A handler class which writes logging records to environ['wsgi.errors']."
 
     def flush(self):
@@ -452,6 +451,8 @@ class WSGIErrorHandler(logging.Handler):
 
 class LazyRfc3339UtcTime(object):
     def __str__(self):
-        """Return now() in RFC3339 UTC Format."""
-        now = datetime.datetime.now()
-        return now.isoformat('T') + 'Z'
+        """Return datetime in RFC3339 UTC Format."""
+        iso_formatted_now = datetime.datetime.now(
+            datetime.timezone.utc,
+        ).isoformat('T')
+        return f'{iso_formatted_now!s}Z'

lib/cherrypy/_cpmodpy.py

@@ -1,4 +1,4 @@
-"""Native adapter for serving CherryPy via mod_python
+"""Native adapter for serving CherryPy via mod_python.
 
 Basic usage:
 
@@ -339,11 +339,8 @@ LoadModule python_module modules/mod_python.so
                                      }
 
         mpconf = os.path.join(os.path.dirname(__file__), 'cpmodpy.conf')
-        f = open(mpconf, 'wb')
-        try:
+        with open(mpconf, 'wb') as f:
             f.write(conf_data)
-        finally:
-            f.close()
 
         response = read_process(self.apache_path, '-k start -f %s' % mpconf)
         self.ready = True

lib/cherrypy/_cpnative_server.py

@@ -120,10 +120,10 @@ class NativeGateway(cheroot.server.Gateway):
 class CPHTTPServer(cheroot.server.HTTPServer):
     """Wrapper for cheroot.server.HTTPServer.
 
-    cheroot has been designed to not reference CherryPy in any way,
-    so that it can be used in other frameworks and applications.
-    Therefore, we wrap it here, so we can apply some attributes
-    from config -> cherrypy.server -> HTTPServer.
+    cheroot has been designed to not reference CherryPy in any way, so
+    that it can be used in other frameworks and applications. Therefore,
+    we wrap it here, so we can apply some attributes from config ->
+    cherrypy.server -> HTTPServer.
     """
 
     def __init__(self, server_adapter=cherrypy.server):

lib/cherrypy/_cpreqbody.py

@@ -248,7 +248,10 @@ def process_multipart_form_data(entity):
 
 
 def _old_process_multipart(entity):
-    """The behavior of 3.2 and lower. Deprecated and will be changed in 3.3."""
+    """The behavior of 3.2 and lower.
+
+    Deprecated and will be changed in 3.3.
+    """
     process_multipart(entity)
 
     params = entity.params
@@ -277,7 +280,6 @@ def _old_process_multipart(entity):
 
 # -------------------------------- Entities --------------------------------- #
 class Entity(object):
-
     """An HTTP request body, or MIME multipart body.
 
     This class collects information about the HTTP request entity. When a
@@ -346,13 +348,15 @@ class Entity(object):
     content_type = None
     """The value of the Content-Type request header.
 
-    If the Entity is part of a multipart payload, this will be the Content-Type
-    given in the MIME headers for this part.
+    If the Entity is part of a multipart payload, this will be the
+    Content-Type given in the MIME headers for this part.
     """
 
     default_content_type = 'application/x-www-form-urlencoded'
     """This defines a default ``Content-Type`` to use if no Content-Type header
-    is given. The empty string is used for RequestBody, which results in the
+    is given.
+
+    The empty string is used for RequestBody, which results in the
     request body not being read or parsed at all. This is by design; a missing
     ``Content-Type`` header in the HTTP request entity is an error at best,
     and a security hole at worst. For multipart parts, however, the MIME spec
@@ -402,8 +406,8 @@ class Entity(object):
     part_class = None
     """The class used for multipart parts.
 
-    You can replace this with custom subclasses to alter the processing of
-    multipart parts.
+    You can replace this with custom subclasses to alter the processing
+    of multipart parts.
     """
 
     def __init__(self, fp, headers, params=None, parts=None):
@@ -509,7 +513,8 @@ class Entity(object):
         """Return a file-like object into which the request body will be read.
 
         By default, this will return a TemporaryFile. Override as needed.
-        See also :attr:`cherrypy._cpreqbody.Part.maxrambytes`."""
+        See also :attr:`cherrypy._cpreqbody.Part.maxrambytes`.
+        """
         return tempfile.TemporaryFile()
 
     def fullvalue(self):
@@ -525,7 +530,7 @@ class Entity(object):
         return value
 
     def decode_entity(self, value):
-        """Return a given byte encoded value as a string"""
+        """Return a given byte encoded value as a string."""
         for charset in self.attempt_charsets:
             try:
                 value = value.decode(charset)
@@ -569,7 +574,6 @@ class Entity(object):
 
 
 class Part(Entity):
-
     """A MIME part entity, part of a multipart entity."""
 
     # "The default character set, which must be assumed in the absence of a
@@ -653,8 +657,8 @@ class Part(Entity):
     def read_lines_to_boundary(self, fp_out=None):
         """Read bytes from self.fp and return or write them to a file.
 
-        If the 'fp_out' argument is None (the default), all bytes read are
-        returned in a single byte string.
+        If the 'fp_out' argument is None (the default), all bytes read
+        are returned in a single byte string.
 
         If the 'fp_out' argument is not None, it must be a file-like
         object that supports the 'write' method; all bytes read will be
@@ -755,15 +759,15 @@ class SizedReader:
     def read(self, size=None, fp_out=None):
         """Read bytes from the request body and return or write them to a file.
 
-        A number of bytes less than or equal to the 'size' argument are read
-        off the socket. The actual number of bytes read are tracked in
-        self.bytes_read. The number may be smaller than 'size' when 1) the
-        client sends fewer bytes, 2) the 'Content-Length' request header
-        specifies fewer bytes than requested, or 3) the number of bytes read
-        exceeds self.maxbytes (in which case, 413 is raised).
+        A number of bytes less than or equal to the 'size' argument are
+        read off the socket. The actual number of bytes read are tracked
+        in self.bytes_read. The number may be smaller than 'size' when
+        1) the client sends fewer bytes, 2) the 'Content-Length' request
+        header specifies fewer bytes than requested, or 3) the number of
+        bytes read exceeds self.maxbytes (in which case, 413 is raised).
 
-        If the 'fp_out' argument is None (the default), all bytes read are
-        returned in a single byte string.
+        If the 'fp_out' argument is None (the default), all bytes read
+        are returned in a single byte string.
 
         If the 'fp_out' argument is not None, it must be a file-like
         object that supports the 'write' method; all bytes read will be
@@ -918,7 +922,6 @@ class SizedReader:
 
 
 class RequestBody(Entity):
-
     """The entity of the HTTP request."""
 
     bufsize = 8 * 1024

lib/cherrypy/_cprequest.py

@@ -16,7 +16,6 @@ from cherrypy.lib import httputil, reprconf, encoding
 
 
 class Hook(object):
-
     """A callback and its metadata: failsafe, priority, and kwargs."""
 
     callback = None
@@ -30,10 +29,12 @@ class Hook(object):
     from the same call point raise exceptions."""
 
     priority = 50
+    """Defines the order of execution for a list of Hooks.
+
+    Priority numbers should be limited to the closed interval [0, 100],
+    but values outside this range are acceptable, as are fractional
+    values.
     """
-    Defines the order of execution for a list of Hooks. Priority numbers
-    should be limited to the closed interval [0, 100], but values outside
-    this range are acceptable, as are fractional values."""
 
     kwargs = {}
     """
@@ -74,7 +75,6 @@ class Hook(object):
 
 
 class HookMap(dict):
-
     """A map of call points to lists of callbacks (Hook objects)."""
 
     def __new__(cls, points=None):
@@ -169,7 +169,7 @@ def request_namespace(k, v):
 def response_namespace(k, v):
     """Attach response attributes declared in config."""
     # Provides config entries to set default response headers
-    # http://cherrypy.org/ticket/889
+    # http://cherrypy.dev/ticket/889
     if k[:8] == 'headers.':
         cherrypy.serving.response.headers[k.split('.', 1)[1]] = v
     else:
@@ -190,23 +190,23 @@ hookpoints = ['on_start_resource', 'before_request_body',
 
 
 class Request(object):
-
     """An HTTP request.
 
-    This object represents the metadata of an HTTP request message;
-    that is, it contains attributes which describe the environment
-    in which the request URL, headers, and body were sent (if you
-    want tools to interpret the headers and body, those are elsewhere,
-    mostly in Tools). This 'metadata' consists of socket data,
-    transport characteristics, and the Request-Line. This object
-    also contains data regarding the configuration in effect for
-    the given URL, and the execution plan for generating a response.
+    This object represents the metadata of an HTTP request message; that
+    is, it contains attributes which describe the environment in which
+    the request URL, headers, and body were sent (if you want tools to
+    interpret the headers and body, those are elsewhere, mostly in
+    Tools). This 'metadata' consists of socket data, transport
+    characteristics, and the Request-Line. This object also contains
+    data regarding the configuration in effect for the given URL, and
+    the execution plan for generating a response.
     """
 
     prev = None
+    """The previous Request object (if any).
+
+    This should be None unless we are processing an InternalRedirect.
     """
-    The previous Request object (if any). This should be None
-    unless we are processing an InternalRedirect."""
 
     # Conversation/connection attributes
     local = httputil.Host('127.0.0.1', 80)
@@ -216,9 +216,10 @@ class Request(object):
     'An httputil.Host(ip, port, hostname) object for the client socket.'
 
     scheme = 'http'
+    """The protocol used between client and server.
+
+    In most cases, this will be either 'http' or 'https'.
     """
-    The protocol used between client and server. In most cases,
-    this will be either 'http' or 'https'."""
 
     server_protocol = 'HTTP/1.1'
     """
@@ -227,32 +228,37 @@ class Request(object):
 
     base = ''
     """The (scheme://host) portion of the requested URL.
+
     In some cases (e.g. when proxying via mod_rewrite), this may contain
     path segments which cherrypy.url uses when constructing url's, but
-    which otherwise are ignored by CherryPy. Regardless, this value
-    MUST NOT end in a slash."""
+    which otherwise are ignored by CherryPy. Regardless, this value MUST
+    NOT end in a slash.
+    """
 
     # Request-Line attributes
     request_line = ''
+    """The complete Request-Line received from the client.
+
+    This is a single string consisting of the request method, URI, and
+    protocol version (joined by spaces). Any final CRLF is removed.
     """
-    The complete Request-Line received from the client. This is a
-    single string consisting of the request method, URI, and protocol
-    version (joined by spaces). Any final CRLF is removed."""
 
     method = 'GET'
+    """Indicates the HTTP method to be performed on the resource identified by
+    the Request-URI.
+
+    Common methods include GET, HEAD, POST, PUT, and DELETE. CherryPy
+    allows any extension method; however, various HTTP servers and
+    gateways may restrict the set of allowable methods. CherryPy
+    applications SHOULD restrict the set (on a per-URI basis).
     """
-    Indicates the HTTP method to be performed on the resource identified
-    by the Request-URI. Common methods include GET, HEAD, POST, PUT, and
-    DELETE. CherryPy allows any extension method; however, various HTTP
-    servers and gateways may restrict the set of allowable methods.
-    CherryPy applications SHOULD restrict the set (on a per-URI basis)."""
 
     query_string = ''
     """
     The query component of the Request-URI, a string of information to be
     interpreted by the resource. The query portion of a URI follows the
     path component, and is separated by a '?'. For example, the URI
-    'http://www.cherrypy.org/wiki?a=3&b=4' has the query component,
+    'http://www.cherrypy.dev/wiki?a=3&b=4' has the query component,
     'a=3&b=4'."""
 
     query_string_encoding = 'utf8'
@@ -277,22 +283,26 @@ class Request(object):
     A dict which combines query string (GET) and request entity (POST)
     variables. This is populated in two stages: GET params are added
     before the 'on_start_resource' hook, and POST params are added
-    between the 'before_request_body' and 'before_handler' hooks."""
+    between the 'before_request_body' and 'before_handler' hooks.
+    """
 
     # Message attributes
     header_list = []
+    """A list of the HTTP request headers as (name, value) tuples.
+
+    In general, you should use request.headers (a dict) instead.
     """
-    A list of the HTTP request headers as (name, value) tuples.
-    In general, you should use request.headers (a dict) instead."""
 
     headers = httputil.HeaderMap()
-    """
-    A dict-like object containing the request headers. Keys are header
+    """A dict-like object containing the request headers.
+
+    Keys are header
     names (in Title-Case format); however, you may get and set them in
     a case-insensitive manner. That is, headers['Content-Type'] and
     headers['content-type'] refer to the same value. Values are header
     values (decoded according to :rfc:`2047` if necessary). See also:
-    httputil.HeaderMap, httputil.HeaderElement."""
+    httputil.HeaderMap, httputil.HeaderElement.
+    """
 
     cookie = SimpleCookie()
     """See help(Cookie)."""
@@ -336,7 +346,8 @@ class Request(object):
     or multipart, this will be None. Otherwise, this will be an instance
     of :class:`RequestBody<cherrypy._cpreqbody.RequestBody>` (which you
     can .read()); this value is set between the 'before_request_body' and
-    'before_handler' hooks (assuming that process_request_body is True)."""
+    'before_handler' hooks (assuming that process_request_body is True).
+    """
 
     # Dispatch attributes
     dispatch = cherrypy.dispatch.Dispatcher()
@@ -347,23 +358,24 @@ class Request(object):
     calls the dispatcher as early as possible, passing it a 'path_info'
     argument.
 
-    The default dispatcher discovers the page handler by matching path_info
-    to a hierarchical arrangement of objects, starting at request.app.root.
-    See help(cherrypy.dispatch) for more information."""
+    The default dispatcher discovers the page handler by matching
+    path_info to a hierarchical arrangement of objects, starting at
+    request.app.root. See help(cherrypy.dispatch) for more information.
+    """
 
     script_name = ''
-    """
-    The 'mount point' of the application which is handling this request.
+    """The 'mount point' of the application which is handling this request.
 
     This attribute MUST NOT end in a slash. If the script_name refers to
     the root of the URI, it MUST be an empty string (not "/").
     """
 
     path_info = '/'
+    """The 'relative path' portion of the Request-URI.
+
+    This is relative to the script_name ('mount point') of the
+    application which is handling this request.
     """
-    The 'relative path' portion of the Request-URI. This is relative
-    to the script_name ('mount point') of the application which is
-    handling this request."""
 
     login = None
     """
@@ -391,14 +403,16 @@ class Request(object):
     of the form: {Toolbox.namespace: {Tool.name: config dict}}."""
 
     config = None
+    """A flat dict of all configuration entries which apply to the current
+    request.
+
+    These entries are collected from global config, application config
+    (based on request.path_info), and from handler config (exactly how
+    is governed by the request.dispatch object in effect for this
+    request; by default, handler config can be attached anywhere in the
+    tree between request.app.root and the final handler, and inherits
+    downward).
     """
-    A flat dict of all configuration entries which apply to the
-    current request. These entries are collected from global config,
-    application config (based on request.path_info), and from handler
-    config (exactly how is governed by the request.dispatch object in
-    effect for this request; by default, handler config can be attached
-    anywhere in the tree between request.app.root and the final handler,
-    and inherits downward)."""
 
     is_index = None
     """
@@ -409,13 +423,14 @@ class Request(object):
     the trailing slash. See cherrypy.tools.trailing_slash."""
 
     hooks = HookMap(hookpoints)
-    """
-    A HookMap (dict-like object) of the form: {hookpoint: [hook, ...]}.
+    """A HookMap (dict-like object) of the form: {hookpoint: [hook, ...]}.
+
     Each key is a str naming the hook point, and each value is a list
     of hooks which will be called at that hook point during this request.
     The list of hooks is generally populated as early as possible (mostly
     from Tools specified in config), but may be extended at any time.
-    See also: _cprequest.Hook, _cprequest.HookMap, and cherrypy.tools."""
+    See also: _cprequest.Hook, _cprequest.HookMap, and cherrypy.tools.
+    """
 
     error_response = cherrypy.HTTPError(500).set_response
     """
@@ -428,12 +443,11 @@ class Request(object):
     error response to the user-agent."""
 
     error_page = {}
-    """
-    A dict of {error code: response filename or callable} pairs.
+    """A dict of {error code: response filename or callable} pairs.
 
     The error code must be an int representing a given HTTP error code,
-    or the string 'default', which will be used if no matching entry
-    is found for a given numeric code.
+    or the string 'default', which will be used if no matching entry is
+    found for a given numeric code.
 
     If a filename is provided, the file should contain a Python string-
     formatting template, and can expect by default to receive format
@@ -447,8 +461,8 @@ class Request(object):
     iterable of strings which will be set to response.body. It may also
     override headers or perform any other processing.
 
-    If no entry is given for an error code, and no 'default' entry exists,
-    a default template will be used.
+    If no entry is given for an error code, and no 'default' entry
+    exists, a default template will be used.
     """
 
     show_tracebacks = True
@@ -473,9 +487,10 @@ class Request(object):
     """True once the close method has been called, False otherwise."""
 
     stage = None
+    """A string containing the stage reached in the request-handling process.
+
+    This is useful when debugging a live server with hung requests.
     """
-    A string containing the stage reached in the request-handling process.
-    This is useful when debugging a live server with hung requests."""
 
     unique_id = None
     """A lazy object generating and memorizing UUID4 on ``str()`` render."""
@@ -492,9 +507,10 @@ class Request(object):
                  server_protocol='HTTP/1.1'):
         """Populate a new Request object.
 
-        local_host should be an httputil.Host object with the server info.
-        remote_host should be an httputil.Host object with the client info.
-        scheme should be a string, either "http" or "https".
+        local_host should be an httputil.Host object with the server
+        info. remote_host should be an httputil.Host object with the
+        client info. scheme should be a string, either "http" or
+        "https".
         """
         self.local = local_host
         self.remote = remote_host
@@ -514,7 +530,10 @@ class Request(object):
         self.unique_id = LazyUUID4()
 
     def close(self):
-        """Run cleanup code. (Core)"""
+        """Run cleanup code.
+
+        (Core)
+        """
         if not self.closed:
             self.closed = True
             self.stage = 'on_end_request'
@@ -551,7 +570,6 @@ class Request(object):
 
         Consumer code (HTTP servers) should then access these response
         attributes to build the outbound stream.
-
         """
         response = cherrypy.serving.response
         self.stage = 'run'
@@ -631,7 +649,10 @@ class Request(object):
         return response
 
     def respond(self, path_info):
-        """Generate a response for the resource at self.path_info. (Core)"""
+        """Generate a response for the resource at self.path_info.
+
+        (Core)
+        """
         try:
             try:
                 try:
@@ -702,7 +723,10 @@ class Request(object):
         response.finalize()
 
     def process_query_string(self):
-        """Parse the query string into Python structures. (Core)"""
+        """Parse the query string into Python structures.
+
+        (Core)
+        """
         try:
             p = httputil.parse_query_string(
                 self.query_string, encoding=self.query_string_encoding)
@@ -715,7 +739,10 @@ class Request(object):
         self.params.update(p)
 
     def process_headers(self):
-        """Parse HTTP header data into Python structures. (Core)"""
+        """Parse HTTP header data into Python structures.
+
+        (Core)
+        """
         # Process the headers into self.headers
         headers = self.headers
         for name, value in self.header_list:
@@ -742,13 +769,19 @@ class Request(object):
             if self.protocol >= (1, 1):
                 msg = "HTTP/1.1 requires a 'Host' request header."
                 raise cherrypy.HTTPError(400, msg)
+        else:
+            headers['Host'] = httputil.SanitizedHost(dict.get(headers, 'Host'))
+
         host = dict.get(headers, 'Host')
         if not host:
             host = self.local.name or self.local.ip
         self.base = '%s://%s' % (self.scheme, host)
 
     def get_resource(self, path):
-        """Call a dispatcher (which sets self.handler and .config). (Core)"""
+        """Call a dispatcher (which sets self.handler and .config).
+
+        (Core)
+        """
         # First, see if there is a custom dispatch at this URI. Custom
         # dispatchers can only be specified in app.config, not in _cp_config
         # (since custom dispatchers may not even have an app.root).
@@ -759,7 +792,10 @@ class Request(object):
         dispatch(path)
 
     def handle_error(self):
-        """Handle the last unanticipated exception. (Core)"""
+        """Handle the last unanticipated exception.
+
+        (Core)
+        """
         try:
             self.hooks.run('before_error_response')
             if self.error_response:
@@ -773,7 +809,6 @@ class Request(object):
 
 
 class ResponseBody(object):
-
     """The body of the HTTP response (the response entity)."""
 
     unicode_err = ('Page handlers MUST return bytes. Use tools.encode '
@@ -799,18 +834,18 @@ class ResponseBody(object):
 
 
 class Response(object):
-
     """An HTTP Response, including status, headers, and body."""
 
     status = ''
     """The HTTP Status-Code and Reason-Phrase."""
 
     header_list = []
-    """
-    A list of the HTTP response headers as (name, value) tuples.
+    """A list of the HTTP response headers as (name, value) tuples.
+
     In general, you should use response.headers (a dict) instead. This
     attribute is generated from response.headers and is not valid until
-    after the finalize phase."""
+    after the finalize phase.
+    """
 
     headers = httputil.HeaderMap()
     """
@@ -830,7 +865,10 @@ class Response(object):
     """The body (entity) of the HTTP response."""
 
     time = None
-    """The value of time.time() when created. Use in HTTP dates."""
+    """The value of time.time() when created.
+
+    Use in HTTP dates.
+    """
 
     stream = False
     """If False, buffer the response body."""
@@ -858,15 +896,15 @@ class Response(object):
         return new_body
 
     def _flush_body(self):
-        """
-        Discard self.body but consume any generator such that
-        any finalization can occur, such as is required by
-        caching.tee_output().
-        """
+        """Discard self.body but consume any generator such that any
+        finalization can occur, such as is required by caching.tee_output()."""
         consume(iter(self.body))
 
     def finalize(self):
-        """Transform headers (and cookies) into self.header_list. (Core)"""
+        """Transform headers (and cookies) into self.header_list.
+
+        (Core)
+        """
         try:
             code, reason, _ = httputil.valid_status(self.status)
         except ValueError:

lib/cherrypy/_cpserver.py

@@ -50,7 +50,8 @@ class Server(ServerAdapter):
     """If given, the name of the UNIX socket to use instead of TCP/IP.
 
     When this option is not None, the `socket_host` and `socket_port` options
-    are ignored."""
+    are ignored.
+    """
 
     socket_queue_size = 5
     """The 'backlog' argument to socket.listen(); specifies the maximum number
@@ -79,17 +80,24 @@ class Server(ServerAdapter):
     """The number of worker threads to start up in the pool."""
 
     thread_pool_max = -1
-    """The maximum size of the worker-thread pool. Use -1 to indicate no limit.
+    """The maximum size of the worker-thread pool.
+
+    Use -1 to indicate no limit.
     """
 
     max_request_header_size = 500 * 1024
     """The maximum number of bytes allowable in the request headers.
-    If exceeded, the HTTP server should return "413 Request Entity Too Large".
+
+    If exceeded, the HTTP server should return "413 Request Entity Too
+    Large".
     """
 
     max_request_body_size = 100 * 1024 * 1024
-    """The maximum number of bytes allowable in the request body. If exceeded,
-    the HTTP server should return "413 Request Entity Too Large"."""
+    """The maximum number of bytes allowable in the request body.
+
+    If exceeded, the HTTP server should return "413 Request Entity Too
+    Large".
+    """
 
     instance = None
     """If not None, this should be an HTTP server instance (such as
@@ -119,7 +127,8 @@ class Server(ServerAdapter):
     the builtin WSGI server. Builtin options are: 'builtin' (to
     use the SSL library built into recent versions of Python).
     You may also register your own classes in the
-    cheroot.server.ssl_adapters dict."""
+    cheroot.server.ssl_adapters dict.
+    """
 
     statistics = False
     """Turns statistics-gathering on or off for aware HTTP servers."""
@@ -129,11 +138,13 @@ class Server(ServerAdapter):
 
     wsgi_version = (1, 0)
     """The WSGI version tuple to use with the builtin WSGI server.
-    The provided options are (1, 0) [which includes support for PEP 3333,
-    which declares it covers WSGI version 1.0.1 but still mandates the
-    wsgi.version (1, 0)] and ('u', 0), an experimental unicode version.
-    You may create and register your own experimental versions of the WSGI
-    protocol by adding custom classes to the cheroot.server.wsgi_gateways dict.
+
+    The provided options are (1, 0) [which includes support for PEP
+    3333, which declares it covers WSGI version 1.0.1 but still mandates
+    the wsgi.version (1, 0)] and ('u', 0), an experimental unicode
+    version. You may create and register your own experimental versions
+    of the WSGI protocol by adding custom classes to the
+    cheroot.server.wsgi_gateways dict.
     """
 
     peercreds = False
@@ -184,7 +195,8 @@ class Server(ServerAdapter):
     def bind_addr(self):
         """Return bind address.
 
-        A (host, port) tuple for TCP sockets or a str for Unix domain sockts.
+        A (host, port) tuple for TCP sockets or a str for Unix domain
+        sockets.
         """
         if self.socket_file:
             return self.socket_file

lib/cherrypy/_cptools.py

@@ -1,7 +1,7 @@
 """CherryPy tools. A "tool" is any helper, adapted to CP.
 
-Tools are usually designed to be used in a variety of ways (although some
-may only offer one if they choose):
+Tools are usually designed to be used in a variety of ways (although
+some may only offer one if they choose):
 
     Library calls
         All tools are callables that can be used wherever needed.
@@ -48,10 +48,10 @@ _attr_error = (
 
 
 class Tool(object):
-
     """A registered function for use with CherryPy request-processing hooks.
 
-    help(tool.callable) should give you more information about this Tool.
+    help(tool.callable) should give you more information about this
+    Tool.
     """
 
     namespace = 'tools'
@@ -135,8 +135,8 @@ class Tool(object):
     def _setup(self):
         """Hook this tool into cherrypy.request.
 
-        The standard CherryPy request object will automatically call this
-        method when the tool is "turned on" in config.
+        The standard CherryPy request object will automatically call
+        this method when the tool is "turned on" in config.
         """
         conf = self._merged_args()
         p = conf.pop('priority', None)
@@ -147,15 +147,15 @@ class Tool(object):
 
 
 class HandlerTool(Tool):
-
     """Tool which is called 'before main', that may skip normal handlers.
 
-    If the tool successfully handles the request (by setting response.body),
-    if should return True. This will cause CherryPy to skip any 'normal' page
-    handler. If the tool did not handle the request, it should return False
-    to tell CherryPy to continue on and call the normal page handler. If the
-    tool is declared AS a page handler (see the 'handler' method), returning
-    False will raise NotFound.
+    If the tool successfully handles the request (by setting
+    response.body), if should return True. This will cause CherryPy to
+    skip any 'normal' page handler. If the tool did not handle the
+    request, it should return False to tell CherryPy to continue on and
+    call the normal page handler. If the tool is declared AS a page
+    handler (see the 'handler' method), returning False will raise
+    NotFound.
     """
 
     def __init__(self, callable, name=None):
@@ -185,8 +185,8 @@ class HandlerTool(Tool):
     def _setup(self):
         """Hook this tool into cherrypy.request.
 
-        The standard CherryPy request object will automatically call this
-        method when the tool is "turned on" in config.
+        The standard CherryPy request object will automatically call
+        this method when the tool is "turned on" in config.
         """
         conf = self._merged_args()
         p = conf.pop('priority', None)
@@ -197,7 +197,6 @@ class HandlerTool(Tool):
 
 
 class HandlerWrapperTool(Tool):
-
     """Tool which wraps request.handler in a provided wrapper function.
 
     The 'newhandler' arg must be a handler wrapper function that takes a
@@ -232,7 +231,6 @@ class HandlerWrapperTool(Tool):
 
 
 class ErrorTool(Tool):
-
     """Tool which is used to replace the default request.error_response."""
 
     def __init__(self, callable, name=None):
@@ -244,8 +242,8 @@ class ErrorTool(Tool):
     def _setup(self):
         """Hook this tool into cherrypy.request.
 
-        The standard CherryPy request object will automatically call this
-        method when the tool is "turned on" in config.
+        The standard CherryPy request object will automatically call
+        this method when the tool is "turned on" in config.
         """
         cherrypy.serving.request.error_response = self._wrapper
 
@@ -254,7 +252,6 @@ class ErrorTool(Tool):
 
 
 class SessionTool(Tool):
-
     """Session Tool for CherryPy.
 
     sessions.locking
@@ -282,8 +279,8 @@ class SessionTool(Tool):
     def _setup(self):
         """Hook this tool into cherrypy.request.
 
-        The standard CherryPy request object will automatically call this
-        method when the tool is "turned on" in config.
+        The standard CherryPy request object will automatically call
+        this method when the tool is "turned on" in config.
         """
         hooks = cherrypy.serving.request.hooks
 
@@ -325,7 +322,6 @@ class SessionTool(Tool):
 
 
 class XMLRPCController(object):
-
     """A Controller (page handler collection) for XML-RPC.
 
     To use it, have your controllers subclass this base class (it will
@@ -392,7 +388,6 @@ class SessionAuthTool(HandlerTool):
 
 
 class CachingTool(Tool):
-
     """Caching Tool for CherryPy."""
 
     def _wrapper(self, **kwargs):
@@ -416,11 +411,11 @@ class CachingTool(Tool):
 
 
 class Toolbox(object):
-
     """A collection of Tools.
 
     This object also functions as a config namespace handler for itself.
-    Custom toolboxes should be added to each Application's toolboxes dict.
+    Custom toolboxes should be added to each Application's toolboxes
+    dict.
     """
 
     def __init__(self, namespace):

lib/cherrypy/_cptree.py

@@ -10,19 +10,22 @@ from cherrypy.lib import httputil, reprconf
 class Application(object):
     """A CherryPy Application.
 
-    Servers and gateways should not instantiate Request objects directly.
-    Instead, they should ask an Application object for a request object.
+    Servers and gateways should not instantiate Request objects
+    directly. Instead, they should ask an Application object for a
+    request object.
 
-    An instance of this class may also be used as a WSGI callable
-    (WSGI application object) for itself.
+    An instance of this class may also be used as a WSGI callable (WSGI
+    application object) for itself.
     """
 
     root = None
-    """The top-most container of page handlers for this app. Handlers should
-    be arranged in a hierarchy of attributes, matching the expected URI
-    hierarchy; the default dispatcher then searches this hierarchy for a
-    matching handler. When using a dispatcher other than the default,
-    this value may be None."""
+    """The top-most container of page handlers for this app.
+
+    Handlers should be arranged in a hierarchy of attributes, matching
+    the expected URI hierarchy; the default dispatcher then searches
+    this hierarchy for a matching handler. When using a dispatcher other
+    than the default, this value may be None.
+    """
 
     config = {}
     """A dict of {path: pathconf} pairs, where 'pathconf' is itself a dict
@@ -32,10 +35,16 @@ class Application(object):
     toolboxes = {'tools': cherrypy.tools}
 
     log = None
-    """A LogManager instance. See _cplogging."""
+    """A LogManager instance.
+
+    See _cplogging.
+    """
 
     wsgiapp = None
-    """A CPWSGIApp instance. See _cpwsgi."""
+    """A CPWSGIApp instance.
+
+    See _cpwsgi.
+    """
 
     request_class = _cprequest.Request
     response_class = _cprequest.Response
@@ -82,12 +91,15 @@ class Application(object):
     def script_name(self):  # noqa: D401; irrelevant for properties
         """The URI "mount point" for this app.
 
-        A mount point is that portion of the URI which is constant for all URIs
-        that are serviced by this application; it does not include scheme,
-        host, or proxy ("virtual host") portions of the URI.
+        A mount point is that portion of the URI which is constant for
+        all URIs that are serviced by this application; it does not
+        include scheme, host, or proxy ("virtual host") portions of the
+        URI.
 
-        For example, if script_name is "/my/cool/app", then the URL
-        "http://www.example.com/my/cool/app/page1" might be handled by a
+        For example, if script_name is "/my/cool/app", then the URL "
+
+        http://www.example.com/my/cool/app/page1"
+        might be handled by a
         "page1" method on the root object.
 
         The value of script_name MUST NOT end in a slash. If the script_name
@@ -171,9 +183,9 @@ class Application(object):
 class Tree(object):
     """A registry of CherryPy applications, mounted at diverse points.
 
-    An instance of this class may also be used as a WSGI callable
-    (WSGI application object), in which case it dispatches to all
-    mounted apps.
+    An instance of this class may also be used as a WSGI callable (WSGI
+    application object), in which case it dispatches to all mounted
+    apps.
     """
 
     apps = {}

lib/cherrypy/_cpwsgi.py

@@ -1,10 +1,10 @@
 """WSGI interface (see PEP 333 and 3333).
 
 Note that WSGI environ keys and values are 'native strings'; that is,
-whatever the type of "" is. For Python 2, that's a byte string; for Python 3,
-it's a unicode string. But PEP 3333 says: "even if Python's str type is
-actually Unicode "under the hood", the content of native strings must
-still be translatable to bytes via the Latin-1 encoding!"
+whatever the type of "" is. For Python 2, that's a byte string; for
+Python 3, it's a unicode string. But PEP 3333 says: "even if Python's
+str type is actually Unicode "under the hood", the content of native
+strings must still be translatable to bytes via the Latin-1 encoding!"
 """
 
 import sys as _sys
@@ -34,7 +34,6 @@ def downgrade_wsgi_ux_to_1x(environ):
 
 
 class VirtualHost(object):
-
     """Select a different WSGI application based on the Host header.
 
     This can be useful when running multiple sites within one CP server.
@@ -56,7 +55,10 @@ class VirtualHost(object):
         cherrypy.tree.graft(vhost)
     """
     default = None
-    """Required. The default WSGI application."""
+    """Required.
+
+    The default WSGI application.
+    """
 
     use_x_forwarded_host = True
     """If True (the default), any "X-Forwarded-Host"
@@ -65,11 +67,12 @@ class VirtualHost(object):
 
     domains = {}
     """A dict of {host header value: application} pairs.
-    The incoming "Host" request header is looked up in this dict,
-    and, if a match is found, the corresponding WSGI application
-    will be called instead of the default. Note that you often need
-    separate entries for "example.com" and "www.example.com".
-    In addition, "Host" headers may contain the port number.
+
+    The incoming "Host" request header is looked up in this dict, and,
+    if a match is found, the corresponding WSGI application will be
+    called instead of the default. Note that you often need separate
+    entries for "example.com" and "www.example.com". In addition, "Host"
+    headers may contain the port number.
     """
 
     def __init__(self, default, domains=None, use_x_forwarded_host=True):
@@ -89,7 +92,6 @@ class VirtualHost(object):
 
 
 class InternalRedirector(object):
-
     """WSGI middleware that handles raised cherrypy.InternalRedirect."""
 
     def __init__(self, nextapp, recursive=False):
@@ -137,7 +139,6 @@ class InternalRedirector(object):
 
 
 class ExceptionTrapper(object):
-
     """WSGI middleware that traps exceptions."""
 
     def __init__(self, nextapp, throws=(KeyboardInterrupt, SystemExit)):
@@ -226,7 +227,6 @@ class _TrappedResponse(object):
 
 
 class AppResponse(object):
-
     """WSGI response iterable for CherryPy applications."""
 
     def __init__(self, environ, start_response, cpapp):
@@ -277,7 +277,10 @@ class AppResponse(object):
         return next(self.iter_response)
 
     def close(self):
-        """Close and de-reference the current request and response. (Core)"""
+        """Close and de-reference the current request and response.
+
+        (Core)
+        """
         streaming = _cherrypy.serving.response.stream
         self.cpapp.release_serving()
 
@@ -380,18 +383,20 @@ class AppResponse(object):
 
 
 class CPWSGIApp(object):
-
     """A WSGI application object for a CherryPy Application."""
 
     pipeline = [
         ('ExceptionTrapper', ExceptionTrapper),
         ('InternalRedirector', InternalRedirector),
     ]
-    """A list of (name, wsgiapp) pairs. Each 'wsgiapp' MUST be a
-    constructor that takes an initial, positional 'nextapp' argument,
-    plus optional keyword arguments, and returns a WSGI application
-    (that takes environ and start_response arguments). The 'name' can
-    be any you choose, and will correspond to keys in self.config."""
+    """A list of (name, wsgiapp) pairs.
+
+    Each 'wsgiapp' MUST be a constructor that takes an initial,
+    positional 'nextapp' argument, plus optional keyword arguments, and
+    returns a WSGI application (that takes environ and start_response
+    arguments). The 'name' can be any you choose, and will correspond to
+    keys in self.config.
+    """
 
     head = None
     """Rather than nest all apps in the pipeline on each call, it's only
@@ -399,9 +404,12 @@ class CPWSGIApp(object):
     this to None again if you change self.pipeline after calling self."""
 
     config = {}
-    """A dict whose keys match names listed in the pipeline. Each
-    value is a further dict which will be passed to the corresponding
-    named WSGI callable (from the pipeline) as keyword arguments."""
+    """A dict whose keys match names listed in the pipeline.
+
+    Each value is a further dict which will be passed to the
+    corresponding named WSGI callable (from the pipeline) as keyword
+    arguments.
+    """
 
     response_class = AppResponse
     """The class to instantiate and return as the next app in the WSGI chain.
@@ -417,8 +425,8 @@ class CPWSGIApp(object):
     def tail(self, environ, start_response):
         """WSGI application callable for the actual CherryPy application.
 
-        You probably shouldn't call this; call self.__call__ instead,
-        so that any WSGI middleware in self.pipeline can run first.
+        You probably shouldn't call this; call self.__call__ instead, so
+        that any WSGI middleware in self.pipeline can run first.
         """
         return self.response_class(environ, start_response, self.cpapp)
 

lib/cherrypy/_cpwsgi_server.py

@@ -1,7 +1,7 @@
-"""
-WSGI server interface (see PEP 333).
+"""WSGI server interface (see PEP 333).
 
-This adds some CP-specific bits to the framework-agnostic cheroot package.
+This adds some CP-specific bits to the framework-agnostic cheroot
+package.
 """
 import sys
 
@@ -35,10 +35,11 @@ class CPWSGIHTTPRequest(cheroot.server.HTTPRequest):
 class CPWSGIServer(cheroot.wsgi.Server):
     """Wrapper for cheroot.wsgi.Server.
 
-    cheroot has been designed to not reference CherryPy in any way,
-    so that it can be used in other frameworks and applications. Therefore,
-    we wrap it here, so we can set our own mount points from cherrypy.tree
-    and apply some attributes from config -> cherrypy.server -> wsgi.Server.
+    cheroot has been designed to not reference CherryPy in any way, so
+    that it can be used in other frameworks and applications. Therefore,
+    we wrap it here, so we can set our own mount points from
+    cherrypy.tree and apply some attributes from config ->
+    cherrypy.server -> wsgi.Server.
     """
 
     fmt = 'CherryPy/{cherrypy.__version__} {cheroot.wsgi.Server.version}'

lib/cherrypy/_helper.py

@@ -137,7 +137,6 @@ def popargs(*args, **kwargs):
         class Root:
             def index(self):
                 #...
-
     """
     # Since keyword arg comes after *args, we have to process it ourselves
     # for lower versions of python.
@@ -201,16 +200,17 @@ def url(path='', qs='', script_name=None, base=None, relative=None):
     If it does not start with a slash, this returns
         (base + script_name [+ request.path_info] + path + qs).
 
-    If script_name is None, cherrypy.request will be used
-    to find a script_name, if available.
+    If script_name is None, cherrypy.request will be used to find a
+    script_name, if available.
 
     If base is None, cherrypy.request.base will be used (if available).
     Note that you can use cherrypy.tools.proxy to change this.
 
-    Finally, note that this function can be used to obtain an absolute URL
-    for the current request path (minus the querystring) by passing no args.
-    If you call url(qs=cherrypy.request.query_string), you should get the
-    original browser URL (assuming no internal redirections).
+    Finally, note that this function can be used to obtain an absolute
+    URL for the current request path (minus the querystring) by passing
+    no args. If you call url(qs=cherrypy.request.query_string), you
+    should get the original browser URL (assuming no internal
+    redirections).
 
     If relative is None or not provided, request.app.relative_urls will
     be used (if available, else False). If False, the output will be an
@@ -320,8 +320,8 @@ def normalize_path(path):
 class _ClassPropertyDescriptor(object):
     """Descript for read-only class-based property.
 
-    Turns a classmethod-decorated func into a read-only property of that class
-    type (means the value cannot be set).
+    Turns a classmethod-decorated func into a read-only property of that
+    class type (means the value cannot be set).
     """
 
     def __init__(self, fget, fset=None):

lib/cherrypy/_json.py

@@ -1,5 +1,4 @@
-"""
-JSON support.
+"""JSON support.
 
 Expose preferred json module as json and provide encode/decode
 convenience functions.

lib/cherrypy/lib/__init__.py

@@ -6,8 +6,8 @@ def is_iterator(obj):
 
     (i.e. like a generator).
 
-    This will return False for objects which are iterable,
-    but not iterators themselves.
+    This will return False for objects which are iterable, but not
+    iterators themselves.
     """
     from types import GeneratorType
     if isinstance(obj, GeneratorType):

lib/cherrypy/lib/auth_basic.py

@@ -18,7 +18,6 @@ as the credentials store::
                   'tools.auth_basic.accept_charset': 'UTF-8',
     }
     app_config = { '/' : basic_auth }
-
 """
 
 import binascii

lib/cherrypy/lib/auth_digest.py

@@ -55,7 +55,7 @@ def TRACE(msg):
 
 
 def get_ha1_dict_plain(user_password_dict):
-    """Returns a get_ha1 function which obtains a plaintext password from a
+    """Return a get_ha1 function which obtains a plaintext password from a
     dictionary of the form: {username : password}.
 
     If you want a simple dictionary-based authentication scheme, with plaintext
@@ -72,7 +72,7 @@ def get_ha1_dict_plain(user_password_dict):
 
 
 def get_ha1_dict(user_ha1_dict):
-    """Returns a get_ha1 function which obtains a HA1 password hash from a
+    """Return a get_ha1 function which obtains a HA1 password hash from a
     dictionary of the form: {username : HA1}.
 
     If you want a dictionary-based authentication scheme, but with
@@ -87,7 +87,7 @@ def get_ha1_dict(user_ha1_dict):
 
 
 def get_ha1_file_htdigest(filename):
-    """Returns a get_ha1 function which obtains a HA1 password hash from a
+    """Return a get_ha1 function which obtains a HA1 password hash from a
     flat file with lines of the same format as that produced by the Apache
     htdigest utility. For example, for realm 'wonderland', username 'alice',
     and password '4x5istwelve', the htdigest line would be::
@@ -101,13 +101,12 @@ def get_ha1_file_htdigest(filename):
     """
     def get_ha1(realm, username):
         result = None
-        f = open(filename, 'r')
-        for line in f:
-            u, r, ha1 = line.rstrip().split(':')
-            if u == username and r == realm:
-                result = ha1
-                break
-        f.close()
+        with open(filename, 'r') as f:
+            for line in f:
+                u, r, ha1 = line.rstrip().split(':')
+                if u == username and r == realm:
+                    result = ha1
+                    break
         return result
 
     return get_ha1
@@ -136,7 +135,7 @@ def synthesize_nonce(s, key, timestamp=None):
 
 
 def H(s):
-    """The hash function H"""
+    """The hash function H."""
     return md5_hex(s)
 
 
@@ -260,10 +259,11 @@ class HttpDigestAuthorization(object):
         return False
 
     def is_nonce_stale(self, max_age_seconds=600):
-        """Returns True if a validated nonce is stale. The nonce contains a
-        timestamp in plaintext and also a secure hash of the timestamp.
-        You should first validate the nonce to ensure the plaintext
-        timestamp is not spoofed.
+        """Return True if a validated nonce is stale.
+
+        The nonce contains a timestamp in plaintext and also a secure
+        hash of the timestamp. You should first validate the nonce to
+        ensure the plaintext timestamp is not spoofed.
         """
         try:
             timestamp, hashpart = self.nonce.split(':', 1)
@@ -276,7 +276,10 @@ class HttpDigestAuthorization(object):
         return True
 
     def HA2(self, entity_body=''):
-        """Returns the H(A2) string. See :rfc:`2617` section 3.2.2.3."""
+        """Return the H(A2) string.
+
+        See :rfc:`2617` section 3.2.2.3.
+        """
         # RFC 2617 3.2.2.3
         # If the "qop" directive's value is "auth" or is unspecified,
         # then A2 is:
@@ -307,7 +310,6 @@ class HttpDigestAuthorization(object):
             4.3.  This refers to the entity the user agent sent in the
             request which has the Authorization header. Typically GET
             requests don't have an entity, and POST requests do.
-
         """
         ha2 = self.HA2(entity_body)
         # Request-Digest -- RFC 2617 3.2.2.1
@@ -396,7 +398,6 @@ def digest_auth(realm, get_ha1, key, debug=False, accept_charset='utf-8'):
     key
         A secret string known only to the server, used in the synthesis
         of nonces.
-
     """
     request = cherrypy.serving.request
 
@@ -448,9 +449,7 @@ def digest_auth(realm, get_ha1, key, debug=False, accept_charset='utf-8'):
 
 
 def _respond_401(realm, key, accept_charset, debug, **kwargs):
-    """
-    Respond with 401 status and a WWW-Authenticate header
-    """
+    """Respond with 401 status and a WWW-Authenticate header."""
     header = www_authenticate(
         realm, key,
         accept_charset=accept_charset,

lib/cherrypy/lib/caching.py

@@ -42,7 +42,6 @@ from cherrypy.lib import cptools, httputil
 
 
 class Cache(object):
-
     """Base class for Cache implementations."""
 
     def get(self):
@@ -64,17 +63,16 @@ class Cache(object):
 
 # ------------------------------ Memory Cache ------------------------------- #
 class AntiStampedeCache(dict):
-
     """A storage system for cached items which reduces stampede collisions."""
 
     def wait(self, key, timeout=5, debug=False):
         """Return the cached value for the given key, or None.
 
-        If timeout is not None, and the value is already
-        being calculated by another thread, wait until the given timeout has
-        elapsed. If the value is available before the timeout expires, it is
-        returned. If not, None is returned, and a sentinel placed in the cache
-        to signal other threads to wait.
+        If timeout is not None, and the value is already being
+        calculated by another thread, wait until the given timeout has
+        elapsed. If the value is available before the timeout expires,
+        it is returned. If not, None is returned, and a sentinel placed
+        in the cache to signal other threads to wait.
 
         If timeout is None, no waiting is performed nor sentinels used.
         """
@@ -127,7 +125,6 @@ class AntiStampedeCache(dict):
 
 
 class MemoryCache(Cache):
-
     """An in-memory cache for varying response content.
 
     Each key in self.store is a URI, and each value is an AntiStampedeCache.
@@ -381,7 +378,10 @@ def get(invalid_methods=('POST', 'PUT', 'DELETE'), debug=False, **kwargs):
 
 
 def tee_output():
-    """Tee response output to cache storage. Internal."""
+    """Tee response output to cache storage.
+
+    Internal.
+    """
     # Used by CachingTool by attaching to request.hooks
 
     request = cherrypy.serving.request
@@ -441,7 +441,6 @@ def expires(secs=0, force=False, debug=False):
             * Expires
 
         If any are already present, none of the above response headers are set.
-
     """
 
     response = cherrypy.serving.response

lib/cherrypy/lib/covercp.py

@@ -334,9 +334,10 @@ class CoverStats(object):
         yield '</body></html>'
 
     def annotated_file(self, filename, statements, excluded, missing):
-        source = open(filename, 'r')
+        with open(filename, 'r') as source:
+            lines = source.readlines()
         buffer = []
-        for lineno, line in enumerate(source.readlines()):
+        for lineno, line in enumerate(lines):
             lineno += 1
             line = line.strip('\n\r')
             empty_the_buffer = True

lib/cherrypy/lib/cpstats.py

@@ -184,7 +184,6 @@ To report statistics::
 To format statistics reports::
 
     See 'Reporting', above.
-
 """
 
 import logging
@@ -254,7 +253,6 @@ def proc_time(s):
 
 
 class ByteCountWrapper(object):
-
     """Wraps a file-like object, counting the number of bytes read."""
 
     def __init__(self, rfile):
@@ -307,7 +305,6 @@ def _get_threading_ident():
 
 
 class StatsTool(cherrypy.Tool):
-
     """Record various information about the current request."""
 
     def __init__(self):
@@ -316,8 +313,8 @@ class StatsTool(cherrypy.Tool):
     def _setup(self):
         """Hook this tool into cherrypy.request.
 
-        The standard CherryPy request object will automatically call this
-        method when the tool is "turned on" in config.
+        The standard CherryPy request object will automatically call
+        this method when the tool is "turned on" in config.
         """
         if appstats.get('Enabled', False):
             cherrypy.Tool._setup(self)

lib/cherrypy/lib/cptools.py

@@ -94,8 +94,8 @@ def validate_etags(autotags=False, debug=False):
 def validate_since():
     """Validate the current Last-Modified against If-Modified-Since headers.
 
-    If no code has set the Last-Modified response header, then no validation
-    will be performed.
+    If no code has set the Last-Modified response header, then no
+    validation will be performed.
     """
     response = cherrypy.serving.response
     lastmod = response.headers.get('Last-Modified')
@@ -123,9 +123,9 @@ def validate_since():
 def allow(methods=None, debug=False):
     """Raise 405 if request.method not in methods (default ['GET', 'HEAD']).
 
-    The given methods are case-insensitive, and may be in any order.
-    If only one method is allowed, you may supply a single string;
-    if more than one, supply a list of strings.
+    The given methods are case-insensitive, and may be in any order. If
+    only one method is allowed, you may supply a single string; if more
+    than one, supply a list of strings.
 
     Regardless of whether the current method is allowed or not, this
     also emits an 'Allow' response header, containing the given methods.
@@ -154,22 +154,23 @@ def proxy(base=None, local='X-Forwarded-Host', remote='X-Forwarded-For',
           scheme='X-Forwarded-Proto', debug=False):
     """Change the base URL (scheme://host[:port][/path]).
 
-    For running a CP server behind Apache, lighttpd, or other HTTP server.
+    For running a CP server behind Apache, lighttpd, or other HTTP
+    server.
 
-    For Apache and lighttpd, you should leave the 'local' argument at the
-    default value of 'X-Forwarded-Host'. For Squid, you probably want to set
-    tools.proxy.local = 'Origin'.
+    For Apache and lighttpd, you should leave the 'local' argument at
+    the default value of 'X-Forwarded-Host'. For Squid, you probably
+    want to set tools.proxy.local = 'Origin'.
 
-    If you want the new request.base to include path info (not just the host),
-    you must explicitly set base to the full base path, and ALSO set 'local'
-    to '', so that the X-Forwarded-Host request header (which never includes
-    path info) does not override it. Regardless, the value for 'base' MUST
-    NOT end in a slash.
+    If you want the new request.base to include path info (not just the
+    host), you must explicitly set base to the full base path, and ALSO
+    set 'local' to '', so that the X-Forwarded-Host request header
+    (which never includes path info) does not override it. Regardless,
+    the value for 'base' MUST NOT end in a slash.
 
     cherrypy.request.remote.ip (the IP address of the client) will be
-    rewritten if the header specified by the 'remote' arg is valid.
-    By default, 'remote' is set to 'X-Forwarded-For'. If you do not
-    want to rewrite remote.ip, set the 'remote' arg to an empty string.
+    rewritten if the header specified by the 'remote' arg is valid. By
+    default, 'remote' is set to 'X-Forwarded-For'. If you do not want to
+    rewrite remote.ip, set the 'remote' arg to an empty string.
     """
 
     request = cherrypy.serving.request
@@ -217,8 +218,8 @@ def proxy(base=None, local='X-Forwarded-Host', remote='X-Forwarded-For',
 def ignore_headers(headers=('Range',), debug=False):
     """Delete request headers whose field names are included in 'headers'.
 
-    This is a useful tool for working behind certain HTTP servers;
-    for example, Apache duplicates the work that CP does for 'Range'
+    This is a useful tool for working behind certain HTTP servers; for
+    example, Apache duplicates the work that CP does for 'Range'
     headers, and will doubly-truncate the response.
     """
     request = cherrypy.serving.request
@@ -281,7 +282,6 @@ def referer(pattern, accept=True, accept_missing=False, error=403,
 
 
 class SessionAuth(object):
-
     """Assert that the user is logged in."""
 
     session_key = 'username'
@@ -319,7 +319,10 @@ Message: %(error_msg)s
 </body></html>""") % vars()).encode('utf-8')
 
     def do_login(self, username, password, from_page='..', **kwargs):
-        """Login. May raise redirect, or return True if request handled."""
+        """Login.
+
+        May raise redirect, or return True if request handled.
+        """
         response = cherrypy.serving.response
         error_msg = self.check_username_and_password(username, password)
         if error_msg:
@@ -336,7 +339,10 @@ Message: %(error_msg)s
             raise cherrypy.HTTPRedirect(from_page or '/')
 
     def do_logout(self, from_page='..', **kwargs):
-        """Logout. May raise redirect, or return True if request handled."""
+        """Logout.
+
+        May raise redirect, or return True if request handled.
+        """
         sess = cherrypy.session
         username = sess.get(self.session_key)
         sess[self.session_key] = None
@@ -346,7 +352,9 @@ Message: %(error_msg)s
         raise cherrypy.HTTPRedirect(from_page)
 
     def do_check(self):
-        """Assert username. Raise redirect, or return True if request handled.
+        """Assert username.
+
+        Raise redirect, or return True if request handled.
         """
         sess = cherrypy.session
         request = cherrypy.serving.request
@@ -408,8 +416,7 @@ def session_auth(**kwargs):
 
     Any attribute of the SessionAuth class may be overridden
     via a keyword arg to this function:
-
-    """ + '\n    '.join(
+    """ + '\n' + '\n    '.join(
         '{!s}: {!s}'.format(k, type(getattr(SessionAuth, k)).__name__)
         for k in dir(SessionAuth)
         if not k.startswith('__')
@@ -490,8 +497,8 @@ def trailing_slash(missing=True, extra=False, status=None, debug=False):
 def flatten(debug=False):
     """Wrap response.body in a generator that recursively iterates over body.
 
-    This allows cherrypy.response.body to consist of 'nested generators';
-    that is, a set of generators that yield generators.
+    This allows cherrypy.response.body to consist of 'nested
+    generators'; that is, a set of generators that yield generators.
     """
     def flattener(input):
         numchunks = 0
@@ -622,13 +629,15 @@ def autovary(ignore=None, debug=False):
 
 
 def convert_params(exception=ValueError, error=400):
-    """Convert request params based on function annotations, with error handling.
+    """Convert request params based on function annotations.
 
-    exception
-        Exception class to catch.
+    This function also processes errors that are subclasses of ``exception``.
 
-    status
-        The HTTP error code to return to the client on failure.
+    :param BaseException exception: Exception class to catch.
+    :type exception: BaseException
+
+    :param error: The HTTP status code to return to the client on failure.
+    :type error: int
     """
     request = cherrypy.serving.request
     types = request.handler.callable.__annotations__

lib/cherrypy/lib/encoding.py

@@ -261,9 +261,7 @@ class ResponseEncoder:
 
 
 def prepare_iter(value):
-    """
-    Ensure response body is iterable and resolves to False when empty.
-    """
+    """Ensure response body is iterable and resolves to False when empty."""
     if isinstance(value, text_or_bytes):
         # strings get wrapped in a list because iterating over a single
         # item list is much faster than iterating over every character
@@ -360,7 +358,6 @@ def gzip(compress_level=5, mime_types=['text/html', 'text/plain'],
         * No 'gzip' or 'x-gzip' is present in the Accept-Encoding header
         * No 'gzip' or 'x-gzip' with a qvalue > 0 is present
         * The 'identity' value is given with a qvalue > 0.
-
     """
     request = cherrypy.serving.request
     response = cherrypy.serving.response

lib/cherrypy/lib/gctools.py

@@ -14,7 +14,6 @@ from cherrypy.process.plugins import SimplePlugin
 
 
 class ReferrerTree(object):
-
     """An object which gathers all referrers of an object to a given depth."""
 
     peek_length = 40
@@ -132,7 +131,6 @@ def get_context(obj):
 
 
 class GCRoot(object):
-
     """A CherryPy page handler for testing reference leaks."""
 
     classes = [

lib/cherrypy/lib/httputil.py

@@ -71,10 +71,10 @@ def protocol_from_http(protocol_str):
 def get_ranges(headervalue, content_length):
     """Return a list of (start, stop) indices from a Range header, or None.
 
-    Each (start, stop) tuple will be composed of two ints, which are suitable
-    for use in a slicing operation. That is, the header "Range: bytes=3-6",
-    if applied against a Python string, is requesting resource[3:7]. This
-    function will return the list [(3, 7)].
+    Each (start, stop) tuple will be composed of two ints, which are
+    suitable for use in a slicing operation. That is, the header "Range:
+    bytes=3-6", if applied against a Python string, is requesting
+    resource[3:7]. This function will return the list [(3, 7)].
 
     If this function returns an empty list, you should return HTTP 416.
     """
@@ -127,7 +127,6 @@ def get_ranges(headervalue, content_length):
 
 
 class HeaderElement(object):
-
     """An element (with parameters) from an HTTP header's element list."""
 
     def __init__(self, value, params=None):
@@ -169,14 +168,14 @@ q_separator = re.compile(r'; *q *=')
 
 
 class AcceptElement(HeaderElement):
-
     """An element (with parameters) from an Accept* header's element list.
 
-    AcceptElement objects are comparable; the more-preferred object will be
-    "less than" the less-preferred object. They are also therefore sortable;
-    if you sort a list of AcceptElement objects, they will be listed in
-    priority order; the most preferred value will be first. Yes, it should
-    have been the other way around, but it's too late to fix now.
+    AcceptElement objects are comparable; the more-preferred object will
+    be "less than" the less-preferred object. They are also therefore
+    sortable; if you sort a list of AcceptElement objects, they will be
+    listed in priority order; the most preferred value will be first.
+    Yes, it should have been the other way around, but it's too late to
+    fix now.
     """
 
     @classmethod
@@ -249,8 +248,7 @@ def header_elements(fieldname, fieldvalue):
 
 
 def decode_TEXT(value):
-    r"""
-    Decode :rfc:`2047` TEXT
+    r"""Decode :rfc:`2047` TEXT.
 
     >>> decode_TEXT("=?utf-8?q?f=C3=BCr?=") == b'f\xfcr'.decode('latin-1')
     True
@@ -265,9 +263,7 @@ def decode_TEXT(value):
 
 
 def decode_TEXT_maybe(value):
-    """
-    Decode the text but only if '=?' appears in it.
-    """
+    """Decode the text but only if '=?' appears in it."""
     return decode_TEXT(value) if '=?' in value else value
 
 
@@ -388,7 +384,6 @@ def parse_query_string(query_string, keep_blank_values=True, encoding='utf-8'):
 
 
 class CaseInsensitiveDict(jaraco.collections.KeyTransformingDict):
-
     """A case-insensitive dict subclass.
 
     Each key is changed on entry to title case.
@@ -417,7 +412,6 @@ else:
 
 
 class HeaderMap(CaseInsensitiveDict):
-
     """A dict subclass for HTTP request and response headers.
 
     Each key is changed on entry to str(key).title(). This allows headers
@@ -494,7 +488,6 @@ class HeaderMap(CaseInsensitiveDict):
 
 
 class Host(object):
-
     """An internet address.
 
     name
@@ -516,3 +509,33 @@ class Host(object):
 
     def __repr__(self):
         return 'httputil.Host(%r, %r, %r)' % (self.ip, self.port, self.name)
+
+
+class SanitizedHost(str):
+    r"""
+    Wraps a raw host header received from the network in
+    a sanitized version that elides dangerous characters.
+
+    >>> SanitizedHost('foo\nbar')
+    'foobar'
+    >>> SanitizedHost('foo\nbar').raw
+    'foo\nbar'
+
+    A SanitizedInstance is only returned if sanitization was performed.
+
+    >>> isinstance(SanitizedHost('foobar'), SanitizedHost)
+    False
+    """
+    dangerous = re.compile(r'[\n\r]')
+
+    def __new__(cls, raw):
+        sanitized = cls._sanitize(raw)
+        if sanitized == raw:
+            return raw
+        instance = super().__new__(cls, sanitized)
+        instance.raw = raw
+        return instance
+
+    @classmethod
+    def _sanitize(cls, raw):
+        return cls.dangerous.sub('', raw)

lib/cherrypy/lib/locking.py

@@ -7,22 +7,22 @@ class NeverExpires(object):
 
 
 class Timer(object):
-    """
-    A simple timer that will indicate when an expiration time has passed.
-    """
+    """A simple timer that will indicate when an expiration time has passed."""
     def __init__(self, expiration):
         'Create a timer that expires at `expiration` (UTC datetime)'
         self.expiration = expiration
 
     @classmethod
     def after(cls, elapsed):
-        """
-        Return a timer that will expire after `elapsed` passes.
-        """
-        return cls(datetime.datetime.utcnow() + elapsed)
+        """Return a timer that will expire after `elapsed` passes."""
+        return cls(
+            datetime.datetime.now(datetime.timezone.utc) + elapsed,
+        )
 
     def expired(self):
-        return datetime.datetime.utcnow() >= self.expiration
+        return datetime.datetime.now(
+            datetime.timezone.utc,
+        ) >= self.expiration
 
 
 class LockTimeout(Exception):
@@ -30,9 +30,7 @@ class LockTimeout(Exception):
 
 
 class LockChecker(object):
-    """
-    Keep track of the time and detect if a timeout has expired
-    """
+    """Keep track of the time and detect if a timeout has expired."""
     def __init__(self, session_id, timeout):
         self.session_id = session_id
         if timeout:

lib/cherrypy/lib/profiler.py

@@ -30,7 +30,6 @@ to get a quick sanity-check on overall CP performance. Use the
 ``--profile`` flag when running the test suite. Then, use the ``serve()``
 function to browse the results in a web browser. If you run this
 module from the command line, it will call ``serve()`` for you.
-
 """
 
 import io
@@ -47,7 +46,9 @@ try:
     import pstats
 
     def new_func_strip_path(func_name):
-        """Make profiler output more readable by adding `__init__` modules' parents
+        """Add ``__init__`` modules' parents.
+
+        This makes the profiler output more readable.
         """
         filename, line, name = func_name
         if filename.endswith('__init__.py'):

lib/cherrypy/lib/reprconf.py

@@ -27,18 +27,17 @@ from cherrypy._cpcompat import text_or_bytes
 
 
 class NamespaceSet(dict):
-
     """A dict of config namespace names and handlers.
 
-    Each config entry should begin with a namespace name; the corresponding
-    namespace handler will be called once for each config entry in that
-    namespace, and will be passed two arguments: the config key (with the
-    namespace removed) and the config value.
+    Each config entry should begin with a namespace name; the
+    corresponding namespace handler will be called once for each config
+    entry in that namespace, and will be passed two arguments: the
+    config key (with the namespace removed) and the config value.
 
     Namespace handlers may be any Python callable; they may also be
-    context managers, in which case their __enter__
-    method should return a callable to be used as the handler.
-    See cherrypy.tools (the Toolbox class) for an example.
+    context managers, in which case their __enter__ method should return
+    a callable to be used as the handler. See cherrypy.tools (the
+    Toolbox class) for an example.
     """
 
     def __call__(self, config):
@@ -48,9 +47,10 @@ class NamespaceSet(dict):
             A flat dict, where keys use dots to separate
             namespaces, and values are arbitrary.
 
-        The first name in each config key is used to look up the corresponding
-        namespace handler. For example, a config entry of {'tools.gzip.on': v}
-        will call the 'tools' namespace handler with the args: ('gzip.on', v)
+        The first name in each config key is used to look up the
+        corresponding namespace handler. For example, a config entry of
+        {'tools.gzip.on': v} will call the 'tools' namespace handler
+        with the args: ('gzip.on', v)
         """
         # Separate the given config into namespaces
         ns_confs = {}
@@ -103,7 +103,6 @@ class NamespaceSet(dict):
 
 
 class Config(dict):
-
     """A dict-like set of configuration data, with defaults and namespaces.
 
     May take a file, filename, or dict.
@@ -163,14 +162,11 @@ class Parser(configparser.ConfigParser):
             #     fp = open(filename)
             # except IOError:
             #     continue
-            fp = open(filename)
-            try:
+            with open(filename) as fp:
                 self._read(fp, filename)
-            finally:
-                fp.close()
 
     def as_dict(self, raw=False, vars=None):
-        """Convert an INI file to a dictionary"""
+        """Convert an INI file to a dictionary."""
         # Load INI file into a dict
         result = {}
         for section in self.sections():
@@ -191,7 +187,7 @@ class Parser(configparser.ConfigParser):
 
     def dict_from_file(self, file):
         if hasattr(file, 'read'):
-            self.readfp(file)
+            self.read_file(file)
         else:
             self.read(file)
         return self.as_dict()

lib/cherrypy/lib/sessions.py

@@ -120,7 +120,6 @@ missing = object()
 
 
 class Session(object):
-
     """A CherryPy dict-like Session object (one per request)."""
 
     _id = None
@@ -148,9 +147,11 @@ class Session(object):
     to session data."""
 
     loaded = False
+    """If True, data has been retrieved from storage.
+
+    This should happen automatically on the first attempt to access
+    session data.
     """
-    If True, data has been retrieved from storage. This should happen
-    automatically on the first attempt to access session data."""
 
     clean_thread = None
     'Class-level Monitor which calls self.clean_up.'
@@ -165,9 +166,10 @@ class Session(object):
     'True if the session requested by the client did not exist.'
 
     regenerated = False
+    """True if the application called session.regenerate().
+
+    This is not set by internal calls to regenerate the session id.
     """
-    True if the application called session.regenerate(). This is not set by
-    internal calls to regenerate the session id."""
 
     debug = False
     'If True, log debug information.'
@@ -335,8 +337,9 @@ class Session(object):
 
     def pop(self, key, default=missing):
         """Remove the specified key and return the corresponding value.
-        If key is not found, default is returned if given,
-        otherwise KeyError is raised.
+
+        If key is not found, default is returned if given, otherwise
+        KeyError is raised.
         """
         if not self.loaded:
             self.load()
@@ -351,13 +354,19 @@ class Session(object):
         return key in self._data
 
     def get(self, key, default=None):
-        """D.get(k[,d]) -> D[k] if k in D, else d.  d defaults to None."""
+        """D.get(k[,d]) -> D[k] if k in D, else d.
+
+        d defaults to None.
+        """
         if not self.loaded:
             self.load()
         return self._data.get(key, default)
 
     def update(self, d):
-        """D.update(E) -> None.  Update D from E: for k in E: D[k] = E[k]."""
+        """D.update(E) -> None.
+
+        Update D from E: for k in E: D[k] = E[k].
+        """
         if not self.loaded:
             self.load()
         self._data.update(d)
@@ -369,7 +378,10 @@ class Session(object):
         return self._data.setdefault(key, default)
 
     def clear(self):
-        """D.clear() -> None.  Remove all items from D."""
+        """D.clear() -> None.
+
+        Remove all items from D.
+        """
         if not self.loaded:
             self.load()
         self._data.clear()
@@ -492,7 +504,8 @@ class FileSession(Session):
         """Set up the storage system for file-based sessions.
 
         This should only be called once per process; this will be done
-        automatically when using sessions.init (as the built-in Tool does).
+        automatically when using sessions.init (as the built-in Tool
+        does).
         """
         # The 'storage_path' arg is required for file-based sessions.
         kwargs['storage_path'] = os.path.abspath(kwargs['storage_path'])
@@ -516,11 +529,8 @@ class FileSession(Session):
         if path is None:
             path = self._get_file_path()
         try:
-            f = open(path, 'rb')
-            try:
+            with open(path, 'rb') as f:
                 return pickle.load(f)
-            finally:
-                f.close()
         except (IOError, EOFError):
             e = sys.exc_info()[1]
             if self.debug:
@@ -531,11 +541,8 @@ class FileSession(Session):
     def _save(self, expiration_time):
         assert self.locked, ('The session was saved without being locked.  '
                              "Check your tools' priority levels.")
-        f = open(self._get_file_path(), 'wb')
-        try:
+        with open(self._get_file_path(), 'wb') as f:
             pickle.dump((self._data, expiration_time), f, self.pickle_protocol)
-        finally:
-            f.close()
 
     def _delete(self):
         assert self.locked, ('The session deletion without being locked.  '
@@ -622,7 +629,8 @@ class MemcachedSession(Session):
         """Set up the storage system for memcached-based sessions.
 
         This should only be called once per process; this will be done
-        automatically when using sessions.init (as the built-in Tool does).
+        automatically when using sessions.init (as the built-in Tool
+        does).
         """
         for k, v in kwargs.items():
             setattr(cls, k, v)

lib/cherrypy/lib/static.py

@@ -1,19 +1,18 @@
 """Module with helpers for serving static files."""
 
+import mimetypes
 import os
 import platform
 import re
 import stat
-import mimetypes
-import urllib.parse
 import unicodedata
-
+import urllib.parse
 from email.generator import _make_boundary as make_boundary
 from io import UnsupportedOperation
 
 import cherrypy
 from cherrypy._cpcompat import ntob
-from cherrypy.lib import cptools, httputil, file_generator_limited
+from cherrypy.lib import cptools, file_generator_limited, httputil
 
 
 def _setup_mimetypes():
@@ -57,15 +56,15 @@ def serve_file(path, content_type=None, disposition=None, name=None,
                debug=False):
     """Set status, headers, and body in order to serve the given path.
 
-    The Content-Type header will be set to the content_type arg, if provided.
-    If not provided, the Content-Type will be guessed by the file extension
-    of the 'path' argument.
+    The Content-Type header will be set to the content_type arg, if
+    provided. If not provided, the Content-Type will be guessed by the
+    file extension of the 'path' argument.
 
-    If disposition is not None, the Content-Disposition header will be set
-    to "<disposition>; filename=<name>; filename*=utf-8''<name>"
-    as described in :rfc:`6266#appendix-D`.
-    If name is None, it will be set to the basename of path.
-    If disposition is None, no Content-Disposition header will be written.
+    If disposition is not None, the Content-Disposition header will be
+    set to "<disposition>; filename=<name>; filename*=utf-8''<name>" as
+    described in :rfc:`6266#appendix-D`. If name is None, it will be set
+    to the basename of path. If disposition is None, no Content-
+    Disposition header will be written.
     """
     response = cherrypy.serving.response
 
@@ -185,7 +184,10 @@ def serve_fileobj(fileobj, content_type=None, disposition=None, name=None,
 
 
 def _serve_fileobj(fileobj, content_type, content_length, debug=False):
-    """Internal. Set response.body to the given file object, perhaps ranged."""
+    """Set ``response.body`` to the given file object, perhaps ranged.
+
+    Internal helper.
+    """
     response = cherrypy.serving.response
 
     # HTTP/1.0 didn't have Range/Accept-Ranges headers, or the 206 code

lib/cherrypy/process/plugins.py

@@ -31,7 +31,6 @@ _module__file__base = os.getcwd()
 
 
 class SimplePlugin(object):
-
     """Plugin base class which auto-subscribes methods for known channels."""
 
     bus = None
@@ -59,7 +58,6 @@ class SimplePlugin(object):
 
 
 class SignalHandler(object):
-
     """Register bus channels (and listeners) for system signals.
 
     You can modify what signals your application listens for, and what it does
@@ -171,8 +169,8 @@ class SignalHandler(object):
         If the optional 'listener' argument is provided, it will be
         subscribed as a listener for the given signal's channel.
 
-        If the given signal name or number is not available on the current
-        platform, ValueError is raised.
+        If the given signal name or number is not available on the
+        current platform, ValueError is raised.
         """
         if isinstance(signal, text_or_bytes):
             signum = getattr(_signal, signal, None)
@@ -218,11 +216,10 @@ except ImportError:
 
 
 class DropPrivileges(SimplePlugin):
-
     """Drop privileges. uid/gid arguments not available on Windows.
 
     Special thanks to `Gavin Baker
-    <http://antonym.org/2005/12/dropping-privileges-in-python.html>`_
+    <http://antonym.org/2005/12/dropping-privileges-in-python.html>`_.
     """
 
     def __init__(self, bus, umask=None, uid=None, gid=None):
@@ -234,7 +231,10 @@ class DropPrivileges(SimplePlugin):
 
     @property
     def uid(self):
-        """The uid under which to run. Availability: Unix."""
+        """The uid under which to run.
+
+        Availability: Unix.
+        """
         return self._uid
 
     @uid.setter
@@ -250,7 +250,10 @@ class DropPrivileges(SimplePlugin):
 
     @property
     def gid(self):
-        """The gid under which to run. Availability: Unix."""
+        """The gid under which to run.
+
+        Availability: Unix.
+        """
         return self._gid
 
     @gid.setter
@@ -332,7 +335,6 @@ class DropPrivileges(SimplePlugin):
 
 
 class Daemonizer(SimplePlugin):
-
     """Daemonize the running script.
 
     Use this with a Web Site Process Bus via::
@@ -423,7 +425,6 @@ class Daemonizer(SimplePlugin):
 
 
 class PIDFile(SimplePlugin):
-
     """Maintain a PID file via a WSPBus."""
 
     def __init__(self, bus, pidfile):
@@ -436,7 +437,8 @@ class PIDFile(SimplePlugin):
         if self.finalized:
             self.bus.log('PID %r already written to %r.' % (pid, self.pidfile))
         else:
-            open(self.pidfile, 'wb').write(ntob('%s\n' % pid, 'utf8'))
+            with open(self.pidfile, 'wb') as f:
+                f.write(ntob('%s\n' % pid, 'utf8'))
             self.bus.log('PID %r written to %r.' % (pid, self.pidfile))
             self.finalized = True
     start.priority = 70
@@ -452,12 +454,11 @@ class PIDFile(SimplePlugin):
 
 
 class PerpetualTimer(threading.Timer):
-
     """A responsive subclass of threading.Timer whose run() method repeats.
 
     Use this timer only when you really need a very interruptible timer;
-    this checks its 'finished' condition up to 20 times a second, which can
-    results in pretty high CPU usage
+    this checks its 'finished' condition up to 20 times a second, which
+    can results in pretty high CPU usage
     """
 
     def __init__(self, *args, **kwargs):
@@ -482,14 +483,14 @@ class PerpetualTimer(threading.Timer):
 
 
 class BackgroundTask(threading.Thread):
-
     """A subclass of threading.Thread whose run() method repeats.
 
-    Use this class for most repeating tasks. It uses time.sleep() to wait
-    for each interval, which isn't very responsive; that is, even if you call
-    self.cancel(), you'll have to wait until the sleep() call finishes before
-    the thread stops. To compensate, it defaults to being daemonic, which means
-    it won't delay stopping the whole process.
+    Use this class for most repeating tasks. It uses time.sleep() to
+    wait for each interval, which isn't very responsive; that is, even
+    if you call self.cancel(), you'll have to wait until the sleep()
+    call finishes before the thread stops. To compensate, it defaults to
+    being daemonic, which means it won't delay stopping the whole
+    process.
     """
 
     def __init__(self, interval, function, args=[], kwargs={}, bus=None):
@@ -524,7 +525,6 @@ class BackgroundTask(threading.Thread):
 
 
 class Monitor(SimplePlugin):
-
     """WSPBus listener to periodically run a callback in its own thread."""
 
     callback = None
@@ -581,7 +581,6 @@ class Monitor(SimplePlugin):
 
 
 class Autoreloader(Monitor):
-
     """Monitor which re-executes the process when files change.
 
     This :ref:`plugin<plugins>` restarts the process (via :func:`os.execv`)
@@ -698,20 +697,20 @@ class Autoreloader(Monitor):
 
 
 class ThreadManager(SimplePlugin):
-
     """Manager for HTTP request threads.
 
     If you have control over thread creation and destruction, publish to
-    the 'acquire_thread' and 'release_thread' channels (for each thread).
-    This will register/unregister the current thread and publish to
-    'start_thread' and 'stop_thread' listeners in the bus as needed.
+    the 'acquire_thread' and 'release_thread' channels (for each
+    thread). This will register/unregister the current thread and
+    publish to 'start_thread' and 'stop_thread' listeners in the bus as
+    needed.
 
     If threads are created and destroyed by code you do not control
     (e.g., Apache), then, at the beginning of every HTTP request,
     publish to 'acquire_thread' only. You should not publish to
-    'release_thread' in this case, since you do not know whether
-    the thread will be re-used or not. The bus will call
-    'stop_thread' listeners for you when it stops.
+    'release_thread' in this case, since you do not know whether the
+    thread will be re-used or not. The bus will call 'stop_thread'
+    listeners for you when it stops.
     """
 
     threads = None

lib/cherrypy/process/servers.py

@@ -132,7 +132,6 @@ class Timeouts:
 
 
 class ServerAdapter(object):
-
     """Adapter for an HTTP server.
 
     If you need to start more than one HTTP server (to serve on multiple
@@ -188,9 +187,7 @@ class ServerAdapter(object):
 
     @property
     def description(self):
-        """
-        A description about where this server is bound.
-        """
+        """A description about where this server is bound."""
         if self.bind_addr is None:
             on_what = 'unknown interface (dynamic?)'
         elif isinstance(self.bind_addr, tuple):
@@ -292,7 +289,6 @@ class ServerAdapter(object):
 
 
 class FlupCGIServer(object):
-
     """Adapter for a flup.server.cgi.WSGIServer."""
 
     def __init__(self, *args, **kwargs):
@@ -316,7 +312,6 @@ class FlupCGIServer(object):
 
 
 class FlupFCGIServer(object):
-
     """Adapter for a flup.server.fcgi.WSGIServer."""
 
     def __init__(self, *args, **kwargs):
@@ -362,7 +357,6 @@ class FlupFCGIServer(object):
 
 
 class FlupSCGIServer(object):
-
     """Adapter for a flup.server.scgi.WSGIServer."""
 
     def __init__(self, *args, **kwargs):

lib/cherrypy/process/win32.py

@@ -1,4 +1,7 @@
-"""Windows service. Requires pywin32."""
+"""Windows service.
+
+Requires pywin32.
+"""
 
 import os
 import win32api
@@ -11,7 +14,6 @@ from cherrypy.process import wspbus, plugins
 
 
 class ConsoleCtrlHandler(plugins.SimplePlugin):
-
     """A WSPBus plugin for handling Win32 console events (like Ctrl-C)."""
 
     def __init__(self, bus):
@@ -69,10 +71,10 @@ class ConsoleCtrlHandler(plugins.SimplePlugin):
 
 
 class Win32Bus(wspbus.Bus):
-
     """A Web Site Process Bus implementation for Win32.
 
-    Instead of time.sleep, this bus blocks using native win32event objects.
+    Instead of time.sleep, this bus blocks using native win32event
+    objects.
     """
 
     def __init__(self):
@@ -120,7 +122,6 @@ class Win32Bus(wspbus.Bus):
 
 
 class _ControlCodes(dict):
-
     """Control codes used to "signal" a service via ControlService.
 
     User-defined control codes are in the range 128-255. We generally use
@@ -152,7 +153,6 @@ def signal_child(service, command):
 
 
 class PyWebService(win32serviceutil.ServiceFramework):
-
     """Python Web Service."""
 
     _svc_name_ = 'Python Web Service'

lib/cherrypy/process/wspbus.py

@@ -57,7 +57,6 @@ the new state.::
           |        \    |
           |         V   V
         STARTED <-- STARTING
-
 """
 
 import atexit
@@ -65,7 +64,7 @@ import atexit
 try:
     import ctypes
 except ImportError:
-    """Google AppEngine is shipped without ctypes
+    """Google AppEngine is shipped without ctypes.
 
     :seealso: http://stackoverflow.com/a/6523777/70170
     """
@@ -165,8 +164,8 @@ class Bus(object):
     All listeners for a given channel are guaranteed to be called even
     if others at the same channel fail. Each failure is logged, but
     execution proceeds on to the next listener. The only way to stop all
-    processing from inside a listener is to raise SystemExit and stop the
-    whole server.
+    processing from inside a listener is to raise SystemExit and stop
+    the whole server.
     """
 
     states = states
@@ -312,8 +311,9 @@ class Bus(object):
     def restart(self):
         """Restart the process (may close connections).
 
-        This method does not restart the process from the calling thread;
-        instead, it stops the bus and asks the main thread to call execv.
+        This method does not restart the process from the calling
+        thread; instead, it stops the bus and asks the main thread to
+        call execv.
         """
         self.execv = True
         self.exit()
@@ -327,10 +327,11 @@ class Bus(object):
         """Wait for the EXITING state, KeyboardInterrupt or SystemExit.
 
         This function is intended to be called only by the main thread.
-        After waiting for the EXITING state, it also waits for all threads
-        to terminate, and then calls os.execv if self.execv is True. This
-        design allows another thread to call bus.restart, yet have the main
-        thread perform the actual execv call (required on some platforms).
+        After waiting for the EXITING state, it also waits for all
+        threads to terminate, and then calls os.execv if self.execv is
+        True. This design allows another thread to call bus.restart, yet
+        have the main thread perform the actual execv call (required on
+        some platforms).
         """
         try:
             self.wait(states.EXITING, interval=interval, channel='main')
@@ -379,13 +380,14 @@ class Bus(object):
     def _do_execv(self):
         """Re-execute the current process.
 
-        This must be called from the main thread, because certain platforms
-        (OS X) don't allow execv to be called in a child thread very well.
+        This must be called from the main thread, because certain
+        platforms (OS X) don't allow execv to be called in a child
+        thread very well.
         """
         try:
             args = self._get_true_argv()
         except NotImplementedError:
-            """It's probably win32 or GAE"""
+            """It's probably win32 or GAE."""
             args = [sys.executable] + self._get_interpreter_argv() + sys.argv
 
         self.log('Re-spawning %s' % ' '.join(args))
@@ -472,7 +474,7 @@ class Bus(object):
                 c_ind = None
 
             if is_module:
-                """It's containing `-m -m` sequence of arguments"""
+                """It's containing `-m -m` sequence of arguments."""
                 if is_command and c_ind < m_ind:
                     """There's `-c -c` before `-m`"""
                     raise RuntimeError(
@@ -481,7 +483,7 @@ class Bus(object):
                 # Survive module argument here
                 original_module = sys.argv[0]
                 if not os.access(original_module, os.R_OK):
-                    """There's no such module exist"""
+                    """There's no such module exist."""
                     raise AttributeError(
                         "{} doesn't seem to be a module "
                         'accessible by current user'.format(original_module))
@@ -489,12 +491,12 @@ class Bus(object):
                 # ... and substitute it with the original module path:
                 _argv.insert(m_ind, original_module)
             elif is_command:
-                """It's containing just `-c -c` sequence of arguments"""
+                """It's containing just `-c -c` sequence of arguments."""
                 raise RuntimeError(
                     "Cannot reconstruct command from '-c'. "
                     'Ref: https://github.com/cherrypy/cherrypy/issues/1545')
         except AttributeError:
-            """It looks Py_GetArgcArgv is completely absent in some environments
+            """It looks Py_GetArgcArgv's completely absent in some environments
 
             It is known, that there's no Py_GetArgcArgv in MS Windows and
             ``ctypes`` module is completely absent in Google AppEngine
@@ -512,13 +514,13 @@ class Bus(object):
         """Prepend current working dir to PATH environment variable if needed.
 
         If sys.path[0] is an empty string, the interpreter was likely
-        invoked with -m and the effective path is about to change on
-        re-exec.  Add the current directory to $PYTHONPATH to ensure
-        that the new process sees the same path.
+        invoked with -m and the effective path is about to change on re-
+        exec.  Add the current directory to $PYTHONPATH to ensure that
+        the new process sees the same path.
 
         This issue cannot be addressed in the general case because
-        Python cannot reliably reconstruct the
-        original command line (http://bugs.python.org/issue14208).
+        Python cannot reliably reconstruct the original command line (
+        http://bugs.python.org/issue14208).
 
         (This idea filched from tornado.autoreload)
         """
@@ -536,10 +538,10 @@ class Bus(object):
         """Set the CLOEXEC flag on all open files (except stdin/out/err).
 
         If self.max_cloexec_files is an integer (the default), then on
-        platforms which support it, it represents the max open files setting
-        for the operating system. This function will be called just before
-        the process is restarted via os.execv() to prevent open files
-        from persisting into the new process.
+        platforms which support it, it represents the max open files
+        setting for the operating system. This function will be called
+        just before the process is restarted via os.execv() to prevent
+        open files from persisting into the new process.
 
         Set self.max_cloexec_files to 0 to disable this behavior.
         """
@@ -578,7 +580,10 @@ class Bus(object):
         return t
 
     def log(self, msg='', level=20, traceback=False):
-        """Log the given message. Append the last traceback if requested."""
+        """Log the given message.
+
+        Append the last traceback if requested.
+        """
         if traceback:
             msg += '\n' + ''.join(_traceback.format_exception(*sys.exc_info()))
         self.publish('log', msg, level)

lib/cherrypy/scaffold/__init__.py

@@ -9,7 +9,6 @@ Even before any tweaking, this should serve a few demonstration pages.
 Change to this directory and run:
 
     cherryd -c site.conf
-
 """
 
 import cherrypy

lib/importlib_resources/__init__.py

@@ -0,0 +1,36 @@
+"""Read resources contained within a package."""
+
+from ._common import (
+    as_file,
+    files,
+    Package,
+)
+
+from ._legacy import (
+    contents,
+    open_binary,
+    read_binary,
+    open_text,
+    read_text,
+    is_resource,
+    path,
+    Resource,
+)
+
+from importlib_resources.abc import ResourceReader
+
+
+__all__ = [
+    'Package',
+    'Resource',
+    'ResourceReader',
+    'as_file',
+    'contents',
+    'files',
+    'is_resource',
+    'open_binary',
+    'open_text',
+    'path',
+    'read_binary',
+    'read_text',
+]

lib/importlib_resources/_adapters.py

@@ -0,0 +1,170 @@
+from contextlib import suppress
+from io import TextIOWrapper
+
+from . import abc
+
+
+class SpecLoaderAdapter:
+    """
+    Adapt a package spec to adapt the underlying loader.
+    """
+
+    def __init__(self, spec, adapter=lambda spec: spec.loader):
+        self.spec = spec
+        self.loader = adapter(spec)
+
+    def __getattr__(self, name):
+        return getattr(self.spec, name)
+
+
+class TraversableResourcesLoader:
+    """
+    Adapt a loader to provide TraversableResources.
+    """
+
+    def __init__(self, spec):
+        self.spec = spec
+
+    def get_resource_reader(self, name):
+        return CompatibilityFiles(self.spec)._native()
+
+
+def _io_wrapper(file, mode='r', *args, **kwargs):
+    if mode == 'r':
+        return TextIOWrapper(file, *args, **kwargs)
+    elif mode == 'rb':
+        return file
+    raise ValueError(
+        "Invalid mode value '{}', only 'r' and 'rb' are supported".format(mode)
+    )
+
+
+class CompatibilityFiles:
+    """
+    Adapter for an existing or non-existent resource reader
+    to provide a compatibility .files().
+    """
+
+    class SpecPath(abc.Traversable):
+        """
+        Path tied to a module spec.
+        Can be read and exposes the resource reader children.
+        """
+
+        def __init__(self, spec, reader):
+            self._spec = spec
+            self._reader = reader
+
+        def iterdir(self):
+            if not self._reader:
+                return iter(())
+            return iter(
+                CompatibilityFiles.ChildPath(self._reader, path)
+                for path in self._reader.contents()
+            )
+
+        def is_file(self):
+            return False
+
+        is_dir = is_file
+
+        def joinpath(self, other):
+            if not self._reader:
+                return CompatibilityFiles.OrphanPath(other)
+            return CompatibilityFiles.ChildPath(self._reader, other)
+
+        @property
+        def name(self):
+            return self._spec.name
+
+        def open(self, mode='r', *args, **kwargs):
+            return _io_wrapper(self._reader.open_resource(None), mode, *args, **kwargs)
+
+    class ChildPath(abc.Traversable):
+        """
+        Path tied to a resource reader child.
+        Can be read but doesn't expose any meaningful children.
+        """
+
+        def __init__(self, reader, name):
+            self._reader = reader
+            self._name = name
+
+        def iterdir(self):
+            return iter(())
+
+        def is_file(self):
+            return self._reader.is_resource(self.name)
+
+        def is_dir(self):
+            return not self.is_file()
+
+        def joinpath(self, other):
+            return CompatibilityFiles.OrphanPath(self.name, other)
+
+        @property
+        def name(self):
+            return self._name
+
+        def open(self, mode='r', *args, **kwargs):
+            return _io_wrapper(
+                self._reader.open_resource(self.name), mode, *args, **kwargs
+            )
+
+    class OrphanPath(abc.Traversable):
+        """
+        Orphan path, not tied to a module spec or resource reader.
+        Can't be read and doesn't expose any meaningful children.
+        """
+
+        def __init__(self, *path_parts):
+            if len(path_parts) < 1:
+                raise ValueError('Need at least one path part to construct a path')
+            self._path = path_parts
+
+        def iterdir(self):
+            return iter(())
+
+        def is_file(self):
+            return False
+
+        is_dir = is_file
+
+        def joinpath(self, other):
+            return CompatibilityFiles.OrphanPath(*self._path, other)
+
+        @property
+        def name(self):
+            return self._path[-1]
+
+        def open(self, mode='r', *args, **kwargs):
+            raise FileNotFoundError("Can't open orphan path")
+
+    def __init__(self, spec):
+        self.spec = spec
+
+    @property
+    def _reader(self):
+        with suppress(AttributeError):
+            return self.spec.loader.get_resource_reader(self.spec.name)
+
+    def _native(self):
+        """
+        Return the native reader if it supports files().
+        """
+        reader = self._reader
+        return reader if hasattr(reader, 'files') else self
+
+    def __getattr__(self, attr):
+        return getattr(self._reader, attr)
+
+    def files(self):
+        return CompatibilityFiles.SpecPath(self.spec, self._reader)
+
+
+def wrap_spec(package):
+    """
+    Construct a package spec with traversable compatibility
+    on the spec/loader/reader.
+    """
+    return SpecLoaderAdapter(package.__spec__, TraversableResourcesLoader)